From e76b03f4af7ab1d300d206c246f736b0c5cb2241 Mon Sep 17 00:00:00 2001 From: David Date: Wed, 31 May 2023 05:10:08 +0800 Subject: [PATCH] feat(all): check L1 reorg before each operation (#252) --- driver/chain_syncer/calldata/syncer.go | 141 ++++-------------- driver/chain_syncer/calldata/syncer_test.go | 45 ------ pkg/chain_iterator/block_batch_iterator.go | 50 ++----- .../block_batch_iterator_test.go | 70 +-------- .../event_iterator/block_proposed_iterator.go | 17 +-- .../event_iterator/block_proven_iterator.go | 17 +-- pkg/rpc/methods.go | 69 +++++++++ prover/proof_submitter/util.go | 31 +++- prover/proof_submitter/util_test.go | 56 ++++--- .../proof_submitter/valid_proof_submitter.go | 10 +- prover/prover.go | 30 ++++ 11 files changed, 231 insertions(+), 305 deletions(-) diff --git a/driver/chain_syncer/calldata/syncer.go b/driver/chain_syncer/calldata/syncer.go index 8fda23c18..7e2cb809f 100644 --- a/driver/chain_syncer/calldata/syncer.go +++ b/driver/chain_syncer/calldata/syncer.go @@ -7,7 +7,6 @@ import ( "math/big" "time" - "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/beacon/engine" "github.com/ethereum/go-ethereum/common" @@ -104,8 +103,39 @@ func (s *Syncer) onBlockProposed( event *bindings.TaikoL1ClientBlockProposed, endIter eventIterator.EndBlockProposedEventIterFunc, ) error { + if event.Id.Cmp(common.Big0) == 0 { + return nil + } + + if !s.progressTracker.Triggered() { + // Check whteher we need to reorg the L2 chain at first. + reorged, l1CurrentToReset, lastInsertedBlockIDToReset, err := s.rpc.CheckL1Reorg( + ctx, + new(big.Int).Sub(event.Id, common.Big1), + ) + if err != nil { + return fmt.Errorf("failed to check whether L1 chain has been reorged: %w", err) + } + + if reorged { + log.Info( + "Reset L1Current cursor due to L1 reorg", + "l1CurrentHeightOld", s.state.GetL1Current().Number, + "l1CurrentHashOld", s.state.GetL1Current().Hash(), + "l1CurrentHeightNew", l1CurrentToReset.Number, + "l1CurrentHashNew", l1CurrentToReset.Hash(), + "lastInsertedBlockIDOld", s.lastInsertedBlockID, + "lastInsertedBlockIDNew", lastInsertedBlockIDToReset, + ) + s.state.SetL1Current(l1CurrentToReset) + s.lastInsertedBlockID = lastInsertedBlockIDToReset + + return fmt.Errorf("reorg detected, reset l1Current cursor to %d", l1CurrentToReset.Number) + } + } + // Ignore those already inserted blocks. - if event.Id.Cmp(common.Big0) == 0 || (s.lastInsertedBlockID != nil && event.Id.Cmp(s.lastInsertedBlockID) <= 0) { + if s.lastInsertedBlockID != nil && event.Id.Cmp(s.lastInsertedBlockID) <= 0 { return nil } @@ -117,11 +147,6 @@ func (s *Syncer) onBlockProposed( "Removed", event.Raw.Removed, ) - // handle reorg - if event.Raw.Removed { - return s.handleReorg(ctx, event) - } - // Fetch the L2 parent block. var ( parent *types.Header @@ -229,108 +254,6 @@ func (s *Syncer) onBlockProposed( return nil } -// handleReorg detects reorg and rewinds the chain by 1 until we find a block that is still in the chain, -// then inserts that block as the new head. -func (s *Syncer) handleReorg(ctx context.Context, event *bindings.TaikoL1ClientBlockProposed) error { - log.Info( - "Reorg detected", - "L1Height", event.Raw.BlockNumber, - "L1Hash", event.Raw.BlockHash, - "BlockID", event.Id, - "Removed", event.Raw.Removed, - ) - - // rewind chain by 1 until we find a block that is still in the chain - var ( - lastKnownGoodBlockID *big.Int - blockID *big.Int = s.lastInsertedBlockID - block *types.Block - err error - ) - - // if `lastInsertedBlockID` has not been set, we use current L2 chain head as blockID instead - if blockID == nil { - l2Head, err := s.rpc.L2.BlockByNumber(ctx, nil) - if err != nil { - return err - } - blockID = l2Head.Number() - } - - stateVars, err := s.rpc.GetProtocolStateVariables(nil) - if err != nil { - return fmt.Errorf("failed to get state variables: %w", err) - } - - for { - if blockID.Cmp(common.Big0) == 0 { - if block, err = s.rpc.L2.BlockByNumber(ctx, common.Big0); err != nil { - return err - } - lastKnownGoodBlockID = common.Big0 - break - } - - if block, err = s.rpc.L2.BlockByNumber(ctx, blockID); err != nil && !errors.Is(err, ethereum.NotFound) { - return err - } - - if block != nil && blockID.Uint64() < stateVars.NumBlocks { - // block exists, we can rewind to this block - lastKnownGoodBlockID = blockID - break - } else { - // otherwise, sub 1 from blockId and try again - blockID = new(big.Int).Sub(blockID, common.Big1) - } - } - - // shouldn't be able to reach this error because of the 0 check above - // but just in case - if lastKnownGoodBlockID == nil { - return fmt.Errorf("failed to find last known good block ID after reorg") - } - - log.Info( - "🔗 Last known good block ID before reorg found", - "blockID", lastKnownGoodBlockID, - ) - - fcRes, err := s.rpc.L2Engine.ForkchoiceUpdate(ctx, &engine.ForkchoiceStateV1{HeadBlockHash: block.Hash()}, nil) - if err != nil { - return err - } - if fcRes.PayloadStatus.Status != engine.VALID { - return fmt.Errorf("unexpected ForkchoiceUpdate response status: %s", fcRes.PayloadStatus.Status) - } - - // reset l1 current to when the last known good block was inserted, and return the event. - if _, _, err := s.state.ResetL1Current(ctx, &state.HeightOrID{ID: lastKnownGoodBlockID}); err != nil { - return fmt.Errorf("failed to reset L1 current: %w", err) - } - - log.Info( - "🔗 Rewound chain and inserted last known good block as new head", - "blockID", event.Id, - "height", block.Number(), - "hash", block.Hash(), - "latestVerifiedBlockHeight", s.state.GetLatestVerifiedBlock().Height, - "latestVerifiedBlockHash", s.state.GetLatestVerifiedBlock().Hash, - "transactions", len(block.Transactions()), - "baseFee", block.BaseFee(), - "withdrawals", len(block.Withdrawals()), - ) - - metrics.DriverL1CurrentHeightGauge.Update(int64(event.Raw.BlockNumber)) - s.lastInsertedBlockID = block.Number() - - if s.progressTracker.Triggered() { - s.progressTracker.ClearMeta() - } - - return nil -} - // insertNewHead tries to insert a new head block to the L2 execution engine's local // block chain through Engine APIs. func (s *Syncer) insertNewHead( diff --git a/driver/chain_syncer/calldata/syncer_test.go b/driver/chain_syncer/calldata/syncer_test.go index 09e9636f7..9a703f61c 100644 --- a/driver/chain_syncer/calldata/syncer_test.go +++ b/driver/chain_syncer/calldata/syncer_test.go @@ -10,7 +10,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/stretchr/testify/suite" "github.com/taikoxyz/taiko-client/bindings" @@ -103,50 +102,6 @@ func (s *CalldataSyncerTestSuite) TestInsertNewHead() { s.Nil(payloadErr) } -func (s *CalldataSyncerTestSuite) TestHandleReorgToGenesis() { - testutils.ProposeAndInsertEmptyBlocks(&s.ClientTestSuite, s.p, s.s) - - l2Head1, err := s.s.rpc.L2.BlockByNumber(context.Background(), nil) - s.Nil(err) - s.Greater(l2Head1.NumberU64(), uint64(0)) - s.NotZero(s.s.lastInsertedBlockID.Uint64()) - s.s.lastInsertedBlockID = common.Big0 // let the chain reorg to genesis - - s.Nil(s.s.handleReorg(context.Background(), &bindings.TaikoL1ClientBlockProposed{ - Id: l2Head1.Number(), - Raw: types.Log{Removed: true}, - })) - - l2Head2, err := s.s.rpc.L2.BlockByNumber(context.Background(), nil) - s.Nil(err) - s.Equal(uint64(0), l2Head2.NumberU64()) -} - -func (s *CalldataSyncerTestSuite) TestHandleReorgToNoneGenesis() { - testutils.ProposeAndInsertEmptyBlocks(&s.ClientTestSuite, s.p, s.s) - - l2Head1, err := s.s.rpc.L2.BlockByNumber(context.Background(), nil) - s.Nil(err) - s.Greater(l2Head1.NumberU64(), uint64(0)) - s.NotZero(s.s.lastInsertedBlockID.Uint64()) - s.s.lastInsertedBlockID = common.Big1 // let the chain reorg to height 1 - - s.Nil(s.s.handleReorg(context.Background(), &bindings.TaikoL1ClientBlockProposed{ - Id: l2Head1.Number(), - Raw: types.Log{Removed: true}, - })) - - l2Head2, err := s.s.rpc.L2.BlockByNumber(context.Background(), nil) - s.Nil(err) - s.Equal(uint64(1), l2Head2.NumberU64()) - - testutils.ProposeAndInsertEmptyBlocks(&s.ClientTestSuite, s.p, s.s) - l2Head3, err := s.s.rpc.L2.BlockByNumber(context.Background(), nil) - s.Nil(err) - s.Greater(l2Head3.NumberU64(), l2Head2.NumberU64()) - s.Greater(s.s.lastInsertedBlockID.Uint64(), uint64(1)) -} - func (s *CalldataSyncerTestSuite) TestTreasuryIncomeAllAnchors() { treasury := common.HexToAddress(os.Getenv("TREASURY")) s.NotZero(treasury.Big().Uint64()) diff --git a/pkg/chain_iterator/block_batch_iterator.go b/pkg/chain_iterator/block_batch_iterator.go index a7d15c311..56a363156 100644 --- a/pkg/chain_iterator/block_batch_iterator.go +++ b/pkg/chain_iterator/block_batch_iterator.go @@ -16,7 +16,6 @@ import ( const ( DefaultBlocksReadPerEpoch = 1000 - DefaultReorgRewindDepth = 20 ) var ( @@ -29,9 +28,8 @@ type OnBlocksFunc func( ctx context.Context, start, end *types.Header, updateCurrentFunc UpdateCurrentFunc, - onReorgFunc OnReorgFunc, endIterFunc EndIterFunc, -) (bool, error) +) error // UpdateCurrentFunc updates the iterator.current cursor in the iterator. type UpdateCurrentFunc func(*types.Header) @@ -39,9 +37,6 @@ type UpdateCurrentFunc func(*types.Header) // EndIterFunc ends the current iteration. type EndIterFunc func() -// OnReorgFunc handles a reorganization from the source chain. -type OnReorgFunc func() error - // BlockBatchIterator iterates the blocks in batches between the given start and end heights, // with the awareness of reorganization. type BlockBatchIterator struct { @@ -56,7 +51,6 @@ type BlockBatchIterator struct { isEnd bool reverse bool reorgRewindDepth uint64 - onReorg OnReorgFunc } // BlockBatchIteratorConfig represents the configs of a block batch iterator. @@ -68,7 +62,6 @@ type BlockBatchIteratorConfig struct { OnBlocks OnBlocksFunc Reverse bool ReorgRewindDepth *uint64 - OnReorg OnReorgFunc } // NewBlockBatchIterator creates a new block batch iterator instance. @@ -108,27 +101,13 @@ func NewBlockBatchIterator(ctx context.Context, cfg *BlockBatchIteratorConfig) ( } } - var reorgRewindDepth uint64 - if cfg.ReorgRewindDepth != nil { - reorgRewindDepth = *cfg.ReorgRewindDepth - } else { - reorgRewindDepth = DefaultReorgRewindDepth - } - iterator := &BlockBatchIterator{ - ctx: ctx, - client: cfg.Client, - chainID: chainID, - startHeight: cfg.StartHeight.Uint64(), - onBlocks: cfg.OnBlocks, - reverse: cfg.Reverse, - reorgRewindDepth: reorgRewindDepth, - } - - if cfg.OnReorg != nil { - iterator.onReorg = cfg.OnReorg - } else { - iterator.onReorg = iterator.rewindOnReorgDetected + ctx: ctx, + client: cfg.Client, + chainID: chainID, + startHeight: cfg.StartHeight.Uint64(), + onBlocks: cfg.OnBlocks, + reverse: cfg.Reverse, } if cfg.Reverse { @@ -227,16 +206,10 @@ func (i *BlockBatchIterator) iter() (err error) { return err } - reorged, err := i.onBlocks(i.ctx, i.current, endHeader, i.updateCurrent, i.onReorg, i.end) - if err != nil { + if err := i.onBlocks(i.ctx, i.current, endHeader, i.updateCurrent, i.end); err != nil { return err } - // if we reorged, we want to skip checking if we are at the end, and also skip updating i.current - if reorged { - return nil - } - if i.isEnd { return io.EOF } @@ -280,15 +253,10 @@ func (i *BlockBatchIterator) reverseIter() (err error) { return err } - reorged, err := i.onBlocks(i.ctx, startHeader, i.current, i.updateCurrent, i.onReorg, i.end) - if err != nil { + if err := i.onBlocks(i.ctx, startHeader, i.current, i.updateCurrent, i.end); err != nil { return err } - if reorged { - return nil - } - i.current = startHeader if !isLastEpoch && !i.isEnd { diff --git a/pkg/chain_iterator/block_batch_iterator_test.go b/pkg/chain_iterator/block_batch_iterator_test.go index ebb243cae..fd809936d 100644 --- a/pkg/chain_iterator/block_batch_iterator_test.go +++ b/pkg/chain_iterator/block_batch_iterator_test.go @@ -33,12 +33,11 @@ func (s *BlockBatchIteratorTestSuite) TestIter() { ctx context.Context, start, end *types.Header, updateCurrentFunc UpdateCurrentFunc, - onReorgFunc OnReorgFunc, endIterFunc EndIterFunc, - ) (bool, error) { + ) error { s.Equal(lastEnd.Uint64(), start.Number.Uint64()) lastEnd = end.Number - return false, nil + return nil }, }) @@ -69,12 +68,11 @@ func (s *BlockBatchIteratorTestSuite) TestIterReverse() { ctx context.Context, start, end *types.Header, updateCurrentFunc UpdateCurrentFunc, - onReorgFunc OnReorgFunc, endIterFunc EndIterFunc, - ) (bool, error) { + ) error { s.Equal(lastStart.Uint64(), end.Number.Uint64()) lastStart = start.Number - return false, nil + return nil }, }) @@ -101,74 +99,18 @@ func (s *BlockBatchIteratorTestSuite) TestIterEndFunc() { ctx context.Context, start, end *types.Header, updateCurrentFunc UpdateCurrentFunc, - onReorgFunc OnReorgFunc, endIterFunc EndIterFunc, - ) (bool, error) { + ) error { s.Equal(lastEnd.Uint64(), start.Number.Uint64()) lastEnd = end.Number endIterFunc() - return false, nil - }, - }) - - s.Nil(err) - s.Nil(iter.Iter()) - s.Equal(lastEnd.Uint64(), maxBlocksReadPerEpoch) -} - -func (s *BlockBatchIteratorTestSuite) TestIter_ReorgEncountered() { - var maxBlocksReadPerEpoch uint64 = 1 - var reorgRewindDepth uint64 = 1 - var rewindEveryNBlocks uint64 = 2 - var lastBlockReorged bool = false - - headHeight, err := s.RpcClient.L1.BlockNumber(context.Background()) - s.Nil(err) - s.Greater(headHeight, uint64(0)) - - lastEnd := common.Big0 - - iter, err := NewBlockBatchIterator(context.Background(), &BlockBatchIteratorConfig{ - Client: s.RpcClient.L1, - MaxBlocksReadPerEpoch: &maxBlocksReadPerEpoch, - StartHeight: common.Big0, - EndHeight: new(big.Int).SetUint64(headHeight), - ReorgRewindDepth: &reorgRewindDepth, - OnReorg: func() error { - if lastEnd.Uint64() < reorgRewindDepth { - lastEnd = common.Big0 - } else { - lastEnd = new(big.Int).Sub(lastEnd, new(big.Int).SetUint64(reorgRewindDepth)) - } - lastBlockReorged = true return nil }, - OnBlocks: func( - ctx context.Context, - start, end *types.Header, - updateCurrentFunc UpdateCurrentFunc, - onReorgFunc OnReorgFunc, - endIterFunc EndIterFunc, - ) (bool, error) { - // reorg every 2 blocks but not the first block - if lastEnd != common.Big0 && lastEnd.Uint64()%rewindEveryNBlocks == 0 { - return true, onReorgFunc() - } - - if lastBlockReorged { - s.Equal(start.Number.Uint64(), lastEnd.Uint64()+reorgRewindDepth) - } else { - s.Equal(start.Number.Uint64(), lastEnd.Uint64()) - } - - lastEnd = end.Number - lastBlockReorged = false - return false, nil - }, }) s.Nil(err) s.Nil(iter.Iter()) + s.Equal(lastEnd.Uint64(), maxBlocksReadPerEpoch) } func TestBlockBatchIteratorTestSuite(t *testing.T) { diff --git a/pkg/chain_iterator/event_iterator/block_proposed_iterator.go b/pkg/chain_iterator/event_iterator/block_proposed_iterator.go index ff7b2bd69..5002367d1 100644 --- a/pkg/chain_iterator/event_iterator/block_proposed_iterator.go +++ b/pkg/chain_iterator/event_iterator/block_proposed_iterator.go @@ -105,43 +105,38 @@ func assembleBlockProposedIteratorCallback( ctx context.Context, start, end *types.Header, updateCurrentFunc chainIterator.UpdateCurrentFunc, - onReorgFunc chainIterator.OnReorgFunc, endFunc chainIterator.EndIterFunc, - ) (bool, error) { + ) error { endHeight := end.Number.Uint64() iter, err := taikoL1Client.FilterBlockProposed( &bind.FilterOpts{Start: start.Number.Uint64(), End: &endHeight, Context: ctx}, filterQuery, ) if err != nil { - return false, err + return err } defer iter.Close() for iter.Next() { event := iter.Event - if event.Raw.Removed { - return true, onReorgFunc() - } - if err := callback(ctx, event, eventIter.end); err != nil { - return false, err + return err } if eventIter.isEnd { endFunc() - return false, nil + return nil } current, err := client.HeaderByHash(ctx, event.Raw.BlockHash) if err != nil { - return false, err + return err } updateCurrentFunc(current) } - return false, nil + return nil } } diff --git a/pkg/chain_iterator/event_iterator/block_proven_iterator.go b/pkg/chain_iterator/event_iterator/block_proven_iterator.go index 43b4f5f12..e8e44dbe3 100644 --- a/pkg/chain_iterator/event_iterator/block_proven_iterator.go +++ b/pkg/chain_iterator/event_iterator/block_proven_iterator.go @@ -101,43 +101,38 @@ func assembleBlockProvenIteratorCallback( ctx context.Context, start, end *types.Header, updateCurrentFunc chainIterator.UpdateCurrentFunc, - onReorgFunc chainIterator.OnReorgFunc, endFunc chainIterator.EndIterFunc, - ) (bool, error) { + ) error { endHeight := end.Number.Uint64() iter, err := taikoL1Client.FilterBlockProven( &bind.FilterOpts{Start: start.Number.Uint64(), End: &endHeight, Context: ctx}, filterQuery, ) if err != nil { - return false, err + return err } defer iter.Close() for iter.Next() { event := iter.Event - if event.Raw.Removed { - return true, onReorgFunc() - } - if err := callback(ctx, event, eventIter.end); err != nil { - return false, err + return err } if eventIter.isEnd { endFunc() - return false, nil + return nil } current, err := client.HeaderByHash(ctx, event.Raw.BlockHash) if err != nil { - return false, err + return err } updateCurrentFunc(current) } - return false, nil + return nil } } diff --git a/pkg/rpc/methods.go b/pkg/rpc/methods.go index d6c2402bc..6a7cd264e 100644 --- a/pkg/rpc/methods.go +++ b/pkg/rpc/methods.go @@ -129,6 +129,8 @@ func (c *Client) LatestL2KnownL1Header(ctx context.Context) (*types.Header, erro } } + log.Info("Latest L2 known L1 header", "height", header.Number, "hash", header.Hash()) + return header, nil } @@ -314,3 +316,70 @@ func (c *Client) GetStorageRoot( return proof.StorageHash, nil } + +// CheckL1Reorg checks whether the L1 chain has been reorged, if so, returns the l1Current cursor and L2 blockID +// that need to reset to. +func (c *Client) CheckL1Reorg(ctx context.Context, blockID *big.Int) (bool, *types.Header, *big.Int, error) { + var ( + reorged bool + l1CurrentToReset *types.Header + blockIDToReset *big.Int + ) + for { + if blockID.Cmp(common.Big0) == 0 { + stateVars, err := c.TaikoL1.GetStateVariables(nil) + if err != nil { + return false, nil, nil, err + } + + if l1CurrentToReset, err = c.L1.HeaderByNumber( + ctx, + new(big.Int).SetUint64(stateVars.GenesisHeight), + ); err != nil { + return false, nil, nil, err + } + + break + } + + l1Origin, err := c.L2.L1OriginByID(ctx, blockID) + if err != nil { + return false, nil, nil, err + } + + l1Header, err := c.L1.HeaderByNumber(ctx, l1Origin.L1BlockHeight) + if err != nil { + if errors.Is(err, ethereum.NotFound) { + continue + } + return false, nil, nil, fmt.Errorf("failed to fetch L1 header (%d): %w", l1Origin.L1BlockHeight, err) + } + + if l1Header.Hash() != l1Origin.L1BlockHash { + log.Info( + "Reorg detected", + "blockID", blockID, + "l1Height", l1Origin.L1BlockHeight, + "l1HashOld", l1Origin.L1BlockHash, + "l1HashNew", l1Header.Hash(), + ) + reorged = true + blockID = new(big.Int).Sub(blockID, common.Big1) + continue + } + + l1CurrentToReset = l1Header + blockIDToReset = l1Origin.BlockID + break + } + + log.Debug( + "Check L1 reorg", + "reorged", reorged, + "l1CurrentToResetNumber", l1CurrentToReset.Number, + "l1CurrentToResetHash", l1CurrentToReset.Hash(), + "blockIDToReset", blockIDToReset, + ) + + return reorged, l1CurrentToReset, blockIDToReset, nil +} diff --git a/prover/proof_submitter/util.go b/prover/proof_submitter/util.go index 0c558289b..d3a26ae86 100644 --- a/prover/proof_submitter/util.go +++ b/prover/proof_submitter/util.go @@ -15,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" + "github.com/taikoxyz/taiko-client/bindings" "github.com/taikoxyz/taiko-client/bindings/encoding" "github.com/taikoxyz/taiko-client/pkg/rpc" ) @@ -27,8 +28,7 @@ var ( // isSubmitProofTxErrorRetryable checks whether the error returned by a proof submission transaction // is retryable. func isSubmitProofTxErrorRetryable(err error, blockID *big.Int) bool { - if strings.HasPrefix(err.Error(), "L1_NOT_SPECIAL_PROVER") || - !strings.HasPrefix(err.Error(), "L1_") { + if !strings.HasPrefix(err.Error(), "L1_") { return true } @@ -69,6 +69,7 @@ func sendTxWithBackoff( blockID *big.Int, proposedAt uint64, expectedReward uint64, + meta *bindings.TaikoDataBlockMetadata, sendTxFunc func() (*types.Transaction, error), ) error { var ( @@ -81,6 +82,29 @@ func sendTxWithBackoff( return nil } + // Check if the corresponding L1 block is still in the canonical chain. + l1Header, err := cli.L1.HeaderByNumber(ctx, new(big.Int).SetUint64(meta.L1Height)) + if err != nil { + log.Warn( + "Failed to fetch L1 block", + "blockID", blockID, + "l1Height", meta.L1Height, + "l1Hash", common.BytesToHash(meta.L1Hash[:]), + "error", err, + ) + return err + } + if l1Header.Hash() != meta.L1Hash { + log.Warn( + "Reorg detected, skip the current proof submission", + "blockID", blockID, + "l1Height", meta.L1Height, + "l1HashOld", common.BytesToHash(meta.L1Hash[:]), + "l1HashNew", l1Header.Hash(), + ) + return nil + } + // Check the expected reward. if expectedReward != 0 { // Check if this proof is still needed at first. @@ -124,6 +148,9 @@ func sendTxWithBackoff( if time.Now().Before(proposedTime.Add(time.Duration(targetDelay) * time.Second)) { return errNeedWaiting } + } else { + log.Info("Proof was submitted another prover, skip the current proof submission", "blockID", blockID) + return nil } } diff --git a/prover/proof_submitter/util_test.go b/prover/proof_submitter/util_test.go index 86fa6314f..a005e894f 100644 --- a/prover/proof_submitter/util_test.go +++ b/prover/proof_submitter/util_test.go @@ -7,11 +7,12 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/taikoxyz/taiko-client/bindings" ) func (s *ProofSubmitterTestSuite) TestIsSubmitProofTxErrorRetryable() { s.True(isSubmitProofTxErrorRetryable(errors.New(testAddr.String()), common.Big0)) - s.True(isSubmitProofTxErrorRetryable(errors.New("L1_NOT_SPECIAL_PROVER"), common.Big0)) + s.False(isSubmitProofTxErrorRetryable(errors.New("L1_NOT_SPECIAL_PROVER"), common.Big0)) s.False(isSubmitProofTxErrorRetryable(errors.New("L1_DUP_PROVERS"), common.Big0)) s.False(isSubmitProofTxErrorRetryable(errors.New("L1_"+testAddr.String()), common.Big0)) } @@ -27,28 +28,41 @@ func (s *ProofSubmitterTestSuite) TestGetProveBlocksTxOpts() { } func (s *ProofSubmitterTestSuite) TestSendTxWithBackoff() { - err := sendTxWithBackoff(context.Background(), s.RpcClient, common.Big1, 0, 0, func() (*types.Transaction, error) { - return nil, errors.New("L1_TEST") - }) - - s.NotNil(err) - - err = sendTxWithBackoff(context.Background(), s.RpcClient, common.Big1, 0, 0, func() (*types.Transaction, error) { - height, err := s.RpcClient.L1.BlockNumber(context.Background()) - s.Nil(err) + l1Head, err := s.RpcClient.L1.HeaderByNumber(context.Background(), nil) + s.Nil(err) + meta := &bindings.TaikoDataBlockMetadata{L1Height: l1Head.Number.Uint64(), L1Hash: l1Head.Hash()} + s.NotNil(sendTxWithBackoff( + context.Background(), + s.RpcClient, + common.Big1, + 0, + 0, + meta, + func() (*types.Transaction, error) { + return nil, errors.New("L1_TEST") + })) - var block *types.Block - for { - block, err = s.RpcClient.L1.BlockByNumber(context.Background(), new(big.Int).SetUint64(height)) + s.Nil(sendTxWithBackoff( + context.Background(), + s.RpcClient, + common.Big1, + 0, + 0, + meta, + func() (*types.Transaction, error) { + height, err := s.RpcClient.L1.BlockNumber(context.Background()) s.Nil(err) - if block.Transactions().Len() != 0 { - break - } - height -= 1 - } - return block.Transactions()[0], nil - }) + var block *types.Block + for { + block, err = s.RpcClient.L1.BlockByNumber(context.Background(), new(big.Int).SetUint64(height)) + s.Nil(err) + if block.Transactions().Len() != 0 { + break + } + height -= 1 + } - s.Nil(err) + return block.Transactions()[0], nil + })) } diff --git a/prover/proof_submitter/valid_proof_submitter.go b/prover/proof_submitter/valid_proof_submitter.go index e86f3d668..90b6c8baf 100644 --- a/prover/proof_submitter/valid_proof_submitter.go +++ b/prover/proof_submitter/valid_proof_submitter.go @@ -254,7 +254,15 @@ func (s *ValidProofSubmitter) SubmitProof( return s.rpc.TaikoL1.ProveBlock(txOpts, blockID, input) } - if err := sendTxWithBackoff(ctx, s.rpc, blockID, block.Header().Time, s.expectedReward, sendTx); err != nil { + if err := sendTxWithBackoff( + ctx, + s.rpc, + blockID, + block.Header().Time, + s.expectedReward, + proofWithHeader.Meta, + sendTx, + ); err != nil { if errors.Is(err, errUnretryable) { return nil } diff --git a/prover/prover.go b/prover/prover.go index 91616cc94..cf868e3e7 100644 --- a/prover/prover.go +++ b/prover/prover.go @@ -304,6 +304,30 @@ func (p *Prover) onBlockProposed( end() return nil } + + // Check whteher the L1 chain has been reorged. + reorged, l1CurrentToReset, lastHandledBlockIDToReset, err := p.rpc.CheckL1Reorg( + ctx, + new(big.Int).Sub(event.Id, common.Big1), + ) + if err != nil { + return fmt.Errorf("failed to check whether L1 chain was reorged: %w", err) + } + + if reorged { + log.Info( + "Reset L1Current cursor due to reorg", + "l1CurrentHeightOld", p.l1Current, + "l1CurrentHeightNew", l1CurrentToReset.Number, + "lastHandledBlockIDOld", p.lastHandledBlockID, + "lastHandledBlockIDNew", lastHandledBlockIDToReset, + ) + p.l1Current = l1CurrentToReset.Number.Uint64() + p.lastHandledBlockID = lastHandledBlockIDToReset.Uint64() + + return fmt.Errorf("reorg detected, reset l1Current cursor to %d", l1CurrentToReset.Number) + } + if event.Id.Uint64() <= p.lastHandledBlockID { return nil } @@ -342,6 +366,12 @@ func (p *Prover) onBlockProposed( } } + // Check if the current prover has seen this block ID before, there was probably + // a L1 reorg, we need to cancel that reorged block's proof generation task at first. + if p.currentBlocksBeingProven[event.Meta.Id] != nil { + p.cancelProof(ctx, event.Meta.Id) + } + ctx, cancelCtx := context.WithCancel(ctx) p.currentBlocksBeingProvenMutex.Lock() p.currentBlocksBeingProven[event.Id.Uint64()] = cancelFunc(func() {