From 2f0f190f36a2f082dc7a2366c16c19b80de264a1 Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Fri, 8 Mar 2024 14:03:14 +0800 Subject: [PATCH 01/12] core: improve chain rewinding mechanism --- core/blockchain.go | 202 +++++++++++++++++++++++++++++++++------------ 1 file changed, 147 insertions(+), 55 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 67b49cfe0262..ddb283560ce9 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -616,6 +616,136 @@ func (bc *BlockChain) SetSafe(header *types.Header) { } } +// rewindPathHead implements the logic of rewindHead in the context of hash scheme. +func (bc *BlockChain) rewindHashHead(root common.Hash) (*types.Header, uint64) { + var ( + limit uint64 // The oldest block that will be searched for this rewinding + beyondRoot = root == common.Hash{} // Flag whether we're beyond the requested root (no root, always true) + pivot = rawdb.ReadLastPivotNumber(bc.db) // Associated block number of pivot point state + headBlock = bc.CurrentBlock() // Head block of the chain + rootNumber uint64 // Associated block number of requested root + ) + // The oldest block to be searched is determined by the pivot block or a soft + // finality threshold. The rationale behind this is as follows: + // + // - Snap sync is selected if the pivot block is available. The earliest available + // state is the pivot block itself, so there is no sense in going further back. + // + // - Full sync is selected if the pivot block does not exist. The hash database + // periodically flushes the state to disk, and the soft finality threshold is + // considered sufficient to find a persistent state, even for the testnet. It + // might be not enough for a chain that is nearly empty. In the worst case, + // the entire chain is reset to genesis, and snap sync is re-enabled on top, + // which is still acceptable. + if pivot != nil { + limit = *pivot + } else if headBlock.Number.Uint64() > params.FullImmutabilityThreshold { + limit = headBlock.Number.Uint64() - params.FullImmutabilityThreshold + } + for { + // If a root threshold was requested but not yet crossed, check + if !beyondRoot && headBlock.Root == root { + beyondRoot, rootNumber = true, headBlock.Number.Uint64() + } + // If the associated state is not reachable, continue searching + // backwards until an available state is found. + if !bc.HasState(headBlock.Root) { + log.Trace("Block state missing, rewinding further", "number", headBlock.Number, "hash", headBlock.Hash()) + + // If search limit is reached, return the genesis block as + // the new chain head. + if headBlock.Number.Uint64() < limit { + log.Info("Rewinding limit is reached, aiming genesis", "number", headBlock.Number, "hash", headBlock.Hash(), "limit", limit) + return bc.genesisBlock.Header(), rootNumber + } + // If the chain is gapped in the middle, return the genesis + // block as the new chain head. + parent := bc.GetHeader(headBlock.ParentHash, headBlock.Number.Uint64()-1) + if parent == nil { + log.Error("Missing block in the middle, aiming genesis", "number", headBlock.Number.Uint64()-1, "hash", headBlock.ParentHash) + return bc.genesisBlock.Header(), rootNumber + } + headBlock = parent + + // If the genesis block is reached, stop searching. + if headBlock.Number.Uint64() == 0 { + log.Info("Genesis block is reached", "number", headBlock.Number, "hash", headBlock.Hash()) + return headBlock, rootNumber + } + continue // keep rewinding + } + // Once the available state is found, ensure that the requested root has already + // been crossed. If not, continue rewinding. + if beyondRoot || headBlock.Number.Uint64() == 0 { + log.Debug("Rewound to block with state", "number", headBlock.Number, "hash", headBlock.Hash()) + return headBlock, rootNumber + } + log.Debug("Skipping block with threshold state", "number", headBlock.Number, "hash", headBlock.Hash(), "root", headBlock.Root) + headBlock = bc.GetHeader(headBlock.ParentHash, headBlock.Number.Uint64()-1) // Keep rewinding + } +} + +// rewindPathHead implements the logic of rewindHead in the context of path scheme. +func (bc *BlockChain) rewindPathHead(root common.Hash) (*types.Header, uint64) { + var ( + beyondRoot = root == common.Hash{} // Flag whether we're beyond the requested root (no root, always true) + headBlock = bc.CurrentBlock() // Head block of the chain + pivot = rawdb.ReadLastPivotNumber(bc.db) // Associated block number of pivot point state + rootNumber uint64 // Associated block number of requested root + ) + // Rewind the head block tag until an available state is found. + for !bc.HasState(headBlock.Root) { + // If a root threshold was requested but not yet crossed, check + if !beyondRoot && headBlock.Root == root { + beyondRoot, rootNumber = true, headBlock.Number.Uint64() + } + // If pivot block is reached, return the genesis block as the + // new chain head. Theoretically there must be a persistent + // state before or at the pivot block, prevent endless rewinding + // towards the genesis just in case. + if pivot != nil && *pivot >= headBlock.Number.Uint64() { + log.Info("Pivot block is reached, aiming genesis", "number", headBlock.Number, "hash", headBlock.Hash()) + return bc.genesisBlock.Header(), rootNumber + } + headBlock = bc.GetHeader(headBlock.ParentHash, headBlock.Number.Uint64()-1) // Keep rewinding + } + // The requested root has already been crossed. Return the block + // corresponding to the available state as the new chain head. + if beyondRoot { + return headBlock, rootNumber + } + // If the requested root has not yet been crossed, prevent further + // backward rewinding if reaching the root is deemed impossible. + if !bc.HasState(root) && !bc.stateRecoverable(root) { + return headBlock, 0 + } + // Rewind the head block tag until the requested root is reached. + for !beyondRoot { + headBlock = bc.GetHeader(headBlock.ParentHash, headBlock.Number.Uint64()-1) // Keep rewinding + beyondRoot = headBlock.Root == root + } + // Recover if the target state is not available. + if !bc.HasState(root) { + if err := bc.triedb.Recover(root); err != nil { + log.Crit("Failed to rollback state", "err", err) // Shouldn't happen + } + } + return headBlock, rootNumber +} + +// rewindHead searches the available states in the database and returns the associated +// block as the new head block. +// +// If the given root is not empty, then the rewind should attempt to pass the specified +// state root and return the associated block number as well. If the root is deemed +// impossible to pass, 0 is returned. +func (bc *BlockChain) rewindHead(root common.Hash) (*types.Header, uint64) { + if bc.triedb.Scheme() == rawdb.PathScheme { + return bc.rewindPathHead(root) + } + return bc.rewindHashHead(root) +} + // setHeadBeyondRoot rewinds the local chain to a new head with the extra condition // that the rewind must pass the specified state root. This method is meant to be // used when rewinding with snapshots enabled to ensure that we go back further than @@ -634,79 +764,40 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha } defer bc.chainmu.Unlock() - // Track the block number of the requested root hash - var rootNumber uint64 // (no root == always 0) - - // Retrieve the last pivot block to short circuit rollbacks beyond it and the - // current freezer limit to start nuking id underflown - pivot := rawdb.ReadLastPivotNumber(bc.db) - frozen, _ := bc.db.Ancients() + var ( + // Track the block number of the requested root hash + rootNumber uint64 // (no root == always 0) + // Retrieve the last pivot block to short circuit rollbacks beyond it + // and the current freezer limit to start nuking it's underflown. + pivot = rawdb.ReadLastPivotNumber(bc.db) + ) updateFn := func(db ethdb.KeyValueWriter, header *types.Header) (*types.Header, bool) { // Rewind the blockchain, ensuring we don't end up with a stateless head // block. Note, depth equality is permitted to allow using SetHead as a // chain reparation mechanism without deleting any data! if currentBlock := bc.CurrentBlock(); currentBlock != nil && header.Number.Uint64() <= currentBlock.Number.Uint64() { - newHeadBlock := bc.GetBlock(header.Hash(), header.Number.Uint64()) - if newHeadBlock == nil { - log.Error("Gap in the chain, rewinding to genesis", "number", header.Number, "hash", header.Hash()) - newHeadBlock = bc.genesisBlock - } else { - // Block exists. Keep rewinding until either we find one with state - // or until we exceed the optional threshold root hash - beyondRoot := (root == common.Hash{}) // Flag whether we're beyond the requested root (no root, always true) - - for { - // If a root threshold was requested but not yet crossed, check - if root != (common.Hash{}) && !beyondRoot && newHeadBlock.Root() == root { - beyondRoot, rootNumber = true, newHeadBlock.NumberU64() - } - if !bc.HasState(newHeadBlock.Root()) && !bc.stateRecoverable(newHeadBlock.Root()) { - log.Trace("Block state missing, rewinding further", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash()) - if pivot == nil || newHeadBlock.NumberU64() > *pivot { - parent := bc.GetBlock(newHeadBlock.ParentHash(), newHeadBlock.NumberU64()-1) - if parent != nil { - newHeadBlock = parent - continue - } - log.Error("Missing block in the middle, aiming genesis", "number", newHeadBlock.NumberU64()-1, "hash", newHeadBlock.ParentHash()) - newHeadBlock = bc.genesisBlock - } else { - log.Trace("Rewind passed pivot, aiming genesis", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash(), "pivot", *pivot) - newHeadBlock = bc.genesisBlock - } - } - if beyondRoot || newHeadBlock.NumberU64() == 0 { - if !bc.HasState(newHeadBlock.Root()) && bc.stateRecoverable(newHeadBlock.Root()) { - // Rewind to a block with recoverable state. If the state is - // missing, run the state recovery here. - if err := bc.triedb.Recover(newHeadBlock.Root()); err != nil { - log.Crit("Failed to rollback state", "err", err) // Shouldn't happen - } - log.Debug("Rewound to block with state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash()) - } - break - } - log.Debug("Skipping block with threshold state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash(), "root", newHeadBlock.Root()) - newHeadBlock = bc.GetBlock(newHeadBlock.ParentHash(), newHeadBlock.NumberU64()-1) // Keep rewinding - } - } + var newHeadBlock *types.Header + newHeadBlock, rootNumber = bc.rewindHead(root) rawdb.WriteHeadBlockHash(db, newHeadBlock.Hash()) // Degrade the chain markers if they are explicitly reverted. // In theory we should update all in-memory markers in the // last step, however the direction of SetHead is from high // to low, so it's safe to update in-memory markers directly. - bc.currentBlock.Store(newHeadBlock.Header()) - headBlockGauge.Update(int64(newHeadBlock.NumberU64())) + bc.currentBlock.Store(newHeadBlock) + headBlockGauge.Update(int64(newHeadBlock.Number.Uint64())) // The head state is missing, which is only possible in the path-based // scheme. This situation occurs when the chain head is rewound below // the pivot point. In this scenario, there is no possible recovery // approach except for rerunning a snap sync. Do nothing here until the // state syncer picks it up. - if !bc.HasState(newHeadBlock.Root()) { - log.Info("Chain is stateless, wait state sync", "number", newHeadBlock.Number(), "hash", newHeadBlock.Hash()) + if !bc.HasState(newHeadBlock.Root) { + if newHeadBlock.Number.Uint64() != 0 { + log.Crit("Chain is stateless at a non-genesis block") + } + log.Info("Chain is stateless, wait state sync", "number", newHeadBlock.Number, "hash", newHeadBlock.Hash()) } } // Rewind the snap block in a simpleton way to the target head @@ -733,6 +824,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha // intent afterwards is full block importing, delete the chain segment // between the stateful-block and the sethead target. var wipe bool + frozen, _ := bc.db.Ancients() if headNumber+1 < frozen { wipe = pivot == nil || headNumber >= *pivot } From d131a9583c994c45f96ca0765149390538470c28 Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Mon, 11 Mar 2024 16:58:52 +0800 Subject: [PATCH 02/12] core: address comment --- core/blockchain.go | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index ddb283560ce9..80bac5b6b9c6 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -688,17 +688,28 @@ func (bc *BlockChain) rewindHashHead(root common.Hash) (*types.Header, uint64) { // rewindPathHead implements the logic of rewindHead in the context of path scheme. func (bc *BlockChain) rewindPathHead(root common.Hash) (*types.Header, uint64) { var ( - beyondRoot = root == common.Hash{} // Flag whether we're beyond the requested root (no root, always true) headBlock = bc.CurrentBlock() // Head block of the chain - pivot = rawdb.ReadLastPivotNumber(bc.db) // Associated block number of pivot point state + pivot = rawdb.ReadLastPivotNumber(bc.db) // Associated block number of pivot block rootNumber uint64 // Associated block number of requested root ) + // BeyondRoot represents whether the requested root is already crossed. + // The flag value is set to true if the root is empty, or reaching the + // root is deemed impossible. + beyondRoot := root == common.Hash{} + if !beyondRoot && !bc.HasState(root) && !bc.stateRecoverable(root) { + beyondRoot = true + } // Rewind the head block tag until an available state is found. - for !bc.HasState(headBlock.Root) { + for { // If a root threshold was requested but not yet crossed, check if !beyondRoot && headBlock.Root == root { beyondRoot, rootNumber = true, headBlock.Number.Uint64() } + // Check if the associated state is available or recoverable if + // the requested root has already been crossed. + if beyondRoot && (bc.HasState(headBlock.Root) || bc.stateRecoverable(headBlock.Root)) { + break + } // If pivot block is reached, return the genesis block as the // new chain head. Theoretically there must be a persistent // state before or at the pivot block, prevent endless rewinding @@ -709,25 +720,10 @@ func (bc *BlockChain) rewindPathHead(root common.Hash) (*types.Header, uint64) { } headBlock = bc.GetHeader(headBlock.ParentHash, headBlock.Number.Uint64()-1) // Keep rewinding } - // The requested root has already been crossed. Return the block - // corresponding to the available state as the new chain head. - if beyondRoot { - return headBlock, rootNumber - } - // If the requested root has not yet been crossed, prevent further - // backward rewinding if reaching the root is deemed impossible. - if !bc.HasState(root) && !bc.stateRecoverable(root) { - return headBlock, 0 - } - // Rewind the head block tag until the requested root is reached. - for !beyondRoot { - headBlock = bc.GetHeader(headBlock.ParentHash, headBlock.Number.Uint64()-1) // Keep rewinding - beyondRoot = headBlock.Root == root - } - // Recover if the target state is not available. + // Recover if the target state if it's not available yet. if !bc.HasState(root) { if err := bc.triedb.Recover(root); err != nil { - log.Crit("Failed to rollback state", "err", err) // Shouldn't happen + log.Crit("Failed to rollback state", "err", err) } } return headBlock, rootNumber From 88a6dc84935bc53eec48d961bba98ca5b53923f3 Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Mon, 11 Mar 2024 17:58:09 +0800 Subject: [PATCH 03/12] core: periodically print progress log --- core/blockchain.go | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 80bac5b6b9c6..b929709e2433 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -624,6 +624,9 @@ func (bc *BlockChain) rewindHashHead(root common.Hash) (*types.Header, uint64) { pivot = rawdb.ReadLastPivotNumber(bc.db) // Associated block number of pivot point state headBlock = bc.CurrentBlock() // Head block of the chain rootNumber uint64 // Associated block number of requested root + + start = time.Now() // Timestamp the rewinding is restarted + logged = time.Now() // Timestamp last progress log was printed ) // The oldest block to be searched is determined by the pivot block or a soft // finality threshold. The rationale behind this is as follows: @@ -643,6 +646,13 @@ func (bc *BlockChain) rewindHashHead(root common.Hash) (*types.Header, uint64) { limit = headBlock.Number.Uint64() - params.FullImmutabilityThreshold } for { + logger := log.Trace + if time.Since(logged) > time.Second*8 { + logged = time.Now() + logger = log.Info + } + logger("Block state missing, rewinding further", "number", headBlock.Number, "hash", headBlock.Hash(), "elapsed", common.PrettyDuration(time.Since(start))) + // If a root threshold was requested but not yet crossed, check if !beyondRoot && headBlock.Root == root { beyondRoot, rootNumber = true, headBlock.Number.Uint64() @@ -650,8 +660,6 @@ func (bc *BlockChain) rewindHashHead(root common.Hash) (*types.Header, uint64) { // If the associated state is not reachable, continue searching // backwards until an available state is found. if !bc.HasState(headBlock.Root) { - log.Trace("Block state missing, rewinding further", "number", headBlock.Number, "hash", headBlock.Hash()) - // If search limit is reached, return the genesis block as // the new chain head. if headBlock.Number.Uint64() < limit { @@ -691,6 +699,9 @@ func (bc *BlockChain) rewindPathHead(root common.Hash) (*types.Header, uint64) { headBlock = bc.CurrentBlock() // Head block of the chain pivot = rawdb.ReadLastPivotNumber(bc.db) // Associated block number of pivot block rootNumber uint64 // Associated block number of requested root + + start = time.Now() // Timestamp the rewinding is restarted + logged = time.Now() // Timestamp last progress log was printed ) // BeyondRoot represents whether the requested root is already crossed. // The flag value is set to true if the root is empty, or reaching the @@ -701,6 +712,13 @@ func (bc *BlockChain) rewindPathHead(root common.Hash) (*types.Header, uint64) { } // Rewind the head block tag until an available state is found. for { + logger := log.Trace + if time.Since(logged) > time.Second*8 { + logged = time.Now() + logger = log.Info + } + logger("Block state missing, rewinding further", "number", headBlock.Number, "hash", headBlock.Hash(), "elapsed", common.PrettyDuration(time.Since(start))) + // If a root threshold was requested but not yet crossed, check if !beyondRoot && headBlock.Root == root { beyondRoot, rootNumber = true, headBlock.Number.Uint64() From 9d7a4dbb14d88aefdce49daa380659953275e97f Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Mon, 11 Mar 2024 18:01:22 +0800 Subject: [PATCH 04/12] core: address comments --- core/blockchain.go | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index b929709e2433..33e302b03c28 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -622,7 +622,7 @@ func (bc *BlockChain) rewindHashHead(root common.Hash) (*types.Header, uint64) { limit uint64 // The oldest block that will be searched for this rewinding beyondRoot = root == common.Hash{} // Flag whether we're beyond the requested root (no root, always true) pivot = rawdb.ReadLastPivotNumber(bc.db) // Associated block number of pivot point state - headBlock = bc.CurrentBlock() // Head block of the chain + head = bc.CurrentBlock() // Head block of the chain rootNumber uint64 // Associated block number of requested root start = time.Now() // Timestamp the rewinding is restarted @@ -642,8 +642,8 @@ func (bc *BlockChain) rewindHashHead(root common.Hash) (*types.Header, uint64) { // which is still acceptable. if pivot != nil { limit = *pivot - } else if headBlock.Number.Uint64() > params.FullImmutabilityThreshold { - limit = headBlock.Number.Uint64() - params.FullImmutabilityThreshold + } else if head.Number.Uint64() > params.FullImmutabilityThreshold { + limit = head.Number.Uint64() - params.FullImmutabilityThreshold } for { logger := log.Trace @@ -651,45 +651,45 @@ func (bc *BlockChain) rewindHashHead(root common.Hash) (*types.Header, uint64) { logged = time.Now() logger = log.Info } - logger("Block state missing, rewinding further", "number", headBlock.Number, "hash", headBlock.Hash(), "elapsed", common.PrettyDuration(time.Since(start))) + logger("Block state missing, rewinding further", "number", head.Number, "hash", head.Hash(), "elapsed", common.PrettyDuration(time.Since(start))) // If a root threshold was requested but not yet crossed, check - if !beyondRoot && headBlock.Root == root { - beyondRoot, rootNumber = true, headBlock.Number.Uint64() + if !beyondRoot && head.Root == root { + beyondRoot, rootNumber = true, head.Number.Uint64() } // If the associated state is not reachable, continue searching // backwards until an available state is found. - if !bc.HasState(headBlock.Root) { + if !bc.HasState(head.Root) { // If search limit is reached, return the genesis block as // the new chain head. - if headBlock.Number.Uint64() < limit { - log.Info("Rewinding limit is reached, aiming genesis", "number", headBlock.Number, "hash", headBlock.Hash(), "limit", limit) + if head.Number.Uint64() < limit { + log.Info("Rewinding limit reached, resetting to genesis", "number", head.Number, "hash", head.Hash(), "limit", limit) return bc.genesisBlock.Header(), rootNumber } // If the chain is gapped in the middle, return the genesis // block as the new chain head. - parent := bc.GetHeader(headBlock.ParentHash, headBlock.Number.Uint64()-1) + parent := bc.GetHeader(head.ParentHash, head.Number.Uint64()-1) if parent == nil { - log.Error("Missing block in the middle, aiming genesis", "number", headBlock.Number.Uint64()-1, "hash", headBlock.ParentHash) + log.Error("Missing block in the middle, resetting to genesis", "number", head.Number.Uint64()-1, "hash", head.ParentHash) return bc.genesisBlock.Header(), rootNumber } - headBlock = parent + head = parent // If the genesis block is reached, stop searching. - if headBlock.Number.Uint64() == 0 { - log.Info("Genesis block is reached", "number", headBlock.Number, "hash", headBlock.Hash()) - return headBlock, rootNumber + if head.Number.Uint64() == 0 { + log.Info("Genesis block reached", "number", head.Number, "hash", head.Hash()) + return head, rootNumber } continue // keep rewinding } // Once the available state is found, ensure that the requested root has already // been crossed. If not, continue rewinding. - if beyondRoot || headBlock.Number.Uint64() == 0 { - log.Debug("Rewound to block with state", "number", headBlock.Number, "hash", headBlock.Hash()) - return headBlock, rootNumber + if beyondRoot || head.Number.Uint64() == 0 { + log.Info("Rewound to block with state", "number", head.Number, "hash", head.Hash()) + return head, rootNumber } - log.Debug("Skipping block with threshold state", "number", headBlock.Number, "hash", headBlock.Hash(), "root", headBlock.Root) - headBlock = bc.GetHeader(headBlock.ParentHash, headBlock.Number.Uint64()-1) // Keep rewinding + log.Debug("Skipping block with threshold state", "number", head.Number, "hash", head.Hash(), "root", head.Root) + head = bc.GetHeader(head.ParentHash, head.Number.Uint64()-1) // Keep rewinding } } @@ -733,7 +733,7 @@ func (bc *BlockChain) rewindPathHead(root common.Hash) (*types.Header, uint64) { // state before or at the pivot block, prevent endless rewinding // towards the genesis just in case. if pivot != nil && *pivot >= headBlock.Number.Uint64() { - log.Info("Pivot block is reached, aiming genesis", "number", headBlock.Number, "hash", headBlock.Hash()) + log.Info("Pivot block reached, resetting to genesis", "number", headBlock.Number, "hash", headBlock.Hash()) return bc.genesisBlock.Header(), rootNumber } headBlock = bc.GetHeader(headBlock.ParentHash, headBlock.Number.Uint64()-1) // Keep rewinding From 470916de461e46b8770a0a36ebffe674a5d2fca5 Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Mon, 11 Mar 2024 18:05:34 +0800 Subject: [PATCH 05/12] core: fix comment --- core/blockchain.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 33e302b03c28..4fd344616b40 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -628,14 +628,14 @@ func (bc *BlockChain) rewindHashHead(root common.Hash) (*types.Header, uint64) { start = time.Now() // Timestamp the rewinding is restarted logged = time.Now() // Timestamp last progress log was printed ) - // The oldest block to be searched is determined by the pivot block or a soft - // finality threshold. The rationale behind this is as follows: + // The oldest block to be searched is determined by the pivot block or a constant + // searching threshold. The rationale behind this is as follows: // // - Snap sync is selected if the pivot block is available. The earliest available // state is the pivot block itself, so there is no sense in going further back. // // - Full sync is selected if the pivot block does not exist. The hash database - // periodically flushes the state to disk, and the soft finality threshold is + // periodically flushes the state to disk, and the used searching threshold is // considered sufficient to find a persistent state, even for the testnet. It // might be not enough for a chain that is nearly empty. In the worst case, // the entire chain is reset to genesis, and snap sync is re-enabled on top, From c6b0eb75fad4dd9aab17dbbfb889da41136a2e0f Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Mon, 11 Mar 2024 18:10:12 +0800 Subject: [PATCH 06/12] core: fix rewinding in path --- core/blockchain.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 4fd344616b40..88727e7cd51a 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -739,8 +739,8 @@ func (bc *BlockChain) rewindPathHead(root common.Hash) (*types.Header, uint64) { headBlock = bc.GetHeader(headBlock.ParentHash, headBlock.Number.Uint64()-1) // Keep rewinding } // Recover if the target state if it's not available yet. - if !bc.HasState(root) { - if err := bc.triedb.Recover(root); err != nil { + if !bc.HasState(headBlock.Root) { + if err := bc.triedb.Recover(headBlock.Root); err != nil { log.Crit("Failed to rollback state", "err", err) } } From 60fb704da451e02c10fcf95b9a6f13f78fcc06a0 Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Mon, 11 Mar 2024 18:25:04 +0800 Subject: [PATCH 07/12] core: fix beyondRoot condition --- core/blockchain.go | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 88727e7cd51a..059ffd51bdc9 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -696,20 +696,17 @@ func (bc *BlockChain) rewindHashHead(root common.Hash) (*types.Header, uint64) { // rewindPathHead implements the logic of rewindHead in the context of path scheme. func (bc *BlockChain) rewindPathHead(root common.Hash) (*types.Header, uint64) { var ( - headBlock = bc.CurrentBlock() // Head block of the chain + head = bc.CurrentBlock() // Head block of the chain pivot = rawdb.ReadLastPivotNumber(bc.db) // Associated block number of pivot block rootNumber uint64 // Associated block number of requested root + // BeyondRoot represents whether the requested root is already crossed. + // The flag value is set to true if the root is empty. + beyondRoot = root == common.Hash{} + start = time.Now() // Timestamp the rewinding is restarted logged = time.Now() // Timestamp last progress log was printed ) - // BeyondRoot represents whether the requested root is already crossed. - // The flag value is set to true if the root is empty, or reaching the - // root is deemed impossible. - beyondRoot := root == common.Hash{} - if !beyondRoot && !bc.HasState(root) && !bc.stateRecoverable(root) { - beyondRoot = true - } // Rewind the head block tag until an available state is found. for { logger := log.Trace @@ -717,34 +714,41 @@ func (bc *BlockChain) rewindPathHead(root common.Hash) (*types.Header, uint64) { logged = time.Now() logger = log.Info } - logger("Block state missing, rewinding further", "number", headBlock.Number, "hash", headBlock.Hash(), "elapsed", common.PrettyDuration(time.Since(start))) + logger("Block state missing, rewinding further", "number", head.Number, "hash", head.Hash(), "elapsed", common.PrettyDuration(time.Since(start))) // If a root threshold was requested but not yet crossed, check - if !beyondRoot && headBlock.Root == root { - beyondRoot, rootNumber = true, headBlock.Number.Uint64() + if !beyondRoot && head.Root == root { + beyondRoot, rootNumber = true, head.Number.Uint64() + } + // If the root threshold hasn't been crossed but the available + // state is found, disable root searching if it's regarded + // impossible to reach. + if !beyondRoot && bc.HasState(head.Root) { + beyondRoot = !bc.HasState(root) && !bc.stateRecoverable(root) } // Check if the associated state is available or recoverable if // the requested root has already been crossed. - if beyondRoot && (bc.HasState(headBlock.Root) || bc.stateRecoverable(headBlock.Root)) { + if beyondRoot && (bc.HasState(head.Root) || bc.stateRecoverable(head.Root)) { break } // If pivot block is reached, return the genesis block as the // new chain head. Theoretically there must be a persistent // state before or at the pivot block, prevent endless rewinding // towards the genesis just in case. - if pivot != nil && *pivot >= headBlock.Number.Uint64() { - log.Info("Pivot block reached, resetting to genesis", "number", headBlock.Number, "hash", headBlock.Hash()) + if pivot != nil && *pivot >= head.Number.Uint64() { + log.Info("Pivot block reached, resetting to genesis", "number", head.Number, "hash", head.Hash()) return bc.genesisBlock.Header(), rootNumber } - headBlock = bc.GetHeader(headBlock.ParentHash, headBlock.Number.Uint64()-1) // Keep rewinding + head = bc.GetHeader(head.ParentHash, head.Number.Uint64()-1) // Keep rewinding } // Recover if the target state if it's not available yet. - if !bc.HasState(headBlock.Root) { - if err := bc.triedb.Recover(headBlock.Root); err != nil { + if !bc.HasState(head.Root) { + if err := bc.triedb.Recover(head.Root); err != nil { log.Crit("Failed to rollback state", "err", err) } } - return headBlock, rootNumber + log.Info("Rewound to block with state", "number", head.Number, "hash", head.Hash()) + return head, rootNumber } // rewindHead searches the available states in the database and returns the associated From b5df29c693581535c9a1f529ee83d134d366199b Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Mon, 11 Mar 2024 18:32:18 +0800 Subject: [PATCH 08/12] core: polish code --- core/blockchain.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 059ffd51bdc9..7cf5db31d70e 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -700,8 +700,8 @@ func (bc *BlockChain) rewindPathHead(root common.Hash) (*types.Header, uint64) { pivot = rawdb.ReadLastPivotNumber(bc.db) // Associated block number of pivot block rootNumber uint64 // Associated block number of requested root - // BeyondRoot represents whether the requested root is already crossed. - // The flag value is set to true if the root is empty. + // BeyondRoot represents whether the requested root is already + // crossed. The flag value is set to true if the root is empty. beyondRoot = root == common.Hash{} start = time.Now() // Timestamp the rewinding is restarted @@ -721,10 +721,11 @@ func (bc *BlockChain) rewindPathHead(root common.Hash) (*types.Header, uint64) { beyondRoot, rootNumber = true, head.Number.Uint64() } // If the root threshold hasn't been crossed but the available - // state is found, disable root searching if it's regarded - // impossible to reach. + // state is reached, quickly determine if the target state is + // possible to be reached or not. if !beyondRoot && bc.HasState(head.Root) { beyondRoot = !bc.HasState(root) && !bc.stateRecoverable(root) + log.Info("Disable the search for unattainable state", "root", root) } // Check if the associated state is available or recoverable if // the requested root has already been crossed. From e3884f146b834525ec92666498d91ff530d9350c Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Mon, 11 Mar 2024 18:36:33 +0800 Subject: [PATCH 09/12] core: polish code --- core/blockchain.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 7cf5db31d70e..eed828f1e002 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -704,6 +704,10 @@ func (bc *BlockChain) rewindPathHead(root common.Hash) (*types.Header, uint64) { // crossed. The flag value is set to true if the root is empty. beyondRoot = root == common.Hash{} + // noState represents if the target state requested for search + // is unavailable and impossible to be recovered. + noState = !bc.HasState(root) && !bc.stateRecoverable(root) + start = time.Now() // Timestamp the rewinding is restarted logged = time.Now() // Timestamp last progress log was printed ) @@ -723,8 +727,8 @@ func (bc *BlockChain) rewindPathHead(root common.Hash) (*types.Header, uint64) { // If the root threshold hasn't been crossed but the available // state is reached, quickly determine if the target state is // possible to be reached or not. - if !beyondRoot && bc.HasState(head.Root) { - beyondRoot = !bc.HasState(root) && !bc.stateRecoverable(root) + if !beyondRoot && noState && bc.HasState(head.Root) { + beyondRoot = true log.Info("Disable the search for unattainable state", "root", root) } // Check if the associated state is available or recoverable if From 3784c6caee5238e727b1fa17cefabc7c70c1aebc Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Mon, 11 Mar 2024 23:28:30 +0800 Subject: [PATCH 10/12] core: extend code comment --- core/blockchain.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index eed828f1e002..55da62132e75 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -760,8 +760,10 @@ func (bc *BlockChain) rewindPathHead(root common.Hash) (*types.Header, uint64) { // block as the new head block. // // If the given root is not empty, then the rewind should attempt to pass the specified -// state root and return the associated block number as well. If the root is deemed -// impossible to pass, 0 is returned. +// state root and return the associated block number as well. If the root, typically +// representing the state corresponding to snapshot disk layer, is deemed impassable, +// then block number zero is returned, indicating that snapshot recovery is disabled +// and the whole snapshot should be auto-generated in case of head mismatch. func (bc *BlockChain) rewindHead(root common.Hash) (*types.Header, uint64) { if bc.triedb.Scheme() == rawdb.PathScheme { return bc.rewindPathHead(root) From ad7ac99e70d2987440a48719b6c04a27e6b63973 Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Tue, 12 Mar 2024 10:07:20 +0800 Subject: [PATCH 11/12] core: stop rewinding if chain is gapped or genesis is reached --- core/blockchain.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 55da62132e75..bc2b51988bbf 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -662,7 +662,7 @@ func (bc *BlockChain) rewindHashHead(root common.Hash) (*types.Header, uint64) { if !bc.HasState(head.Root) { // If search limit is reached, return the genesis block as // the new chain head. - if head.Number.Uint64() < limit { + if head.Number.Uint64() <= limit { log.Info("Rewinding limit reached, resetting to genesis", "number", head.Number, "hash", head.Hash(), "limit", limit) return bc.genesisBlock.Header(), rootNumber } @@ -744,7 +744,20 @@ func (bc *BlockChain) rewindPathHead(root common.Hash) (*types.Header, uint64) { log.Info("Pivot block reached, resetting to genesis", "number", head.Number, "hash", head.Hash()) return bc.genesisBlock.Header(), rootNumber } - head = bc.GetHeader(head.ParentHash, head.Number.Uint64()-1) // Keep rewinding + // If the chain is gapped in the middle, return the genesis + // block as the new chain head + parent := bc.GetHeader(head.ParentHash, head.Number.Uint64()-1) // Keep rewinding + if parent == nil { + log.Error("Missing block in the middle, resetting to genesis", "number", head.Number.Uint64()-1, "hash", head.ParentHash) + return bc.genesisBlock.Header(), rootNumber + } + head = parent + + // If the genesis block is reached, stop searching. + if head.Number.Uint64() == 0 { + log.Info("Genesis block reached", "number", head.Number, "hash", head.Hash()) + return head, rootNumber + } } // Recover if the target state if it's not available yet. if !bc.HasState(head.Root) { From 48bbb38c3435a753377b863a0e09c955a439cfba Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Tue, 12 Mar 2024 10:56:55 +0800 Subject: [PATCH 12/12] core: fix broken tests --- core/blockchain.go | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index bc2b51988bbf..ab71e4fdb50a 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -617,12 +617,11 @@ func (bc *BlockChain) SetSafe(header *types.Header) { } // rewindPathHead implements the logic of rewindHead in the context of hash scheme. -func (bc *BlockChain) rewindHashHead(root common.Hash) (*types.Header, uint64) { +func (bc *BlockChain) rewindHashHead(head *types.Header, root common.Hash) (*types.Header, uint64) { var ( limit uint64 // The oldest block that will be searched for this rewinding beyondRoot = root == common.Hash{} // Flag whether we're beyond the requested root (no root, always true) pivot = rawdb.ReadLastPivotNumber(bc.db) // Associated block number of pivot point state - head = bc.CurrentBlock() // Head block of the chain rootNumber uint64 // Associated block number of requested root start = time.Now() // Timestamp the rewinding is restarted @@ -657,15 +656,15 @@ func (bc *BlockChain) rewindHashHead(root common.Hash) (*types.Header, uint64) { if !beyondRoot && head.Root == root { beyondRoot, rootNumber = true, head.Number.Uint64() } + // If search limit is reached, return the genesis block as the + // new chain head. + if head.Number.Uint64() < limit { + log.Info("Rewinding limit reached, resetting to genesis", "number", head.Number, "hash", head.Hash(), "limit", limit) + return bc.genesisBlock.Header(), rootNumber + } // If the associated state is not reachable, continue searching // backwards until an available state is found. if !bc.HasState(head.Root) { - // If search limit is reached, return the genesis block as - // the new chain head. - if head.Number.Uint64() <= limit { - log.Info("Rewinding limit reached, resetting to genesis", "number", head.Number, "hash", head.Hash(), "limit", limit) - return bc.genesisBlock.Header(), rootNumber - } // If the chain is gapped in the middle, return the genesis // block as the new chain head. parent := bc.GetHeader(head.ParentHash, head.Number.Uint64()-1) @@ -682,8 +681,8 @@ func (bc *BlockChain) rewindHashHead(root common.Hash) (*types.Header, uint64) { } continue // keep rewinding } - // Once the available state is found, ensure that the requested root has already - // been crossed. If not, continue rewinding. + // Once the available state is found, ensure that the requested root + // has already been crossed. If not, continue rewinding. if beyondRoot || head.Number.Uint64() == 0 { log.Info("Rewound to block with state", "number", head.Number, "hash", head.Hash()) return head, rootNumber @@ -694,9 +693,8 @@ func (bc *BlockChain) rewindHashHead(root common.Hash) (*types.Header, uint64) { } // rewindPathHead implements the logic of rewindHead in the context of path scheme. -func (bc *BlockChain) rewindPathHead(root common.Hash) (*types.Header, uint64) { +func (bc *BlockChain) rewindPathHead(head *types.Header, root common.Hash) (*types.Header, uint64) { var ( - head = bc.CurrentBlock() // Head block of the chain pivot = rawdb.ReadLastPivotNumber(bc.db) // Associated block number of pivot block rootNumber uint64 // Associated block number of requested root @@ -777,11 +775,11 @@ func (bc *BlockChain) rewindPathHead(root common.Hash) (*types.Header, uint64) { // representing the state corresponding to snapshot disk layer, is deemed impassable, // then block number zero is returned, indicating that snapshot recovery is disabled // and the whole snapshot should be auto-generated in case of head mismatch. -func (bc *BlockChain) rewindHead(root common.Hash) (*types.Header, uint64) { +func (bc *BlockChain) rewindHead(head *types.Header, root common.Hash) (*types.Header, uint64) { if bc.triedb.Scheme() == rawdb.PathScheme { - return bc.rewindPathHead(root) + return bc.rewindPathHead(head, root) } - return bc.rewindHashHead(root) + return bc.rewindHashHead(head, root) } // setHeadBeyondRoot rewinds the local chain to a new head with the extra condition @@ -816,7 +814,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha // chain reparation mechanism without deleting any data! if currentBlock := bc.CurrentBlock(); currentBlock != nil && header.Number.Uint64() <= currentBlock.Number.Uint64() { var newHeadBlock *types.Header - newHeadBlock, rootNumber = bc.rewindHead(root) + newHeadBlock, rootNumber = bc.rewindHead(header, root) rawdb.WriteHeadBlockHash(db, newHeadBlock.Hash()) // Degrade the chain markers if they are explicitly reverted.