From 88ba737a46163330cfce1fa0072227ffc6450b13 Mon Sep 17 00:00:00 2001 From: yutianwu Date: Thu, 23 Sep 2021 16:05:09 +0800 Subject: [PATCH 01/18] add block proccess backoff time when validator is not in turn --- consensus/consensus.go | 3 +++ consensus/parlia/parlia.go | 22 ++++++++++++++++++++++ core/blockchain.go | 28 +++++++++++++++++++++++++--- core/chain_makers.go | 1 + core/headerchain.go | 7 ++++++- light/lightchain.go | 7 ++++++- 6 files changed, 63 insertions(+), 5 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index fc161390f9..a9e79359f9 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -49,6 +49,9 @@ type ChainHeaderReader interface { // GetHeaderByHash retrieves a block header from the database by its hash. GetHeaderByHash(hash common.Hash) *types.Header + + // GetHighestVerifiedHeader retrieves the highest header verified. + GetHighestVerifiedHeader() *types.Header } // ChainReader defines a small collection of methods needed to access the local diff --git a/consensus/parlia/parlia.go b/consensus/parlia/parlia.go index 14304fe2bc..004f872716 100644 --- a/consensus/parlia/parlia.go +++ b/consensus/parlia/parlia.go @@ -56,6 +56,7 @@ const ( validatorBytesLength = common.AddressLength wiggleTime = uint64(1) // second, Random delay (per signer) to allow concurrent signers initialBackOffTime = uint64(1) // second + processBackOffTime = uint64(1) // second systemRewardPercent = 4 // it means 1/2^4 = 1/16 percentage of gas fee incoming will be distributed to system @@ -863,6 +864,14 @@ func (p *Parlia) Seal(chain consensus.ChainHeaderReader, block *types.Block, res return case <-time.After(delay): } + if p.shouldWaitForCurrentBlockProcess(chain, header, snap) { + log.Info("not in turn and received current block, wait for current block process") + select { + case <-stop: + return + case <-time.After(time.Duration(processBackOffTime) * time.Second): + } + } select { case results <- block.WithSeal(header): @@ -874,6 +883,19 @@ func (p *Parlia) Seal(chain consensus.ChainHeaderReader, block *types.Block, res return nil } +func (p *Parlia) shouldWaitForCurrentBlockProcess(chain consensus.ChainHeaderReader, header *types.Header, snap *Snapshot) bool { + highestVerifiedHeader := chain.GetHighestVerifiedHeader() + if highestVerifiedHeader == nil || highestVerifiedHeader.Number == nil { + return false + } + + if !snap.inturn(p.val) && header.Number.Cmp(highestVerifiedHeader.Number) == 0 { + return true + } + + return false +} + func (p *Parlia) EnoughDistance(chain consensus.ChainReader, header *types.Header) bool { snap, err := p.snapshot(chain, header.Number.Uint64()-1, header.ParentHash, nil) if err != nil { diff --git a/core/blockchain.go b/core/blockchain.go index be0b0f04ae..d4ebbbfcbc 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -28,6 +28,8 @@ import ( "sync/atomic" "time" + lru "github.com/hashicorp/golang-lru" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/common/prque" @@ -44,7 +46,6 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - lru "github.com/hashicorp/golang-lru" ) var ( @@ -185,8 +186,9 @@ type BlockChain struct { chainmu sync.RWMutex // blockchain insertion lock - currentBlock atomic.Value // Current head of the block chain - currentFastBlock atomic.Value // Current head of the fast-sync chain (may be above the block chain!) + currentBlock atomic.Value // Current head of the block chain + currentFastBlock atomic.Value // Current head of the fast-sync chain (may be above the block chain!) + highestVerifiedHeader atomic.Value stateCache state.Database // State database to reuse between imports (contains state cache) bodyCache *lru.Cache // Cache for the most recent block bodies @@ -266,6 +268,9 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par bc.currentBlock.Store(nilBlock) bc.currentFastBlock.Store(nilBlock) + var nilHeader *types.Header + bc.highestVerifiedHeader.Store(nilHeader) + // Initialize the chain with ancient data if it isn't empty. var txIndexBlock uint64 @@ -1883,6 +1888,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er if err != nil { return it.index, err } + bc.updateHighestVerifiedHeader(block.Header()) + // Enable prefetching to pull in trie node paths while processing transactions statedb.StartPrefetcher("chain") activeState = statedb @@ -1989,6 +1996,21 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er return it.index, err } +func (bc *BlockChain) updateHighestVerifiedHeader(header *types.Header) { + if header == nil || header.Number == nil { + return + } + currentHeader := bc.highestVerifiedHeader.Load().(*types.Header) + if currentHeader == nil || currentHeader.Number == nil || currentHeader.Number.Cmp(header.Number) < 0 { + bc.highestVerifiedHeader.Store(types.CopyHeader(header)) + return + } +} + +func (bc *BlockChain) GetHighestVerifiedHeader() *types.Header { + return bc.highestVerifiedHeader.Load().(*types.Header) +} + // insertSideChain is called when an import batch hits upon a pruned ancestor // error, which happens when a sidechain with a sufficiently old fork-block is // found. diff --git a/core/chain_makers.go b/core/chain_makers.go index 6cb74d51be..7f62e41ad8 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -303,3 +303,4 @@ func (cr *fakeChainReader) GetHeaderByNumber(number uint64) *types.Header func (cr *fakeChainReader) GetHeaderByHash(hash common.Hash) *types.Header { return nil } func (cr *fakeChainReader) GetHeader(hash common.Hash, number uint64) *types.Header { return nil } func (cr *fakeChainReader) GetBlock(hash common.Hash, number uint64) *types.Block { return nil } +func (cr *fakeChainReader) GetHighestVerifiedHeader() *types.Header { return nil } diff --git a/core/headerchain.go b/core/headerchain.go index 1dbf958786..fe4770a469 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -26,6 +26,8 @@ import ( "sync/atomic" "time" + lru "github.com/hashicorp/golang-lru" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core/rawdb" @@ -33,7 +35,6 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" - lru "github.com/hashicorp/golang-lru" ) const ( @@ -413,6 +414,10 @@ func (hc *HeaderChain) GetBlockHashesFromHash(hash common.Hash, max uint64) []co return chain } +func (hc *HeaderChain) GetHighestVerifiedHeader() *types.Header { + return nil +} + // GetAncestor retrieves the Nth ancestor of a given block. It assumes that either the given block or // a close ancestor of it is canonical. maxNonCanonical points to a downwards counter limiting the // number of blocks to be individually checked before we reach the canonical chain. diff --git a/light/lightchain.go b/light/lightchain.go index ca6fbfac49..99644355ac 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -26,6 +26,8 @@ import ( "sync/atomic" "time" + lru "github.com/hashicorp/golang-lru" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" @@ -37,7 +39,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - lru "github.com/hashicorp/golang-lru" ) var ( @@ -148,6 +149,10 @@ func (lc *LightChain) HeaderChain() *core.HeaderChain { return lc.hc } +func (lc *LightChain) GetHighestVerifiedHeader() *types.Header { + return nil +} + // loadLastState loads the last known chain state from the database. This method // assumes that the chain manager mutex is held. func (lc *LightChain) loadLastState() error { From e0e58b23451eee6b0c8c29d6353ee641c2858e0c Mon Sep 17 00:00:00 2001 From: yutianwu Date: Thu, 23 Sep 2021 19:12:34 +0800 Subject: [PATCH 02/18] update comment --- consensus/parlia/parlia.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/parlia/parlia.go b/consensus/parlia/parlia.go index 004f872716..267be62889 100644 --- a/consensus/parlia/parlia.go +++ b/consensus/parlia/parlia.go @@ -865,7 +865,7 @@ func (p *Parlia) Seal(chain consensus.ChainHeaderReader, block *types.Block, res case <-time.After(delay): } if p.shouldWaitForCurrentBlockProcess(chain, header, snap) { - log.Info("not in turn and received current block, wait for current block process") + log.Info("Waiting for received in turn block to process") select { case <-stop: return From 4a84cfdf625976856138cd0635d2c223ab785a11 Mon Sep 17 00:00:00 2001 From: yutianwu Date: Fri, 24 Sep 2021 14:29:16 +0800 Subject: [PATCH 03/18] add more log --- consensus/parlia/parlia.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/consensus/parlia/parlia.go b/consensus/parlia/parlia.go index 267be62889..789ca84ce6 100644 --- a/consensus/parlia/parlia.go +++ b/consensus/parlia/parlia.go @@ -868,8 +868,10 @@ func (p *Parlia) Seal(chain consensus.ChainHeaderReader, block *types.Block, res log.Info("Waiting for received in turn block to process") select { case <-stop: + log.Info("Received block process finished, abort block seal") return case <-time.After(time.Duration(processBackOffTime) * time.Second): + log.Info("Process backoff time exhausted, start to seal block") } } From 065e69e3205e3d66f4888221324b303564dde6b5 Mon Sep 17 00:00:00 2001 From: yutianwu Date: Sun, 26 Sep 2021 18:23:45 +0800 Subject: [PATCH 04/18] fix comments --- consensus/parlia/parlia.go | 9 ++++++--- core/blockchain.go | 10 +++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/consensus/parlia/parlia.go b/consensus/parlia/parlia.go index 789ca84ce6..3f76082010 100644 --- a/consensus/parlia/parlia.go +++ b/consensus/parlia/parlia.go @@ -886,15 +886,18 @@ func (p *Parlia) Seal(chain consensus.ChainHeaderReader, block *types.Block, res } func (p *Parlia) shouldWaitForCurrentBlockProcess(chain consensus.ChainHeaderReader, header *types.Header, snap *Snapshot) bool { + if snap.inturn(p.val) { + return false + } + highestVerifiedHeader := chain.GetHighestVerifiedHeader() - if highestVerifiedHeader == nil || highestVerifiedHeader.Number == nil { + if highestVerifiedHeader == nil { return false } - if !snap.inturn(p.val) && header.Number.Cmp(highestVerifiedHeader.Number) == 0 { + if header.ParentHash == highestVerifiedHeader.ParentHash && header.Difficulty.Cmp(highestVerifiedHeader.Difficulty) < 0 { return true } - return false } diff --git a/core/blockchain.go b/core/blockchain.go index d4ebbbfcbc..a09cc5ff62 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -2001,7 +2001,15 @@ func (bc *BlockChain) updateHighestVerifiedHeader(header *types.Header) { return } currentHeader := bc.highestVerifiedHeader.Load().(*types.Header) - if currentHeader == nil || currentHeader.Number == nil || currentHeader.Number.Cmp(header.Number) < 0 { + if currentHeader == nil { + bc.highestVerifiedHeader.Store(types.CopyHeader(header)) + return + } + + newTD := big.NewInt(0).Add(bc.GetTdByHash(header.ParentHash), header.Difficulty) + oldTD := big.NewInt(0).Add(bc.GetTdByHash(currentHeader.ParentHash), currentHeader.Difficulty) + + if newTD.Cmp(oldTD) > 0 { bc.highestVerifiedHeader.Store(types.CopyHeader(header)) return } From 77f19d435dd6bef1ef7575428d3bac9625fa8e6e Mon Sep 17 00:00:00 2001 From: yutianwu Date: Sun, 26 Sep 2021 19:09:31 +0800 Subject: [PATCH 05/18] wait if td is the same --- consensus/parlia/parlia.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/parlia/parlia.go b/consensus/parlia/parlia.go index 3f76082010..68ade2e6ff 100644 --- a/consensus/parlia/parlia.go +++ b/consensus/parlia/parlia.go @@ -895,7 +895,7 @@ func (p *Parlia) shouldWaitForCurrentBlockProcess(chain consensus.ChainHeaderRea return false } - if header.ParentHash == highestVerifiedHeader.ParentHash && header.Difficulty.Cmp(highestVerifiedHeader.Difficulty) < 0 { + if header.ParentHash == highestVerifiedHeader.ParentHash && header.Difficulty.Cmp(highestVerifiedHeader.Difficulty) <= 0 { return true } return false From 7f9cdc4ae5832d58e833b34cc7d43c35c77fb962 Mon Sep 17 00:00:00 2001 From: yutianwu Date: Mon, 27 Sep 2021 10:50:13 +0800 Subject: [PATCH 06/18] minor update --- consensus/parlia/parlia.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/parlia/parlia.go b/consensus/parlia/parlia.go index 68ade2e6ff..7cb352d1f2 100644 --- a/consensus/parlia/parlia.go +++ b/consensus/parlia/parlia.go @@ -886,7 +886,7 @@ func (p *Parlia) Seal(chain consensus.ChainHeaderReader, block *types.Block, res } func (p *Parlia) shouldWaitForCurrentBlockProcess(chain consensus.ChainHeaderReader, header *types.Header, snap *Snapshot) bool { - if snap.inturn(p.val) { + if header.Difficulty.Cmp(diffInTurn) == 0 { return false } @@ -895,7 +895,7 @@ func (p *Parlia) shouldWaitForCurrentBlockProcess(chain consensus.ChainHeaderRea return false } - if header.ParentHash == highestVerifiedHeader.ParentHash && header.Difficulty.Cmp(highestVerifiedHeader.Difficulty) <= 0 { + if header.ParentHash == highestVerifiedHeader.ParentHash { return true } return false From 1ded097733aa44537f2a18dd48c56451366b8e1b Mon Sep 17 00:00:00 2001 From: zjubfd <296179868@qq.com> Date: Tue, 28 Sep 2021 16:03:38 +0800 Subject: [PATCH 07/18] [R4R]implement diff sync (#376) * implement block process part of light sync * add difflayer protocol * handle difflayer and refine light processor * add testcase for diff protocol * make it faster * allow validator to light sync * change into diff sync * ligth sync: download difflayer (#2) * ligth sync: download difflayer Signed-off-by: kyrie-yl * download diff layer: fix according to the comments Signed-off-by: kyrie-yl * download diff layer: update Signed-off-by: kyrie-yl * download diff layer: fix accroding comments Signed-off-by: kyrie-yl Co-authored-by: kyrie-yl * update light sync to diff sync * raise the max diff limit * add switcher of snap protocol * fix test case * make commit concurrently * remove peer for diff cache when peer closed * consensus tuning * add test code * remove extra message * fix testcase and lint make diff block configable wait code write fix testcase resolve comments resolve comment * resolve comments * resolve comments * resolve comment * fix mistake Co-authored-by: kyrie-yl <83150977+kyrie-yl@users.noreply.github.com> Co-authored-by: kyrie-yl --- cmd/evm/internal/t8ntool/execution.go | 4 +- cmd/faucet/faucet.go | 2 +- cmd/geth/main.go | 4 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 41 ++- common/gopool/pool.go | 14 +- consensus/consensus.go | 1 + consensus/parlia/parlia.go | 16 + consensus/parlia/ramanujanfork.go | 2 +- consensus/parlia/snapshot.go | 2 +- core/block_validator.go | 3 - core/blockchain.go | 465 ++++++++++++++++++++++++-- core/blockchain_diff_test.go | 263 +++++++++++++++ core/blockchain_test.go | 6 +- core/chain_makers.go | 2 +- core/rawdb/accessors_chain.go | 38 +++ core/rawdb/database.go | 26 ++ core/rawdb/freezer_table_test.go | 60 +--- core/rawdb/schema.go | 8 + core/rawdb/table.go | 8 + core/state/database.go | 1 - core/state/journal.go | 2 +- core/state/snapshot/disklayer_test.go | 8 +- core/state/snapshot/iterator_test.go | 86 ++--- core/state/snapshot/snapshot.go | 33 +- core/state/snapshot/snapshot_test.go | 24 +- core/state/state_object.go | 13 +- core/state/state_test.go | 2 +- core/state/statedb.go | 309 +++++++++++++++-- core/state/statedb_test.go | 14 +- core/state/sync_test.go | 2 +- core/state_prefetcher.go | 9 - core/state_processor.go | 329 +++++++++++++++++- core/types.go | 2 +- core/types/block.go | 89 ++++- core/vm/contracts_lightclient_test.go | 2 +- core/vm/lightclient/types.go | 2 +- eth/backend.go | 18 +- eth/downloader/downloader.go | 56 +++- eth/downloader/downloader_test.go | 4 +- eth/ethconfig/config.go | 12 +- eth/ethconfig/gen_config.go | 18 + eth/fetcher/block_fetcher.go | 13 +- eth/fetcher/block_fetcher_test.go | 36 +- eth/handler.go | 41 ++- eth/handler_diff.go | 87 +++++ eth/handler_diff_test.go | 203 +++++++++++ eth/handler_eth.go | 13 +- eth/peer.go | 22 ++ eth/peerset.go | 87 ++++- eth/protocols/diff/discovery.go | 32 ++ eth/protocols/diff/handler.go | 180 ++++++++++ eth/protocols/diff/handler_test.go | 192 +++++++++++ eth/protocols/diff/handshake.go | 82 +++++ eth/protocols/diff/peer.go | 107 ++++++ eth/protocols/diff/peer_test.go | 61 ++++ eth/protocols/diff/protocol.go | 122 +++++++ eth/protocols/diff/protocol_test.go | 131 ++++++++ eth/protocols/diff/tracker.go | 161 +++++++++ eth/state_accessor.go | 4 +- eth/tracers/tracers_test.go | 2 +- ethclient/ethclient_test.go | 70 +--- ethdb/database.go | 6 + les/fetcher.go | 4 +- les/peer.go | 2 +- light/trie.go | 12 +- miner/worker.go | 19 -- node/config.go | 3 + node/node.go | 41 +++ rlp/typecache.go | 10 - tests/state_test_util.go | 2 +- 71 files changed, 3352 insertions(+), 394 deletions(-) create mode 100644 core/blockchain_diff_test.go create mode 100644 eth/handler_diff.go create mode 100644 eth/handler_diff_test.go create mode 100644 eth/protocols/diff/discovery.go create mode 100644 eth/protocols/diff/handler.go create mode 100644 eth/protocols/diff/handler_test.go create mode 100644 eth/protocols/diff/handshake.go create mode 100644 eth/protocols/diff/peer.go create mode 100644 eth/protocols/diff/peer_test.go create mode 100644 eth/protocols/diff/protocol.go create mode 100644 eth/protocols/diff/protocol_test.go create mode 100644 eth/protocols/diff/tracker.go diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index c3f1b16efc..0471037ed8 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -223,7 +223,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, statedb.AddBalance(pre.Env.Coinbase, minerReward) } // Commit block - root, err := statedb.Commit(chainConfig.IsEIP158(vmContext.BlockNumber)) + root, _, err := statedb.Commit(chainConfig.IsEIP158(vmContext.BlockNumber)) if err != nil { fmt.Fprintf(os.Stderr, "Could not commit state: %v", err) return nil, nil, NewError(ErrorEVM, fmt.Errorf("could not commit state: %v", err)) @@ -252,7 +252,7 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc) *state.StateDB } } // Commit and re-open to start with a clean state. - root, _ := statedb.Commit(false) + root, _, _ := statedb.Commit(false) statedb, _ = state.New(root, sdb, nil) return statedb } diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 5aab7598a1..500a1e920f 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -139,7 +139,7 @@ func main() { log.Crit("Length of bep2eContracts, bep2eSymbols, bep2eAmounts mismatch") } - bep2eInfos := make(map[string]bep2eInfo, 0) + bep2eInfos := make(map[string]bep2eInfo, len(symbols)) for idx, s := range symbols { n, ok := big.NewInt(0).SetString(bep2eNumAmounts[idx], 10) if !ok { diff --git a/cmd/geth/main.go b/cmd/geth/main.go index d383d99b7e..8be8d20bf4 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -65,6 +65,8 @@ var ( utils.ExternalSignerFlag, utils.NoUSBFlag, utils.DirectBroadcastFlag, + utils.DisableSnapProtocolFlag, + utils.DiffSyncFlag, utils.RangeLimitFlag, utils.USBFlag, utils.SmartCardDaemonPathFlag, @@ -114,6 +116,8 @@ var ( utils.CacheGCFlag, utils.CacheSnapshotFlag, utils.CachePreimagesFlag, + utils.PersistDiffFlag, + utils.DiffBlockFlag, utils.ListenPortFlag, utils.MaxPeersFlag, utils.MaxPendingPeersFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 1450c29e84..2a208c827b 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -40,6 +40,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.KeyStoreDirFlag, utils.NoUSBFlag, utils.DirectBroadcastFlag, + utils.DisableSnapProtocolFlag, utils.RangeLimitFlag, utils.SmartCardDaemonPathFlag, utils.NetworkIdFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index b4d93fc1f1..c4dbd84911 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -117,6 +117,15 @@ var ( Name: "directbroadcast", Usage: "Enable directly broadcast mined block to all peers", } + DisableSnapProtocolFlag = cli.BoolFlag{ + Name: "disablesnapprotocol", + Usage: "Disable snap protocol", + } + DiffSyncFlag = cli.BoolFlag{ + Name: "diffsync", + Usage: "Enable diffy sync, Please note that enable diffsync will improve the syncing speed, " + + "but will degrade the security to light client level", + } RangeLimitFlag = cli.BoolFlag{ Name: "rangelimit", Usage: "Enable 5000 blocks limit for range query", @@ -125,6 +134,10 @@ var ( Name: "datadir.ancient", Usage: "Data directory for ancient chain segments (default = inside chaindata)", } + DiffFlag = DirectoryFlag{ + Name: "datadir.diff", + Usage: "Data directory for difflayer segments (default = inside chaindata)", + } MinFreeDiskSpaceFlag = DirectoryFlag{ Name: "datadir.minfreedisk", Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)", @@ -425,6 +438,15 @@ var ( Name: "cache.preimages", Usage: "Enable recording the SHA3/keccak preimages of trie keys", } + PersistDiffFlag = cli.BoolFlag{ + Name: "persistdiff", + Usage: "Enable persistence of the diff layer", + } + DiffBlockFlag = cli.Uint64Flag{ + Name: "diffblock", + Usage: "The number of blocks should be persisted in db (default = 864000 )", + Value: uint64(864000), + } // Miner settings MiningEnabledFlag = cli.BoolFlag{ Name: "mine", @@ -1271,6 +1293,9 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { if ctx.GlobalIsSet(DirectBroadcastFlag.Name) { cfg.DirectBroadcast = ctx.GlobalBool(DirectBroadcastFlag.Name) } + if ctx.GlobalIsSet(DisableSnapProtocolFlag.Name) { + cfg.DisableSnapProtocol = ctx.GlobalBool(DisableSnapProtocolFlag.Name) + } if ctx.GlobalIsSet(RangeLimitFlag.Name) { cfg.RangeLimit = ctx.GlobalBool(RangeLimitFlag.Name) } @@ -1564,7 +1589,15 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.GlobalIsSet(AncientFlag.Name) { cfg.DatabaseFreezer = ctx.GlobalString(AncientFlag.Name) } - + if ctx.GlobalIsSet(DiffFlag.Name) { + cfg.DatabaseDiff = ctx.GlobalString(DiffFlag.Name) + } + if ctx.GlobalIsSet(PersistDiffFlag.Name) { + cfg.PersistDiff = ctx.GlobalBool(PersistDiffFlag.Name) + } + if ctx.GlobalIsSet(DiffBlockFlag.Name) { + cfg.DiffBlock = ctx.GlobalUint64(DiffBlockFlag.Name) + } if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) } @@ -1574,6 +1607,12 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.GlobalIsSet(DirectBroadcastFlag.Name) { cfg.DirectBroadcast = ctx.GlobalBool(DirectBroadcastFlag.Name) } + if ctx.GlobalIsSet(DisableSnapProtocolFlag.Name) { + cfg.DisableSnapProtocol = ctx.GlobalBool(DisableSnapProtocolFlag.Name) + } + if ctx.GlobalIsSet(DiffSyncFlag.Name) { + cfg.DiffSync = ctx.GlobalBool(DiffSyncFlag.Name) + } if ctx.GlobalIsSet(RangeLimitFlag.Name) { cfg.RangeLimit = ctx.GlobalBool(RangeLimitFlag.Name) } diff --git a/common/gopool/pool.go b/common/gopool/pool.go index b4fc1c459d..bfcc618e46 100644 --- a/common/gopool/pool.go +++ b/common/gopool/pool.go @@ -1,6 +1,7 @@ package gopool import ( + "runtime" "time" "github.com/panjf2000/ants/v2" @@ -8,7 +9,8 @@ import ( var ( // Init a instance pool when importing ants. - defaultPool, _ = ants.NewPool(ants.DefaultAntsPoolSize, ants.WithExpiryDuration(10*time.Second)) + defaultPool, _ = ants.NewPool(ants.DefaultAntsPoolSize, ants.WithExpiryDuration(10*time.Second)) + minNumberPerTask = 5 ) // Logger is used for logging formatted messages. @@ -46,3 +48,13 @@ func Release() { func Reboot() { defaultPool.Reboot() } + +func Threads(tasks int) int { + threads := tasks / minNumberPerTask + if threads > runtime.NumCPU() { + threads = runtime.NumCPU() + } else if threads == 0 { + threads = 1 + } + return threads +} diff --git a/consensus/consensus.go b/consensus/consensus.go index fc161390f9..04a9924383 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -141,4 +141,5 @@ type PoSA interface { IsSystemContract(to *common.Address) bool EnoughDistance(chain ChainReader, header *types.Header) bool IsLocalBlock(header *types.Header) bool + AllowLightProcess(chain ChainReader, currentHeader *types.Header) bool } diff --git a/consensus/parlia/parlia.go b/consensus/parlia/parlia.go index 14304fe2bc..729c6f3730 100644 --- a/consensus/parlia/parlia.go +++ b/consensus/parlia/parlia.go @@ -799,6 +799,11 @@ func (p *Parlia) Delay(chain consensus.ChainReader, header *types.Header) *time. return nil } delay := p.delayForRamanujanFork(snap, header) + // The blocking time should be no more than half of period + half := time.Duration(p.config.Period) * time.Second / 2 + if delay > half { + delay = half + } return &delay } @@ -882,6 +887,17 @@ func (p *Parlia) EnoughDistance(chain consensus.ChainReader, header *types.Heade return snap.enoughDistance(p.val, header) } +func (p *Parlia) AllowLightProcess(chain consensus.ChainReader, currentHeader *types.Header) bool { + snap, err := p.snapshot(chain, currentHeader.Number.Uint64()-1, currentHeader.ParentHash, nil) + if err != nil { + return true + } + + idx := snap.indexOfVal(p.val) + // validator is not allowed to diff sync + return idx < 0 +} + func (p *Parlia) IsLocalBlock(header *types.Header) bool { return p.val == header.Coinbase } diff --git a/consensus/parlia/ramanujanfork.go b/consensus/parlia/ramanujanfork.go index 903c678f42..9b702ca6c5 100644 --- a/consensus/parlia/ramanujanfork.go +++ b/consensus/parlia/ramanujanfork.go @@ -21,7 +21,7 @@ func (p *Parlia) delayForRamanujanFork(snap *Snapshot, header *types.Header) tim if header.Difficulty.Cmp(diffNoTurn) == 0 { // It's not our turn explicitly to sign, delay it a bit wiggle := time.Duration(len(snap.Validators)/2+1) * wiggleTimeBeforeFork - delay += time.Duration(fixedBackOffTimeBeforeFork) + time.Duration(rand.Int63n(int64(wiggle))) + delay += fixedBackOffTimeBeforeFork + time.Duration(rand.Int63n(int64(wiggle))) } return delay } diff --git a/consensus/parlia/snapshot.go b/consensus/parlia/snapshot.go index dc83d92c8d..95aaf861de 100644 --- a/consensus/parlia/snapshot.go +++ b/consensus/parlia/snapshot.go @@ -256,7 +256,7 @@ func (s *Snapshot) enoughDistance(validator common.Address, header *types.Header if validator == header.Coinbase { return false } - offset := (int64(s.Number) + 1) % int64(validatorNum) + offset := (int64(s.Number) + 1) % validatorNum if int64(idx) >= offset { return int64(idx)-offset >= validatorNum-2 } else { diff --git a/core/block_validator.go b/core/block_validator.go index 92be755199..7ef440b4b7 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -17,9 +17,7 @@ package core import ( - "encoding/json" "fmt" - "os" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core/state" @@ -133,7 +131,6 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD }, func() error { if root := statedb.IntermediateRoot(v.config.IsEIP158(header.Number)); header.Root != root { - statedb.IterativeDump(true, true, true, json.NewEncoder(os.Stdout)) return fmt.Errorf("invalid merkle root (remote: %x local: %x)", header.Root, root) } else { return nil diff --git a/core/blockchain.go b/core/blockchain.go index be0b0f04ae..3cfa21654f 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -80,14 +80,22 @@ var ( ) const ( - bodyCacheLimit = 256 - blockCacheLimit = 256 - receiptsCacheLimit = 10000 - txLookupCacheLimit = 1024 - maxFutureBlocks = 256 - maxTimeFutureBlocks = 30 - badBlockLimit = 10 - maxBeyondBlocks = 2048 + bodyCacheLimit = 256 + blockCacheLimit = 256 + diffLayerCacheLimit = 1024 + diffLayerRLPCacheLimit = 256 + receiptsCacheLimit = 10000 + txLookupCacheLimit = 1024 + maxFutureBlocks = 256 + maxTimeFutureBlocks = 30 + maxBeyondBlocks = 2048 + + diffLayerFreezerRecheckInterval = 3 * time.Second + diffLayerPruneRecheckInterval = 1 * time.Second // The interval to prune unverified diff layers + maxDiffQueueDist = 2048 // Maximum allowed distance from the chain head to queue diffLayers + maxDiffLimit = 2048 // Maximum number of unique diff layers a peer may have responded + maxDiffForkDist = 11 // Maximum allowed backward distance from the chain head + maxDiffLimitForBroadcast = 128 // Maximum number of unique diff layers a peer may have broadcasted // BlockChainVersion ensures that an incompatible database forces a resync from scratch. // @@ -131,6 +139,11 @@ type CacheConfig struct { SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it } +// To avoid cycle import +type PeerIDer interface { + ID() string +} + // defaultCacheConfig are the default caching values if none are specified by the // user (also used during testing). var defaultCacheConfig = &CacheConfig{ @@ -142,6 +155,8 @@ var defaultCacheConfig = &CacheConfig{ SnapshotWait: true, } +type BlockChainOption func(*BlockChain) *BlockChain + // BlockChain represents the canonical chain given a database with a genesis // block. The Blockchain manages chain imports, reverts, chain reorganisations. // @@ -196,6 +211,21 @@ type BlockChain struct { txLookupCache *lru.Cache // Cache for the most recent transaction lookup data. futureBlocks *lru.Cache // future blocks are blocks added for later processing + // trusted diff layers + diffLayerCache *lru.Cache // Cache for the diffLayers + diffLayerRLPCache *lru.Cache // Cache for the rlp encoded diffLayers + diffQueue *prque.Prque // A Priority queue to store recent diff layer + diffQueueBuffer chan *types.DiffLayer + diffLayerFreezerBlockLimit uint64 + + // untrusted diff layers + diffMux sync.RWMutex + blockHashToDiffLayers map[common.Hash]map[common.Hash]*types.DiffLayer // map[blockHash] map[DiffHash]Diff + diffHashToBlockHash map[common.Hash]common.Hash // map[diffHash]blockHash + diffHashToPeers map[common.Hash]map[string]struct{} // map[diffHash]map[pid] + diffNumToBlockHashes map[uint64]map[common.Hash]struct{} // map[number]map[blockHash] + diffPeersToDiffHashes map[string]map[common.Hash]struct{} // map[pid]map[diffHash] + quit chan struct{} // blockchain quit channel wg sync.WaitGroup // chain processing wait group for shutting down running int32 // 0 if chain is running, 1 when stopped @@ -213,12 +243,15 @@ type BlockChain struct { // NewBlockChain returns a fully initialised block chain using information // available in the database. It initialises the default Ethereum Validator and // Processor. -func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool, txLookupLimit *uint64) (*BlockChain, error) { +func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, + vmConfig vm.Config, shouldPreserve func(block *types.Block) bool, txLookupLimit *uint64, + options ...BlockChainOption) (*BlockChain, error) { if cacheConfig == nil { cacheConfig = defaultCacheConfig } if cacheConfig.TriesInMemory != 128 { - log.Warn("TriesInMemory isn't the default value(128), you need specify exact same TriesInMemory when prune data", "triesInMemory", cacheConfig.TriesInMemory) + log.Warn("TriesInMemory isn't the default value(128), you need specify exact same TriesInMemory when prune data", + "triesInMemory", cacheConfig.TriesInMemory) } bodyCache, _ := lru.New(bodyCacheLimit) bodyRLPCache, _ := lru.New(bodyCacheLimit) @@ -226,6 +259,8 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par blockCache, _ := lru.New(blockCacheLimit) txLookupCache, _ := lru.New(txLookupCacheLimit) futureBlocks, _ := lru.New(maxFutureBlocks) + diffLayerCache, _ := lru.New(diffLayerCacheLimit) + diffLayerRLPCache, _ := lru.New(diffLayerRLPCacheLimit) bc := &BlockChain{ chainConfig: chainConfig, @@ -237,17 +272,26 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par Journal: cacheConfig.TrieCleanJournal, Preimages: cacheConfig.Preimages, }), - triesInMemory: cacheConfig.TriesInMemory, - quit: make(chan struct{}), - shouldPreserve: shouldPreserve, - bodyCache: bodyCache, - bodyRLPCache: bodyRLPCache, - receiptsCache: receiptsCache, - blockCache: blockCache, - txLookupCache: txLookupCache, - futureBlocks: futureBlocks, - engine: engine, - vmConfig: vmConfig, + triesInMemory: cacheConfig.TriesInMemory, + quit: make(chan struct{}), + shouldPreserve: shouldPreserve, + bodyCache: bodyCache, + bodyRLPCache: bodyRLPCache, + receiptsCache: receiptsCache, + blockCache: blockCache, + diffLayerCache: diffLayerCache, + diffLayerRLPCache: diffLayerRLPCache, + txLookupCache: txLookupCache, + futureBlocks: futureBlocks, + engine: engine, + vmConfig: vmConfig, + diffQueue: prque.New(nil), + diffQueueBuffer: make(chan *types.DiffLayer), + blockHashToDiffLayers: make(map[common.Hash]map[common.Hash]*types.DiffLayer), + diffHashToBlockHash: make(map[common.Hash]common.Hash), + diffHashToPeers: make(map[common.Hash]map[string]struct{}), + diffNumToBlockHashes: make(map[uint64]map[common.Hash]struct{}), + diffPeersToDiffHashes: make(map[string]map[common.Hash]struct{}), } bc.validator = NewBlockValidator(chainConfig, bc, engine) bc.processor = NewStateProcessor(chainConfig, bc, engine) @@ -375,6 +419,10 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par } bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, int(bc.cacheConfig.TriesInMemory), head.Root(), !bc.cacheConfig.SnapshotWait, true, recover) } + // do options before start any routine + for _, option := range options { + bc = option(bc) + } // Take ownership of this particular state go bc.update() if txLookupLimit != nil { @@ -396,6 +444,12 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par triedb.SaveCachePeriodically(bc.cacheConfig.TrieCleanJournal, bc.cacheConfig.TrieCleanRejournal, bc.quit) }() } + // Need persist and prune diff layer + if bc.db.DiffStore() != nil { + go bc.trustedDiffLayerLoop() + } + go bc.untrustedDiffLayerPruneLoop() + return bc, nil } @@ -404,11 +458,19 @@ func (bc *BlockChain) GetVMConfig() *vm.Config { return &bc.vmConfig } -func (bc *BlockChain) CacheReceipts(hash common.Hash, receipts types.Receipts) { +func (bc *BlockChain) cacheReceipts(hash common.Hash, receipts types.Receipts) { bc.receiptsCache.Add(hash, receipts) } -func (bc *BlockChain) CacheBlock(hash common.Hash, block *types.Block) { +func (bc *BlockChain) cacheDiffLayer(diffLayer *types.DiffLayer) { + bc.diffLayerCache.Add(diffLayer.BlockHash, diffLayer) + if bc.db.DiffStore() != nil { + // push to priority queue before persisting + bc.diffQueueBuffer <- diffLayer + } +} + +func (bc *BlockChain) cacheBlock(hash common.Hash, block *types.Block) { bc.blockCache.Add(hash, block) } @@ -873,6 +935,45 @@ func (bc *BlockChain) GetBodyRLP(hash common.Hash) rlp.RawValue { return body } +// GetDiffLayerRLP retrieves a diff layer in RLP encoding from the cache or database by blockHash +func (bc *BlockChain) GetDiffLayerRLP(blockHash common.Hash) rlp.RawValue { + // Short circuit if the diffLayer's already in the cache, retrieve otherwise + if cached, ok := bc.diffLayerRLPCache.Get(blockHash); ok { + return cached.(rlp.RawValue) + } + if cached, ok := bc.diffLayerCache.Get(blockHash); ok { + diff := cached.(*types.DiffLayer) + bz, err := rlp.EncodeToBytes(diff) + if err != nil { + return nil + } + bc.diffLayerRLPCache.Add(blockHash, rlp.RawValue(bz)) + return bz + } + + // fallback to untrusted sources. + diff := bc.GetUnTrustedDiffLayer(blockHash, "") + if diff != nil { + bz, err := rlp.EncodeToBytes(diff) + if err != nil { + return nil + } + // No need to cache untrusted data + return bz + } + + // fallback to disk + diffStore := bc.db.DiffStore() + if diffStore == nil { + return nil + } + rawData := rawdb.ReadDiffLayerRLP(diffStore, blockHash) + if len(rawData) != 0 { + bc.diffLayerRLPCache.Add(blockHash, rawData) + } + return rawData +} + // HasBlock checks if a block is fully present in the database or not. func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool { if bc.blockCache.Contains(hash) { @@ -1506,10 +1607,19 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. wg.Done() }() // Commit all cached state changes into underlying memory database. - root, err := state.Commit(bc.chainConfig.IsEIP158(block.Number())) + root, diffLayer, err := state.Commit(bc.chainConfig.IsEIP158(block.Number())) if err != nil { return NonStatTy, err } + + // Ensure no empty block body + if diffLayer != nil && block.Header().TxHash != types.EmptyRootHash { + // Filling necessary field + diffLayer.Receipts = receipts + diffLayer.BlockHash = block.Hash() + diffLayer.Number = block.NumberU64() + bc.cacheDiffLayer(diffLayer) + } triedb := bc.stateCache.TrieDB() // If we're running an archive node, always flush @@ -1885,18 +1995,15 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er } // Enable prefetching to pull in trie node paths while processing transactions statedb.StartPrefetcher("chain") - activeState = statedb - statedb.TryPreload(block, signer) //Process block using the parent state as reference point substart := time.Now() - receipts, logs, usedGas, err := bc.processor.Process(block, statedb, bc.vmConfig) + statedb, receipts, logs, usedGas, err := bc.processor.Process(block, statedb, bc.vmConfig) + activeState = statedb if err != nil { bc.reportBlock(block, receipts, err) return it.index, err } - bc.CacheReceipts(block.Hash(), receipts) - bc.CacheBlock(block.Hash(), block) // Update the metrics touched during block processing accountReadTimer.Update(statedb.AccountReads) // Account reads are complete, we can mark them storageReadTimer.Update(statedb.StorageReads) // Storage reads are complete, we can mark them @@ -1904,18 +2011,20 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er storageUpdateTimer.Update(statedb.StorageUpdates) // Storage updates are complete, we can mark them snapshotAccountReadTimer.Update(statedb.SnapshotAccountReads) // Account reads are complete, we can mark them snapshotStorageReadTimer.Update(statedb.SnapshotStorageReads) // Storage reads are complete, we can mark them - trieproc := statedb.SnapshotAccountReads + statedb.AccountReads + statedb.AccountUpdates - trieproc += statedb.SnapshotStorageReads + statedb.StorageReads + statedb.StorageUpdates blockExecutionTimer.Update(time.Since(substart)) // Validate the state using the default validator substart = time.Now() - if err := bc.validator.ValidateState(block, statedb, receipts, usedGas); err != nil { - bc.reportBlock(block, receipts, err) - log.Error("validate state failed", "error", err) - return it.index, err + if !statedb.IsLightProcessed() { + if err := bc.validator.ValidateState(block, statedb, receipts, usedGas); err != nil { + log.Error("validate state failed", "error", err) + bc.reportBlock(block, receipts, err) + return it.index, err + } } + bc.cacheReceipts(block.Hash(), receipts) + bc.cacheBlock(block.Hash(), block) proctime := time.Since(start) // Update the metrics touched during block validation @@ -2292,6 +2401,279 @@ func (bc *BlockChain) update() { } } +func (bc *BlockChain) trustedDiffLayerLoop() { + recheck := time.Tick(diffLayerFreezerRecheckInterval) + bc.wg.Add(1) + defer bc.wg.Done() + for { + select { + case diff := <-bc.diffQueueBuffer: + bc.diffQueue.Push(diff, -(int64(diff.Number))) + case <-bc.quit: + // Persist all diffLayers when shutdown, it will introduce redundant storage, but it is acceptable. + // If the client been ungracefully shutdown, it will missing all cached diff layers, it is acceptable as well. + var batch ethdb.Batch + for !bc.diffQueue.Empty() { + diff, _ := bc.diffQueue.Pop() + diffLayer := diff.(*types.DiffLayer) + if batch == nil { + batch = bc.db.DiffStore().NewBatch() + } + rawdb.WriteDiffLayer(batch, diffLayer.BlockHash, diffLayer) + if batch.ValueSize() > ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + log.Error("Failed to write diff layer", "err", err) + return + } + batch.Reset() + } + } + if batch != nil { + // flush data + if err := batch.Write(); err != nil { + log.Error("Failed to write diff layer", "err", err) + return + } + batch.Reset() + } + return + case <-recheck: + currentHeight := bc.CurrentBlock().NumberU64() + var batch ethdb.Batch + for !bc.diffQueue.Empty() { + diff, prio := bc.diffQueue.Pop() + diffLayer := diff.(*types.DiffLayer) + + // if the block old enough + if int64(currentHeight)+prio >= int64(bc.triesInMemory) { + canonicalHash := bc.GetCanonicalHash(uint64(-prio)) + // on the canonical chain + if canonicalHash == diffLayer.BlockHash { + if batch == nil { + batch = bc.db.DiffStore().NewBatch() + } + rawdb.WriteDiffLayer(batch, diffLayer.BlockHash, diffLayer) + staleHash := bc.GetCanonicalHash(uint64(-prio) - bc.diffLayerFreezerBlockLimit) + rawdb.DeleteDiffLayer(batch, staleHash) + } + } else { + bc.diffQueue.Push(diffLayer, prio) + break + } + if batch != nil && batch.ValueSize() > ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + panic(fmt.Sprintf("Failed to write diff layer, error %v", err)) + } + batch.Reset() + } + } + if batch != nil { + if err := batch.Write(); err != nil { + panic(fmt.Sprintf("Failed to write diff layer, error %v", err)) + } + batch.Reset() + } + } + } +} + +func (bc *BlockChain) GetUnTrustedDiffLayer(blockHash common.Hash, pid string) *types.DiffLayer { + bc.diffMux.RLock() + defer bc.diffMux.RUnlock() + if diffs, exist := bc.blockHashToDiffLayers[blockHash]; exist && len(diffs) != 0 { + if len(diffs) == 1 { + // return the only one diff layer + for _, diff := range diffs { + return diff + } + } + // pick the one from exact same peer if we know where the block comes from + if pid != "" { + if diffHashes, exist := bc.diffPeersToDiffHashes[pid]; exist { + for diff := range diffs { + if _, overlap := diffHashes[diff]; overlap { + return bc.blockHashToDiffLayers[blockHash][diff] + } + } + } + } + // Do not find overlap, do random pick + for _, diff := range diffs { + return diff + } + } + return nil +} + +func (bc *BlockChain) removeDiffLayers(diffHash common.Hash) { + bc.diffMux.Lock() + defer bc.diffMux.Unlock() + + // Untrusted peers + pids := bc.diffHashToPeers[diffHash] + invalidDiffHashes := make(map[common.Hash]struct{}) + for pid := range pids { + invaliDiffHashesPeer := bc.diffPeersToDiffHashes[pid] + for invaliDiffHash := range invaliDiffHashesPeer { + invalidDiffHashes[invaliDiffHash] = struct{}{} + } + delete(bc.diffPeersToDiffHashes, pid) + } + for invalidDiffHash := range invalidDiffHashes { + delete(bc.diffHashToPeers, invalidDiffHash) + affectedBlockHash := bc.diffHashToBlockHash[invalidDiffHash] + if diffs, exist := bc.blockHashToDiffLayers[affectedBlockHash]; exist { + delete(diffs, invalidDiffHash) + if len(diffs) == 0 { + delete(bc.blockHashToDiffLayers, affectedBlockHash) + } + } + delete(bc.diffHashToBlockHash, invalidDiffHash) + } +} + +func (bc *BlockChain) RemoveDiffPeer(pid string) { + bc.diffMux.Lock() + defer bc.diffMux.Unlock() + if invaliDiffHashes := bc.diffPeersToDiffHashes[pid]; invaliDiffHashes != nil { + for invalidDiffHash := range invaliDiffHashes { + lastDiffHash := false + if peers, ok := bc.diffHashToPeers[invalidDiffHash]; ok { + delete(peers, pid) + if len(peers) == 0 { + lastDiffHash = true + delete(bc.diffHashToPeers, invalidDiffHash) + } + } + if lastDiffHash { + affectedBlockHash := bc.diffHashToBlockHash[invalidDiffHash] + if diffs, exist := bc.blockHashToDiffLayers[affectedBlockHash]; exist { + delete(diffs, invalidDiffHash) + if len(diffs) == 0 { + delete(bc.blockHashToDiffLayers, affectedBlockHash) + } + } + delete(bc.diffHashToBlockHash, invalidDiffHash) + } + } + delete(bc.diffPeersToDiffHashes, pid) + } +} + +func (bc *BlockChain) untrustedDiffLayerPruneLoop() { + recheck := time.NewTicker(diffLayerPruneRecheckInterval) + bc.wg.Add(1) + defer func() { + bc.wg.Done() + recheck.Stop() + }() + for { + select { + case <-bc.quit: + return + case <-recheck.C: + bc.pruneDiffLayer() + } + } +} + +func (bc *BlockChain) pruneDiffLayer() { + currentHeight := bc.CurrentBlock().NumberU64() + bc.diffMux.Lock() + defer bc.diffMux.Unlock() + sortNumbers := make([]uint64, 0, len(bc.diffNumToBlockHashes)) + for number := range bc.diffNumToBlockHashes { + sortNumbers = append(sortNumbers, number) + } + sort.Slice(sortNumbers, func(i, j int) bool { + return sortNumbers[i] <= sortNumbers[j] + }) + staleBlockHashes := make(map[common.Hash]struct{}) + for _, number := range sortNumbers { + if number >= currentHeight-maxDiffForkDist { + break + } + affectedHashes := bc.diffNumToBlockHashes[number] + if affectedHashes != nil { + for affectedHash := range affectedHashes { + staleBlockHashes[affectedHash] = struct{}{} + } + delete(bc.diffNumToBlockHashes, number) + } + } + staleDiffHashes := make(map[common.Hash]struct{}) + for blockHash := range staleBlockHashes { + if diffHashes, exist := bc.blockHashToDiffLayers[blockHash]; exist { + for diffHash := range diffHashes { + staleDiffHashes[diffHash] = struct{}{} + delete(bc.diffHashToBlockHash, diffHash) + delete(bc.diffHashToPeers, diffHash) + } + } + delete(bc.blockHashToDiffLayers, blockHash) + } + for diffHash := range staleDiffHashes { + for p, diffHashes := range bc.diffPeersToDiffHashes { + delete(diffHashes, diffHash) + if len(diffHashes) == 0 { + delete(bc.diffPeersToDiffHashes, p) + } + } + } +} + +// Process received diff layers +func (bc *BlockChain) HandleDiffLayer(diffLayer *types.DiffLayer, pid string, fulfilled bool) error { + // Basic check + currentHeight := bc.CurrentBlock().NumberU64() + if diffLayer.Number > currentHeight && diffLayer.Number-currentHeight > maxDiffQueueDist { + log.Error("diff layers too new from current", "pid", pid) + return nil + } + if diffLayer.Number < currentHeight && currentHeight-diffLayer.Number > maxDiffForkDist { + log.Error("diff layers too old from current", "pid", pid) + return nil + } + + bc.diffMux.Lock() + defer bc.diffMux.Unlock() + + if !fulfilled && len(bc.diffPeersToDiffHashes[pid]) > maxDiffLimitForBroadcast { + log.Error("too many accumulated diffLayers", "pid", pid) + return nil + } + + if len(bc.diffPeersToDiffHashes[pid]) > maxDiffLimit { + log.Error("too many accumulated diffLayers", "pid", pid) + return nil + } + if _, exist := bc.diffPeersToDiffHashes[pid]; exist { + if _, alreadyHas := bc.diffPeersToDiffHashes[pid][diffLayer.DiffHash]; alreadyHas { + return nil + } + } else { + bc.diffPeersToDiffHashes[pid] = make(map[common.Hash]struct{}) + } + bc.diffPeersToDiffHashes[pid][diffLayer.DiffHash] = struct{}{} + if _, exist := bc.diffNumToBlockHashes[diffLayer.Number]; !exist { + bc.diffNumToBlockHashes[diffLayer.Number] = make(map[common.Hash]struct{}) + } + bc.diffNumToBlockHashes[diffLayer.Number][diffLayer.BlockHash] = struct{}{} + + if _, exist := bc.diffHashToPeers[diffLayer.DiffHash]; !exist { + bc.diffHashToPeers[diffLayer.DiffHash] = make(map[string]struct{}) + } + bc.diffHashToPeers[diffLayer.DiffHash][pid] = struct{}{} + + if _, exist := bc.blockHashToDiffLayers[diffLayer.BlockHash]; !exist { + bc.blockHashToDiffLayers[diffLayer.BlockHash] = make(map[common.Hash]*types.DiffLayer) + } + bc.blockHashToDiffLayers[diffLayer.BlockHash][diffLayer.DiffHash] = diffLayer + bc.diffHashToBlockHash[diffLayer.DiffHash] = diffLayer.BlockHash + + return nil +} + // maintainTxIndex is responsible for the construction and deletion of the // transaction index. // @@ -2541,3 +2923,16 @@ func (bc *BlockChain) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscript func (bc *BlockChain) SubscribeBlockProcessingEvent(ch chan<- bool) event.Subscription { return bc.scope.Track(bc.blockProcFeed.Subscribe(ch)) } + +// Options +func EnableLightProcessor(bc *BlockChain) *BlockChain { + bc.processor = NewLightStateProcessor(bc.Config(), bc, bc.engine) + return bc +} + +func EnablePersistDiff(limit uint64) BlockChainOption { + return func(chain *BlockChain) *BlockChain { + chain.diffLayerFreezerBlockLimit = limit + return chain + } +} diff --git a/core/blockchain_diff_test.go b/core/blockchain_diff_test.go new file mode 100644 index 0000000000..717b4039ed --- /dev/null +++ b/core/blockchain_diff_test.go @@ -0,0 +1,263 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Tests that abnormal program termination (i.e.crash) and restart doesn't leave +// the database in some strange state with gaps in the chain, nor with block data +// dangling in the future. + +package core + +import ( + "math/big" + "testing" + "time" + + "golang.org/x/crypto/sha3" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" +) + +var ( + // testKey is a private key to use for funding a tester account. + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + // testAddr is the Ethereum address of the tester account. + testAddr = crypto.PubkeyToAddress(testKey.PublicKey) +) + +// testBackend is a mock implementation of the live Ethereum message handler. Its +// purpose is to allow testing the request/reply workflows and wire serialization +// in the `eth` protocol without actually doing any data processing. +type testBackend struct { + db ethdb.Database + chain *BlockChain +} + +// newTestBackend creates an empty chain and wraps it into a mock backend. +func newTestBackend(blocks int, light bool) *testBackend { + return newTestBackendWithGenerator(blocks, light) +} + +// newTestBackend creates a chain with a number of explicitly defined blocks and +// wraps it into a mock backend. +func newTestBackendWithGenerator(blocks int, lightProcess bool) *testBackend { + signer := types.HomesteadSigner{} + // Create a database pre-initialize with a genesis block + db := rawdb.NewMemoryDatabase() + db.SetDiffStore(memorydb.New()) + (&Genesis{ + Config: params.TestChainConfig, + Alloc: GenesisAlloc{testAddr: {Balance: big.NewInt(100000000000000000)}}, + }).MustCommit(db) + + chain, _ := NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, EnablePersistDiff(860000)) + generator := func(i int, block *BlockGen) { + // The chain maker doesn't have access to a chain, so the difficulty will be + // lets unset (nil). Set it here to the correct value. + block.SetCoinbase(testAddr) + + // We want to simulate an empty middle block, having the same state as the + // first one. The last is needs a state change again to force a reorg. + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), common.Address{0x01}, big.NewInt(1), params.TxGas, big.NewInt(1), nil), signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(chain, tx) + } + bs, _ := GenerateChain(params.TestChainConfig, chain.Genesis(), ethash.NewFaker(), db, blocks, generator) + if _, err := chain.InsertChain(bs); err != nil { + panic(err) + } + if lightProcess { + EnableLightProcessor(chain) + } + + return &testBackend{ + db: db, + chain: chain, + } +} + +// close tears down the transaction pool and chain behind the mock backend. +func (b *testBackend) close() { + b.chain.Stop() +} + +func (b *testBackend) Chain() *BlockChain { return b.chain } + +func rawDataToDiffLayer(data rlp.RawValue) (*types.DiffLayer, error) { + var diff types.DiffLayer + hasher := sha3.NewLegacyKeccak256() + err := rlp.DecodeBytes(data, &diff) + if err != nil { + return nil, err + } + hasher.Write(data) + var diffHash common.Hash + hasher.Sum(diffHash[:0]) + diff.DiffHash = diffHash + hasher.Reset() + return &diff, nil +} + +func TestProcessDiffLayer(t *testing.T) { + t.Parallel() + + blockNum := 128 + fullBackend := newTestBackend(blockNum, false) + falseDiff := 5 + defer fullBackend.close() + + lightBackend := newTestBackend(0, true) + defer lightBackend.close() + for i := 1; i <= blockNum-falseDiff; i++ { + block := fullBackend.chain.GetBlockByNumber(uint64(i)) + if block == nil { + t.Fatal("block should not be nil") + } + blockHash := block.Hash() + rawDiff := fullBackend.chain.GetDiffLayerRLP(blockHash) + diff, err := rawDataToDiffLayer(rawDiff) + if err != nil { + t.Errorf("failed to decode rawdata %v", err) + } + lightBackend.Chain().HandleDiffLayer(diff, "testpid", true) + _, err = lightBackend.chain.insertChain([]*types.Block{block}, true) + if err != nil { + t.Errorf("failed to insert block %v", err) + } + } + currentBlock := lightBackend.chain.CurrentBlock() + nextBlock := fullBackend.chain.GetBlockByNumber(currentBlock.NumberU64() + 1) + rawDiff := fullBackend.chain.GetDiffLayerRLP(nextBlock.Hash()) + diff, _ := rawDataToDiffLayer(rawDiff) + latestAccount, _ := snapshot.FullAccount(diff.Accounts[0].Blob) + latestAccount.Balance = big.NewInt(0) + bz, _ := rlp.EncodeToBytes(&latestAccount) + diff.Accounts[0].Blob = bz + + lightBackend.Chain().HandleDiffLayer(diff, "testpid", true) + + _, err := lightBackend.chain.insertChain([]*types.Block{nextBlock}, true) + if err != nil { + t.Errorf("failed to process block %v", err) + } + + // the diff cache should be cleared + if len(lightBackend.chain.diffPeersToDiffHashes) != 0 { + t.Errorf("the size of diffPeersToDiffHashes should be 0, but get %d", len(lightBackend.chain.diffPeersToDiffHashes)) + } + if len(lightBackend.chain.diffHashToPeers) != 0 { + t.Errorf("the size of diffHashToPeers should be 0, but get %d", len(lightBackend.chain.diffHashToPeers)) + } + if len(lightBackend.chain.diffHashToBlockHash) != 0 { + t.Errorf("the size of diffHashToBlockHash should be 0, but get %d", len(lightBackend.chain.diffHashToBlockHash)) + } + if len(lightBackend.chain.blockHashToDiffLayers) != 0 { + t.Errorf("the size of blockHashToDiffLayers should be 0, but get %d", len(lightBackend.chain.blockHashToDiffLayers)) + } +} + +func TestFreezeDiffLayer(t *testing.T) { + t.Parallel() + + blockNum := 1024 + fullBackend := newTestBackend(blockNum, true) + defer fullBackend.close() + if fullBackend.chain.diffQueue.Size() != blockNum { + t.Errorf("size of diff queue is wrong, expected: %d, get: %d", blockNum, fullBackend.chain.diffQueue.Size()) + } + time.Sleep(diffLayerFreezerRecheckInterval + 1*time.Second) + if fullBackend.chain.diffQueue.Size() != int(fullBackend.chain.triesInMemory) { + t.Errorf("size of diff queue is wrong, expected: %d, get: %d", blockNum, fullBackend.chain.diffQueue.Size()) + } + + block := fullBackend.chain.GetBlockByNumber(uint64(blockNum / 2)) + diffStore := fullBackend.chain.db.DiffStore() + rawData := rawdb.ReadDiffLayerRLP(diffStore, block.Hash()) + if len(rawData) == 0 { + t.Error("do not find diff layer in db") + } +} + +func TestPruneDiffLayer(t *testing.T) { + t.Parallel() + + blockNum := 1024 + fullBackend := newTestBackend(blockNum, true) + defer fullBackend.close() + + anotherFullBackend := newTestBackend(2*blockNum, true) + defer anotherFullBackend.close() + + for num := uint64(1); num < uint64(blockNum); num++ { + header := fullBackend.chain.GetHeaderByNumber(num) + rawDiff := fullBackend.chain.GetDiffLayerRLP(header.Hash()) + diff, _ := rawDataToDiffLayer(rawDiff) + fullBackend.Chain().HandleDiffLayer(diff, "testpid1", true) + fullBackend.Chain().HandleDiffLayer(diff, "testpid2", true) + + } + fullBackend.chain.pruneDiffLayer() + if len(fullBackend.chain.diffNumToBlockHashes) != maxDiffForkDist { + t.Error("unexpected size of diffNumToBlockHashes") + } + if len(fullBackend.chain.diffPeersToDiffHashes) != 2 { + t.Error("unexpected size of diffPeersToDiffHashes") + } + if len(fullBackend.chain.blockHashToDiffLayers) != maxDiffForkDist { + t.Error("unexpected size of diffNumToBlockHashes") + } + if len(fullBackend.chain.diffHashToBlockHash) != maxDiffForkDist { + t.Error("unexpected size of diffHashToBlockHash") + } + if len(fullBackend.chain.diffHashToPeers) != maxDiffForkDist { + t.Error("unexpected size of diffHashToPeers") + } + + blocks := make([]*types.Block, 0, blockNum) + for i := blockNum + 1; i <= 2*blockNum; i++ { + b := anotherFullBackend.chain.GetBlockByNumber(uint64(i)) + blocks = append(blocks, b) + } + fullBackend.chain.insertChain(blocks, true) + fullBackend.chain.pruneDiffLayer() + if len(fullBackend.chain.diffNumToBlockHashes) != 0 { + t.Error("unexpected size of diffNumToBlockHashes") + } + if len(fullBackend.chain.diffPeersToDiffHashes) != 0 { + t.Error("unexpected size of diffPeersToDiffHashes") + } + if len(fullBackend.chain.blockHashToDiffLayers) != 0 { + t.Error("unexpected size of diffNumToBlockHashes") + } + if len(fullBackend.chain.diffHashToBlockHash) != 0 { + t.Error("unexpected size of diffHashToBlockHash") + } + if len(fullBackend.chain.diffHashToPeers) != 0 { + t.Error("unexpected size of diffHashToPeers") + } + +} diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 9395a379f5..4314bd45a9 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -151,7 +151,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { if err != nil { return err } - receipts, _, usedGas, err := blockchain.processor.Process(block, statedb, vm.Config{}) + statedb, receipts, _, usedGas, err := blockchain.processor.Process(block, statedb, vm.Config{}) if err != nil { blockchain.reportBlock(block, receipts, err) return err @@ -1769,7 +1769,7 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon } lastPrunedIndex := len(blocks) - TestTriesInMemory - 1 - lastPrunedBlock := blocks[lastPrunedIndex] + lastPrunedBlock := blocks[lastPrunedIndex-1] firstNonPrunedBlock := blocks[len(blocks)-TestTriesInMemory] // Verify pruning of lastPrunedBlock @@ -2420,7 +2420,7 @@ func TestSideImportPrunedBlocks(t *testing.T) { } lastPrunedIndex := len(blocks) - TestTriesInMemory - 1 - lastPrunedBlock := blocks[lastPrunedIndex] + lastPrunedBlock := blocks[lastPrunedIndex-1] // Verify pruning of lastPrunedBlock if chain.HasBlockAndState(lastPrunedBlock.Hash(), lastPrunedBlock.NumberU64()) { diff --git a/core/chain_makers.go b/core/chain_makers.go index 6cb74d51be..9ded01a433 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -223,7 +223,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse block, _, _ := b.engine.FinalizeAndAssemble(chainreader, b.header, statedb, b.txs, b.uncles, b.receipts) // Write state changes to db - root, err := statedb.Commit(config.IsEIP158(b.header.Number)) + root, _, err := statedb.Commit(config.IsEIP158(b.header.Number)) if err != nil { panic(fmt.Sprintf("state write error: %v", err)) } diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 76132bf37e..6489a600fb 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -447,6 +447,44 @@ func WriteBody(db ethdb.KeyValueWriter, hash common.Hash, number uint64, body *t WriteBodyRLP(db, hash, number, data) } +func WriteDiffLayer(db ethdb.KeyValueWriter, hash common.Hash, layer *types.DiffLayer) { + data, err := rlp.EncodeToBytes(layer) + if err != nil { + log.Crit("Failed to RLP encode diff layer", "err", err) + } + WriteDiffLayerRLP(db, hash, data) +} + +func WriteDiffLayerRLP(db ethdb.KeyValueWriter, blockHash common.Hash, rlp rlp.RawValue) { + if err := db.Put(diffLayerKey(blockHash), rlp); err != nil { + log.Crit("Failed to store diff layer", "err", err) + } +} + +func ReadDiffLayer(db ethdb.KeyValueReader, blockHash common.Hash) *types.DiffLayer { + data := ReadDiffLayerRLP(db, blockHash) + if len(data) == 0 { + return nil + } + diff := new(types.DiffLayer) + if err := rlp.Decode(bytes.NewReader(data), diff); err != nil { + log.Error("Invalid diff layer RLP", "hash", blockHash, "err", err) + return nil + } + return diff +} + +func ReadDiffLayerRLP(db ethdb.KeyValueReader, blockHash common.Hash) rlp.RawValue { + data, _ := db.Get(diffLayerKey(blockHash)) + return data +} + +func DeleteDiffLayer(db ethdb.KeyValueWriter, blockHash common.Hash) { + if err := db.Delete(diffLayerKey(blockHash)); err != nil { + log.Crit("Failed to delete diffLayer", "err", err) + } +} + // DeleteBody removes all block body data associated with a hash. func DeleteBody(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { if err := db.Delete(blockBodyKey(number, hash)); err != nil { diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 725972f9ba..82d5df06ce 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -36,6 +36,7 @@ import ( type freezerdb struct { ethdb.KeyValueStore ethdb.AncientStore + diffStore ethdb.KeyValueStore } // Close implements io.Closer, closing both the fast key-value store as well as @@ -48,12 +49,28 @@ func (frdb *freezerdb) Close() error { if err := frdb.KeyValueStore.Close(); err != nil { errs = append(errs, err) } + if frdb.diffStore != nil { + if err := frdb.diffStore.Close(); err != nil { + errs = append(errs, err) + } + } if len(errs) != 0 { return fmt.Errorf("%v", errs) } return nil } +func (frdb *freezerdb) DiffStore() ethdb.KeyValueStore { + return frdb.diffStore +} + +func (frdb *freezerdb) SetDiffStore(diff ethdb.KeyValueStore) { + if frdb.diffStore != nil { + frdb.diffStore.Close() + } + frdb.diffStore = diff +} + // Freeze is a helper method used for external testing to trigger and block until // a freeze cycle completes, without having to sleep for a minute to trigger the // automatic background run. @@ -77,6 +94,7 @@ func (frdb *freezerdb) Freeze(threshold uint64) error { // nofreezedb is a database wrapper that disables freezer data retrievals. type nofreezedb struct { ethdb.KeyValueStore + diffStore ethdb.KeyValueStore } // HasAncient returns an error as we don't have a backing chain freezer. @@ -114,6 +132,14 @@ func (db *nofreezedb) Sync() error { return errNotSupported } +func (db *nofreezedb) DiffStore() ethdb.KeyValueStore { + return db.diffStore +} + +func (db *nofreezedb) SetDiffStore(diff ethdb.KeyValueStore) { + db.diffStore = diff +} + // NewDatabase creates a high level database on top of a given key-value data // store without a freezer moving immutable chain segments into cold storage. func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index 0df28f236d..8e52b20088 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -18,13 +18,10 @@ package rawdb import ( "bytes" - "encoding/binary" "fmt" - "io/ioutil" "math/rand" "os" "path/filepath" - "sync" "testing" "time" @@ -528,7 +525,6 @@ func TestOffset(t *testing.T) { f.Append(4, getChunk(20, 0xbb)) f.Append(5, getChunk(20, 0xaa)) - f.DumpIndex(0, 100) f.Close() } // Now crop it. @@ -575,7 +571,6 @@ func TestOffset(t *testing.T) { if err != nil { t.Fatal(err) } - f.DumpIndex(0, 100) // It should allow writing item 6 f.Append(numDeleted+2, getChunk(20, 0x99)) @@ -640,55 +635,6 @@ func TestOffset(t *testing.T) { // 1. have data files d0, d1, d2, d3 // 2. remove d2,d3 // -// However, all 'normal' failure modes arising due to failing to sync() or save a file -// should be handled already, and the case described above can only (?) happen if an -// external process/user deletes files from the filesystem. - -// TestAppendTruncateParallel is a test to check if the Append/truncate operations are -// racy. -// -// The reason why it's not a regular fuzzer, within tests/fuzzers, is that it is dependent -// on timing rather than 'clever' input -- there's no determinism. -func TestAppendTruncateParallel(t *testing.T) { - dir, err := ioutil.TempDir("", "freezer") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - f, err := newCustomTable(dir, "tmp", metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, 8, true) - if err != nil { - t.Fatal(err) - } - - fill := func(mark uint64) []byte { - data := make([]byte, 8) - binary.LittleEndian.PutUint64(data, mark) - return data - } - - for i := 0; i < 5000; i++ { - f.truncate(0) - data0 := fill(0) - f.Append(0, data0) - data1 := fill(1) - - var wg sync.WaitGroup - wg.Add(2) - go func() { - f.truncate(0) - wg.Done() - }() - go func() { - f.Append(1, data1) - wg.Done() - }() - wg.Wait() - - if have, err := f.Retrieve(0); err == nil { - if !bytes.Equal(have, data0) { - t.Fatalf("have %x want %x", have, data0) - } - } - } -} +// However, all 'normal' failure modes arising due to failing to sync() or save a file should be +// handled already, and the case described above can only (?) happen if an external process/user +// deletes files from the filesystem. diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 2505ce90b9..b4fb99e451 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -90,6 +90,9 @@ var ( SnapshotStoragePrefix = []byte("o") // SnapshotStoragePrefix + account hash + storage hash -> storage trie value CodePrefix = []byte("c") // CodePrefix + code hash -> account code + // difflayer database + diffLayerPrefix = []byte("d") // diffLayerPrefix + hash -> diffLayer + preimagePrefix = []byte("secure-key-") // preimagePrefix + hash -> preimage configPrefix = []byte("ethereum-config-") // config prefix for the db @@ -177,6 +180,11 @@ func blockReceiptsKey(number uint64, hash common.Hash) []byte { return append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...) } +// diffLayerKey = diffLayerKeyPrefix + hash +func diffLayerKey(hash common.Hash) []byte { + return append(append(diffLayerPrefix, hash.Bytes()...)) +} + // txLookupKey = txLookupPrefix + hash func txLookupKey(hash common.Hash) []byte { return append(txLookupPrefix, hash.Bytes()...) diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 323ef6293c..a27f5f8ed7 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -159,6 +159,14 @@ func (t *table) NewBatch() ethdb.Batch { return &tableBatch{t.db.NewBatch(), t.prefix} } +func (t *table) DiffStore() ethdb.KeyValueStore { + return nil +} + +func (t *table) SetDiffStore(diff ethdb.KeyValueStore) { + panic("not implement") +} + // tableBatch is a wrapper around a database batch that prefixes each key access // with a pre-configured string. type tableBatch struct { diff --git a/core/state/database.go b/core/state/database.go index ce37e73837..0bcde2d5a9 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -243,7 +243,6 @@ func (db *cachingDB) CacheStorage(addrHash common.Hash, root common.Hash, t Trie triesArray := [3]*triePair{{root: root, trie: tr.ResetCopy()}, nil, nil} db.storageTrieCache.Add(addrHash, triesArray) } - return } func (db *cachingDB) Purge() { diff --git a/core/state/journal.go b/core/state/journal.go index 366e0c9c26..d86823c2ca 100644 --- a/core/state/journal.go +++ b/core/state/journal.go @@ -153,7 +153,7 @@ func (ch createObjectChange) dirtied() *common.Address { func (ch resetObjectChange) revert(s *StateDB) { s.SetStateObject(ch.prev) if !ch.prevdestruct && s.snap != nil { - delete(s.snapDestructs, ch.prev.addrHash) + delete(s.snapDestructs, ch.prev.address) } } diff --git a/core/state/snapshot/disklayer_test.go b/core/state/snapshot/disklayer_test.go index ccde2fc094..362edba90d 100644 --- a/core/state/snapshot/disklayer_test.go +++ b/core/state/snapshot/disklayer_test.go @@ -121,7 +121,7 @@ func TestDiskMerge(t *testing.T) { base.Storage(conNukeCache, conNukeCacheSlot) // Modify or delete some accounts, flatten everything onto disk - if err := snaps.Update(diffRoot, baseRoot, map[common.Hash]struct{}{ + if err := snaps.update(diffRoot, baseRoot, map[common.Hash]struct{}{ accDelNoCache: {}, accDelCache: {}, conNukeNoCache: {}, @@ -344,7 +344,7 @@ func TestDiskPartialMerge(t *testing.T) { assertStorage(conNukeCache, conNukeCacheSlot, conNukeCacheSlot[:]) // Modify or delete some accounts, flatten everything onto disk - if err := snaps.Update(diffRoot, baseRoot, map[common.Hash]struct{}{ + if err := snaps.update(diffRoot, baseRoot, map[common.Hash]struct{}{ accDelNoCache: {}, accDelCache: {}, conNukeNoCache: {}, @@ -466,7 +466,7 @@ func TestDiskGeneratorPersistence(t *testing.T) { }, } // Modify or delete some accounts, flatten everything onto disk - if err := snaps.Update(diffRoot, baseRoot, nil, map[common.Hash][]byte{ + if err := snaps.update(diffRoot, baseRoot, nil, map[common.Hash][]byte{ accTwo: accTwo[:], }, nil); err != nil { t.Fatalf("failed to update snapshot tree: %v", err) @@ -484,7 +484,7 @@ func TestDiskGeneratorPersistence(t *testing.T) { } // Test scenario 2, the disk layer is fully generated // Modify or delete some accounts, flatten everything onto disk - if err := snaps.Update(diffTwoRoot, diffRoot, nil, map[common.Hash][]byte{ + if err := snaps.update(diffTwoRoot, diffRoot, nil, map[common.Hash][]byte{ accThree: accThree.Bytes(), }, map[common.Hash]map[common.Hash][]byte{ accThree: {accThreeSlot: accThreeSlot.Bytes()}, diff --git a/core/state/snapshot/iterator_test.go b/core/state/snapshot/iterator_test.go index 2c7e876e08..2a27b01577 100644 --- a/core/state/snapshot/iterator_test.go +++ b/core/state/snapshot/iterator_test.go @@ -221,13 +221,13 @@ func TestAccountIteratorTraversal(t *testing.T) { }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, randomAccountSet("0xbb", "0xdd", "0xf0"), nil) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, randomAccountSet("0xcc", "0xf0", "0xff"), nil) // Verify the single and multi-layer iterators @@ -268,13 +268,13 @@ func TestStorageIteratorTraversal(t *testing.T) { }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x02", "0x03"}}, nil)) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x04", "0x05", "0x06"}}, nil)) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x02", "0x03"}}, nil)) // Verify the single and multi-layer iterators @@ -353,14 +353,14 @@ func TestAccountIteratorTraversalValues(t *testing.T) { } } // Assemble a stack of snapshots from the account layers - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, a, nil) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, b, nil) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, c, nil) - snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, d, nil) - snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), nil, e, nil) - snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), nil, f, nil) - snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), nil, g, nil) - snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), nil, h, nil) + snaps.update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, a, nil) + snaps.update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, b, nil) + snaps.update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, c, nil) + snaps.update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, d, nil) + snaps.update(common.HexToHash("0x06"), common.HexToHash("0x05"), nil, e, nil) + snaps.update(common.HexToHash("0x07"), common.HexToHash("0x06"), nil, f, nil) + snaps.update(common.HexToHash("0x08"), common.HexToHash("0x07"), nil, g, nil) + snaps.update(common.HexToHash("0x09"), common.HexToHash("0x08"), nil, h, nil) it, _ := snaps.AccountIterator(common.HexToHash("0x09"), common.Hash{}) head := snaps.Snapshot(common.HexToHash("0x09")) @@ -452,14 +452,14 @@ func TestStorageIteratorTraversalValues(t *testing.T) { } } // Assemble a stack of snapshots from the account layers - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, randomAccountSet("0xaa"), wrapStorage(a)) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, randomAccountSet("0xaa"), wrapStorage(b)) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, randomAccountSet("0xaa"), wrapStorage(c)) - snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, randomAccountSet("0xaa"), wrapStorage(d)) - snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), nil, randomAccountSet("0xaa"), wrapStorage(e)) - snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), nil, randomAccountSet("0xaa"), wrapStorage(e)) - snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), nil, randomAccountSet("0xaa"), wrapStorage(g)) - snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), nil, randomAccountSet("0xaa"), wrapStorage(h)) + snaps.update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, randomAccountSet("0xaa"), wrapStorage(a)) + snaps.update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, randomAccountSet("0xaa"), wrapStorage(b)) + snaps.update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, randomAccountSet("0xaa"), wrapStorage(c)) + snaps.update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, randomAccountSet("0xaa"), wrapStorage(d)) + snaps.update(common.HexToHash("0x06"), common.HexToHash("0x05"), nil, randomAccountSet("0xaa"), wrapStorage(e)) + snaps.update(common.HexToHash("0x07"), common.HexToHash("0x06"), nil, randomAccountSet("0xaa"), wrapStorage(e)) + snaps.update(common.HexToHash("0x08"), common.HexToHash("0x07"), nil, randomAccountSet("0xaa"), wrapStorage(g)) + snaps.update(common.HexToHash("0x09"), common.HexToHash("0x08"), nil, randomAccountSet("0xaa"), wrapStorage(h)) it, _ := snaps.StorageIterator(common.HexToHash("0x09"), common.HexToHash("0xaa"), common.Hash{}) head := snaps.Snapshot(common.HexToHash("0x09")) @@ -522,7 +522,7 @@ func TestAccountIteratorLargeTraversal(t *testing.T) { }, } for i := 1; i < 128; i++ { - snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(200), nil) + snaps.update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(200), nil) } // Iterate the entire stack and ensure everything is hit only once head := snaps.Snapshot(common.HexToHash("0x80")) @@ -566,13 +566,13 @@ func TestAccountIteratorFlattening(t *testing.T) { }, } // Create a stack of diffs on top - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, randomAccountSet("0xbb", "0xdd", "0xf0"), nil) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, randomAccountSet("0xcc", "0xf0", "0xff"), nil) // Create an iterator and flatten the data from underneath it @@ -597,13 +597,13 @@ func TestAccountIteratorSeek(t *testing.T) { base.root: base, }, } - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, randomAccountSet("0xbb", "0xdd", "0xf0"), nil) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, randomAccountSet("0xcc", "0xf0", "0xff"), nil) // Account set is now @@ -661,13 +661,13 @@ func TestStorageIteratorSeek(t *testing.T) { }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x03", "0x05"}}, nil)) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x02", "0x05", "0x06"}}, nil)) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x05", "0x08"}}, nil)) // Account set is now @@ -724,17 +724,17 @@ func TestAccountIteratorDeletions(t *testing.T) { }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), + snaps.update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, randomAccountSet("0x11", "0x22", "0x33"), nil) deleted := common.HexToHash("0x22") destructed := map[common.Hash]struct{}{ deleted: {}, } - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), + snaps.update(common.HexToHash("0x03"), common.HexToHash("0x02"), destructed, randomAccountSet("0x11", "0x33"), nil) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), + snaps.update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, randomAccountSet("0x33", "0x44", "0x55"), nil) // The output should be 11,33,44,55 @@ -770,10 +770,10 @@ func TestStorageIteratorDeletions(t *testing.T) { }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x03", "0x05"}}, nil)) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x02", "0x04", "0x06"}}, [][]string{{"0x01", "0x03"}})) // The output should be 02,04,05,06 @@ -790,14 +790,14 @@ func TestStorageIteratorDeletions(t *testing.T) { destructed := map[common.Hash]struct{}{ common.HexToHash("0xaa"): {}, } - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), destructed, nil, nil) + snaps.update(common.HexToHash("0x04"), common.HexToHash("0x03"), destructed, nil, nil) it, _ = snaps.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.Hash{}) verifyIterator(t, 0, it, verifyStorage) it.Release() // Re-insert the slots of the same account - snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, + snaps.update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x07", "0x08", "0x09"}}, nil)) // The output should be 07,08,09 @@ -806,7 +806,7 @@ func TestStorageIteratorDeletions(t *testing.T) { it.Release() // Destruct the whole storage but re-create the account in the same layer - snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), destructed, randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x11", "0x12"}}, nil)) + snaps.update(common.HexToHash("0x06"), common.HexToHash("0x05"), destructed, randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x11", "0x12"}}, nil)) it, _ = snaps.StorageIterator(common.HexToHash("0x06"), common.HexToHash("0xaa"), common.Hash{}) verifyIterator(t, 2, it, verifyStorage) // The output should be 11,12 it.Release() @@ -848,7 +848,7 @@ func BenchmarkAccountIteratorTraversal(b *testing.B) { }, } for i := 1; i <= 100; i++ { - snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(200), nil) + snaps.update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(200), nil) } // We call this once before the benchmark, so the creation of // sorted accountlists are not included in the results. @@ -943,9 +943,9 @@ func BenchmarkAccountIteratorLargeBaselayer(b *testing.B) { base.root: base, }, } - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, makeAccounts(2000), nil) + snaps.update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, makeAccounts(2000), nil) for i := 2; i <= 100; i++ { - snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(20), nil) + snaps.update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(20), nil) } // We call this once before the benchmark, so the creation of // sorted accountlists are not included in the results. diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 1b0d883439..46d1b06def 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -59,7 +60,6 @@ var ( snapshotDirtyStorageWriteMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/storage/write", nil) snapshotDirtyAccountHitDepthHist = metrics.NewRegisteredHistogram("state/snapshot/dirty/account/hit/depth", nil, metrics.NewExpDecaySample(1028, 0.015)) - snapshotDirtyStorageHitDepthHist = metrics.NewRegisteredHistogram("state/snapshot/dirty/storage/hit/depth", nil, metrics.NewExpDecaySample(1028, 0.015)) snapshotFlushAccountItemMeter = metrics.NewRegisteredMeter("state/snapshot/flush/account/item", nil) snapshotFlushAccountSizeMeter = metrics.NewRegisteredMeter("state/snapshot/flush/account/size", nil) @@ -322,9 +322,14 @@ func (t *Tree) Snapshots(root common.Hash, limits int, nodisk bool) []Snapshot { return ret } +func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, destructs map[common.Address]struct{}, accounts map[common.Address][]byte, storage map[common.Address]map[string][]byte) error { + hashDestructs, hashAccounts, hashStorage := transformSnapData(destructs, accounts, storage) + return t.update(blockRoot, parentRoot, hashDestructs, hashAccounts, hashStorage) +} + // Update adds a new snapshot into the tree, if that can be linked to an existing // old parent. It is disallowed to insert a disk layer (the origin of all). -func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { +func (t *Tree) update(blockRoot common.Hash, parentRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { // Reject noop updates to avoid self-loops in the snapshot tree. This is a // special case that can only happen for Clique networks where empty blocks // don't modify the state (0 block subsidy). @@ -836,3 +841,27 @@ func (t *Tree) DiskRoot() common.Hash { return t.diskRoot() } + +// TODO we can further improve it when the set is very large +func transformSnapData(destructs map[common.Address]struct{}, accounts map[common.Address][]byte, + storage map[common.Address]map[string][]byte) (map[common.Hash]struct{}, map[common.Hash][]byte, + map[common.Hash]map[common.Hash][]byte) { + hasher := crypto.NewKeccakState() + hashDestructs := make(map[common.Hash]struct{}, len(destructs)) + hashAccounts := make(map[common.Hash][]byte, len(accounts)) + hashStorages := make(map[common.Hash]map[common.Hash][]byte, len(storage)) + for addr := range destructs { + hashDestructs[crypto.Keccak256Hash(addr[:])] = struct{}{} + } + for addr, account := range accounts { + hashAccounts[crypto.Keccak256Hash(addr[:])] = account + } + for addr, accountStore := range storage { + hashStorage := make(map[common.Hash][]byte, len(accountStore)) + for k, v := range accountStore { + hashStorage[crypto.HashData(hasher, []byte(k))] = v + } + hashStorages[crypto.Keccak256Hash(addr[:])] = hashStorage + } + return hashDestructs, hashAccounts, hashStorages +} diff --git a/core/state/snapshot/snapshot_test.go b/core/state/snapshot/snapshot_test.go index 4b787cfe2e..f8ced63665 100644 --- a/core/state/snapshot/snapshot_test.go +++ b/core/state/snapshot/snapshot_test.go @@ -105,7 +105,7 @@ func TestDiskLayerExternalInvalidationFullFlatten(t *testing.T) { accounts := map[common.Hash][]byte{ common.HexToHash("0xa1"): randomAccount(), } - if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { + if err := snaps.update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } if n := len(snaps.layers); n != 2 { @@ -149,10 +149,10 @@ func TestDiskLayerExternalInvalidationPartialFlatten(t *testing.T) { accounts := map[common.Hash][]byte{ common.HexToHash("0xa1"): randomAccount(), } - if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { + if err := snaps.update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } - if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil { + if err := snaps.update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } if n := len(snaps.layers); n != 3 { @@ -197,13 +197,13 @@ func TestDiffLayerExternalInvalidationPartialFlatten(t *testing.T) { accounts := map[common.Hash][]byte{ common.HexToHash("0xa1"): randomAccount(), } - if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { + if err := snaps.update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } - if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil { + if err := snaps.update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } - if err := snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, accounts, nil); err != nil { + if err := snaps.update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } if n := len(snaps.layers); n != 4 { @@ -257,12 +257,12 @@ func TestPostCapBasicDataAccess(t *testing.T) { }, } // The lowest difflayer - snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), nil, setAccount("0xa1"), nil) - snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), nil, setAccount("0xa2"), nil) - snaps.Update(common.HexToHash("0xb2"), common.HexToHash("0xa1"), nil, setAccount("0xb2"), nil) + snaps.update(common.HexToHash("0xa1"), common.HexToHash("0x01"), nil, setAccount("0xa1"), nil) + snaps.update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), nil, setAccount("0xa2"), nil) + snaps.update(common.HexToHash("0xb2"), common.HexToHash("0xa1"), nil, setAccount("0xb2"), nil) - snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil) - snaps.Update(common.HexToHash("0xb3"), common.HexToHash("0xb2"), nil, setAccount("0xb3"), nil) + snaps.update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil) + snaps.update(common.HexToHash("0xb3"), common.HexToHash("0xb2"), nil, setAccount("0xb3"), nil) // checkExist verifies if an account exiss in a snapshot checkExist := func(layer *diffLayer, key string) error { @@ -357,7 +357,7 @@ func TestSnaphots(t *testing.T) { ) for i := 0; i < 129; i++ { head = makeRoot(uint64(i + 2)) - snaps.Update(head, last, nil, setAccount(fmt.Sprintf("%d", i+2)), nil) + snaps.update(head, last, nil, setAccount(fmt.Sprintf("%d", i+2)), nil) last = head snaps.Cap(head, 128) // 130 layers (128 diffs + 1 accumulator + 1 disk) } diff --git a/core/state/state_object.go b/core/state/state_object.go index 623d07ac13..298f4305ba 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -234,7 +234,7 @@ func (s *StateObject) GetCommittedState(db Database, key common.Hash) common.Has // 1) resurrect happened, and new slot values were set -- those should // have been handles via pendingStorage above. // 2) we don't have new values, and can deliver empty response back - if _, destructed := s.db.snapDestructs[s.addrHash]; destructed { + if _, destructed := s.db.snapDestructs[s.address]; destructed { return common.Hash{} } enc, err = s.db.snap.Storage(s.addrHash, crypto.Keccak256Hash(key.Bytes())) @@ -345,10 +345,9 @@ func (s *StateObject) updateTrie(db Database) Trie { }(time.Now()) } // The snapshot storage map for the object - var storage map[common.Hash][]byte + var storage map[string][]byte // Insert all the pending updates into the trie tr := s.getTrie(db) - hasher := s.db.hasher usedStorage := make([][]byte, 0, len(s.pendingStorage)) for key, value := range s.pendingStorage { @@ -371,12 +370,12 @@ func (s *StateObject) updateTrie(db Database) Trie { s.db.snapMux.Lock() if storage == nil { // Retrieve the old storage map, if available, create a new one otherwise - if storage = s.db.snapStorage[s.addrHash]; storage == nil { - storage = make(map[common.Hash][]byte) - s.db.snapStorage[s.addrHash] = storage + if storage = s.db.snapStorage[s.address]; storage == nil { + storage = make(map[string][]byte) + s.db.snapStorage[s.address] = storage } } - storage[crypto.HashData(hasher, key[:])] = v // v will be nil if value is 0x00 + storage[string(key[:])] = v // v will be nil if value is 0x00 s.db.snapMux.Unlock() } usedStorage = append(usedStorage, common.CopyBytes(key[:])) // Copy needed for closure diff --git a/core/state/state_test.go b/core/state/state_test.go index 9f003fefb5..77847772c6 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -167,7 +167,7 @@ func TestSnapshot2(t *testing.T) { so0.deleted = false state.SetStateObject(so0) - root, _ := state.Commit(false) + root, _, _ := state.Commit(false) state, _ = New(root, state.db, state.snaps) // and one with deleted == true diff --git a/core/state/statedb.go b/core/state/statedb.go index 7940613cd6..c68e09490c 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -27,6 +27,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/gopool" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" @@ -39,7 +40,7 @@ import ( ) const ( - preLoadLimit = 64 + preLoadLimit = 128 defaultNumOfSlots = 100 ) @@ -72,18 +73,22 @@ func (n *proofList) Delete(key []byte) error { // * Contracts // * Accounts type StateDB struct { - db Database - prefetcher *triePrefetcher - originalRoot common.Hash // The pre-state root, before any changes were made - trie Trie - hasher crypto.KeccakState + db Database + prefetcher *triePrefetcher + originalRoot common.Hash // The pre-state root, before any changes were made + trie Trie + hasher crypto.KeccakState + diffLayer *types.DiffLayer + diffTries map[common.Address]Trie + diffCode map[common.Hash][]byte + lightProcessed bool snapMux sync.Mutex snaps *snapshot.Tree snap snapshot.Snapshot - snapDestructs map[common.Hash]struct{} - snapAccounts map[common.Hash][]byte - snapStorage map[common.Hash]map[common.Hash][]byte + snapDestructs map[common.Address]struct{} + snapAccounts map[common.Address][]byte + snapStorage map[common.Address]map[string][]byte // This map holds 'live' objects, which will get modified while processing a state transition. stateObjects map[common.Address]*StateObject @@ -156,9 +161,9 @@ func newStateDB(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, sdb.trie = tr if sdb.snaps != nil { if sdb.snap = sdb.snaps.Snapshot(root); sdb.snap != nil { - sdb.snapDestructs = make(map[common.Hash]struct{}) - sdb.snapAccounts = make(map[common.Hash][]byte) - sdb.snapStorage = make(map[common.Hash]map[common.Hash][]byte) + sdb.snapDestructs = make(map[common.Address]struct{}) + sdb.snapAccounts = make(map[common.Address][]byte) + sdb.snapStorage = make(map[common.Address]map[string][]byte) } } return sdb, nil @@ -186,6 +191,15 @@ func (s *StateDB) StopPrefetcher() { } } +// Mark that the block is processed by diff layer +func (s *StateDB) MarkLightProcessed() { + s.lightProcessed = true +} + +func (s *StateDB) IsLightProcessed() bool { + return s.lightProcessed +} + // setError remembers the first non-nil error it is called with. func (s *StateDB) setError(err error) { if s.dbErr == nil { @@ -197,6 +211,19 @@ func (s *StateDB) Error() error { return s.dbErr } +func (s *StateDB) Trie() Trie { + return s.trie +} + +func (s *StateDB) SetDiff(diffLayer *types.DiffLayer, diffTries map[common.Address]Trie, diffCode map[common.Hash][]byte) { + s.diffLayer, s.diffTries, s.diffCode = diffLayer, diffTries, diffCode +} + +func (s *StateDB) SetSnapData(snapDestructs map[common.Address]struct{}, snapAccounts map[common.Address][]byte, + snapStorage map[common.Address]map[string][]byte) { + s.snapDestructs, s.snapAccounts, s.snapStorage = snapDestructs, snapAccounts, snapStorage +} + func (s *StateDB) AddLog(log *types.Log) { s.journal.append(addLogChange{txhash: s.thash}) @@ -532,7 +559,7 @@ func (s *StateDB) TryPreload(block *types.Block, signer types.Signer) { accounts[*tx.To()] = true } } - for account, _ := range accounts { + for account := range accounts { accountsSlice = append(accountsSlice, account) } if len(accountsSlice) >= preLoadLimit && len(accountsSlice) > runtime.NumCPU() { @@ -550,10 +577,8 @@ func (s *StateDB) TryPreload(block *types.Block, signer types.Signer) { } for i := 0; i < runtime.NumCPU(); i++ { objs := <-objsChan - if objs != nil { - for _, obj := range objs { - s.SetStateObject(obj) - } + for _, obj := range objs { + s.SetStateObject(obj) } } } @@ -683,9 +708,9 @@ func (s *StateDB) createObject(addr common.Address) (newobj, prev *StateObject) var prevdestruct bool if s.snap != nil && prev != nil { - _, prevdestruct = s.snapDestructs[prev.addrHash] + _, prevdestruct = s.snapDestructs[prev.address] if !prevdestruct { - s.snapDestructs[prev.addrHash] = struct{}{} + s.snapDestructs[prev.address] = struct{}{} } } newobj = newObject(s, addr, Account{}) @@ -830,17 +855,17 @@ func (s *StateDB) Copy() *StateDB { state.snaps = s.snaps state.snap = s.snap // deep copy needed - state.snapDestructs = make(map[common.Hash]struct{}) + state.snapDestructs = make(map[common.Address]struct{}) for k, v := range s.snapDestructs { state.snapDestructs[k] = v } - state.snapAccounts = make(map[common.Hash][]byte) + state.snapAccounts = make(map[common.Address][]byte) for k, v := range s.snapAccounts { state.snapAccounts[k] = v } - state.snapStorage = make(map[common.Hash]map[common.Hash][]byte) + state.snapStorage = make(map[common.Address]map[string][]byte) for k, v := range s.snapStorage { - temp := make(map[common.Hash][]byte) + temp := make(map[string][]byte) for kk, vv := range v { temp[kk] = vv } @@ -903,9 +928,9 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { // transactions within the same block might self destruct and then // ressurrect an account; but the snapshotter needs both events. if s.snap != nil { - s.snapDestructs[obj.addrHash] = struct{}{} // We need to maintain account deletions explicitly (will remain set indefinitely) - delete(s.snapAccounts, obj.addrHash) // Clear out any previously updated account data (may be recreated via a ressurrect) - delete(s.snapStorage, obj.addrHash) // Clear out any previously updated storage data (may be recreated via a ressurrect) + s.snapDestructs[obj.address] = struct{}{} // We need to maintain account deletions explicitly (will remain set indefinitely) + delete(s.snapAccounts, obj.address) // Clear out any previously updated account data (may be recreated via a ressurrect) + delete(s.snapStorage, obj.address) // Clear out any previously updated storage data (may be recreated via a ressurrect) } } else { obj.finalise(true) // Prefetch slots in the background @@ -932,6 +957,9 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { // It is called in between transactions to get the root hash that // goes into transaction receipts. func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { + if s.lightProcessed { + return s.trie.Hash() + } // Finalise all the dirty storage states and write them into the tries s.Finalise(deleteEmptyObjects) @@ -983,7 +1011,8 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // at transaction boundary level to ensure we capture state clearing. if s.snap != nil && !obj.deleted { s.snapMux.Lock() - s.snapAccounts[obj.addrHash] = snapshot.SlimAccountRLP(obj.data.Nonce, obj.data.Balance, obj.data.Root, obj.data.CodeHash) + // It is possible to add unnecessary change, but it is fine. + s.snapAccounts[obj.address] = snapshot.SlimAccountRLP(obj.data.Nonce, obj.data.Balance, obj.data.Root, obj.data.CodeHash) s.snapMux.Unlock() } data, err := rlp.EncodeToBytes(obj) @@ -1007,7 +1036,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { if s.trie == nil { tr, err := s.db.OpenTrie(s.originalRoot) if err != nil { - panic(fmt.Sprintf("Failed to open trie tree")) + panic("Failed to open trie tree") } s.trie = tr } @@ -1051,14 +1080,143 @@ func (s *StateDB) clearJournalAndRefund() { s.validRevisions = s.validRevisions[:0] // Snapshots can be created without journal entires } +func (s *StateDB) LightCommit(root common.Hash) (common.Hash, *types.DiffLayer, error) { + codeWriter := s.db.TrieDB().DiskDB().NewBatch() + + commitFuncs := []func() error{ + func() error { + for codeHash, code := range s.diffCode { + rawdb.WriteCode(codeWriter, codeHash, code) + if codeWriter.ValueSize() >= ethdb.IdealBatchSize { + if err := codeWriter.Write(); err != nil { + return err + } + codeWriter.Reset() + } + } + if codeWriter.ValueSize() > 0 { + if err := codeWriter.Write(); err != nil { + return err + } + } + return nil + }, + func() error { + tasks := make(chan func()) + taskResults := make(chan error, len(s.diffTries)) + tasksNum := 0 + finishCh := make(chan struct{}) + defer close(finishCh) + threads := gopool.Threads(len(s.diffTries)) + + for i := 0; i < threads; i++ { + go func() { + for { + select { + case task := <-tasks: + task() + case <-finishCh: + return + } + } + }() + } + + for account, diff := range s.diffTries { + tmpAccount := account + tmpDiff := diff + tasks <- func() { + root, err := tmpDiff.Commit(nil) + if err != nil { + taskResults <- err + return + } + s.db.CacheStorage(crypto.Keccak256Hash(tmpAccount[:]), root, tmpDiff) + taskResults <- nil + } + tasksNum++ + } + + for i := 0; i < tasksNum; i++ { + err := <-taskResults + if err != nil { + return err + } + } + + // commit account trie + var account Account + root, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error { + if err := rlp.DecodeBytes(leaf, &account); err != nil { + return nil + } + if account.Root != emptyRoot { + s.db.TrieDB().Reference(account.Root, parent) + } + return nil + }) + if err != nil { + return err + } + if root != emptyRoot { + s.db.CacheAccount(root, s.trie) + } + return nil + }, + func() error { + if s.snap != nil { + if metrics.EnabledExpensive { + defer func(start time.Time) { s.SnapshotCommits += time.Since(start) }(time.Now()) + } + // Only update if there's a state transition (skip empty Clique blocks) + if parent := s.snap.Root(); parent != root { + if err := s.snaps.Update(root, parent, s.snapDestructs, s.snapAccounts, s.snapStorage); err != nil { + log.Warn("Failed to update snapshot tree", "from", parent, "to", root, "err", err) + } + // Keep n diff layers in the memory + // - head layer is paired with HEAD state + // - head-1 layer is paired with HEAD-1 state + // - head-(n-1) layer(bottom-most diff layer) is paired with HEAD-(n-1)state + if err := s.snaps.Cap(root, s.snaps.CapLimit()); err != nil { + log.Warn("Failed to cap snapshot tree", "root", root, "layers", s.snaps.CapLimit(), "err", err) + } + } + } + return nil + }, + } + commitRes := make(chan error, len(commitFuncs)) + for _, f := range commitFuncs { + tmpFunc := f + go func() { + commitRes <- tmpFunc() + }() + } + for i := 0; i < len(commitFuncs); i++ { + r := <-commitRes + if r != nil { + return common.Hash{}, nil, r + } + } + s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil + s.diffTries, s.diffCode = nil, nil + return root, s.diffLayer, nil +} + // Commit writes the state to the underlying in-memory trie database. -func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { +func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, *types.DiffLayer, error) { if s.dbErr != nil { - return common.Hash{}, fmt.Errorf("commit aborted due to earlier error: %v", s.dbErr) + return common.Hash{}, nil, fmt.Errorf("commit aborted due to earlier error: %v", s.dbErr) } // Finalize any pending changes and merge everything into the tries root := s.IntermediateRoot(deleteEmptyObjects) - + if s.lightProcessed { + return s.LightCommit(root) + } + var diffLayer *types.DiffLayer + if s.snap != nil { + diffLayer = &types.DiffLayer{} + } commitFuncs := []func() error{ func() error { // Commit objects to the trie, measuring the elapsed time @@ -1066,9 +1224,13 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { taskResults := make(chan error, len(s.stateObjectsDirty)) tasksNum := 0 finishCh := make(chan struct{}) - defer close(finishCh) - for i := 0; i < runtime.NumCPU(); i++ { + + threads := gopool.Threads(len(s.stateObjectsDirty)) + wg := sync.WaitGroup{} + for i := 0; i < threads; i++ { + wg.Add(1) go func() { + defer wg.Done() codeWriter := s.db.TrieDB().DiskDB().NewBatch() for { select { @@ -1086,6 +1248,19 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { }() } + if s.snap != nil { + for addr := range s.stateObjectsDirty { + if obj := s.stateObjects[addr]; !obj.deleted { + if obj.code != nil && obj.dirtyCode { + diffLayer.Codes = append(diffLayer.Codes, types.DiffCode{ + Hash: common.BytesToHash(obj.CodeHash()), + Code: obj.code, + }) + } + } + } + } + for addr := range s.stateObjectsDirty { if obj := s.stateObjects[addr]; !obj.deleted { // Write any contract code associated with the state object @@ -1107,9 +1282,11 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { for i := 0; i < tasksNum; i++ { err := <-taskResults if err != nil { + close(finishCh) return err } } + close(finishCh) if len(s.stateObjectsDirty) > 0 { s.stateObjectsDirty = make(map[common.Address]struct{}, len(s.stateObjectsDirty)/2) @@ -1140,6 +1317,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { if root != emptyRoot { s.db.CacheAccount(root, s.trie) } + wg.Wait() return nil }, func() error { @@ -1161,7 +1339,12 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { log.Warn("Failed to cap snapshot tree", "root", root, "layers", s.snaps.CapLimit(), "err", err) } } - s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil + } + return nil + }, + func() error { + if s.snap != nil { + diffLayer.Destructs, diffLayer.Accounts, diffLayer.Storages = s.SnapToDiffLayer() } return nil }, @@ -1176,11 +1359,65 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { for i := 0; i < len(commitFuncs); i++ { r := <-commitRes if r != nil { - return common.Hash{}, r + return common.Hash{}, nil, r } } + s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil + return root, diffLayer, nil +} - return root, nil +func (s *StateDB) DiffLayerToSnap(diffLayer *types.DiffLayer) (map[common.Address]struct{}, map[common.Address][]byte, map[common.Address]map[string][]byte, error) { + snapDestructs := make(map[common.Address]struct{}) + snapAccounts := make(map[common.Address][]byte) + snapStorage := make(map[common.Address]map[string][]byte) + + for _, des := range diffLayer.Destructs { + snapDestructs[des] = struct{}{} + } + for _, account := range diffLayer.Accounts { + snapAccounts[account.Account] = account.Blob + } + for _, storage := range diffLayer.Storages { + // should never happen + if len(storage.Keys) != len(storage.Vals) { + return nil, nil, nil, errors.New("invalid diffLayer: length of keys and values mismatch") + } + snapStorage[storage.Account] = make(map[string][]byte, len(storage.Keys)) + n := len(storage.Keys) + for i := 0; i < n; i++ { + snapStorage[storage.Account][storage.Keys[i]] = storage.Vals[i] + } + } + return snapDestructs, snapAccounts, snapStorage, nil +} + +func (s *StateDB) SnapToDiffLayer() ([]common.Address, []types.DiffAccount, []types.DiffStorage) { + destructs := make([]common.Address, 0, len(s.snapDestructs)) + for account := range s.snapDestructs { + destructs = append(destructs, account) + } + accounts := make([]types.DiffAccount, 0, len(s.snapAccounts)) + for accountHash, account := range s.snapAccounts { + accounts = append(accounts, types.DiffAccount{ + Account: accountHash, + Blob: account, + }) + } + storages := make([]types.DiffStorage, 0, len(s.snapStorage)) + for accountHash, storage := range s.snapStorage { + keys := make([]string, 0, len(storage)) + values := make([][]byte, 0, len(storage)) + for k, v := range storage { + keys = append(keys, k) + values = append(values, v) + } + storages = append(storages, types.DiffStorage{ + Account: accountHash, + Keys: keys, + Vals: values, + }) + } + return destructs, accounts, storages } // PrepareAccessList handles the preparatory steps for executing a state transition with diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 9524e3730d..2c0b9296ff 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -102,7 +102,7 @@ func TestIntermediateLeaks(t *testing.T) { } // Commit and cross check the databases. - transRoot, err := transState.Commit(false) + transRoot, _, err := transState.Commit(false) if err != nil { t.Fatalf("failed to commit transition state: %v", err) } @@ -110,7 +110,7 @@ func TestIntermediateLeaks(t *testing.T) { t.Errorf("can not commit trie %v to persistent database", transRoot.Hex()) } - finalRoot, err := finalState.Commit(false) + finalRoot, _, err := finalState.Commit(false) if err != nil { t.Fatalf("failed to commit final state: %v", err) } @@ -473,7 +473,7 @@ func (test *snapshotTest) checkEqual(state, checkstate *StateDB) error { func TestTouchDelete(t *testing.T) { s := newStateTest() s.state.GetOrNewStateObject(common.Address{}) - root, _ := s.state.Commit(false) + root, _, _ := s.state.Commit(false) s.state, _ = New(root, s.state.db, s.state.snaps) snapshot := s.state.Snapshot() @@ -675,7 +675,7 @@ func TestDeleteCreateRevert(t *testing.T) { addr := common.BytesToAddress([]byte("so")) state.SetBalance(addr, big.NewInt(1)) - root, _ := state.Commit(false) + root, _, _ := state.Commit(false) state, _ = New(root, state.db, state.snaps) // Simulate self-destructing in one transaction, then create-reverting in another @@ -687,7 +687,7 @@ func TestDeleteCreateRevert(t *testing.T) { state.RevertToSnapshot(id) // Commit the entire state and make sure we don't crash and have the correct state - root, _ = state.Commit(true) + root, _, _ = state.Commit(true) state, _ = New(root, state.db, state.snaps) if state.getStateObject(addr) != nil { @@ -712,7 +712,7 @@ func TestMissingTrieNodes(t *testing.T) { a2 := common.BytesToAddress([]byte("another")) state.SetBalance(a2, big.NewInt(100)) state.SetCode(a2, []byte{1, 2, 4}) - root, _ = state.Commit(false) + root, _, _ = state.Commit(false) t.Logf("root: %x", root) // force-flush state.Database().TrieDB().Cap(0) @@ -736,7 +736,7 @@ func TestMissingTrieNodes(t *testing.T) { } // Modify the state state.SetBalance(addr, big.NewInt(2)) - root, err := state.Commit(false) + root, _, err := state.Commit(false) if err == nil { t.Fatalf("expected error, got root :%x", root) } diff --git a/core/state/sync_test.go b/core/state/sync_test.go index a13fcf56a3..24cae59004 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -69,7 +69,7 @@ func makeTestState() (Database, common.Hash, []*testAccount) { state.updateStateObject(obj) accounts = append(accounts, acc) } - root, _ := state.Commit(false) + root, _, _ := state.Commit(false) // Return the generated state return db, root, accounts diff --git a/core/state_prefetcher.go b/core/state_prefetcher.go index 05394321f7..dacd8df404 100644 --- a/core/state_prefetcher.go +++ b/core/state_prefetcher.go @@ -35,15 +35,6 @@ type statePrefetcher struct { engine consensus.Engine // Consensus engine used for block rewards } -// newStatePrefetcher initialises a new statePrefetcher. -func newStatePrefetcher(config *params.ChainConfig, bc *BlockChain, engine consensus.Engine) *statePrefetcher { - return &statePrefetcher{ - config: config, - bc: bc, - engine: engine, - } -} - // Prefetch processes the state changes according to the Ethereum rules by running // the transaction messages using the statedb, but any changes are discarded. The // only goal is to pre-cache transaction signatures and state trie nodes. diff --git a/core/state_processor.go b/core/state_processor.go index 858796b67a..973fd27b54 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -17,17 +17,36 @@ package core import ( + "bytes" + "errors" "fmt" + "math/big" + "math/rand" + "sync" + "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/gopool" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/systemcontracts" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" +) + +const ( + fullProcessCheck = 21 // On diff sync mode, will do full process every fullProcessCheck randomly + minNumberOfAccountPerTask = 5 + recentTime = 2048 * 3 + recentDiffLayerTimeout = 20 + farDiffLayerTimeout = 2 ) // StateProcessor is a basic Processor, which takes care of transitioning @@ -49,6 +68,301 @@ func NewStateProcessor(config *params.ChainConfig, bc *BlockChain, engine consen } } +type LightStateProcessor struct { + randomGenerator *rand.Rand + StateProcessor +} + +func NewLightStateProcessor(config *params.ChainConfig, bc *BlockChain, engine consensus.Engine) *LightStateProcessor { + randomGenerator := rand.New(rand.NewSource(int64(time.Now().Nanosecond()))) + return &LightStateProcessor{ + randomGenerator: randomGenerator, + StateProcessor: *NewStateProcessor(config, bc, engine), + } +} + +func (p *LightStateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg vm.Config) (*state.StateDB, types.Receipts, []*types.Log, uint64, error) { + allowLightProcess := true + if posa, ok := p.engine.(consensus.PoSA); ok { + allowLightProcess = posa.AllowLightProcess(p.bc, block.Header()) + } + // random fallback to full process + if check := p.randomGenerator.Int63n(fullProcessCheck); allowLightProcess && check != 0 && len(block.Transactions()) != 0 { + var pid string + if peer, ok := block.ReceivedFrom.(PeerIDer); ok { + pid = peer.ID() + } + var diffLayer *types.DiffLayer + var diffLayerTimeout = recentDiffLayerTimeout + if time.Now().Unix()-int64(block.Time()) > recentTime { + diffLayerTimeout = farDiffLayerTimeout + } + for tried := 0; tried < diffLayerTimeout; tried++ { + // wait a bit for the diff layer + diffLayer = p.bc.GetUnTrustedDiffLayer(block.Hash(), pid) + if diffLayer != nil { + break + } + time.Sleep(time.Millisecond) + } + if diffLayer != nil { + if err := diffLayer.Receipts.DeriveFields(p.bc.chainConfig, block.Hash(), block.NumberU64(), block.Transactions()); err != nil { + log.Error("Failed to derive block receipts fields", "hash", block.Hash(), "number", block.NumberU64(), "err", err) + // fallback to full process + return p.StateProcessor.Process(block, statedb, cfg) + } + + receipts, logs, gasUsed, err := p.LightProcess(diffLayer, block, statedb) + if err == nil { + log.Info("do light process success at block", "num", block.NumberU64()) + return statedb, receipts, logs, gasUsed, nil + } + log.Error("do light process err at block", "num", block.NumberU64(), "err", err) + p.bc.removeDiffLayers(diffLayer.DiffHash) + // prepare new statedb + statedb.StopPrefetcher() + parent := p.bc.GetHeader(block.ParentHash(), block.NumberU64()-1) + statedb, err = state.New(parent.Root, p.bc.stateCache, p.bc.snaps) + if err != nil { + return statedb, nil, nil, 0, err + } + // Enable prefetching to pull in trie node paths while processing transactions + statedb.StartPrefetcher("chain") + } + } + // fallback to full process + return p.StateProcessor.Process(block, statedb, cfg) +} + +func (p *LightStateProcessor) LightProcess(diffLayer *types.DiffLayer, block *types.Block, statedb *state.StateDB) (types.Receipts, []*types.Log, uint64, error) { + statedb.MarkLightProcessed() + fullDiffCode := make(map[common.Hash][]byte, len(diffLayer.Codes)) + diffTries := make(map[common.Address]state.Trie) + diffCode := make(map[common.Hash][]byte) + + snapDestructs, snapAccounts, snapStorage, err := statedb.DiffLayerToSnap(diffLayer) + if err != nil { + return nil, nil, 0, err + } + + for _, c := range diffLayer.Codes { + fullDiffCode[c.Hash] = c.Code + } + + for des := range snapDestructs { + statedb.Trie().TryDelete(des[:]) + } + threads := gopool.Threads(len(snapAccounts)) + + iteAccounts := make([]common.Address, 0, len(snapAccounts)) + for diffAccount := range snapAccounts { + iteAccounts = append(iteAccounts, diffAccount) + } + + errChan := make(chan error, threads) + exitChan := make(chan struct{}) + var snapMux sync.RWMutex + var stateMux, diffMux sync.Mutex + for i := 0; i < threads; i++ { + start := i * len(iteAccounts) / threads + end := (i + 1) * len(iteAccounts) / threads + if i+1 == threads { + end = len(iteAccounts) + } + go func(start, end int) { + for index := start; index < end; index++ { + select { + // fast fail + case <-exitChan: + return + default: + } + diffAccount := iteAccounts[index] + snapMux.RLock() + blob := snapAccounts[diffAccount] + snapMux.RUnlock() + addrHash := crypto.Keccak256Hash(diffAccount[:]) + latestAccount, err := snapshot.FullAccount(blob) + if err != nil { + errChan <- err + return + } + + // fetch previous state + var previousAccount state.Account + stateMux.Lock() + enc, err := statedb.Trie().TryGet(diffAccount[:]) + stateMux.Unlock() + if err != nil { + errChan <- err + return + } + if len(enc) != 0 { + if err := rlp.DecodeBytes(enc, &previousAccount); err != nil { + errChan <- err + return + } + } + if latestAccount.Balance == nil { + latestAccount.Balance = new(big.Int) + } + if previousAccount.Balance == nil { + previousAccount.Balance = new(big.Int) + } + if previousAccount.Root == (common.Hash{}) { + previousAccount.Root = types.EmptyRootHash + } + if len(previousAccount.CodeHash) == 0 { + previousAccount.CodeHash = types.EmptyCodeHash + } + + // skip no change account + if previousAccount.Nonce == latestAccount.Nonce && + bytes.Equal(previousAccount.CodeHash, latestAccount.CodeHash) && + previousAccount.Balance.Cmp(latestAccount.Balance) == 0 && + previousAccount.Root == common.BytesToHash(latestAccount.Root) { + // It is normal to receive redundant message since the collected message is redundant. + log.Debug("receive redundant account change in diff layer", "account", diffAccount, "num", block.NumberU64()) + snapMux.Lock() + delete(snapAccounts, diffAccount) + delete(snapStorage, diffAccount) + snapMux.Unlock() + continue + } + + // update code + codeHash := common.BytesToHash(latestAccount.CodeHash) + if !bytes.Equal(latestAccount.CodeHash, previousAccount.CodeHash) && + !bytes.Equal(latestAccount.CodeHash, types.EmptyCodeHash) { + if code, exist := fullDiffCode[codeHash]; exist { + if crypto.Keccak256Hash(code) != codeHash { + errChan <- fmt.Errorf("code and code hash mismatch, account %s", diffAccount.String()) + return + } + diffMux.Lock() + diffCode[codeHash] = code + diffMux.Unlock() + } else { + rawCode := rawdb.ReadCode(p.bc.db, codeHash) + if len(rawCode) == 0 { + errChan <- fmt.Errorf("missing code, account %s", diffAccount.String()) + return + } + } + } + + //update storage + latestRoot := common.BytesToHash(latestAccount.Root) + if latestRoot != previousAccount.Root && latestRoot != types.EmptyRootHash { + accountTrie, err := statedb.Database().OpenStorageTrie(addrHash, previousAccount.Root) + if err != nil { + errChan <- err + return + } + snapMux.RLock() + storageChange, exist := snapStorage[diffAccount] + snapMux.RUnlock() + + if !exist { + errChan <- errors.New("missing storage change in difflayer") + return + } + for k, v := range storageChange { + if len(v) != 0 { + accountTrie.TryUpdate([]byte(k), v) + } else { + accountTrie.TryDelete([]byte(k)) + } + } + + // check storage root + accountRootHash := accountTrie.Hash() + if latestRoot != accountRootHash { + errChan <- errors.New("account storage root mismatch") + return + } + diffMux.Lock() + diffTries[diffAccount] = accountTrie + diffMux.Unlock() + } else { + snapMux.Lock() + delete(snapStorage, diffAccount) + snapMux.Unlock() + } + + // can't trust the blob, need encode by our-self. + latestStateAccount := state.Account{ + Nonce: latestAccount.Nonce, + Balance: latestAccount.Balance, + Root: common.BytesToHash(latestAccount.Root), + CodeHash: latestAccount.CodeHash, + } + bz, err := rlp.EncodeToBytes(&latestStateAccount) + if err != nil { + errChan <- err + return + } + stateMux.Lock() + err = statedb.Trie().TryUpdate(diffAccount[:], bz) + stateMux.Unlock() + if err != nil { + errChan <- err + return + } + } + errChan <- nil + }(start, end) + } + + for i := 0; i < threads; i++ { + err := <-errChan + if err != nil { + close(exitChan) + return nil, nil, 0, err + } + } + + var allLogs []*types.Log + var gasUsed uint64 + for _, receipt := range diffLayer.Receipts { + allLogs = append(allLogs, receipt.Logs...) + gasUsed += receipt.GasUsed + } + + // Do validate in advance so that we can fall back to full process + if err := p.bc.validator.ValidateState(block, statedb, diffLayer.Receipts, gasUsed); err != nil { + log.Error("validate state failed during diff sync", "error", err) + return nil, nil, 0, err + } + + // remove redundant storage change + for account := range snapStorage { + if _, exist := snapAccounts[account]; !exist { + log.Warn("receive redundant storage change in diff layer") + delete(snapStorage, account) + } + } + + // remove redundant code + if len(fullDiffCode) != len(diffLayer.Codes) { + diffLayer.Codes = make([]types.DiffCode, 0, len(diffCode)) + for hash, code := range diffCode { + diffLayer.Codes = append(diffLayer.Codes, types.DiffCode{ + Hash: hash, + Code: code, + }) + } + } + + statedb.SetSnapData(snapDestructs, snapAccounts, snapStorage) + if len(snapAccounts) != len(diffLayer.Accounts) || len(snapStorage) != len(diffLayer.Storages) { + diffLayer.Destructs, diffLayer.Accounts, diffLayer.Storages = statedb.SnapToDiffLayer() + } + statedb.SetDiff(diffLayer, diffTries, diffCode) + + return diffLayer.Receipts, allLogs, gasUsed, nil +} + // Process processes the state changes according to the Ethereum rules by running // the transaction messages using the statedb and applying any rewards to both // the processor (coinbase) and any included uncles. @@ -56,13 +370,15 @@ func NewStateProcessor(config *params.ChainConfig, bc *BlockChain, engine consen // Process returns the receipts and logs accumulated during the process and // returns the amount of gas that was used in the process. If any of the // transactions failed to execute due to insufficient gas it will return an error. -func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg vm.Config) (types.Receipts, []*types.Log, uint64, error) { +func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg vm.Config) (*state.StateDB, types.Receipts, []*types.Log, uint64, error) { var ( usedGas = new(uint64) header = block.Header() allLogs []*types.Log gp = new(GasPool).AddGas(block.GasLimit()) ) + signer := types.MakeSigner(p.bc.chainConfig, block.Number()) + statedb.TryPreload(block, signer) var receipts = make([]*types.Receipt, 0) // Mutate the block and state according to any hard-fork specs if p.config.DAOForkSupport && p.config.DAOForkBlock != nil && p.config.DAOForkBlock.Cmp(block.Number()) == 0 { @@ -79,11 +395,10 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg commonTxs := make([]*types.Transaction, 0, len(block.Transactions())) // usually do have two tx, one for validator set contract, another for system reward contract. systemTxs := make([]*types.Transaction, 0, 2) - signer := types.MakeSigner(p.config, header.Number) for i, tx := range block.Transactions() { if isPoSA { if isSystemTx, err := posa.IsSystemTransaction(tx, block.Header()); err != nil { - return nil, nil, 0, err + return statedb, nil, nil, 0, err } else if isSystemTx { systemTxs = append(systemTxs, tx) continue @@ -92,12 +407,12 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg msg, err := tx.AsMessage(signer) if err != nil { - return nil, nil, 0, err + return statedb, nil, nil, 0, err } statedb.Prepare(tx.Hash(), block.Hash(), i) receipt, err := applyTransaction(msg, p.config, p.bc, nil, gp, statedb, header, tx, usedGas, vmenv) if err != nil { - return nil, nil, 0, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) + return statedb, nil, nil, 0, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } commonTxs = append(commonTxs, tx) @@ -107,13 +422,13 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // Finalize the block, applying any consensus engine specific extras (e.g. block rewards) err := p.engine.Finalize(p.bc, header, statedb, &commonTxs, block.Uncles(), &receipts, &systemTxs, usedGas) if err != nil { - return receipts, allLogs, *usedGas, err + return statedb, receipts, allLogs, *usedGas, err } for _, receipt := range receipts { allLogs = append(allLogs, receipt.Logs...) } - return receipts, allLogs, *usedGas, nil + return statedb, receipts, allLogs, *usedGas, nil } func applyTransaction(msg types.Message, config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64, evm *vm.EVM) (*types.Receipt, error) { diff --git a/core/types.go b/core/types.go index 4c5b74a498..49bd58e086 100644 --- a/core/types.go +++ b/core/types.go @@ -47,5 +47,5 @@ type Processor interface { // Process processes the state changes according to the Ethereum rules by running // the transaction messages using the statedb and applying any rewards to both // the processor (coinbase) and any included uncles. - Process(block *types.Block, statedb *state.StateDB, cfg vm.Config) (types.Receipts, []*types.Log, uint64, error) + Process(block *types.Block, statedb *state.StateDB, cfg vm.Config) (*state.StateDB, types.Receipts, []*types.Log, uint64, error) } diff --git a/core/types/block.go b/core/types/block.go index b33493ef7d..a577e60516 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -19,6 +19,7 @@ package types import ( "encoding/binary" + "errors" "fmt" "io" "math/big" @@ -28,11 +29,14 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" ) var ( - EmptyRootHash = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + EmptyRootHash = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + EmptyCodeHash = crypto.Keccak256(nil) + EmptyUncleHash = rlpHash([]*Header(nil)) ) @@ -366,3 +370,86 @@ func (b *Block) Hash() common.Hash { } type Blocks []*Block + +type DiffLayer struct { + BlockHash common.Hash + Number uint64 + Receipts Receipts // Receipts are duplicated stored to simplify the logic + Codes []DiffCode + Destructs []common.Address + Accounts []DiffAccount + Storages []DiffStorage + + DiffHash common.Hash +} + +type extDiffLayer struct { + BlockHash common.Hash + Number uint64 + Receipts []*ReceiptForStorage // Receipts are duplicated stored to simplify the logic + Codes []DiffCode + Destructs []common.Address + Accounts []DiffAccount + Storages []DiffStorage +} + +// DecodeRLP decodes the Ethereum +func (d *DiffLayer) DecodeRLP(s *rlp.Stream) error { + var ed extDiffLayer + if err := s.Decode(&ed); err != nil { + return err + } + d.BlockHash, d.Number, d.Codes, d.Destructs, d.Accounts, d.Storages = + ed.BlockHash, ed.Number, ed.Codes, ed.Destructs, ed.Accounts, ed.Storages + + d.Receipts = make([]*Receipt, len(ed.Receipts)) + for i, storageReceipt := range ed.Receipts { + d.Receipts[i] = (*Receipt)(storageReceipt) + } + return nil +} + +// EncodeRLP serializes b into the Ethereum RLP block format. +func (d *DiffLayer) EncodeRLP(w io.Writer) error { + storageReceipts := make([]*ReceiptForStorage, len(d.Receipts)) + for i, receipt := range d.Receipts { + storageReceipts[i] = (*ReceiptForStorage)(receipt) + } + return rlp.Encode(w, extDiffLayer{ + BlockHash: d.BlockHash, + Number: d.Number, + Receipts: storageReceipts, + Codes: d.Codes, + Destructs: d.Destructs, + Accounts: d.Accounts, + Storages: d.Storages, + }) +} + +func (d *DiffLayer) Validate() error { + if d.BlockHash == (common.Hash{}) { + return errors.New("blockHash can't be empty") + } + for _, storage := range d.Storages { + if len(storage.Keys) != len(storage.Vals) { + return errors.New("the length of keys and values mismatch in storage") + } + } + return nil +} + +type DiffCode struct { + Hash common.Hash + Code []byte +} + +type DiffAccount struct { + Account common.Address + Blob []byte +} + +type DiffStorage struct { + Account common.Address + Keys []string + Vals [][]byte +} diff --git a/core/vm/contracts_lightclient_test.go b/core/vm/contracts_lightclient_test.go index cf17c03559..e1634be809 100644 --- a/core/vm/contracts_lightclient_test.go +++ b/core/vm/contracts_lightclient_test.go @@ -10,7 +10,7 @@ import ( ) const ( - testHeight uint64 = 66848226 + testHeight uint64 = 66848226 ) func TestTmHeaderValidateAndMerkleProofValidate(t *testing.T) { diff --git a/core/vm/lightclient/types.go b/core/vm/lightclient/types.go index 406d428e74..6006fe04d4 100644 --- a/core/vm/lightclient/types.go +++ b/core/vm/lightclient/types.go @@ -103,7 +103,7 @@ func (cs ConsensusState) EncodeConsensusState() ([]byte, error) { copy(encodingBytes[pos:pos+chainIDLength], cs.ChainID) pos += chainIDLength - binary.BigEndian.PutUint64(encodingBytes[pos:pos+heightLength], uint64(cs.Height)) + binary.BigEndian.PutUint64(encodingBytes[pos:pos+heightLength], cs.Height) pos += heightLength copy(encodingBytes[pos:pos+appHashLength], cs.AppHash) diff --git a/eth/backend.go b/eth/backend.go index b52591fd71..bdae4b4235 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -42,6 +42,7 @@ import ( "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" + "github.com/ethereum/go-ethereum/eth/protocols/diff" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/ethdb" @@ -128,7 +129,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { ethashConfig.NotifyFull = config.Miner.NotifyFull // Assemble the Ethereum object - chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false) + chainDb, err := stack.OpenAndMergeDatabase("chaindata", config.DatabaseCache, config.DatabaseHandles, + config.DatabaseFreezer, config.DatabaseDiff, "eth/db/chaindata/", false, config.PersistDiff) if err != nil { return nil, err } @@ -197,7 +199,14 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { Preimages: config.Preimages, } ) - eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit) + bcOps := make([]core.BlockChainOption, 0) + if config.DiffSync { + bcOps = append(bcOps, core.EnableLightProcessor) + } + if config.PersistDiff { + bcOps = append(bcOps, core.EnablePersistDiff(config.DiffBlock)) + } + eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit, bcOps...) if err != nil { return nil, err } @@ -232,6 +241,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { Checkpoint: checkpoint, Whitelist: config.Whitelist, DirectBroadcast: config.DirectBroadcast, + DiffSync: config.DiffSync, }); err != nil { return nil, err } @@ -534,9 +544,11 @@ func (s *Ethereum) BloomIndexer() *core.ChainIndexer { return s.bloomIndexer } // network protocols to start. func (s *Ethereum) Protocols() []p2p.Protocol { protos := eth.MakeProtocols((*ethHandler)(s.handler), s.networkID, s.ethDialCandidates) - if s.config.SnapshotCache > 0 { + if !s.config.DisableSnapProtocol && s.config.SnapshotCache > 0 { protos = append(protos, snap.MakeProtocols((*snapHandler)(s.handler), s.snapDialCandidates)...) } + // diff protocol can still open without snap protocol + protos = append(protos, diff.MakeProtocols((*diffHandler)(s.handler), s.snapDialCandidates)...) return protos } diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 3dee90c424..55be200d59 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -161,10 +161,10 @@ type Downloader struct { quitLock sync.Mutex // Lock to prevent double closes // Testing hooks - syncInitHook func(uint64, uint64) // Method to call upon initiating a new sync run - bodyFetchHook func([]*types.Header) // Method to call upon starting a block body fetch - receiptFetchHook func([]*types.Header) // Method to call upon starting a receipt fetch - chainInsertHook func([]*fetchResult) // Method to call upon inserting a chain of blocks (possibly in multiple invocations) + syncInitHook func(uint64, uint64) // Method to call upon initiating a new sync run + bodyFetchHook func([]*types.Header, ...interface{}) // Method to call upon starting a block body fetch + receiptFetchHook func([]*types.Header, ...interface{}) // Method to call upon starting a receipt fetch + chainInsertHook func([]*fetchResult) // Method to call upon inserting a chain of blocks (possibly in multiple invocations) } // LightChain encapsulates functions required to synchronise a light chain. @@ -220,8 +220,45 @@ type BlockChain interface { Snapshots() *snapshot.Tree } +type DownloadOption func(downloader *Downloader) *Downloader + +type IDiffPeer interface { + RequestDiffLayers([]common.Hash) error +} + +type IPeerSet interface { + GetDiffPeer(string) IDiffPeer +} + +func EnableDiffFetchOp(peers IPeerSet) DownloadOption { + return func(dl *Downloader) *Downloader { + var hook = func(headers []*types.Header, args ...interface{}) { + if len(args) < 2 { + return + } + peerID, ok := args[1].(string) + if !ok { + return + } + mode, ok := args[0].(SyncMode) + if !ok { + return + } + if ep := peers.GetDiffPeer(peerID); mode == FullSync && ep != nil { + hashes := make([]common.Hash, 0, len(headers)) + for _, header := range headers { + hashes = append(hashes, header.Hash()) + } + ep.RequestDiffLayers(hashes) + } + } + dl.bodyFetchHook = hook + return dl + } +} + // New creates a new downloader to fetch hashes and blocks from remote peers. -func New(checkpoint uint64, stateDb ethdb.Database, stateBloom *trie.SyncBloom, mux *event.TypeMux, chain BlockChain, lightchain LightChain, dropPeer peerDropFn) *Downloader { +func New(checkpoint uint64, stateDb ethdb.Database, stateBloom *trie.SyncBloom, mux *event.TypeMux, chain BlockChain, lightchain LightChain, dropPeer peerDropFn, options ...DownloadOption) *Downloader { if lightchain == nil { lightchain = chain } @@ -252,6 +289,11 @@ func New(checkpoint uint64, stateDb ethdb.Database, stateBloom *trie.SyncBloom, }, trackStateReq: make(chan *stateReq), } + for _, option := range options { + if dl != nil { + dl = option(dl) + } + } go dl.qosTuner() go dl.stateFetcher() return dl @@ -1363,7 +1405,7 @@ func (d *Downloader) fetchReceipts(from uint64) error { // - kind: textual label of the type being downloaded to display in log messages func (d *Downloader) fetchParts(deliveryCh chan dataPack, deliver func(dataPack) (int, error), wakeCh chan bool, expire func() map[string]int, pending func() int, inFlight func() bool, reserve func(*peerConnection, int) (*fetchRequest, bool, bool), - fetchHook func([]*types.Header), fetch func(*peerConnection, *fetchRequest) error, cancel func(*fetchRequest), capacity func(*peerConnection) int, + fetchHook func([]*types.Header, ...interface{}), fetch func(*peerConnection, *fetchRequest) error, cancel func(*fetchRequest), capacity func(*peerConnection) int, idle func() ([]*peerConnection, int), setIdle func(*peerConnection, int, time.Time), kind string) error { // Create a ticker to detect expired retrieval tasks @@ -1512,7 +1554,7 @@ func (d *Downloader) fetchParts(deliveryCh chan dataPack, deliver func(dataPack) } // Fetch the chunk and make sure any errors return the hashes to the queue if fetchHook != nil { - fetchHook(request.Headers) + fetchHook(request.Headers, d.getMode(), peer.id) } if err := fetch(peer, request); err != nil { // Although we could try and make an attempt to fix this, this error really diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 794160993b..66f6872025 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -921,10 +921,10 @@ func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { // Instrument the downloader to signal body requests bodiesHave, receiptsHave := int32(0), int32(0) - tester.downloader.bodyFetchHook = func(headers []*types.Header) { + tester.downloader.bodyFetchHook = func(headers []*types.Header, _ ...interface{}) { atomic.AddInt32(&bodiesHave, int32(len(headers))) } - tester.downloader.receiptFetchHook = func(headers []*types.Header) { + tester.downloader.receiptFetchHook = func(headers []*types.Header, _ ...interface{}) { atomic.AddInt32(&receiptsHave, int32(len(headers))) } // Synchronise with the peer and make sure all blocks were retrieved diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 40dece429a..bf143ba02c 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -80,6 +80,7 @@ var Defaults = Config{ TrieTimeout: 60 * time.Minute, TriesInMemory: 128, SnapshotCache: 102, + DiffBlock: uint64(864000), Miner: miner.Config{ GasFloor: 8000000, GasCeil: 8000000, @@ -131,9 +132,11 @@ type Config struct { EthDiscoveryURLs []string SnapDiscoveryURLs []string - NoPruning bool // Whether to disable pruning and flush everything to disk - DirectBroadcast bool - RangeLimit bool + NoPruning bool // Whether to disable pruning and flush everything to disk + DirectBroadcast bool + DisableSnapProtocol bool //Whether disable snap protocol + DiffSync bool // Whether support diff sync + RangeLimit bool TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. @@ -159,6 +162,9 @@ type Config struct { DatabaseHandles int `toml:"-"` DatabaseCache int DatabaseFreezer string + DatabaseDiff string + PersistDiff bool + DiffBlock uint64 TrieCleanCache int TrieCleanCacheJournal string `toml:",omitempty"` // Disk journal directory for trie cache to survive node restarts diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index fa31b78335..258ade2293 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -40,6 +40,7 @@ func (c Config) MarshalTOML() (interface{}, error) { DatabaseHandles int `toml:"-"` DatabaseCache int DatabaseFreezer string + DatabaseDiff string TrieCleanCache int TrieCleanCacheJournal string `toml:",omitempty"` TrieCleanCacheRejournal time.Duration `toml:",omitempty"` @@ -48,6 +49,8 @@ func (c Config) MarshalTOML() (interface{}, error) { TriesInMemory uint64 `toml:",omitempty"` SnapshotCache int Preimages bool + PersistDiff bool + DiffBlock uint64 `toml:",omitempty"` Miner miner.Config Ethash ethash.Config TxPool core.TxPoolConfig @@ -84,6 +87,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.DatabaseHandles = c.DatabaseHandles enc.DatabaseCache = c.DatabaseCache enc.DatabaseFreezer = c.DatabaseFreezer + enc.DatabaseDiff = c.DatabaseDiff enc.TrieCleanCache = c.TrieCleanCache enc.TrieCleanCacheJournal = c.TrieCleanCacheJournal enc.TrieCleanCacheRejournal = c.TrieCleanCacheRejournal @@ -92,6 +96,8 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.TriesInMemory = c.TriesInMemory enc.SnapshotCache = c.SnapshotCache enc.Preimages = c.Preimages + enc.PersistDiff = c.PersistDiff + enc.DiffBlock = c.DiffBlock enc.Miner = c.Miner enc.Ethash = c.Ethash enc.TxPool = c.TxPool @@ -133,6 +139,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { DatabaseHandles *int `toml:"-"` DatabaseCache *int DatabaseFreezer *string + DatabaseDiff *string + PersistDiff *bool + DiffBlock *uint64 `toml:",omitempty"` TrieCleanCache *int TrieCleanCacheJournal *string `toml:",omitempty"` TrieCleanCacheRejournal *time.Duration `toml:",omitempty"` @@ -224,6 +233,15 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.DatabaseFreezer != nil { c.DatabaseFreezer = *dec.DatabaseFreezer } + if dec.DatabaseDiff != nil { + c.DatabaseDiff = *dec.DatabaseDiff + } + if dec.PersistDiff != nil { + c.PersistDiff = *dec.PersistDiff + } + if dec.DiffBlock != nil { + c.DiffBlock = *dec.DiffBlock + } if dec.TrieCleanCache != nil { c.TrieCleanCache = *dec.TrieCleanCache } diff --git a/eth/fetcher/block_fetcher.go b/eth/fetcher/block_fetcher.go index 4819e9eab5..4007fee650 100644 --- a/eth/fetcher/block_fetcher.go +++ b/eth/fetcher/block_fetcher.go @@ -82,6 +82,9 @@ type headerRequesterFn func(common.Hash) error // bodyRequesterFn is a callback type for sending a body retrieval request. type bodyRequesterFn func([]common.Hash) error +// DiffRequesterFn is a callback type for sending a diff layer retrieval request. +type DiffRequesterFn func([]common.Hash) error + // headerVerifierFn is a callback type to verify a block's header for fast propagation. type headerVerifierFn func(header *types.Header) error @@ -112,6 +115,8 @@ type blockAnnounce struct { fetchHeader headerRequesterFn // Fetcher function to retrieve the header of an announced block fetchBodies bodyRequesterFn // Fetcher function to retrieve the body of an announced block + fetchDiffs DiffRequesterFn // Fetcher function to retrieve the diff layer of an announced block + } // headerFilterTask represents a batch of headers needing fetcher filtering. @@ -246,7 +251,7 @@ func (f *BlockFetcher) Stop() { // Notify announces the fetcher of the potential availability of a new block in // the network. func (f *BlockFetcher) Notify(peer string, hash common.Hash, number uint64, time time.Time, - headerFetcher headerRequesterFn, bodyFetcher bodyRequesterFn) error { + headerFetcher headerRequesterFn, bodyFetcher bodyRequesterFn, diffFetcher DiffRequesterFn) error { block := &blockAnnounce{ hash: hash, number: number, @@ -254,6 +259,7 @@ func (f *BlockFetcher) Notify(peer string, hash common.Hash, number uint64, time origin: peer, fetchHeader: headerFetcher, fetchBodies: bodyFetcher, + fetchDiffs: diffFetcher, } select { case f.notify <- block: @@ -481,10 +487,15 @@ func (f *BlockFetcher) loop() { // Create a closure of the fetch and schedule in on a new thread fetchHeader, hashes := f.fetching[hashes[0]].fetchHeader, hashes + fetchDiff := f.fetching[hashes[0]].fetchDiffs + gopool.Submit(func() { if f.fetchingHook != nil { f.fetchingHook(hashes) } + if fetchDiff != nil { + fetchDiff(hashes) + } for _, hash := range hashes { headerFetchMeter.Mark(1) fetchHeader(hash) // Suboptimal, but protocol doesn't allow batch header retrievals diff --git a/eth/fetcher/block_fetcher_test.go b/eth/fetcher/block_fetcher_test.go index a6eef71da0..9a40d4cb3e 100644 --- a/eth/fetcher/block_fetcher_test.go +++ b/eth/fetcher/block_fetcher_test.go @@ -343,7 +343,7 @@ func testSequentialAnnouncements(t *testing.T, light bool) { } } for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher, nil) verifyImportEvent(t, imported, true) } verifyImportDone(t, imported) @@ -392,9 +392,9 @@ func testConcurrentAnnouncements(t *testing.T, light bool) { } } for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("first", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), firstHeaderWrapper, firstBodyFetcher) - tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout+time.Millisecond), secondHeaderWrapper, secondBodyFetcher) - tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout-time.Millisecond), secondHeaderWrapper, secondBodyFetcher) + tester.fetcher.Notify("first", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), firstHeaderWrapper, firstBodyFetcher, nil) + tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout+time.Millisecond), secondHeaderWrapper, secondBodyFetcher, nil) + tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout-time.Millisecond), secondHeaderWrapper, secondBodyFetcher, nil) verifyImportEvent(t, imported, true) } verifyImportDone(t, imported) @@ -441,7 +441,7 @@ func testOverlappingAnnouncements(t *testing.T, light bool) { } for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher, nil) select { case <-imported: case <-time.After(time.Second): @@ -488,7 +488,7 @@ func testPendingDeduplication(t *testing.T, light bool) { } // Announce the same block many times until it's fetched (wait for any pending ops) for checkNonExist() { - tester.fetcher.Notify("repeater", hashes[0], 1, time.Now().Add(-arriveTimeout), headerWrapper, bodyFetcher) + tester.fetcher.Notify("repeater", hashes[0], 1, time.Now().Add(-arriveTimeout), headerWrapper, bodyFetcher, nil) time.Sleep(time.Millisecond) } time.Sleep(delay) @@ -532,12 +532,12 @@ func testRandomArrivalImport(t *testing.T, light bool) { } for i := len(hashes) - 1; i >= 0; i-- { if i != skip { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher, nil) time.Sleep(time.Millisecond) } } // Finally announce the skipped entry and check full import - tester.fetcher.Notify("valid", hashes[skip], uint64(len(hashes)-skip-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) + tester.fetcher.Notify("valid", hashes[skip], uint64(len(hashes)-skip-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher, nil) verifyImportCount(t, imported, len(hashes)-1) verifyChainHeight(t, tester, uint64(len(hashes)-1)) } @@ -560,7 +560,7 @@ func TestQueueGapFill(t *testing.T) { for i := len(hashes) - 1; i >= 0; i-- { if i != skip { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher, nil) time.Sleep(time.Millisecond) } } @@ -593,7 +593,7 @@ func TestImportDeduplication(t *testing.T) { tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } // Announce the duplicating block, wait for retrieval, and also propagate directly - tester.fetcher.Notify("valid", hashes[0], 1, time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) + tester.fetcher.Notify("valid", hashes[0], 1, time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher, nil) <-fetching tester.fetcher.Enqueue("valid", blocks[hashes[0]]) @@ -669,14 +669,14 @@ func testDistantAnnouncementDiscarding(t *testing.T, light bool) { tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- struct{}{} } // Ensure that a block with a lower number than the threshold is discarded - tester.fetcher.Notify("lower", hashes[low], blocks[hashes[low]].NumberU64(), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) + tester.fetcher.Notify("lower", hashes[low], blocks[hashes[low]].NumberU64(), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher, nil) select { case <-time.After(50 * time.Millisecond): case <-fetching: t.Fatalf("fetcher requested stale header") } // Ensure that a block with a higher number than the threshold is discarded - tester.fetcher.Notify("higher", hashes[high], blocks[hashes[high]].NumberU64(), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) + tester.fetcher.Notify("higher", hashes[high], blocks[hashes[high]].NumberU64(), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher, nil) select { case <-time.After(50 * time.Millisecond): case <-fetching: @@ -712,7 +712,7 @@ func testInvalidNumberAnnouncement(t *testing.T, light bool) { } } // Announce a block with a bad number, check for immediate drop - tester.fetcher.Notify("bad", hashes[0], 2, time.Now().Add(-arriveTimeout), badHeaderFetcher, badBodyFetcher) + tester.fetcher.Notify("bad", hashes[0], 2, time.Now().Add(-arriveTimeout), badHeaderFetcher, badBodyFetcher, nil) verifyImportEvent(t, imported, false) tester.lock.RLock() @@ -726,7 +726,7 @@ func testInvalidNumberAnnouncement(t *testing.T, light bool) { goodHeaderFetcher := tester.makeHeaderFetcher("good", blocks, -gatherSlack) goodBodyFetcher := tester.makeBodyFetcher("good", blocks, 0) // Make sure a good announcement passes without a drop - tester.fetcher.Notify("good", hashes[0], 1, time.Now().Add(-arriveTimeout), goodHeaderFetcher, goodBodyFetcher) + tester.fetcher.Notify("good", hashes[0], 1, time.Now().Add(-arriveTimeout), goodHeaderFetcher, goodBodyFetcher, nil) verifyImportEvent(t, imported, true) tester.lock.RLock() @@ -765,7 +765,7 @@ func TestEmptyBlockShortCircuit(t *testing.T) { } // Iteratively announce blocks until all are imported for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher, nil) // All announces should fetch the header verifyFetchingEvent(t, fetching, true) @@ -808,9 +808,9 @@ func TestHashMemoryExhaustionAttack(t *testing.T) { // Feed the tester a huge hashset from the attacker, and a limited from the valid peer for i := 0; i < len(attack); i++ { if i < maxQueueDist { - tester.fetcher.Notify("valid", hashes[len(hashes)-2-i], uint64(i+1), time.Now(), validHeaderFetcher, validBodyFetcher) + tester.fetcher.Notify("valid", hashes[len(hashes)-2-i], uint64(i+1), time.Now(), validHeaderFetcher, validBodyFetcher, nil) } - tester.fetcher.Notify("attacker", attack[i], 1 /* don't distance drop */, time.Now(), attackerHeaderFetcher, attackerBodyFetcher) + tester.fetcher.Notify("attacker", attack[i], 1 /* don't distance drop */, time.Now(), attackerHeaderFetcher, attackerBodyFetcher, nil) } if count := atomic.LoadInt32(&announces); count != hashLimit+maxQueueDist { t.Fatalf("queued announce count mismatch: have %d, want %d", count, hashLimit+maxQueueDist) @@ -820,7 +820,7 @@ func TestHashMemoryExhaustionAttack(t *testing.T) { // Feed the remaining valid hashes to ensure DOS protection state remains clean for i := len(hashes) - maxQueueDist - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), validHeaderFetcher, validBodyFetcher) + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), validHeaderFetcher, validBodyFetcher, nil) verifyImportEvent(t, imported, true) } verifyImportDone(t, imported) diff --git a/eth/handler.go b/eth/handler.go index f42109753d..41b459d2d5 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/fetcher" + "github.com/ethereum/go-ethereum/eth/protocols/diff" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/ethdb" @@ -37,6 +38,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) @@ -81,6 +83,7 @@ type handlerConfig struct { TxPool txPool // Transaction pool to propagate from Network uint64 // Network identifier to adfvertise Sync downloader.SyncMode // Whether to fast or full sync + DiffSync bool // Whether to diff sync BloomCache uint64 // Megabytes to alloc for fast sync bloom EventMux *event.TypeMux // Legacy event mux, deprecate for `feed` Checkpoint *params.TrustedCheckpoint // Hard coded checkpoint for sync challenges @@ -96,6 +99,7 @@ type handler struct { snapSync uint32 // Flag whether fast sync should operate on top of the snap protocol acceptTxs uint32 // Flag whether we're considered synchronised (enables transaction processing) directBroadcast bool + diffSync bool // Flag whether diff sync should operate on top of the diff protocol checkpointNumber uint64 // Block number for the sync progress validator to cross reference checkpointHash common.Hash // Block hash for the sync progress validator to cross reference @@ -143,6 +147,7 @@ func newHandler(config *handlerConfig) (*handler, error) { peers: newPeerSet(), whitelist: config.Whitelist, directBroadcast: config.DirectBroadcast, + diffSync: config.DiffSync, txsyncCh: make(chan *txsync), quitSync: make(chan struct{}), } @@ -187,7 +192,11 @@ func newHandler(config *handlerConfig) (*handler, error) { if atomic.LoadUint32(&h.fastSync) == 1 && atomic.LoadUint32(&h.snapSync) == 0 { h.stateBloom = trie.NewSyncBloom(config.BloomCache, config.Database) } - h.downloader = downloader.New(h.checkpointNumber, config.Database, h.stateBloom, h.eventMux, h.chain, nil, h.removePeer) + var downloadOptions []downloader.DownloadOption + if h.diffSync { + downloadOptions = append(downloadOptions, downloader.EnableDiffFetchOp(h.peers)) + } + h.downloader = downloader.New(h.checkpointNumber, config.Database, h.stateBloom, h.eventMux, h.chain, nil, h.removePeer, downloadOptions...) // Construct the fetcher (short sync) validator := func(header *types.Header) error { @@ -246,6 +255,11 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { peer.Log().Error("Snapshot extension barrier failed", "err", err) return err } + diff, err := h.peers.waitDiffExtension(peer) + if err != nil { + peer.Log().Error("Diff extension barrier failed", "err", err) + return err + } // TODO(karalabe): Not sure why this is needed if !h.chainSync.handlePeerEvent(peer) { return p2p.DiscQuitting @@ -286,7 +300,7 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { peer.Log().Debug("Ethereum peer connected", "name", peer.Name()) // Register the peer locally - if err := h.peers.registerPeer(peer, snap); err != nil { + if err := h.peers.registerPeer(peer, snap, diff); err != nil { peer.Log().Error("Ethereum peer registration failed", "err", err) return err } @@ -357,6 +371,21 @@ func (h *handler) runSnapExtension(peer *snap.Peer, handler snap.Handler) error return handler(peer) } +// runDiffExtension registers a `diff` peer into the joint eth/diff peerset and +// starts handling inbound messages. As `diff` is only a satellite protocol to +// `eth`, all subsystem registrations and lifecycle management will be done by +// the main `eth` handler to prevent strange races. +func (h *handler) runDiffExtension(peer *diff.Peer, handler diff.Handler) error { + h.peerWG.Add(1) + defer h.peerWG.Done() + + if err := h.peers.registerDiffExtension(peer); err != nil { + peer.Log().Error("Diff extension registration failed", "err", err) + return err + } + return handler(peer) +} + // removePeer unregisters a peer from the downloader and fetchers, removes it from // the set of tracked peers and closes the network connection to it. func (h *handler) removePeer(id string) { @@ -449,13 +478,19 @@ func (h *handler) BroadcastBlock(block *types.Block, propagate bool) { // Send the block to a subset of our peers var transfer []*ethPeer if h.directBroadcast { - transfer = peers[:int(len(peers))] + transfer = peers[:] } else { transfer = peers[:int(math.Sqrt(float64(len(peers))))] } + diff := h.chain.GetDiffLayerRLP(block.Hash()) for _, peer := range transfer { + if len(diff) != 0 && peer.diffExt != nil { + // difflayer should send before block + peer.diffExt.SendDiffLayers([]rlp.RawValue{diff}) + } peer.AsyncSendNewBlock(block, td) } + log.Trace("Propagated block", "hash", hash, "recipients", len(transfer), "duration", common.PrettyDuration(time.Since(block.ReceivedAt))) return } diff --git a/eth/handler_diff.go b/eth/handler_diff.go new file mode 100644 index 0000000000..c996105f0f --- /dev/null +++ b/eth/handler_diff.go @@ -0,0 +1,87 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/eth/protocols/diff" + "github.com/ethereum/go-ethereum/p2p/enode" +) + +// diffHandler implements the diff.Backend interface to handle the various network +// packets that are sent as replies or broadcasts. +type diffHandler handler + +func (h *diffHandler) Chain() *core.BlockChain { return h.chain } + +// RunPeer is invoked when a peer joins on the `diff` protocol. +func (h *diffHandler) RunPeer(peer *diff.Peer, hand diff.Handler) error { + if err := peer.Handshake(h.diffSync); err != nil { + return err + } + defer h.chain.RemoveDiffPeer(peer.ID()) + return (*handler)(h).runDiffExtension(peer, hand) +} + +// PeerInfo retrieves all known `diff` information about a peer. +func (h *diffHandler) PeerInfo(id enode.ID) interface{} { + if p := h.peers.peer(id.String()); p != nil && p.diffExt != nil { + return p.diffExt.info() + } + return nil +} + +// Handle is invoked from a peer's message handler when it receives a new remote +// message that the handler couldn't consume and serve itself. +func (h *diffHandler) Handle(peer *diff.Peer, packet diff.Packet) error { + // DeliverSnapPacket is invoked from a peer's message handler when it transmits a + // data packet for the local node to consume. + switch packet := packet.(type) { + case *diff.DiffLayersPacket: + return h.handleDiffLayerPackage(packet, peer.ID(), false) + + case *diff.FullDiffLayersPacket: + return h.handleDiffLayerPackage(&packet.DiffLayersPacket, peer.ID(), true) + + default: + return fmt.Errorf("unexpected diff packet type: %T", packet) + } +} + +func (h *diffHandler) handleDiffLayerPackage(packet *diff.DiffLayersPacket, pid string, fulfilled bool) error { + diffs, err := packet.Unpack() + + if err != nil { + return err + } + for _, d := range diffs { + if d != nil { + if err := d.Validate(); err != nil { + return err + } + } + } + for _, diff := range diffs { + err := h.chain.HandleDiffLayer(diff, pid, fulfilled) + if err != nil { + return err + } + } + return nil +} diff --git a/eth/handler_diff_test.go b/eth/handler_diff_test.go new file mode 100644 index 0000000000..f6b967631a --- /dev/null +++ b/eth/handler_diff_test.go @@ -0,0 +1,203 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "crypto/rand" + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/protocols/diff" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" +) + +// testBackend is a mock implementation of the live Ethereum message handler. Its +// purpose is to allow testing the request/reply workflows and wire serialization +// in the `eth` protocol without actually doing any data processing. +type testBackend struct { + db ethdb.Database + chain *core.BlockChain + txpool *core.TxPool + + handler *handler +} + +// newTestBackend creates an empty chain and wraps it into a mock backend. +func newTestBackend(blocks int) *testBackend { + return newTestBackendWithGenerator(blocks) +} + +// newTestBackend creates a chain with a number of explicitly defined blocks and +// wraps it into a mock backend. +func newTestBackendWithGenerator(blocks int) *testBackend { + signer := types.HomesteadSigner{} + // Create a database pre-initialize with a genesis block + db := rawdb.NewMemoryDatabase() + (&core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(100000000000000000)}}, + }).MustCommit(db) + + chain, _ := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + generator := func(i int, block *core.BlockGen) { + // The chain maker doesn't have access to a chain, so the difficulty will be + // lets unset (nil). Set it here to the correct value. + block.SetCoinbase(testAddr) + + // We want to simulate an empty middle block, having the same state as the + // first one. The last is needs a state change again to force a reorg. + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), common.Address{0x01}, big.NewInt(1), params.TxGas, big.NewInt(1), nil), signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(chain, tx) + } + bs, _ := core.GenerateChain(params.TestChainConfig, chain.Genesis(), ethash.NewFaker(), db, blocks, generator) + if _, err := chain.InsertChain(bs); err != nil { + panic(err) + } + txpool := newTestTxPool() + + handler, _ := newHandler(&handlerConfig{ + Database: db, + Chain: chain, + TxPool: txpool, + Network: 1, + Sync: downloader.FullSync, + BloomCache: 1, + }) + handler.Start(100) + + txconfig := core.DefaultTxPoolConfig + txconfig.Journal = "" // Don't litter the disk with test journals + + return &testBackend{ + db: db, + chain: chain, + txpool: core.NewTxPool(txconfig, params.TestChainConfig, chain), + handler: handler, + } +} + +// close tears down the transaction pool and chain behind the mock backend. +func (b *testBackend) close() { + b.txpool.Stop() + b.chain.Stop() + b.handler.Stop() +} + +func (b *testBackend) Chain() *core.BlockChain { return b.chain } + +func (b *testBackend) RunPeer(peer *diff.Peer, handler diff.Handler) error { + // Normally the backend would do peer mainentance and handshakes. All that + // is omitted and we will just give control back to the handler. + return handler(peer) +} +func (b *testBackend) PeerInfo(enode.ID) interface{} { panic("not implemented") } + +func (b *testBackend) Handle(*diff.Peer, diff.Packet) error { + panic("data processing tests should be done in the handler package") +} + +type testPeer struct { + *diff.Peer + + net p2p.MsgReadWriter // Network layer reader/writer to simulate remote messaging + app *p2p.MsgPipeRW // Application layer reader/writer to simulate the local side +} + +// newTestPeer creates a new peer registered at the given data backend. +func newTestPeer(name string, version uint, backend *testBackend) (*testPeer, <-chan error) { + // Create a message pipe to communicate through + app, net := p2p.MsgPipe() + + // Start the peer on a new thread + var id enode.ID + rand.Read(id[:]) + + peer := diff.NewPeer(version, p2p.NewPeer(id, name, nil), net) + errc := make(chan error, 1) + go func() { + errc <- backend.RunPeer(peer, func(peer *diff.Peer) error { + + return diff.Handle((*diffHandler)(backend.handler), peer) + }) + }() + return &testPeer{app: app, net: net, Peer: peer}, errc +} + +// close terminates the local side of the peer, notifying the remote protocol +// manager of termination. +func (p *testPeer) close() { + p.Peer.Close() + p.app.Close() +} + +func TestHandleDiffLayer(t *testing.T) { + t.Parallel() + + blockNum := 1024 + waitInterval := 100 * time.Millisecond + backend := newTestBackend(blockNum) + defer backend.close() + + peer, _ := newTestPeer("peer", diff.Diff1, backend) + defer peer.close() + + tests := []struct { + DiffLayer *types.DiffLayer + Valid bool + }{ + {DiffLayer: &types.DiffLayer{ + BlockHash: common.Hash{0x1}, + Number: 1025, + }, Valid: true}, + {DiffLayer: &types.DiffLayer{ + BlockHash: common.Hash{0x2}, + Number: 3073, + }, Valid: false}, + {DiffLayer: &types.DiffLayer{ + BlockHash: common.Hash{0x3}, + Number: 500, + }, Valid: false}, + } + + for _, tt := range tests { + bz, _ := rlp.EncodeToBytes(tt.DiffLayer) + + p2p.Send(peer.app, diff.DiffLayerMsg, diff.DiffLayersPacket{rlp.RawValue(bz)}) + } + time.Sleep(waitInterval) + for idx, tt := range tests { + diff := backend.chain.GetUnTrustedDiffLayer(tt.DiffLayer.BlockHash, "") + if (tt.Valid && diff == nil) || (!tt.Valid && diff != nil) { + t.Errorf("test: %d, diff layer handle failed", idx) + } + } +} diff --git a/eth/handler_eth.go b/eth/handler_eth.go index 3ff9f2245b..d2cf83fbfe 100644 --- a/eth/handler_eth.go +++ b/eth/handler_eth.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/fetcher" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/enode" @@ -98,7 +99,6 @@ func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error { case *eth.PooledTransactionsPacket: return h.txFetcher.Enqueue(peer.ID(), *packet, true) - default: return fmt.Errorf("unexpected eth packet type: %T", packet) } @@ -191,8 +191,17 @@ func (h *ethHandler) handleBlockAnnounces(peer *eth.Peer, hashes []common.Hash, unknownNumbers = append(unknownNumbers, numbers[i]) } } + // self support diff sync + var diffFetcher fetcher.DiffRequesterFn + if h.diffSync { + // the peer support diff protocol + if ep := h.peers.peer(peer.ID()); ep != nil && ep.diffExt != nil { + diffFetcher = ep.diffExt.RequestDiffLayers + } + } + for i := 0; i < len(unknownHashes); i++ { - h.blockFetcher.Notify(peer.ID(), unknownHashes[i], unknownNumbers[i], time.Now(), peer.RequestOneHeader, peer.RequestBodies) + h.blockFetcher.Notify(peer.ID(), unknownHashes[i], unknownNumbers[i], time.Now(), peer.RequestOneHeader, peer.RequestBodies, diffFetcher) } return nil } diff --git a/eth/peer.go b/eth/peer.go index 1cea9c640e..2fb6fabf26 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -21,6 +21,7 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum/eth/protocols/diff" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" ) @@ -37,6 +38,7 @@ type ethPeerInfo struct { type ethPeer struct { *eth.Peer snapExt *snapPeer // Satellite `snap` connection + diffExt *diffPeer syncDrop *time.Timer // Connection dropper if `eth` sync progress isn't validated in time snapWait chan struct{} // Notification channel for snap connections @@ -60,11 +62,31 @@ type snapPeerInfo struct { Version uint `json:"version"` // Snapshot protocol version negotiated } +// diffPeerInfo represents a short summary of the `diff` sub-protocol metadata known +// about a connected peer. +type diffPeerInfo struct { + Version uint `json:"version"` // diff protocol version negotiated + DiffSync bool `json:"diff_sync"` +} + // snapPeer is a wrapper around snap.Peer to maintain a few extra metadata. type snapPeer struct { *snap.Peer } +// diffPeer is a wrapper around diff.Peer to maintain a few extra metadata. +type diffPeer struct { + *diff.Peer +} + +// info gathers and returns some `diff` protocol metadata known about a peer. +func (p *diffPeer) info() *diffPeerInfo { + return &diffPeerInfo{ + Version: p.Version(), + DiffSync: p.DiffSync(), + } +} + // info gathers and returns some `snap` protocol metadata known about a peer. func (p *snapPeer) info() *snapPeerInfo { return &snapPeerInfo{ diff --git a/eth/peerset.go b/eth/peerset.go index 1e864a8e46..f0955f34c6 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -22,6 +22,8 @@ import ( "sync" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/protocols/diff" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/p2p" @@ -43,6 +45,10 @@ var ( // errSnapWithoutEth is returned if a peer attempts to connect only on the // snap protocol without advertizing the eth main protocol. errSnapWithoutEth = errors.New("peer connected on snap without compatible eth support") + + // errDiffWithoutEth is returned if a peer attempts to connect only on the + // diff protocol without advertizing the eth main protocol. + errDiffWithoutEth = errors.New("peer connected on diff without compatible eth support") ) // peerSet represents the collection of active peers currently participating in @@ -54,6 +60,9 @@ type peerSet struct { snapWait map[string]chan *snap.Peer // Peers connected on `eth` waiting for their snap extension snapPend map[string]*snap.Peer // Peers connected on the `snap` protocol, but not yet on `eth` + diffWait map[string]chan *diff.Peer // Peers connected on `eth` waiting for their diff extension + diffPend map[string]*diff.Peer // Peers connected on the `diff` protocol, but not yet on `eth` + lock sync.RWMutex closed bool } @@ -64,6 +73,8 @@ func newPeerSet() *peerSet { peers: make(map[string]*ethPeer), snapWait: make(map[string]chan *snap.Peer), snapPend: make(map[string]*snap.Peer), + diffWait: make(map[string]chan *diff.Peer), + diffPend: make(map[string]*diff.Peer), } } @@ -97,6 +108,36 @@ func (ps *peerSet) registerSnapExtension(peer *snap.Peer) error { return nil } +// registerDiffExtension unblocks an already connected `eth` peer waiting for its +// `diff` extension, or if no such peer exists, tracks the extension for the time +// being until the `eth` main protocol starts looking for it. +func (ps *peerSet) registerDiffExtension(peer *diff.Peer) error { + // Reject the peer if it advertises `diff` without `eth` as `diff` is only a + // satellite protocol meaningful with the chain selection of `eth` + if !peer.RunningCap(eth.ProtocolName, eth.ProtocolVersions) { + return errDiffWithoutEth + } + // Ensure nobody can double connect + ps.lock.Lock() + defer ps.lock.Unlock() + + id := peer.ID() + if _, ok := ps.peers[id]; ok { + return errPeerAlreadyRegistered // avoid connections with the same id as existing ones + } + if _, ok := ps.diffPend[id]; ok { + return errPeerAlreadyRegistered // avoid connections with the same id as pending ones + } + // Inject the peer into an `eth` counterpart is available, otherwise save for later + if wait, ok := ps.diffWait[id]; ok { + delete(ps.diffWait, id) + wait <- peer + return nil + } + ps.diffPend[id] = peer + return nil +} + // waitExtensions blocks until all satellite protocols are connected and tracked // by the peerset. func (ps *peerSet) waitSnapExtension(peer *eth.Peer) (*snap.Peer, error) { @@ -131,9 +172,50 @@ func (ps *peerSet) waitSnapExtension(peer *eth.Peer) (*snap.Peer, error) { return <-wait, nil } +// waitDiffExtension blocks until all satellite protocols are connected and tracked +// by the peerset. +func (ps *peerSet) waitDiffExtension(peer *eth.Peer) (*diff.Peer, error) { + // If the peer does not support a compatible `diff`, don't wait + if !peer.RunningCap(diff.ProtocolName, diff.ProtocolVersions) { + return nil, nil + } + // Ensure nobody can double connect + ps.lock.Lock() + + id := peer.ID() + if _, ok := ps.peers[id]; ok { + ps.lock.Unlock() + return nil, errPeerAlreadyRegistered // avoid connections with the same id as existing ones + } + if _, ok := ps.diffWait[id]; ok { + ps.lock.Unlock() + return nil, errPeerAlreadyRegistered // avoid connections with the same id as pending ones + } + // If `diff` already connected, retrieve the peer from the pending set + if diff, ok := ps.diffPend[id]; ok { + delete(ps.diffPend, id) + + ps.lock.Unlock() + return diff, nil + } + // Otherwise wait for `diff` to connect concurrently + wait := make(chan *diff.Peer) + ps.diffWait[id] = wait + ps.lock.Unlock() + + return <-wait, nil +} + +func (ps *peerSet) GetDiffPeer(pid string) downloader.IDiffPeer { + if p := ps.peer(pid); p != nil && p.diffExt != nil { + return p.diffExt + } + return nil +} + // registerPeer injects a new `eth` peer into the working set, or returns an error // if the peer is already known. -func (ps *peerSet) registerPeer(peer *eth.Peer, ext *snap.Peer) error { +func (ps *peerSet) registerPeer(peer *eth.Peer, ext *snap.Peer, diffExt *diff.Peer) error { // Start tracking the new peer ps.lock.Lock() defer ps.lock.Unlock() @@ -152,6 +234,9 @@ func (ps *peerSet) registerPeer(peer *eth.Peer, ext *snap.Peer) error { eth.snapExt = &snapPeer{ext} ps.snapPeers++ } + if diffExt != nil { + eth.diffExt = &diffPeer{diffExt} + } ps.peers[id] = eth return nil } diff --git a/eth/protocols/diff/discovery.go b/eth/protocols/diff/discovery.go new file mode 100644 index 0000000000..6a6ff066db --- /dev/null +++ b/eth/protocols/diff/discovery.go @@ -0,0 +1,32 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package diff + +import ( + "github.com/ethereum/go-ethereum/rlp" +) + +// enrEntry is the ENR entry which advertises `diff` protocol on the discovery. +type enrEntry struct { + // Ignore additional fields (for forward compatibility). + Rest []rlp.RawValue `rlp:"tail"` +} + +// ENRKey implements enr.Entry. +func (e enrEntry) ENRKey() string { + return "diff" +} diff --git a/eth/protocols/diff/handler.go b/eth/protocols/diff/handler.go new file mode 100644 index 0000000000..8678ff7f65 --- /dev/null +++ b/eth/protocols/diff/handler.go @@ -0,0 +1,180 @@ +package diff + +import ( + "fmt" + "time" + + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/enr" + "github.com/ethereum/go-ethereum/rlp" +) + +const ( + // softResponseLimit is the target maximum size of replies to data retrievals. + softResponseLimit = 2 * 1024 * 1024 + + // maxDiffLayerServe is the maximum number of diff layers to serve. + maxDiffLayerServe = 1024 +) + +var requestTracker = NewTracker(time.Minute) + +// Handler is a callback to invoke from an outside runner after the boilerplate +// exchanges have passed. +type Handler func(peer *Peer) error + +type Backend interface { + // Chain retrieves the blockchain object to serve data. + Chain() *core.BlockChain + + // RunPeer is invoked when a peer joins on the `eth` protocol. The handler + // should do any peer maintenance work, handshakes and validations. If all + // is passed, control should be given back to the `handler` to process the + // inbound messages going forward. + RunPeer(peer *Peer, handler Handler) error + + PeerInfo(id enode.ID) interface{} + + Handle(peer *Peer, packet Packet) error +} + +// MakeProtocols constructs the P2P protocol definitions for `diff`. +func MakeProtocols(backend Backend, dnsdisc enode.Iterator) []p2p.Protocol { + // Filter the discovery iterator for nodes advertising diff support. + dnsdisc = enode.Filter(dnsdisc, func(n *enode.Node) bool { + var diff enrEntry + return n.Load(&diff) == nil + }) + + protocols := make([]p2p.Protocol, len(ProtocolVersions)) + for i, version := range ProtocolVersions { + version := version // Closure + + protocols[i] = p2p.Protocol{ + Name: ProtocolName, + Version: version, + Length: protocolLengths[version], + Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + return backend.RunPeer(NewPeer(version, p, rw), func(peer *Peer) error { + defer peer.Close() + return Handle(backend, peer) + }) + }, + NodeInfo: func() interface{} { + return nodeInfo(backend.Chain()) + }, + PeerInfo: func(id enode.ID) interface{} { + return backend.PeerInfo(id) + }, + Attributes: []enr.Entry{&enrEntry{}}, + DialCandidates: dnsdisc, + } + } + return protocols +} + +// Handle is the callback invoked to manage the life cycle of a `diff` peer. +// When this function terminates, the peer is disconnected. +func Handle(backend Backend, peer *Peer) error { + for { + if err := handleMessage(backend, peer); err != nil { + peer.Log().Debug("Message handling failed in `diff`", "err", err) + return err + } + } +} + +// handleMessage is invoked whenever an inbound message is received from a +// remote peer on the `diff` protocol. The remote connection is torn down upon +// returning any error. +func handleMessage(backend Backend, peer *Peer) error { + // Read the next message from the remote peer, and ensure it's fully consumed + msg, err := peer.rw.ReadMsg() + if err != nil { + return err + } + if msg.Size > maxMessageSize { + return fmt.Errorf("%w: %v > %v", errMsgTooLarge, msg.Size, maxMessageSize) + } + defer msg.Discard() + start := time.Now() + // Track the emount of time it takes to serve the request and run the handler + if metrics.Enabled { + h := fmt.Sprintf("%s/%s/%d/%#02x", p2p.HandleHistName, ProtocolName, peer.Version(), msg.Code) + defer func(start time.Time) { + sampler := func() metrics.Sample { + return metrics.ResettingSample( + metrics.NewExpDecaySample(1028, 0.015), + ) + } + metrics.GetOrRegisterHistogramLazy(h, nil, sampler).Update(time.Since(start).Microseconds()) + }(start) + } + // Handle the message depending on its contents + switch { + case msg.Code == GetDiffLayerMsg: + res := new(GetDiffLayersPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + diffs := answerDiffLayersQuery(backend, res) + + p2p.Send(peer.rw, FullDiffLayerMsg, &FullDiffLayersPacket{ + RequestId: res.RequestId, + DiffLayersPacket: diffs, + }) + return nil + + case msg.Code == DiffLayerMsg: + // A batch of trie nodes arrived to one of our previous requests + res := new(DiffLayersPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, res) + case msg.Code == FullDiffLayerMsg: + // A batch of trie nodes arrived to one of our previous requests + res := new(FullDiffLayersPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + if fulfilled := requestTracker.Fulfil(peer.id, peer.version, FullDiffLayerMsg, res.RequestId); fulfilled { + return backend.Handle(peer, res) + } + return fmt.Errorf("%w: %v", errUnexpectedMsg, msg.Code) + default: + return fmt.Errorf("%w: %v", errInvalidMsgCode, msg.Code) + } +} + +func answerDiffLayersQuery(backend Backend, query *GetDiffLayersPacket) []rlp.RawValue { + // Gather blocks until the fetch or network limits is reached + var ( + bytes int + diffLayers []rlp.RawValue + ) + // Need avoid transfer huge package + for lookups, hash := range query.BlockHashes { + if bytes >= softResponseLimit || len(diffLayers) >= maxDiffLayerServe || + lookups >= 2*maxDiffLayerServe { + break + } + if data := backend.Chain().GetDiffLayerRLP(hash); len(data) != 0 { + diffLayers = append(diffLayers, data) + bytes += len(data) + } + } + return diffLayers +} + +// NodeInfo represents a short summary of the `diff` sub-protocol metadata +// known about the host peer. +type NodeInfo struct{} + +// nodeInfo retrieves some `diff` protocol metadata about the running host node. +func nodeInfo(_ *core.BlockChain) *NodeInfo { + return &NodeInfo{} +} diff --git a/eth/protocols/diff/handler_test.go b/eth/protocols/diff/handler_test.go new file mode 100644 index 0000000000..fcdca07b9f --- /dev/null +++ b/eth/protocols/diff/handler_test.go @@ -0,0 +1,192 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package diff + +import ( + "math/big" + "math/rand" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" +) + +var ( + // testKey is a private key to use for funding a tester account. + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + + // testAddr is the Ethereum address of the tester account. + testAddr = crypto.PubkeyToAddress(testKey.PublicKey) +) + +// testBackend is a mock implementation of the live Ethereum message handler. Its +// purpose is to allow testing the request/reply workflows and wire serialization +// in the `eth` protocol without actually doing any data processing. +type testBackend struct { + db ethdb.Database + chain *core.BlockChain + txpool *core.TxPool +} + +// newTestBackend creates an empty chain and wraps it into a mock backend. +func newTestBackend(blocks int) *testBackend { + return newTestBackendWithGenerator(blocks) +} + +// newTestBackend creates a chain with a number of explicitly defined blocks and +// wraps it into a mock backend. +func newTestBackendWithGenerator(blocks int) *testBackend { + signer := types.HomesteadSigner{} + // Create a database pre-initialize with a genesis block + db := rawdb.NewMemoryDatabase() + (&core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(100000000000000000)}}, + }).MustCommit(db) + + chain, _ := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + generator := func(i int, block *core.BlockGen) { + // The chain maker doesn't have access to a chain, so the difficulty will be + // lets unset (nil). Set it here to the correct value. + block.SetCoinbase(testAddr) + + // We want to simulate an empty middle block, having the same state as the + // first one. The last is needs a state change again to force a reorg. + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), common.Address{0x01}, big.NewInt(1), params.TxGas, big.NewInt(1), nil), signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(chain, tx) + } + bs, _ := core.GenerateChain(params.TestChainConfig, chain.Genesis(), ethash.NewFaker(), db, blocks, generator) + if _, err := chain.InsertChain(bs); err != nil { + panic(err) + } + txconfig := core.DefaultTxPoolConfig + txconfig.Journal = "" // Don't litter the disk with test journals + + return &testBackend{ + db: db, + chain: chain, + txpool: core.NewTxPool(txconfig, params.TestChainConfig, chain), + } +} + +// close tears down the transaction pool and chain behind the mock backend. +func (b *testBackend) close() { + b.txpool.Stop() + b.chain.Stop() +} + +func (b *testBackend) Chain() *core.BlockChain { return b.chain } + +func (b *testBackend) RunPeer(peer *Peer, handler Handler) error { + // Normally the backend would do peer mainentance and handshakes. All that + // is omitted and we will just give control back to the handler. + return handler(peer) +} +func (b *testBackend) PeerInfo(enode.ID) interface{} { panic("not implemented") } + +func (b *testBackend) Handle(*Peer, Packet) error { + panic("data processing tests should be done in the handler package") +} + +func TestGetDiffLayers(t *testing.T) { testGetDiffLayers(t, Diff1) } + +func testGetDiffLayers(t *testing.T, protocol uint) { + t.Parallel() + + blockNum := 2048 + backend := newTestBackend(blockNum) + defer backend.close() + + peer, _ := newTestPeer("peer", protocol, backend) + defer peer.close() + + foundDiffBlockHashes := make([]common.Hash, 0) + foundDiffPackets := make([]FullDiffLayersPacket, 0) + foundDiffRlps := make([]rlp.RawValue, 0) + missDiffBlockHashes := make([]common.Hash, 0) + missDiffPackets := make([]FullDiffLayersPacket, 0) + + for i := 0; i < 100; i++ { + number := uint64(rand.Int63n(1024)) + if number == 0 { + continue + } + foundHash := backend.chain.GetCanonicalHash(number + 1024) + missHash := backend.chain.GetCanonicalHash(number) + foundRlp := backend.chain.GetDiffLayerRLP(foundHash) + + if len(foundHash) == 0 { + t.Fatalf("Faild to fond rlp encoded diff layer %v", foundHash) + } + foundDiffPackets = append(foundDiffPackets, FullDiffLayersPacket{ + RequestId: uint64(i), + DiffLayersPacket: []rlp.RawValue{foundRlp}, + }) + foundDiffRlps = append(foundDiffRlps, foundRlp) + + missDiffPackets = append(missDiffPackets, FullDiffLayersPacket{ + RequestId: uint64(i), + DiffLayersPacket: []rlp.RawValue{}, + }) + + missDiffBlockHashes = append(missDiffBlockHashes, missHash) + foundDiffBlockHashes = append(foundDiffBlockHashes, foundHash) + } + + for idx, blockHash := range foundDiffBlockHashes { + p2p.Send(peer.app, GetDiffLayerMsg, GetDiffLayersPacket{RequestId: uint64(idx), BlockHashes: []common.Hash{blockHash}}) + if err := p2p.ExpectMsg(peer.app, FullDiffLayerMsg, foundDiffPackets[idx]); err != nil { + t.Errorf("test %d: diff layer mismatch: %v", idx, err) + } + } + + for idx, blockHash := range missDiffBlockHashes { + p2p.Send(peer.app, GetDiffLayerMsg, GetDiffLayersPacket{RequestId: uint64(idx), BlockHashes: []common.Hash{blockHash}}) + if err := p2p.ExpectMsg(peer.app, FullDiffLayerMsg, missDiffPackets[idx]); err != nil { + t.Errorf("test %d: diff layer mismatch: %v", idx, err) + } + } + + p2p.Send(peer.app, GetDiffLayerMsg, GetDiffLayersPacket{RequestId: 111, BlockHashes: foundDiffBlockHashes}) + if err := p2p.ExpectMsg(peer.app, FullDiffLayerMsg, FullDiffLayersPacket{ + 111, + foundDiffRlps, + }); err != nil { + t.Errorf("test: diff layer mismatch: %v", err) + } + + p2p.Send(peer.app, GetDiffLayerMsg, GetDiffLayersPacket{RequestId: 111, BlockHashes: missDiffBlockHashes}) + if err := p2p.ExpectMsg(peer.app, FullDiffLayerMsg, FullDiffLayersPacket{ + 111, + nil, + }); err != nil { + t.Errorf("test: diff layer mismatch: %v", err) + } +} diff --git a/eth/protocols/diff/handshake.go b/eth/protocols/diff/handshake.go new file mode 100644 index 0000000000..4198ea88a1 --- /dev/null +++ b/eth/protocols/diff/handshake.go @@ -0,0 +1,82 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package diff + +import ( + "fmt" + "time" + + "github.com/ethereum/go-ethereum/common/gopool" + "github.com/ethereum/go-ethereum/p2p" +) + +const ( + // handshakeTimeout is the maximum allowed time for the `diff` handshake to + // complete before dropping the connection.= as malicious. + handshakeTimeout = 5 * time.Second +) + +// Handshake executes the diff protocol handshake, +func (p *Peer) Handshake(diffSync bool) error { + // Send out own handshake in a new thread + errc := make(chan error, 2) + + var cap DiffCapPacket // safe to read after two values have been received from errc + + gopool.Submit(func() { + errc <- p2p.Send(p.rw, DiffCapMsg, &DiffCapPacket{ + DiffSync: diffSync, + Extra: defaultExtra, + }) + }) + gopool.Submit(func() { + errc <- p.readCap(&cap) + }) + timeout := time.NewTimer(handshakeTimeout) + defer timeout.Stop() + for i := 0; i < 2; i++ { + select { + case err := <-errc: + if err != nil { + return err + } + case <-timeout.C: + return p2p.DiscReadTimeout + } + } + p.diffSync = cap.DiffSync + return nil +} + +// readStatus reads the remote handshake message. +func (p *Peer) readCap(cap *DiffCapPacket) error { + msg, err := p.rw.ReadMsg() + if err != nil { + return err + } + if msg.Code != DiffCapMsg { + return fmt.Errorf("%w: first msg has code %x (!= %x)", errNoCapMsg, msg.Code, DiffCapMsg) + } + if msg.Size > maxMessageSize { + return fmt.Errorf("%w: %v > %v", errMsgTooLarge, msg.Size, maxMessageSize) + } + // Decode the handshake and make sure everything matches + if err := msg.Decode(cap); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return nil +} diff --git a/eth/protocols/diff/peer.go b/eth/protocols/diff/peer.go new file mode 100644 index 0000000000..f110cab69e --- /dev/null +++ b/eth/protocols/diff/peer.go @@ -0,0 +1,107 @@ +package diff + +import ( + "math/rand" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rlp" +) + +const maxQueuedDiffLayers = 12 + +// Peer is a collection of relevant information we have about a `diff` peer. +type Peer struct { + id string // Unique ID for the peer, cached + diffSync bool // whether the peer can diff sync + queuedDiffLayers chan []rlp.RawValue // Queue of diff layers to broadcast to the peer + + *p2p.Peer // The embedded P2P package peer + rw p2p.MsgReadWriter // Input/output streams for diff + version uint // Protocol version negotiated + logger log.Logger // Contextual logger with the peer id injected + term chan struct{} // Termination channel to stop the broadcasters +} + +// NewPeer create a wrapper for a network connection and negotiated protocol +// version. +func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) *Peer { + id := p.ID().String() + peer := &Peer{ + id: id, + Peer: p, + rw: rw, + diffSync: false, + version: version, + logger: log.New("peer", id[:8]), + queuedDiffLayers: make(chan []rlp.RawValue, maxQueuedDiffLayers), + term: make(chan struct{}), + } + go peer.broadcastDiffLayers() + return peer +} + +func (p *Peer) broadcastDiffLayers() { + for { + select { + case prop := <-p.queuedDiffLayers: + if err := p.SendDiffLayers(prop); err != nil { + p.Log().Error("Failed to propagated diff layer", "err", err) + return + } + case <-p.term: + return + } + } +} + +// ID retrieves the peer's unique identifier. +func (p *Peer) ID() string { + return p.id +} + +// Version retrieves the peer's negoatiated `diff` protocol version. +func (p *Peer) Version() uint { + return p.version +} + +func (p *Peer) DiffSync() bool { + return p.diffSync +} + +// Log overrides the P2P logget with the higher level one containing only the id. +func (p *Peer) Log() log.Logger { + return p.logger +} + +// Close signals the broadcast goroutine to terminate. Only ever call this if +// you created the peer yourself via NewPeer. Otherwise let whoever created it +// clean it up! +func (p *Peer) Close() { + close(p.term) +} + +// RequestDiffLayers fetches a batch of diff layers corresponding to the hashes +// specified. +func (p *Peer) RequestDiffLayers(hashes []common.Hash) error { + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetDiffLayerMsg, FullDiffLayerMsg, id) + return p2p.Send(p.rw, GetDiffLayerMsg, GetDiffLayersPacket{ + RequestId: id, + BlockHashes: hashes, + }) +} + +func (p *Peer) SendDiffLayers(diffs []rlp.RawValue) error { + return p2p.Send(p.rw, DiffLayerMsg, diffs) +} + +func (p *Peer) AsyncSendDiffLayer(diffLayers []rlp.RawValue) { + select { + case p.queuedDiffLayers <- diffLayers: + default: + p.Log().Debug("Dropping diff layers propagation") + } +} diff --git a/eth/protocols/diff/peer_test.go b/eth/protocols/diff/peer_test.go new file mode 100644 index 0000000000..015b86f134 --- /dev/null +++ b/eth/protocols/diff/peer_test.go @@ -0,0 +1,61 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// This file contains some shares testing functionality, common to multiple +// different files and modules being tested. + +package diff + +import ( + "crypto/rand" + + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" +) + +// testPeer is a simulated peer to allow testing direct network calls. +type testPeer struct { + *Peer + + net p2p.MsgReadWriter // Network layer reader/writer to simulate remote messaging + app *p2p.MsgPipeRW // Application layer reader/writer to simulate the local side +} + +// newTestPeer creates a new peer registered at the given data backend. +func newTestPeer(name string, version uint, backend Backend) (*testPeer, <-chan error) { + // Create a message pipe to communicate through + app, net := p2p.MsgPipe() + + // Start the peer on a new thread + var id enode.ID + rand.Read(id[:]) + + peer := NewPeer(version, p2p.NewPeer(id, name, nil), net) + errc := make(chan error, 1) + go func() { + errc <- backend.RunPeer(peer, func(peer *Peer) error { + return Handle(backend, peer) + }) + }() + return &testPeer{app: app, net: net, Peer: peer}, errc +} + +// close terminates the local side of the peer, notifying the remote protocol +// manager of termination. +func (p *testPeer) close() { + p.Peer.Close() + p.app.Close() +} diff --git a/eth/protocols/diff/protocol.go b/eth/protocols/diff/protocol.go new file mode 100644 index 0000000000..4467d0b327 --- /dev/null +++ b/eth/protocols/diff/protocol.go @@ -0,0 +1,122 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package diff + +import ( + "errors" + "fmt" + + "golang.org/x/crypto/sha3" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// Constants to match up protocol versions and messages +const ( + Diff1 = 1 +) + +// ProtocolName is the official short name of the `diff` protocol used during +// devp2p capability negotiation. +const ProtocolName = "diff" + +// ProtocolVersions are the supported versions of the `diff` protocol (first +// is primary). +var ProtocolVersions = []uint{Diff1} + +// protocolLengths are the number of implemented message corresponding to +// different protocol versions. +var protocolLengths = map[uint]uint64{Diff1: 4} + +// maxMessageSize is the maximum cap on the size of a protocol message. +const maxMessageSize = 10 * 1024 * 1024 + +const ( + DiffCapMsg = 0x00 + GetDiffLayerMsg = 0x01 + DiffLayerMsg = 0x02 + FullDiffLayerMsg = 0x03 +) + +var defaultExtra = []byte{0x00} + +var ( + errMsgTooLarge = errors.New("message too long") + errDecode = errors.New("invalid message") + errInvalidMsgCode = errors.New("invalid message code") + errUnexpectedMsg = errors.New("unexpected message code") + errNoCapMsg = errors.New("miss cap message during handshake") +) + +// Packet represents a p2p message in the `diff` protocol. +type Packet interface { + Name() string // Name returns a string corresponding to the message type. + Kind() byte // Kind returns the message type. +} + +type GetDiffLayersPacket struct { + RequestId uint64 + BlockHashes []common.Hash +} + +func (p *DiffLayersPacket) Unpack() ([]*types.DiffLayer, error) { + diffLayers := make([]*types.DiffLayer, 0, len(*p)) + hasher := sha3.NewLegacyKeccak256() + for _, rawData := range *p { + var diff types.DiffLayer + err := rlp.DecodeBytes(rawData, &diff) + if err != nil { + return nil, fmt.Errorf("%w: diff layer %v", errDecode, err) + } + diffLayers = append(diffLayers, &diff) + _, err = hasher.Write(rawData) + if err != nil { + return nil, err + } + var diffHash common.Hash + hasher.Sum(diffHash[:0]) + hasher.Reset() + diff.DiffHash = diffHash + } + return diffLayers, nil +} + +type DiffCapPacket struct { + DiffSync bool + Extra rlp.RawValue // for extension +} + +type DiffLayersPacket []rlp.RawValue + +type FullDiffLayersPacket struct { + RequestId uint64 + DiffLayersPacket +} + +func (*GetDiffLayersPacket) Name() string { return "GetDiffLayers" } +func (*GetDiffLayersPacket) Kind() byte { return GetDiffLayerMsg } + +func (*DiffLayersPacket) Name() string { return "DiffLayers" } +func (*DiffLayersPacket) Kind() byte { return DiffLayerMsg } + +func (*FullDiffLayersPacket) Name() string { return "FullDiffLayers" } +func (*FullDiffLayersPacket) Kind() byte { return FullDiffLayerMsg } + +func (*DiffCapPacket) Name() string { return "DiffCap" } +func (*DiffCapPacket) Kind() byte { return DiffCapMsg } diff --git a/eth/protocols/diff/protocol_test.go b/eth/protocols/diff/protocol_test.go new file mode 100644 index 0000000000..eda96adf1b --- /dev/null +++ b/eth/protocols/diff/protocol_test.go @@ -0,0 +1,131 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package diff + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// Tests that the custom union field encoder and decoder works correctly. +func TestDiffLayersPacketDataEncodeDecode(t *testing.T) { + // Create a "random" hash for testing + var hash common.Hash + for i := range hash { + hash[i] = byte(i) + } + + testDiffLayers := []*types.DiffLayer{ + { + BlockHash: common.HexToHash("0x1e9624dcd0874958723aa3dae1fe299861e93ef32b980143d798c428bdd7a20a"), + Number: 10479133, + Receipts: []*types.Receipt{{ + GasUsed: 100, + TransactionIndex: 1, + }}, + Codes: []types.DiffCode{{ + Hash: common.HexToHash("0xaece2dbf80a726206cf4df299afa09f9d8f3dcd85ff39bb4b3f0402a3a6af2f5"), + Code: []byte{1, 2, 3, 4}, + }}, + Destructs: []common.Address{ + common.HexToAddress("0x0205bb28ece9289d3fb8eb0c9e999bbd5be2b931"), + }, + Accounts: []types.DiffAccount{{ + Account: common.HexToAddress("0x18b2a687610328590bc8f2e5fedde3b582a49cda"), + Blob: []byte{2, 3, 4, 5}, + }}, + Storages: []types.DiffStorage{{ + Account: common.HexToAddress("0x18b2a687610328590bc8f2e5fedde3b582a49cda"), + Keys: []string{"abc"}, + Vals: [][]byte{{1, 2, 3}}, + }}, + }, + } + // Assemble some table driven tests + tests := []struct { + diffLayers []*types.DiffLayer + fail bool + }{ + {fail: false, diffLayers: testDiffLayers}, + } + // Iterate over each of the tests and try to encode and then decode + for i, tt := range tests { + originPacket := make([]rlp.RawValue, 0) + for _, diff := range tt.diffLayers { + bz, err := rlp.EncodeToBytes(diff) + assert.NoError(t, err) + originPacket = append(originPacket, bz) + } + + bz, err := rlp.EncodeToBytes(DiffLayersPacket(originPacket)) + if err != nil && !tt.fail { + t.Fatalf("test %d: failed to encode packet: %v", i, err) + } else if err == nil && tt.fail { + t.Fatalf("test %d: encode should have failed", i) + } + if !tt.fail { + packet := new(DiffLayersPacket) + if err := rlp.DecodeBytes(bz, packet); err != nil { + t.Fatalf("test %d: failed to decode packet: %v", i, err) + } + diffLayers, err := packet.Unpack() + assert.NoError(t, err) + + if len(diffLayers) != len(tt.diffLayers) { + t.Fatalf("test %d: encode length mismatch: have %+v, want %+v", i, len(diffLayers), len(tt.diffLayers)) + } + expectedPacket := make([]rlp.RawValue, 0) + for _, diff := range diffLayers { + bz, err := rlp.EncodeToBytes(diff) + assert.NoError(t, err) + expectedPacket = append(expectedPacket, bz) + } + for i := 0; i < len(expectedPacket); i++ { + if !bytes.Equal(expectedPacket[i], originPacket[i]) { + t.Fatalf("test %d: data change during encode and decode", i) + } + } + } + } +} + +func TestDiffMessages(t *testing.T) { + + for i, tc := range []struct { + message interface{} + want []byte + }{ + { + DiffCapPacket{true, defaultExtra}, + common.FromHex("c20100"), + }, + { + GetDiffLayersPacket{1111, []common.Hash{common.HexToHash("0xaece2dbf80a726206cf4df299afa09f9d8f3dcd85ff39bb4b3f0402a3a6af2f5")}}, + common.FromHex("e5820457e1a0aece2dbf80a726206cf4df299afa09f9d8f3dcd85ff39bb4b3f0402a3a6af2f5"), + }, + } { + if have, _ := rlp.EncodeToBytes(tc.message); !bytes.Equal(have, tc.want) { + t.Errorf("test %d, type %T, have\n\t%x\nwant\n\t%x", i, tc.message, have, tc.want) + } + } +} diff --git a/eth/protocols/diff/tracker.go b/eth/protocols/diff/tracker.go new file mode 100644 index 0000000000..7ee49e6ce2 --- /dev/null +++ b/eth/protocols/diff/tracker.go @@ -0,0 +1,161 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package diff + +import ( + "container/list" + "fmt" + "sync" + "time" + + "github.com/ethereum/go-ethereum/log" +) + +const ( + // maxTrackedPackets is a huge number to act as a failsafe on the number of + // pending requests the node will track. It should never be hit unless an + // attacker figures out a way to spin requests. + maxTrackedPackets = 10000 +) + +// request tracks sent network requests which have not yet received a response. +type request struct { + peer string + version uint // Protocol version + + reqCode uint64 // Protocol message code of the request + resCode uint64 // Protocol message code of the expected response + + time time.Time // Timestamp when the request was made + expire *list.Element // Expiration marker to untrack it +} + +type Tracker struct { + timeout time.Duration // Global timeout after which to drop a tracked packet + + pending map[uint64]*request // Currently pending requests + expire *list.List // Linked list tracking the expiration order + wake *time.Timer // Timer tracking the expiration of the next item + + lock sync.Mutex // Lock protecting from concurrent updates +} + +func NewTracker(timeout time.Duration) *Tracker { + return &Tracker{ + timeout: timeout, + pending: make(map[uint64]*request), + expire: list.New(), + } +} + +// Track adds a network request to the tracker to wait for a response to arrive +// or until the request it cancelled or times out. +func (t *Tracker) Track(peer string, version uint, reqCode uint64, resCode uint64, id uint64) { + t.lock.Lock() + defer t.lock.Unlock() + + // If there's a duplicate request, we've just random-collided (or more probably, + // we have a bug), report it. We could also add a metric, but we're not really + // expecting ourselves to be buggy, so a noisy warning should be enough. + if _, ok := t.pending[id]; ok { + log.Error("Network request id collision", "version", version, "code", reqCode, "id", id) + return + } + // If we have too many pending requests, bail out instead of leaking memory + if pending := len(t.pending); pending >= maxTrackedPackets { + log.Error("Request tracker exceeded allowance", "pending", pending, "peer", peer, "version", version, "code", reqCode) + return + } + // Id doesn't exist yet, start tracking it + t.pending[id] = &request{ + peer: peer, + version: version, + reqCode: reqCode, + resCode: resCode, + time: time.Now(), + expire: t.expire.PushBack(id), + } + + // If we've just inserted the first item, start the expiration timer + if t.wake == nil { + t.wake = time.AfterFunc(t.timeout, t.clean) + } +} + +// clean is called automatically when a preset time passes without a response +// being dleivered for the first network request. +func (t *Tracker) clean() { + t.lock.Lock() + defer t.lock.Unlock() + + // Expire anything within a certain threshold (might be no items at all if + // we raced with the delivery) + for t.expire.Len() > 0 { + // Stop iterating if the next pending request is still alive + var ( + head = t.expire.Front() + id = head.Value.(uint64) + req = t.pending[id] + ) + if time.Since(req.time) < t.timeout+5*time.Millisecond { + break + } + // Nope, dead, drop it + t.expire.Remove(head) + delete(t.pending, id) + } + t.schedule() +} + +// schedule starts a timer to trigger on the expiration of the first network +// packet. +func (t *Tracker) schedule() { + if t.expire.Len() == 0 { + t.wake = nil + return + } + t.wake = time.AfterFunc(time.Until(t.pending[t.expire.Front().Value.(uint64)].time.Add(t.timeout)), t.clean) +} + +// Fulfil fills a pending request, if any is available. +func (t *Tracker) Fulfil(peer string, version uint, code uint64, id uint64) bool { + t.lock.Lock() + defer t.lock.Unlock() + + // If it's a non existing request, track as stale response + req, ok := t.pending[id] + if !ok { + return false + } + // If the response is funky, it might be some active attack + if req.peer != peer || req.version != version || req.resCode != code { + log.Warn("Network response id collision", + "have", fmt.Sprintf("%s:/%d:%d", peer, version, code), + "want", fmt.Sprintf("%s:/%d:%d", peer, req.version, req.resCode), + ) + return false + } + // Everything matches, mark the request serviced + t.expire.Remove(req.expire) + delete(t.pending, id) + if req.expire.Prev() == nil { + if t.wake.Stop() { + t.schedule() + } + } + return true +} diff --git a/eth/state_accessor.go b/eth/state_accessor.go index aaabd1d152..461dc491fb 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -114,12 +114,12 @@ func (eth *Ethereum) stateAtBlock(block *types.Block, reexec uint64, base *state if current = eth.blockchain.GetBlockByNumber(next); current == nil { return nil, fmt.Errorf("block #%d not found", next) } - _, _, _, err := eth.blockchain.Processor().Process(current, statedb, vm.Config{}) + statedb, _, _, _, err := eth.blockchain.Processor().Process(current, statedb, vm.Config{}) if err != nil { return nil, fmt.Errorf("processing block %d failed: %v", current.NumberU64(), err) } // Finalize the state so any modifications are written to the trie - root, err := statedb.Commit(eth.blockchain.Config().IsEIP158(current.Number())) + root, _, err := statedb.Commit(eth.blockchain.Config().IsEIP158(current.Number())) if err != nil { return nil, err } diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index cdffdc6024..0bc4cd81e6 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -357,7 +357,7 @@ func BenchmarkTransactionTrace(b *testing.B) { //DisableReturnData: true, }) evm := vm.NewEVM(context, txContext, statedb, params.AllEthashProtocolChanges, vm.Config{Debug: true, Tracer: tracer}) - msg, err := tx.AsMessage(signer, nil) + msg, err := tx.AsMessage(signer) if err != nil { b.Fatalf("failed to prepare transaction for tracing: %v", err) } diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 9fa5bf87a4..341acf978f 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -17,7 +17,6 @@ package ethclient import ( - "bytes" "context" "errors" "fmt" @@ -262,9 +261,7 @@ func TestEthClient(t *testing.T) { "TestCallContract": { func(t *testing.T) { testCallContract(t, client) }, }, - "TestAtFunctions": { - func(t *testing.T) { testAtFunctions(t, client) }, - }, + // DO not have TestAtFunctions now, because we do not have pending block now } t.Parallel() @@ -490,69 +487,6 @@ func testCallContract(t *testing.T, client *rpc.Client) { } } -func testAtFunctions(t *testing.T, client *rpc.Client) { - ec := NewClient(client) - // send a transaction for some interesting pending status - sendTransaction(ec) - time.Sleep(100 * time.Millisecond) - // Check pending transaction count - pending, err := ec.PendingTransactionCount(context.Background()) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if pending != 1 { - t.Fatalf("unexpected pending, wanted 1 got: %v", pending) - } - // Query balance - balance, err := ec.BalanceAt(context.Background(), testAddr, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - penBalance, err := ec.PendingBalanceAt(context.Background(), testAddr) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if balance.Cmp(penBalance) == 0 { - t.Fatalf("unexpected balance: %v %v", balance, penBalance) - } - // NonceAt - nonce, err := ec.NonceAt(context.Background(), testAddr, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - penNonce, err := ec.PendingNonceAt(context.Background(), testAddr) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if penNonce != nonce+1 { - t.Fatalf("unexpected nonce: %v %v", nonce, penNonce) - } - // StorageAt - storage, err := ec.StorageAt(context.Background(), testAddr, common.Hash{}, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - penStorage, err := ec.PendingStorageAt(context.Background(), testAddr, common.Hash{}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !bytes.Equal(storage, penStorage) { - t.Fatalf("unexpected storage: %v %v", storage, penStorage) - } - // CodeAt - code, err := ec.CodeAt(context.Background(), testAddr, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - penCode, err := ec.PendingCodeAt(context.Background(), testAddr) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !bytes.Equal(code, penCode) { - t.Fatalf("unexpected code: %v %v", code, penCode) - } -} - func sendTransaction(ec *Client) error { // Retrieve chainID chainID, err := ec.ChainID(context.Background()) @@ -560,7 +494,7 @@ func sendTransaction(ec *Client) error { return err } // Create transaction - tx := types.NewTransaction(0, common.Address{1}, big.NewInt(1), 22000, big.NewInt(1), nil) + tx := types.NewTransaction(0, common.Address{1}, big.NewInt(1), 23000, big.NewInt(100000), nil) signer := types.LatestSignerForChainID(chainID) signature, err := crypto.Sign(signer.Hash(tx).Bytes(), testKey) if err != nil { diff --git a/ethdb/database.go b/ethdb/database.go index 0dc14624b9..40d0e01d57 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -118,11 +118,17 @@ type AncientStore interface { io.Closer } +type DiffStore interface { + DiffStore() KeyValueStore + SetDiffStore(diff KeyValueStore) +} + // Database contains all the methods required by the high level database to not // only access the key-value data store but also the chain freezer. type Database interface { Reader Writer + DiffStore Batcher Iteratee Stater diff --git a/les/fetcher.go b/les/fetcher.go index fc4c5e386a..ed2e72a330 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -339,7 +339,7 @@ func (f *lightFetcher) mainloop() { log.Debug("Trigger light sync", "peer", peerid, "local", localHead.Number, "localhash", localHead.Hash(), "remote", data.Number, "remotehash", data.Hash) continue } - f.fetcher.Notify(peerid.String(), data.Hash, data.Number, time.Now(), f.requestHeaderByHash(peerid), nil) + f.fetcher.Notify(peerid.String(), data.Hash, data.Number, time.Now(), f.requestHeaderByHash(peerid), nil, nil) log.Debug("Trigger header retrieval", "peer", peerid, "number", data.Number, "hash", data.Hash) } // Keep collecting announces from trusted server even we are syncing. @@ -355,7 +355,7 @@ func (f *lightFetcher) mainloop() { continue } p := agreed[rand.Intn(len(agreed))] - f.fetcher.Notify(p.String(), data.Hash, data.Number, time.Now(), f.requestHeaderByHash(p), nil) + f.fetcher.Notify(p.String(), data.Hash, data.Number, time.Now(), f.requestHeaderByHash(p), nil, nil) log.Debug("Trigger trusted header retrieval", "number", data.Number, "hash", data.Hash) } } diff --git a/les/peer.go b/les/peer.go index 5cdd557a90..e09b3bc130 100644 --- a/les/peer.go +++ b/les/peer.go @@ -1054,7 +1054,7 @@ func (p *clientPeer) Handshake(td *big.Int, head common.Hash, headNum uint64, ge // If local ethereum node is running in archive mode, advertise ourselves we have // all version state data. Otherwise only recent state is available. - stateRecent := uint64(server.handler.blockchain.TriesInMemory() - blockSafetyMargin) + stateRecent := server.handler.blockchain.TriesInMemory() - blockSafetyMargin if server.archiveMode { stateRecent = 0 } diff --git a/light/trie.go b/light/trie.go index e189634e1c..3896b73c4d 100644 --- a/light/trie.go +++ b/light/trie.go @@ -95,17 +95,11 @@ func (db *odrDatabase) TrieDB() *trie.Database { return nil } -func (db *odrDatabase) CacheAccount(_ common.Hash, _ state.Trie) { - return -} +func (db *odrDatabase) CacheAccount(_ common.Hash, _ state.Trie) {} -func (db *odrDatabase) CacheStorage(_ common.Hash, _ common.Hash, _ state.Trie) { - return -} +func (db *odrDatabase) CacheStorage(_ common.Hash, _ common.Hash, _ state.Trie) {} -func (db *odrDatabase) Purge() { - return -} +func (db *odrDatabase) Purge() {} type odrTrie struct { db *odrDatabase diff --git a/miner/worker.go b/miner/worker.go index 984af7baaf..ef7a8c5630 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -1011,16 +1011,6 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st return nil } -// copyReceipts makes a deep copy of the given receipts. -func copyReceipts(receipts []*types.Receipt) []*types.Receipt { - result := make([]*types.Receipt, len(receipts)) - for i, l := range receipts { - cpy := *l - result[i] = &cpy - } - return result -} - // postSideBlock fires a side chain event, only use it for testing. func (w *worker) postSideBlock(event core.ChainSideEvent) { select { @@ -1028,12 +1018,3 @@ func (w *worker) postSideBlock(event core.ChainSideEvent) { case <-w.exitCh: } } - -// totalFees computes total consumed fees in ETH. Block transactions and receipts have to have the same order. -func totalFees(block *types.Block, receipts []*types.Receipt) *big.Float { - feesWei := new(big.Int) - for i, tx := range block.Transactions() { - feesWei.Add(feesWei, new(big.Int).Mul(new(big.Int).SetUint64(receipts[i].GasUsed), tx.GasPrice())) - } - return new(big.Float).Quo(new(big.Float).SetInt(feesWei), new(big.Float).SetInt(big.NewInt(params.Ether))) -} diff --git a/node/config.go b/node/config.go index 23461c1916..527cd36415 100644 --- a/node/config.go +++ b/node/config.go @@ -98,6 +98,9 @@ type Config struct { // DirectBroadcast enable directly broadcast mined block to all peers DirectBroadcast bool `toml:",omitempty"` + // DisableSnapProtocol disable the snap protocol + DisableSnapProtocol bool `toml:",omitempty"` + // RangeLimit enable 5000 blocks limit when handle range query RangeLimit bool `toml:",omitempty"` diff --git a/node/node.go b/node/node.go index f2602dee47..f1564bea23 100644 --- a/node/node.go +++ b/node/node.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/leveldb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" @@ -578,6 +579,22 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r return db, err } +func (n *Node) OpenAndMergeDatabase(name string, cache, handles int, freezer, diff, namespace string, readonly, persistDiff bool) (ethdb.Database, error) { + chainDB, err := n.OpenDatabaseWithFreezer(name, cache, handles, freezer, namespace, readonly) + if err != nil { + return nil, err + } + if persistDiff { + diffStore, err := n.OpenDiffDatabase(name, handles, diff, namespace, readonly) + if err != nil { + chainDB.Close() + return nil, err + } + chainDB.SetDiffStore(diffStore) + } + return chainDB, nil +} + // OpenDatabaseWithFreezer opens an existing database with the given name (or // creates one if no previous can be found) from within the node's data directory, // also attaching a chain freezer to it that moves ancient chain data from the @@ -611,6 +628,30 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, return db, err } +func (n *Node) OpenDiffDatabase(name string, handles int, diff, namespace string, readonly bool) (*leveldb.Database, error) { + n.lock.Lock() + defer n.lock.Unlock() + if n.state == closedState { + return nil, ErrNodeStopped + } + + var db *leveldb.Database + var err error + if n.config.DataDir == "" { + panic("datadir is missing") + } + root := n.ResolvePath(name) + switch { + case diff == "": + diff = filepath.Join(root, "diff") + case !filepath.IsAbs(diff): + diff = n.ResolvePath(diff) + } + db, err = leveldb.New(diff, 0, handles, namespace, readonly) + + return db, err +} + // ResolvePath returns the absolute path of a resource in the instance directory. func (n *Node) ResolvePath(x string) string { return n.config.ResolvePath(x) diff --git a/rlp/typecache.go b/rlp/typecache.go index 62553d3b55..c21025ad26 100644 --- a/rlp/typecache.go +++ b/rlp/typecache.go @@ -172,16 +172,6 @@ func structFields(typ reflect.Type) (fields []field, err error) { return fields, nil } -// anyOptionalFields returns the index of the first field with "optional" tag. -func firstOptionalField(fields []field) int { - for i, f := range fields { - if f.optional { - return i - } - } - return len(fields) -} - type structFieldError struct { typ reflect.Type field int diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 19c79b6eed..77d4fd08d4 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -226,7 +226,7 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter boo } } // Commit and re-open to start with a clean state. - root, _ := statedb.Commit(false) + root, _, _ := statedb.Commit(false) var snaps *snapshot.Tree if snapshotter { From b2f1d25f83f75ba39d3e310743a01dd4fbde09bf Mon Sep 17 00:00:00 2001 From: KeefeL <90749943+KeefeL@users.noreply.github.com> Date: Fri, 8 Oct 2021 12:29:38 +0800 Subject: [PATCH 08/18] Export get diff accounts in block api (#431) * support get diff accounts Signed-off-by: Keefe-Liu * add testcase for diff accounts Signed-off-by: Keefe-Liu --- core/blockchain.go | 36 ++++++++ core/blockchain_diff_test.go | 174 +++++++++++++++++++++++++++++++---- core/types/block.go | 11 +++ eth/api_backend.go | 4 + ethclient/ethclient.go | 14 +++ ethclient/ethclient_test.go | 169 ++++++++++++++++++++++++++++++++-- internal/ethapi/api.go | 106 +++++++++++++++++++++ internal/ethapi/backend.go | 1 + les/api_backend.go | 4 + 9 files changed, 495 insertions(+), 24 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 3cfa21654f..7157d39455 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -974,6 +974,42 @@ func (bc *BlockChain) GetDiffLayerRLP(blockHash common.Hash) rlp.RawValue { return rawData } +func (bc *BlockChain) GetDiffAccounts(blockHash common.Hash) ([]common.Address, error) { + var ( + accounts []common.Address + diffLayer *types.DiffLayer + ) + + header := bc.GetHeaderByHash(blockHash) + if header == nil { + return nil, fmt.Errorf("no block found") + } + + if cached, ok := bc.diffLayerCache.Get(blockHash); ok { + diffLayer = cached.(*types.DiffLayer) + } else if diffStore := bc.db.DiffStore(); diffStore != nil { + diffLayer = rawdb.ReadDiffLayer(diffStore, blockHash) + } + + if diffLayer == nil { + if header.TxHash != types.EmptyRootHash { + return nil, fmt.Errorf("no diff layer found") + } + + return nil, nil + } + + for _, diffAccounts := range diffLayer.Accounts { + accounts = append(accounts, diffAccounts.Account) + } + + if header.TxHash != types.EmptyRootHash && len(accounts) == 0 { + return nil, fmt.Errorf("no diff account in block, maybe bad diff layer") + } + + return accounts, nil +} + // HasBlock checks if a block is fully present in the database or not. func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool { if bc.blockCache.Contains(hash) { diff --git a/core/blockchain_diff_test.go b/core/blockchain_diff_test.go index 717b4039ed..bec463136e 100644 --- a/core/blockchain_diff_test.go +++ b/core/blockchain_diff_test.go @@ -45,8 +45,79 @@ var ( testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") // testAddr is the Ethereum address of the tester account. testAddr = crypto.PubkeyToAddress(testKey.PublicKey) + // testBlocks is the test parameters array for specific blocks. + testBlocks = []testBlockParam{ + { + // This txs params also used to default block. + blockNr: 11, + txs: []testTransactionParam{ + { + to: common.Address{0x01}, + value: big.NewInt(1), + gasPrice: big.NewInt(1), + data: nil, + }, + }, + }, + { + blockNr: 12, + txs: []testTransactionParam{ + { + to: common.Address{0x01}, + value: big.NewInt(1), + gasPrice: big.NewInt(1), + data: nil, + }, + { + to: common.Address{0x02}, + value: big.NewInt(2), + gasPrice: big.NewInt(2), + data: nil, + }, + }, + }, + { + blockNr: 13, + txs: []testTransactionParam{ + { + to: common.Address{0x01}, + value: big.NewInt(1), + gasPrice: big.NewInt(1), + data: nil, + }, + { + to: common.Address{0x02}, + value: big.NewInt(2), + gasPrice: big.NewInt(2), + data: nil, + }, + { + to: common.Address{0x03}, + value: big.NewInt(3), + gasPrice: big.NewInt(3), + data: nil, + }, + }, + }, + { + blockNr: 14, + txs: []testTransactionParam{}, + }, + } ) +type testTransactionParam struct { + to common.Address + value *big.Int + gasPrice *big.Int + data []byte +} + +type testBlockParam struct { + blockNr int + txs []testTransactionParam +} + // testBackend is a mock implementation of the live Ethereum message handler. Its // purpose is to allow testing the request/reply workflows and wire serialization // in the `eth` protocol without actually doing any data processing. @@ -78,13 +149,35 @@ func newTestBackendWithGenerator(blocks int, lightProcess bool) *testBackend { // lets unset (nil). Set it here to the correct value. block.SetCoinbase(testAddr) - // We want to simulate an empty middle block, having the same state as the - // first one. The last is needs a state change again to force a reorg. - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), common.Address{0x01}, big.NewInt(1), params.TxGas, big.NewInt(1), nil), signer, testKey) - if err != nil { - panic(err) + for idx, testBlock := range testBlocks { + // Specific block setting, the index in this generator has 1 diff from specified blockNr. + if i+1 == testBlock.blockNr { + for _, testTransaction := range testBlock.txs { + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), testTransaction.to, + testTransaction.value, params.TxGas, testTransaction.gasPrice, testTransaction.data), signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(chain, tx) + } + break + } + + // Default block setting. + if idx == len(testBlocks)-1 { + // We want to simulate an empty middle block, having the same state as the + // first one. The last is needs a state change again to force a reorg. + for _, testTransaction := range testBlocks[0].txs { + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), testTransaction.to, + testTransaction.value, params.TxGas, testTransaction.gasPrice, testTransaction.data), signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(chain, tx) + } + } } - block.AddTxWithChain(chain, tx) + } bs, _ := GenerateChain(params.TestChainConfig, chain.Genesis(), ethash.NewFaker(), db, blocks, generator) if _, err := chain.InsertChain(bs); err != nil { @@ -139,12 +232,17 @@ func TestProcessDiffLayer(t *testing.T) { } blockHash := block.Hash() rawDiff := fullBackend.chain.GetDiffLayerRLP(blockHash) - diff, err := rawDataToDiffLayer(rawDiff) - if err != nil { - t.Errorf("failed to decode rawdata %v", err) + if len(rawDiff) != 0 { + diff, err := rawDataToDiffLayer(rawDiff) + if err != nil { + t.Errorf("failed to decode rawdata %v", err) + } + if diff == nil { + continue + } + lightBackend.Chain().HandleDiffLayer(diff, "testpid", true) } - lightBackend.Chain().HandleDiffLayer(diff, "testpid", true) - _, err = lightBackend.chain.insertChain([]*types.Block{block}, true) + _, err := lightBackend.chain.insertChain([]*types.Block{block}, true) if err != nil { t.Errorf("failed to insert block %v", err) } @@ -186,7 +284,8 @@ func TestFreezeDiffLayer(t *testing.T) { blockNum := 1024 fullBackend := newTestBackend(blockNum, true) defer fullBackend.close() - if fullBackend.chain.diffQueue.Size() != blockNum { + // Minus one empty block. + if fullBackend.chain.diffQueue.Size() != blockNum-1 { t.Errorf("size of diff queue is wrong, expected: %d, get: %d", blockNum, fullBackend.chain.diffQueue.Size()) } time.Sleep(diffLayerFreezerRecheckInterval + 1*time.Second) @@ -215,10 +314,11 @@ func TestPruneDiffLayer(t *testing.T) { for num := uint64(1); num < uint64(blockNum); num++ { header := fullBackend.chain.GetHeaderByNumber(num) rawDiff := fullBackend.chain.GetDiffLayerRLP(header.Hash()) - diff, _ := rawDataToDiffLayer(rawDiff) - fullBackend.Chain().HandleDiffLayer(diff, "testpid1", true) - fullBackend.Chain().HandleDiffLayer(diff, "testpid2", true) - + if len(rawDiff) != 0 { + diff, _ := rawDataToDiffLayer(rawDiff) + fullBackend.Chain().HandleDiffLayer(diff, "testpid1", true) + fullBackend.Chain().HandleDiffLayer(diff, "testpid2", true) + } } fullBackend.chain.pruneDiffLayer() if len(fullBackend.chain.diffNumToBlockHashes) != maxDiffForkDist { @@ -261,3 +361,45 @@ func TestPruneDiffLayer(t *testing.T) { } } + +func TestGetDiffAccounts(t *testing.T) { + t.Parallel() + + blockNum := 128 + fullBackend := newTestBackend(blockNum, false) + defer fullBackend.close() + + for _, testBlock := range testBlocks { + block := fullBackend.chain.GetBlockByNumber(uint64(testBlock.blockNr)) + if block == nil { + t.Fatal("block should not be nil") + } + blockHash := block.Hash() + accounts, err := fullBackend.chain.GetDiffAccounts(blockHash) + if err != nil { + t.Errorf("get diff accounts eror for block number (%d): %v", testBlock.blockNr, err) + } + + for idx, account := range accounts { + if testAddr == account { + break + } + + if idx == len(accounts)-1 { + t.Errorf("the diff accounts does't include addr: %v", testAddr) + } + } + + for _, transaction := range testBlock.txs { + for idx, account := range accounts { + if transaction.to == account { + break + } + + if idx == len(accounts)-1 { + t.Errorf("the diff accounts does't include addr: %v", transaction.to) + } + } + } + } +} diff --git a/core/types/block.go b/core/types/block.go index a577e60516..bee5d80cdd 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -453,3 +453,14 @@ type DiffStorage struct { Keys []string Vals [][]byte } + +type DiffAccountsInTx struct { + TxHash common.Hash + Accounts map[common.Address]*big.Int +} + +type DiffAccountsInBlock struct { + Number uint64 + BlockHash common.Hash + Transactions []DiffAccountsInTx +} diff --git a/eth/api_backend.go b/eth/api_backend.go index 7ac1f82a86..5c864a236b 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -279,6 +279,10 @@ func (b *EthAPIBackend) SuggestPrice(ctx context.Context) (*big.Int, error) { return b.gpo.SuggestPrice(ctx) } +func (b *EthAPIBackend) Chain() *core.BlockChain { + return b.eth.BlockChain() +} + func (b *EthAPIBackend) ChainDb() ethdb.Database { return b.eth.ChainDb() } diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 81fbe5b407..578b10f09a 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -186,6 +186,20 @@ func (ec *Client) HeaderByNumber(ctx context.Context, number *big.Int) (*types.H return head, err } +// GetDiffAccounts returns changed accounts in a specific block number. +func (ec *Client) GetDiffAccounts(ctx context.Context, number *big.Int) ([]common.Address, error) { + accounts := make([]common.Address, 0) + err := ec.c.CallContext(ctx, &accounts, "eth_getDiffAccounts", toBlockNumArg(number)) + return accounts, err +} + +// GetDiffAccountsWithScope returns detailed changes of some interested accounts in a specific block number. +func (ec *Client) GetDiffAccountsWithScope(ctx context.Context, number *big.Int, accounts []common.Address) (*types.DiffAccountsInBlock, error) { + var result types.DiffAccountsInBlock + err := ec.c.CallContext(ctx, &result, "eth_getDiffAccountsWithScope", toBlockNumArg(number), accounts) + return &result, err +} + type rpcTransaction struct { tx *types.Transaction txExtraInfo diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 341acf978f..d2fd056041 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -31,9 +31,11 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" @@ -181,11 +183,82 @@ func TestToFilterArg(t *testing.T) { } var ( - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testAddr = crypto.PubkeyToAddress(testKey.PublicKey) - testBalance = big.NewInt(2e10) + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testAddr = crypto.PubkeyToAddress(testKey.PublicKey) + testBalance = big.NewInt(2e10) + testBlockNum = 128 + testBlocks = []testBlockParam{ + { + // This txs params also used to default block. + blockNr: 10, + txs: []testTransactionParam{}, + }, + { + blockNr: 11, + txs: []testTransactionParam{ + { + to: common.Address{0x01}, + value: big.NewInt(1), + gasPrice: big.NewInt(1), + data: nil, + }, + }, + }, + { + blockNr: 12, + txs: []testTransactionParam{ + { + to: common.Address{0x01}, + value: big.NewInt(1), + gasPrice: big.NewInt(1), + data: nil, + }, + { + to: common.Address{0x02}, + value: big.NewInt(2), + gasPrice: big.NewInt(2), + data: nil, + }, + }, + }, + { + blockNr: 13, + txs: []testTransactionParam{ + { + to: common.Address{0x01}, + value: big.NewInt(1), + gasPrice: big.NewInt(1), + data: nil, + }, + { + to: common.Address{0x02}, + value: big.NewInt(2), + gasPrice: big.NewInt(2), + data: nil, + }, + { + to: common.Address{0x03}, + value: big.NewInt(3), + gasPrice: big.NewInt(3), + data: nil, + }, + }, + }, + } ) +type testTransactionParam struct { + to common.Address + value *big.Int + gasPrice *big.Int + data []byte +} + +type testBlockParam struct { + blockNr int + txs []testTransactionParam +} + func newTestBackend(t *testing.T) (*node.Node, []*types.Block) { // Generate test chain. genesis, blocks := generateTestChain() @@ -197,6 +270,7 @@ func newTestBackend(t *testing.T) (*node.Node, []*types.Block) { // Create Ethereum Service config := ðconfig.Config{Genesis: genesis} config.Ethash.PowMode = ethash.ModeFake + config.SnapshotCache = 256 ethservice, err := eth.New(n, config) if err != nil { t.Fatalf("can't create new ethereum service: %v", err) @@ -212,7 +286,10 @@ func newTestBackend(t *testing.T) (*node.Node, []*types.Block) { } func generateTestChain() (*core.Genesis, []*types.Block) { + signer := types.HomesteadSigner{} + // Create a database pre-initialize with a genesis block db := rawdb.NewMemoryDatabase() + db.SetDiffStore(memorydb.New()) config := params.AllEthashProtocolChanges genesis := &core.Genesis{ Config: config, @@ -220,13 +297,45 @@ func generateTestChain() (*core.Genesis, []*types.Block) { ExtraData: []byte("test genesis"), Timestamp: 9000, } - generate := func(i int, g *core.BlockGen) { - g.OffsetTime(5) - g.SetExtra([]byte("test")) + genesis.MustCommit(db) + chain, _ := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, core.EnablePersistDiff(860000)) + generate := func(i int, block *core.BlockGen) { + block.OffsetTime(5) + block.SetExtra([]byte("test")) + //block.SetCoinbase(testAddr) + + for idx, testBlock := range testBlocks { + // Specific block setting, the index in this generator has 1 diff from specified blockNr. + if i+1 == testBlock.blockNr { + for _, testTransaction := range testBlock.txs { + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), testTransaction.to, + testTransaction.value, params.TxGas, testTransaction.gasPrice, testTransaction.data), signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(chain, tx) + } + break + } + + // Default block setting. + if idx == len(testBlocks)-1 { + // We want to simulate an empty middle block, having the same state as the + // first one. The last is needs a state change again to force a reorg. + for _, testTransaction := range testBlocks[0].txs { + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), testTransaction.to, + testTransaction.value, params.TxGas, testTransaction.gasPrice, testTransaction.data), signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(chain, tx) + } + } + } } gblock := genesis.ToBlock(db) engine := ethash.NewFaker() - blocks, _ := core.GenerateChain(config, gblock, engine, db, 1, generate) + blocks, _ := core.GenerateChain(config, gblock, engine, db, testBlockNum, generate) blocks = append([]*types.Block{gblock}, blocks...) return genesis, blocks } @@ -261,6 +370,9 @@ func TestEthClient(t *testing.T) { "TestCallContract": { func(t *testing.T) { testCallContract(t, client) }, }, + "TestDiffAccounts": { + func(t *testing.T) { testDiffAccounts(t, client) }, + }, // DO not have TestAtFunctions now, because we do not have pending block now } @@ -393,7 +505,7 @@ func testGetBlock(t *testing.T, client *rpc.Client) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if blockNumber != 1 { + if blockNumber != uint64(testBlockNum) { t.Fatalf("BlockNumber returned wrong number: %d", blockNumber) } // Get current block by number @@ -507,3 +619,44 @@ func sendTransaction(ec *Client) error { // Send transaction return ec.SendTransaction(context.Background(), signedTx) } + +func testDiffAccounts(t *testing.T, client *rpc.Client) { + ec := NewClient(client) + ctx, cancel := context.WithTimeout(context.Background(), 1000*time.Millisecond) + defer cancel() + + for _, testBlock := range testBlocks { + if testBlock.blockNr == 10 { + continue + } + diffAccounts, err := ec.GetDiffAccounts(ctx, big.NewInt(int64(testBlock.blockNr))) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + accounts := make([]common.Address, 0) + for _, tx := range testBlock.txs { + // tx.to should be in the accounts list. + for idx, account := range diffAccounts { + if tx.to == account { + break + } + + if idx == len(diffAccounts)-1 { + t.Fatalf("address(%v) expected in the diff account list, but not", tx.to) + } + } + + accounts = append(accounts, tx.to) + } + + diffDetail, err := ec.GetDiffAccountsWithScope(ctx, big.NewInt(int64(testBlock.blockNr)), accounts) + if err != nil { + t.Fatalf("get diff accounts in block error: %v", err) + } + // No contract deposit tx, so expect empty transactions. + if len(diffDetail.Transactions) != 0 { + t.Fatalf("expect ignore all transactions, but some transaction has recorded") + } + } +} diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 15e7a8c8f1..e6064ab787 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/common/gopool" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" @@ -1086,6 +1087,111 @@ func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args CallArgs, bl return DoEstimateGas(ctx, s.b, args, bNrOrHash, s.b.RPCGasCap()) } +// GetDiffAccounts returns changed accounts in a specific block number. +func (s *PublicBlockChainAPI) GetDiffAccounts(ctx context.Context, blockNr rpc.BlockNumber) ([]common.Address, error) { + if s.b.Chain() == nil { + return nil, fmt.Errorf("blockchain not support get diff accounts") + } + + header, err := s.b.HeaderByNumber(ctx, blockNr) + if err != nil { + return nil, fmt.Errorf("block not found for block number (%d): %v", blockNr, err) + } + + return s.b.Chain().GetDiffAccounts(header.Hash()) +} + +// GetDiffAccountsWithScope returns detailed changes of some interested accounts in a specific block number. +func (s *PublicBlockChainAPI) GetDiffAccountsWithScope(ctx context.Context, blockNr rpc.BlockNumber, accounts []common.Address) (*types.DiffAccountsInBlock, error) { + if s.b.Chain() == nil { + return nil, fmt.Errorf("blockchain not support get diff accounts") + } + + block, err := s.b.BlockByNumber(ctx, blockNr) + if err != nil { + return nil, fmt.Errorf("block not found for block number (%d): %v", blockNr, err) + } + parent, err := s.b.BlockByHash(ctx, block.ParentHash()) + if err != nil { + return nil, fmt.Errorf("block not found for block number (%d): %v", blockNr-1, err) + } + statedb, err := s.b.Chain().StateAt(parent.Root()) + if err != nil { + return nil, fmt.Errorf("state not found for block number (%d): %v", blockNr-1, err) + } + + result := &types.DiffAccountsInBlock{ + Number: uint64(blockNr), + BlockHash: block.Hash(), + Transactions: make([]types.DiffAccountsInTx, 0), + } + + accountSet := make(map[common.Address]struct{}, len(accounts)) + for _, account := range accounts { + accountSet[account] = struct{}{} + } + + // Recompute transactions. + signer := types.MakeSigner(s.b.ChainConfig(), block.Number()) + for _, tx := range block.Transactions() { + // Skip data empty tx and to is one of the interested accounts tx. + skip := false + if len(tx.Data()) == 0 { + skip = true + } else if to := tx.To(); to != nil { + if _, exists := accountSet[*to]; exists { + skip = true + } + } + + diffTx := types.DiffAccountsInTx{ + TxHash: tx.Hash(), + Accounts: make(map[common.Address]*big.Int, len(accounts)), + } + + if !skip { + // Record account balance + for _, account := range accounts { + diffTx.Accounts[account] = statedb.GetBalance(account) + } + } + + // Apply transaction + msg, _ := tx.AsMessage(signer) + txContext := core.NewEVMTxContext(msg) + context := core.NewEVMBlockContext(block.Header(), s.b.Chain(), nil) + vmenv := vm.NewEVM(context, txContext, statedb, s.b.ChainConfig(), vm.Config{}) + + if posa, ok := s.b.Engine().(consensus.PoSA); ok { + if isSystem, _ := posa.IsSystemTransaction(tx, block.Header()); isSystem { + balance := statedb.GetBalance(consensus.SystemAddress) + if balance.Cmp(common.Big0) > 0 { + statedb.SetBalance(consensus.SystemAddress, big.NewInt(0)) + statedb.AddBalance(block.Header().Coinbase, balance) + } + } + } + + if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { + return nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) + } + statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) + + if !skip { + // Compute account balance diff. + for _, account := range accounts { + diffTx.Accounts[account] = new(big.Int).Sub(statedb.GetBalance(account), diffTx.Accounts[account]) + if diffTx.Accounts[account].Cmp(big.NewInt(0)) == 0 { + delete(diffTx.Accounts, account) + } + } + result.Transactions = append(result.Transactions, diffTx) + } + } + + return result, nil +} + // ExecutionResult groups all structured logs emitted by the EVM // while replaying a transaction in debug mode as well as transaction // execution status, the amount of gas used and the return value diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 07e76583f3..ca5a55d5ed 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -42,6 +42,7 @@ type Backend interface { // General Ethereum API Downloader() *downloader.Downloader SuggestPrice(ctx context.Context) (*big.Int, error) + Chain() *core.BlockChain ChainDb() ethdb.Database AccountManager() *accounts.Manager ExtRPCEnabled() bool diff --git a/les/api_backend.go b/les/api_backend.go index 60c64a8bdf..c8eca2d905 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -255,6 +255,10 @@ func (b *LesApiBackend) SuggestPrice(ctx context.Context) (*big.Int, error) { return b.gpo.SuggestPrice(ctx) } +func (b *LesApiBackend) Chain() *core.BlockChain { + return nil +} + func (b *LesApiBackend) ChainDb() ethdb.Database { return b.eth.chainDb } From 0315f60924eec1256285085389aede47fd278255 Mon Sep 17 00:00:00 2001 From: Keefe-Liu Date: Fri, 8 Oct 2021 17:38:19 +0800 Subject: [PATCH 09/18] ignore empty tx in GetDiffAccountsWithScope Signed-off-by: Keefe-Liu --- internal/ethapi/api.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index e6064ab787..da4cf8cad1 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1185,7 +1185,10 @@ func (s *PublicBlockChainAPI) GetDiffAccountsWithScope(ctx context.Context, bloc delete(diffTx.Accounts, account) } } - result.Transactions = append(result.Transactions, diffTx) + + if len(diffTx.Accounts) != 0 { + result.Transactions = append(result.Transactions, diffTx) + } } } From 03febe17c2151b9607493756524175450f4f86b6 Mon Sep 17 00:00:00 2001 From: zjubfd <296179868@qq.com> Date: Mon, 11 Oct 2021 10:55:52 +0800 Subject: [PATCH 10/18] fix blockhash not correct for the logs of system tx receipt (#444) --- core/blockchain.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/core/blockchain.go b/core/blockchain.go index 7157d39455..491cae3043 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -459,6 +459,16 @@ func (bc *BlockChain) GetVMConfig() *vm.Config { } func (bc *BlockChain) cacheReceipts(hash common.Hash, receipts types.Receipts) { + // TODO, This is a hot fix for the block hash of logs is `0x0000000000000000000000000000000000000000000000000000000000000000` for system tx + // Please check details in https://github.com/binance-chain/bsc/issues/443 + // This is a temporary fix, the official fix should be a hard fork. + const possibleSystemReceipts = 3 // One slash tx, two reward distribute txs. + numOfReceipts := len(receipts) + for i := numOfReceipts - 1; i >= 0 && i >= numOfReceipts-possibleSystemReceipts; i-- { + for j := 0; j < len(receipts[i].Logs); j++ { + receipts[i].Logs[j].BlockHash = hash + } + } bc.receiptsCache.Add(hash, receipts) } From 5e210b311b540c3397c65efbac9b0ecd3de68fcf Mon Sep 17 00:00:00 2001 From: zjubfd <296179868@qq.com> Date: Tue, 12 Oct 2021 11:29:17 +0800 Subject: [PATCH 11/18] fix concurrent write seen of subfetcher (#446) --- core/state/trie_prefetcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state/trie_prefetcher.go b/core/state/trie_prefetcher.go index 116302001d..ddecd7a202 100644 --- a/core/state/trie_prefetcher.go +++ b/core/state/trie_prefetcher.go @@ -105,7 +105,7 @@ func (p *triePrefetcher) abortLoop() { func (p *triePrefetcher) close() { for _, fetcher := range p.fetchers { p.abortChan <- fetcher // safe to do multiple times - + <-fetcher.term if metrics.Enabled { if fetcher.root == p.root { p.accountLoadMeter.Mark(int64(len(fetcher.seen))) From 610f6a5e633638794d1c215c74d87e2ce13e2f42 Mon Sep 17 00:00:00 2001 From: yutianwu Date: Wed, 13 Oct 2021 11:30:19 +0800 Subject: [PATCH 12/18] [R4R] add extension in eth protocol handshake to disable tx broadcast (#412) * add extension for eth protocol handshake * fix comments --- eth/backend.go | 23 ++++++----- eth/downloader/peer.go | 8 ++-- eth/ethconfig/config.go | 5 ++- eth/ethconfig/gen_config.go | 6 +++ eth/handler.go | 55 +++++++++++++------------ eth/handler_eth_test.go | 10 ++--- eth/protocols/eth/broadcast.go | 6 +++ eth/protocols/eth/handshake.go | 62 ++++++++++++++++++++++++++++- eth/protocols/eth/handshake_test.go | 2 +- eth/protocols/eth/peer.go | 33 ++++++++++++--- eth/protocols/eth/protocol.go | 40 ++++++++++++++++++- 11 files changed, 193 insertions(+), 57 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index bdae4b4235..f6599529db 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -231,17 +231,18 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } if eth.handler, err = newHandler(&handlerConfig{ - Database: chainDb, - Chain: eth.blockchain, - TxPool: eth.txPool, - Network: config.NetworkId, - Sync: config.SyncMode, - BloomCache: uint64(cacheLimit), - EventMux: eth.eventMux, - Checkpoint: checkpoint, - Whitelist: config.Whitelist, - DirectBroadcast: config.DirectBroadcast, - DiffSync: config.DiffSync, + Database: chainDb, + Chain: eth.blockchain, + TxPool: eth.txPool, + Network: config.NetworkId, + Sync: config.SyncMode, + BloomCache: uint64(cacheLimit), + EventMux: eth.eventMux, + Checkpoint: checkpoint, + Whitelist: config.Whitelist, + DirectBroadcast: config.DirectBroadcast, + DiffSync: config.DiffSync, + DisablePeerTxBroadcast: config.DisablePeerTxBroadcast, }); err != nil { return nil, err } diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 4d76988f71..297ba2fa55 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -457,7 +457,7 @@ func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.headerThroughput } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH65, eth.ETH67, idle, throughput) } // BodyIdlePeers retrieves a flat list of all the currently body-idle peers within @@ -471,7 +471,7 @@ func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.blockThroughput } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH65, eth.ETH67, idle, throughput) } // ReceiptIdlePeers retrieves a flat list of all the currently receipt-idle peers @@ -485,7 +485,7 @@ func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.receiptThroughput } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH65, eth.ETH67, idle, throughput) } // NodeDataIdlePeers retrieves a flat list of all the currently node-data-idle @@ -499,7 +499,7 @@ func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.stateThroughput } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH65, eth.ETH67, idle, throughput) } // idlePeers retrieves a flat list of all currently idle peers satisfying the diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index bf143ba02c..83db7fc998 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -124,8 +124,9 @@ type Config struct { Genesis *core.Genesis `toml:",omitempty"` // Protocol options - NetworkId uint64 // Network ID to use for selecting peers to connect to - SyncMode downloader.SyncMode + NetworkId uint64 // Network ID to use for selecting peers to connect to + SyncMode downloader.SyncMode + DisablePeerTxBroadcast bool // This can be set to list of enrtree:// URLs which will be queried for // for nodes to connect to. diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 258ade2293..f192a1aace 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -20,6 +20,7 @@ func (c Config) MarshalTOML() (interface{}, error) { Genesis *core.Genesis `toml:",omitempty"` NetworkId uint64 SyncMode downloader.SyncMode + DisablePeerTxBroadcast bool EthDiscoveryURLs []string SnapDiscoveryURLs []string NoPruning bool @@ -68,6 +69,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.Genesis = c.Genesis enc.NetworkId = c.NetworkId enc.SyncMode = c.SyncMode + enc.DisablePeerTxBroadcast = c.DisablePeerTxBroadcast enc.EthDiscoveryURLs = c.EthDiscoveryURLs enc.SnapDiscoveryURLs = c.SnapDiscoveryURLs enc.NoPruning = c.NoPruning @@ -119,6 +121,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { Genesis *core.Genesis `toml:",omitempty"` NetworkId *uint64 SyncMode *downloader.SyncMode + DisablePeerTxBroadcast *bool EthDiscoveryURLs []string SnapDiscoveryURLs []string NoPruning *bool @@ -176,6 +179,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.SyncMode != nil { c.SyncMode = *dec.SyncMode } + if dec.DisablePeerTxBroadcast != nil { + c.DisablePeerTxBroadcast = *dec.DisablePeerTxBroadcast + } if dec.EthDiscoveryURLs != nil { c.EthDiscoveryURLs = dec.EthDiscoveryURLs } diff --git a/eth/handler.go b/eth/handler.go index 41b459d2d5..f00f955b34 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -78,22 +78,24 @@ type txPool interface { // handlerConfig is the collection of initialization parameters to create a full // node network handler. type handlerConfig struct { - Database ethdb.Database // Database for direct sync insertions - Chain *core.BlockChain // Blockchain to serve data from - TxPool txPool // Transaction pool to propagate from - Network uint64 // Network identifier to adfvertise - Sync downloader.SyncMode // Whether to fast or full sync - DiffSync bool // Whether to diff sync - BloomCache uint64 // Megabytes to alloc for fast sync bloom - EventMux *event.TypeMux // Legacy event mux, deprecate for `feed` - Checkpoint *params.TrustedCheckpoint // Hard coded checkpoint for sync challenges - Whitelist map[uint64]common.Hash // Hard coded whitelist for sync challenged - DirectBroadcast bool + Database ethdb.Database // Database for direct sync insertions + Chain *core.BlockChain // Blockchain to serve data from + TxPool txPool // Transaction pool to propagate from + Network uint64 // Network identifier to adfvertise + Sync downloader.SyncMode // Whether to fast or full sync + DiffSync bool // Whether to diff sync + BloomCache uint64 // Megabytes to alloc for fast sync bloom + EventMux *event.TypeMux // Legacy event mux, deprecate for `feed` + Checkpoint *params.TrustedCheckpoint // Hard coded checkpoint for sync challenges + Whitelist map[uint64]common.Hash // Hard coded whitelist for sync challenged + DirectBroadcast bool + DisablePeerTxBroadcast bool } type handler struct { - networkID uint64 - forkFilter forkid.Filter // Fork ID filter, constant across the lifetime of the node + networkID uint64 + forkFilter forkid.Filter // Fork ID filter, constant across the lifetime of the node + disablePeerTxBroadcast bool fastSync uint32 // Flag whether fast sync is enabled (gets disabled if we already have blocks) snapSync uint32 // Flag whether fast sync should operate on top of the snap protocol @@ -138,18 +140,19 @@ func newHandler(config *handlerConfig) (*handler, error) { config.EventMux = new(event.TypeMux) // Nicety initialization for tests } h := &handler{ - networkID: config.Network, - forkFilter: forkid.NewFilter(config.Chain), - eventMux: config.EventMux, - database: config.Database, - txpool: config.TxPool, - chain: config.Chain, - peers: newPeerSet(), - whitelist: config.Whitelist, - directBroadcast: config.DirectBroadcast, - diffSync: config.DiffSync, - txsyncCh: make(chan *txsync), - quitSync: make(chan struct{}), + networkID: config.Network, + forkFilter: forkid.NewFilter(config.Chain), + disablePeerTxBroadcast: config.DisablePeerTxBroadcast, + eventMux: config.EventMux, + database: config.Database, + txpool: config.TxPool, + chain: config.Chain, + peers: newPeerSet(), + whitelist: config.Whitelist, + directBroadcast: config.DirectBroadcast, + diffSync: config.DiffSync, + txsyncCh: make(chan *txsync), + quitSync: make(chan struct{}), } if config.Sync == downloader.FullSync { // The database seems empty as the current block is the genesis. Yet the fast @@ -276,7 +279,7 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { td = h.chain.GetTd(hash, number) ) forkID := forkid.NewID(h.chain.Config(), h.chain.Genesis().Hash(), h.chain.CurrentHeader().Number.Uint64()) - if err := peer.Handshake(h.networkID, td, hash, genesis.Hash(), forkID, h.forkFilter); err != nil { + if err := peer.Handshake(h.networkID, td, hash, genesis.Hash(), forkID, h.forkFilter, ð.UpgradeStatusExtension{DisablePeerTxBroadcast: h.disablePeerTxBroadcast}); err != nil { peer.Log().Debug("Ethereum handshake failed", "err", err) return err } diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 1d38e3b666..271bae07c7 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -271,7 +271,7 @@ func testRecvTransactions(t *testing.T, protocol uint) { head = handler.chain.CurrentBlock() td = handler.chain.GetTd(head.Hash(), head.NumberU64()) ) - if err := src.Handshake(1, td, head.Hash(), genesis.Hash(), forkid.NewIDWithChain(handler.chain), forkid.NewFilter(handler.chain)); err != nil { + if err := src.Handshake(1, td, head.Hash(), genesis.Hash(), forkid.NewIDWithChain(handler.chain), forkid.NewFilter(handler.chain), nil); err != nil { t.Fatalf("failed to run protocol handshake") } // Send the transaction to the sink and verify that it's added to the tx pool @@ -333,7 +333,7 @@ func testSendTransactions(t *testing.T, protocol uint) { head = handler.chain.CurrentBlock() td = handler.chain.GetTd(head.Hash(), head.NumberU64()) ) - if err := sink.Handshake(1, td, head.Hash(), genesis.Hash(), forkid.NewIDWithChain(handler.chain), forkid.NewFilter(handler.chain)); err != nil { + if err := sink.Handshake(1, td, head.Hash(), genesis.Hash(), forkid.NewIDWithChain(handler.chain), forkid.NewFilter(handler.chain), nil); err != nil { t.Fatalf("failed to run protocol handshake") } // After the handshake completes, the source handler should stream the sink @@ -532,7 +532,7 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo head = handler.chain.CurrentBlock() td = handler.chain.GetTd(head.Hash(), head.NumberU64()) ) - if err := remote.Handshake(1, td, head.Hash(), genesis.Hash(), forkid.NewIDWithChain(handler.chain), forkid.NewFilter(handler.chain)); err != nil { + if err := remote.Handshake(1, td, head.Hash(), genesis.Hash(), forkid.NewIDWithChain(handler.chain), forkid.NewFilter(handler.chain), nil); err != nil { t.Fatalf("failed to run protocol handshake") } // Connect a new peer and check that we receive the checkpoint challenge @@ -616,7 +616,7 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { go source.handler.runEthPeer(sourcePeer, func(peer *eth.Peer) error { return eth.Handle((*ethHandler)(source.handler), peer) }) - if err := sinkPeer.Handshake(1, td, genesis.Hash(), genesis.Hash(), forkid.NewIDWithChain(source.chain), forkid.NewFilter(source.chain)); err != nil { + if err := sinkPeer.Handshake(1, td, genesis.Hash(), genesis.Hash(), forkid.NewIDWithChain(source.chain), forkid.NewFilter(source.chain), nil); err != nil { t.Fatalf("failed to run protocol handshake") } go eth.Handle(sink, sinkPeer) @@ -689,7 +689,7 @@ func testBroadcastMalformedBlock(t *testing.T, protocol uint) { genesis = source.chain.Genesis() td = source.chain.GetTd(genesis.Hash(), genesis.NumberU64()) ) - if err := sink.Handshake(1, td, genesis.Hash(), genesis.Hash(), forkid.NewIDWithChain(source.chain), forkid.NewFilter(source.chain)); err != nil { + if err := sink.Handshake(1, td, genesis.Hash(), genesis.Hash(), forkid.NewIDWithChain(source.chain), forkid.NewFilter(source.chain), nil); err != nil { t.Fatalf("failed to run protocol handshake") } // After the handshake completes, the source handler should stream the sink diff --git a/eth/protocols/eth/broadcast.go b/eth/protocols/eth/broadcast.go index e0ee2a1cfa..132eac0102 100644 --- a/eth/protocols/eth/broadcast.go +++ b/eth/protocols/eth/broadcast.go @@ -122,6 +122,9 @@ func (p *Peer) broadcastTransactions() { case <-fail: failed = true + case <-p.txTerm: + return + case <-p.term: return } @@ -189,6 +192,9 @@ func (p *Peer) announceTransactions() { case <-fail: failed = true + case <-p.txTerm: + return + case <-p.term: return } diff --git a/eth/protocols/eth/handshake.go b/eth/protocols/eth/handshake.go index b634f18e00..d604f045f4 100644 --- a/eth/protocols/eth/handshake.go +++ b/eth/protocols/eth/handshake.go @@ -35,7 +35,7 @@ const ( // Handshake executes the eth protocol handshake, negotiating version number, // network IDs, difficulties, head and genesis blocks. -func (p *Peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis common.Hash, forkID forkid.ID, forkFilter forkid.Filter) error { +func (p *Peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis common.Hash, forkID forkid.ID, forkFilter forkid.Filter, extension *UpgradeStatusExtension) error { // Send out own handshake in a new thread errc := make(chan error, 2) @@ -68,6 +68,49 @@ func (p *Peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis } p.td, p.head = status.TD, status.Head + if p.version >= ETH67 { + var upgradeStatus UpgradeStatusPacket // safe to read after two values have been received from errc + if extension == nil { + extension = &UpgradeStatusExtension{} + } + extensionRaw, err := extension.Encode() + if err != nil { + return err + } + + gopool.Submit(func() { + errc <- p2p.Send(p.rw, UpgradeStatusMsg, &UpgradeStatusPacket{ + Extension: extensionRaw, + }) + }) + gopool.Submit(func() { + errc <- p.readUpgradeStatus(&upgradeStatus) + }) + timeout := time.NewTimer(handshakeTimeout) + defer timeout.Stop() + for i := 0; i < 2; i++ { + select { + case err := <-errc: + if err != nil { + return err + } + case <-timeout.C: + return p2p.DiscReadTimeout + } + } + + extension, err := upgradeStatus.GetExtension() + if err != nil { + return err + } + p.statusExtension = extension + + if p.statusExtension.DisablePeerTxBroadcast { + p.Log().Debug("peer does not need broadcast txs, closing broadcast routines") + p.CloseTxBroadcast() + } + } + // TD at mainnet block #7753254 is 76 bits. If it becomes 100 million times // larger, it will still fit within 100 bits if tdlen := p.td.BitLen(); tdlen > 100 { @@ -106,3 +149,20 @@ func (p *Peer) readStatus(network uint64, status *StatusPacket, genesis common.H } return nil } + +func (p *Peer) readUpgradeStatus(status *UpgradeStatusPacket) error { + msg, err := p.rw.ReadMsg() + if err != nil { + return err + } + if msg.Code != UpgradeStatusMsg { + return fmt.Errorf("%w: upgrade status msg has code %x (!= %x)", errNoStatusMsg, msg.Code, UpgradeStatusMsg) + } + if msg.Size > maxMessageSize { + return fmt.Errorf("%w: %v > %v", errMsgTooLarge, msg.Size, maxMessageSize) + } + if err := msg.Decode(&status); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return nil +} diff --git a/eth/protocols/eth/handshake_test.go b/eth/protocols/eth/handshake_test.go index 3bebda2dcc..8a433f3ce4 100644 --- a/eth/protocols/eth/handshake_test.go +++ b/eth/protocols/eth/handshake_test.go @@ -81,7 +81,7 @@ func testHandshake(t *testing.T, protocol uint) { // Send the junk test with one peer, check the handshake failure go p2p.Send(app, test.code, test.data) - err := peer.Handshake(1, td, head.Hash(), genesis.Hash(), forkID, forkid.NewFilter(backend.chain)) + err := peer.Handshake(1, td, head.Hash(), genesis.Hash(), forkID, forkid.NewFilter(backend.chain), nil) if err == nil { t.Errorf("test %d: protocol returned nil error, want %q", i, test.want) } else if !errors.Is(err, test.want) { diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index e619c183ba..7ab4fa1a36 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -22,6 +22,7 @@ import ( "sync" mapset "github.com/deckarep/golang-set" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/p2p" @@ -68,9 +69,10 @@ func max(a, b int) int { type Peer struct { id string // Unique ID for the peer, cached - *p2p.Peer // The embedded P2P package peer - rw p2p.MsgReadWriter // Input/output streams for snap - version uint // Protocol version negotiated + *p2p.Peer // The embedded P2P package peer + rw p2p.MsgReadWriter // Input/output streams for snap + version uint // Protocol version negotiated + statusExtension *UpgradeStatusExtension head common.Hash // Latest advertised head block hash td *big.Int // Latest advertised head block total difficulty @@ -84,8 +86,9 @@ type Peer struct { txBroadcast chan []common.Hash // Channel used to queue transaction propagation requests txAnnounce chan []common.Hash // Channel used to queue transaction announcement requests - term chan struct{} // Termination channel to stop the broadcasters - lock sync.RWMutex // Mutex protecting the internal fields + term chan struct{} // Termination channel to stop the broadcasters + txTerm chan struct{} // Termination channel to stop the tx broadcasters + lock sync.RWMutex // Mutex protecting the internal fields } // NewPeer create a wrapper for a network connection and negotiated protocol @@ -104,6 +107,7 @@ func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Pe txAnnounce: make(chan []common.Hash), txpool: txpool, term: make(chan struct{}), + txTerm: make(chan struct{}), } // Start up all the broadcasters go peer.broadcastBlocks() @@ -119,6 +123,17 @@ func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Pe // clean it up! func (p *Peer) Close() { close(p.term) + + p.CloseTxBroadcast() +} + +// CloseTxBroadcast signals the tx broadcast goroutine to terminate. +func (p *Peer) CloseTxBroadcast() { + select { + case <-p.txTerm: + default: + close(p.txTerm) + } } // ID retrieves the peer's unique identifier. @@ -212,6 +227,10 @@ func (p *Peer) AsyncSendTransactions(hashes []common.Hash) { for _, hash := range hashes { p.knownTxs.Add(hash) } + + case <-p.txTerm: + p.Log().Debug("Dropping transaction propagation", "count", len(hashes)) + case <-p.term: p.Log().Debug("Dropping transaction propagation", "count", len(hashes)) } @@ -247,6 +266,10 @@ func (p *Peer) AsyncSendPooledTransactionHashes(hashes []common.Hash) { for _, hash := range hashes { p.knownTxs.Add(hash) } + + case <-p.txTerm: + p.Log().Debug("Dropping transaction announcement", "count", len(hashes)) + case <-p.term: p.Log().Debug("Dropping transaction announcement", "count", len(hashes)) } diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index de1b0ed1ee..3e0e0cf6ed 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -32,6 +32,7 @@ import ( const ( ETH65 = 65 ETH66 = 66 + ETH67 = 67 ) // ProtocolName is the official short name of the `eth` protocol used during @@ -40,11 +41,11 @@ const ProtocolName = "eth" // ProtocolVersions are the supported versions of the `eth` protocol (first // is primary). -var ProtocolVersions = []uint{ETH66, ETH65} +var ProtocolVersions = []uint{ETH67, ETH66, ETH65} // protocolLengths are the number of implemented message corresponding to // different protocol versions. -var protocolLengths = map[uint]uint64{ETH66: 17, ETH65: 17} +var protocolLengths = map[uint]uint64{ETH67: 18, ETH66: 17, ETH65: 17} // maxMessageSize is the maximum cap on the size of a protocol message. const maxMessageSize = 10 * 1024 * 1024 @@ -68,6 +69,9 @@ const ( NewPooledTransactionHashesMsg = 0x08 GetPooledTransactionsMsg = 0x09 PooledTransactionsMsg = 0x0a + + // Protocol messages overloaded in eth/66 + UpgradeStatusMsg = 0x0b ) var ( @@ -97,6 +101,35 @@ type StatusPacket struct { ForkID forkid.ID } +type UpgradeStatusExtension struct { + DisablePeerTxBroadcast bool +} + +func (e *UpgradeStatusExtension) Encode() (*rlp.RawValue, error) { + rawBytes, err := rlp.EncodeToBytes(e) + if err != nil { + return nil, err + } + raw := rlp.RawValue(rawBytes) + return &raw, nil +} + +type UpgradeStatusPacket struct { + Extension *rlp.RawValue `rlp:"nil"` +} + +func (p *UpgradeStatusPacket) GetExtension() (*UpgradeStatusExtension, error) { + extension := &UpgradeStatusExtension{} + if p.Extension == nil { + return extension, nil + } + err := rlp.DecodeBytes(*p.Extension, extension) + if err != nil { + return nil, err + } + return extension, nil +} + // NewBlockHashesPacket is the network packet for the block announcements. type NewBlockHashesPacket []struct { Hash common.Hash // Hash of one particular block being announced @@ -324,6 +357,9 @@ type PooledTransactionsRLPPacket66 struct { func (*StatusPacket) Name() string { return "Status" } func (*StatusPacket) Kind() byte { return StatusMsg } +func (*UpgradeStatusPacket) Name() string { return "UpgradeStatus" } +func (*UpgradeStatusPacket) Kind() byte { return UpgradeStatusMsg } + func (*NewBlockHashesPacket) Name() string { return "NewBlockHashes" } func (*NewBlockHashesPacket) Kind() byte { return NewBlockHashesMsg } From 33aa77949fc027f43e815a735b212a5dc131fdae Mon Sep 17 00:00:00 2001 From: kyrie-yl <83150977+kyrie-yl@users.noreply.github.com> Date: Fri, 15 Oct 2021 15:12:14 +0800 Subject: [PATCH 13/18] cache bitmap and change the cache type of GetCode (#449) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * change cache type of GetCode from fastcache to lrucache Signed-off-by: kyrie-yl * add cache for contract code bitmap Signed-off-by: kyrie-yl * core/vm: rework jumpdest analysis benchmarks (#23499) * core/vm: rework jumpdest analysis benchmarks For BenchmarkJumpdestOpAnalysis use fixed code size of ~1.2MB and classic benchmark loop. * core/vm: clear bitvec in jumpdest analysis benchmark Co-authored-by: PaweÅ‚ Bylica --- core/state/database.go | 28 ++++++++++++++++++---------- core/vm/analysis_test.go | 18 ++++++++++++++---- core/vm/contract.go | 19 +++++++++++++++---- 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/core/state/database.go b/core/state/database.go index 0bcde2d5a9..24a1474d57 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -21,7 +21,6 @@ import ( "fmt" "time" - "github.com/VictoriaMetrics/fastcache" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" @@ -134,22 +133,24 @@ func NewDatabase(db ethdb.Database) Database { // large memory cache. func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { csc, _ := lru.New(codeSizeCacheSize) + cc, _ := lru.New(codeCacheSize) return &cachingDB{ db: trie.NewDatabaseWithConfig(db, config), codeSizeCache: csc, - codeCache: fastcache.New(codeCacheSize), + codeCache: cc, } } func NewDatabaseWithConfigAndCache(db ethdb.Database, config *trie.Config) Database { csc, _ := lru.New(codeSizeCacheSize) + cc, _ := lru.New(codeCacheSize) atc, _ := lru.New(accountTrieCacheSize) stc, _ := lru.New(storageTrieCacheSize) database := &cachingDB{ db: trie.NewDatabaseWithConfig(db, config), codeSizeCache: csc, - codeCache: fastcache.New(codeCacheSize), + codeCache: cc, accountTrieCache: atc, storageTrieCache: stc, } @@ -160,7 +161,7 @@ func NewDatabaseWithConfigAndCache(db ethdb.Database, config *trie.Config) Datab type cachingDB struct { db *trie.Database codeSizeCache *lru.Cache - codeCache *fastcache.Cache + codeCache *lru.Cache accountTrieCache *lru.Cache storageTrieCache *lru.Cache } @@ -266,12 +267,16 @@ func (db *cachingDB) CopyTrie(t Trie) Trie { // ContractCode retrieves a particular contract's code. func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) { - if code := db.codeCache.Get(nil, codeHash.Bytes()); len(code) > 0 { - return code, nil + if cached, ok := db.codeCache.Get(codeHash.Bytes()); ok { + code := cached.([]byte) + if len(code) > 0 { + return code, nil + } } code := rawdb.ReadCode(db.db.DiskDB(), codeHash) if len(code) > 0 { - db.codeCache.Set(codeHash.Bytes(), code) + + db.codeCache.Add(codeHash.Bytes(), code) db.codeSizeCache.Add(codeHash, len(code)) return code, nil } @@ -282,12 +287,15 @@ func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error // code can't be found in the cache, then check the existence with **new** // db scheme. func (db *cachingDB) ContractCodeWithPrefix(addrHash, codeHash common.Hash) ([]byte, error) { - if code := db.codeCache.Get(nil, codeHash.Bytes()); len(code) > 0 { - return code, nil + if cached, ok := db.codeCache.Get(codeHash.Bytes()); ok { + code := cached.([]byte) + if len(code) > 0 { + return code, nil + } } code := rawdb.ReadCodeWithPrefix(db.db.DiskDB(), codeHash) if len(code) > 0 { - db.codeCache.Set(codeHash.Bytes(), code) + db.codeCache.Add(codeHash.Bytes(), code) db.codeSizeCache.Add(codeHash, len(code)) return code, nil } diff --git a/core/vm/analysis_test.go b/core/vm/analysis_test.go index 585bb3097f..d7f21e04aa 100644 --- a/core/vm/analysis_test.go +++ b/core/vm/analysis_test.go @@ -55,9 +55,12 @@ func TestJumpDestAnalysis(t *testing.T) { } } +const analysisCodeSize = 1200 * 1024 + func BenchmarkJumpdestAnalysis_1200k(bench *testing.B) { // 1.4 ms - code := make([]byte, 1200000) + code := make([]byte, analysisCodeSize) + bench.SetBytes(analysisCodeSize) bench.ResetTimer() for i := 0; i < bench.N; i++ { codeBitmap(code) @@ -66,7 +69,8 @@ func BenchmarkJumpdestAnalysis_1200k(bench *testing.B) { } func BenchmarkJumpdestHashing_1200k(bench *testing.B) { // 4 ms - code := make([]byte, 1200000) + code := make([]byte, analysisCodeSize) + bench.SetBytes(analysisCodeSize) bench.ResetTimer() for i := 0; i < bench.N; i++ { crypto.Keccak256Hash(code) @@ -77,13 +81,19 @@ func BenchmarkJumpdestHashing_1200k(bench *testing.B) { func BenchmarkJumpdestOpAnalysis(bench *testing.B) { var op OpCode bencher := func(b *testing.B) { - code := make([]byte, 32*b.N) + code := make([]byte, analysisCodeSize) + b.SetBytes(analysisCodeSize) for i := range code { code[i] = byte(op) } bits := make(bitvec, len(code)/8+1+4) b.ResetTimer() - codeBitmapInternal(code, bits) + for i := 0; i < b.N; i++ { + for j := range bits { + bits[j] = 0 + } + codeBitmapInternal(code, bits) + } } for op = PUSH1; op <= PUSH32; op++ { bench.Run(op.String(), bencher) diff --git a/core/vm/contract.go b/core/vm/contract.go index 61dbd5007a..6fac7f9858 100644 --- a/core/vm/contract.go +++ b/core/vm/contract.go @@ -19,10 +19,16 @@ package vm import ( "math/big" + lru "github.com/hashicorp/golang-lru" + "github.com/ethereum/go-ethereum/common" "github.com/holiman/uint256" ) +const codeBitmapCacheSize = 2000 + +var codeBitmapCache, _ = lru.New(codeBitmapCacheSize) + // ContractRef is a reference to the contract's backing object type ContractRef interface { Address() common.Address @@ -110,10 +116,15 @@ func (c *Contract) isCode(udest uint64) bool { // Does parent context have the analysis? analysis, exist := c.jumpdests[c.CodeHash] if !exist { - // Do the analysis and save in parent context - // We do not need to store it in c.analysis - analysis = codeBitmap(c.Code) - c.jumpdests[c.CodeHash] = analysis + if cached, ok := codeBitmapCache.Get(c.CodeHash); ok { + analysis = cached.(bitvec) + } else { + // Do the analysis and save in parent context + // We do not need to store it in c.analysis + analysis = codeBitmap(c.Code) + c.jumpdests[c.CodeHash] = analysis + codeBitmapCache.Add(c.CodeHash, analysis) + } } // Also stash it in current contract for faster access c.analysis = analysis From 31463f8dd1ccc8f1db0bd104eb69e5b8d7a775a0 Mon Sep 17 00:00:00 2001 From: Steven Tran Date: Fri, 15 Oct 2021 16:30:44 +0800 Subject: [PATCH 14/18] parallel bloom calculation (#445) * parallel bloom calculation * indent * add condition if bloomJobs not nil * add handler for worker * fix format * bloomWorker should exit when all txs have been processed * rename BloomPair => BloomHash * add size to map * rename & unique variable * bloomJobs => bloomProcessors * fix * only assign bloom if empty * abstraction method for processing receipt bloom * remove duplicate receipt_processor * rename Processor * fix ReceiptProcessor * fix ReceiptBloomGenertor typo * reduce worker to 1 * remove empty wg * add defence code to check if channel is closed * remove nil * format fix * remove thread pool * use max 100 worker capacity * reduce worker size * refactor startWorker --- core/chain_makers.go | 2 +- core/receipt_processor.go | 66 +++++++++++++++++++++++++++++++++++++++ core/state_processor.go | 20 ++++++++---- core/types/bloom9.go | 2 ++ core/types/transaction.go | 4 +++ eth/catalyst/api.go | 2 +- miner/worker.go | 15 +++++++-- 7 files changed, 100 insertions(+), 11 deletions(-) create mode 100644 core/receipt_processor.go diff --git a/core/chain_makers.go b/core/chain_makers.go index 985aa61cd6..d8e3ee012f 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -104,7 +104,7 @@ func (b *BlockGen) AddTxWithChain(bc *BlockChain, tx *types.Transaction) { b.SetCoinbase(common.Address{}) } b.statedb.Prepare(tx.Hash(), common.Hash{}, len(b.txs)) - receipt, err := ApplyTransaction(b.config, bc, &b.header.Coinbase, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed, vm.Config{}) + receipt, err := ApplyTransaction(b.config, bc, &b.header.Coinbase, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed, vm.Config{}, NewReceiptBloomGenerator()) if err != nil { panic(err) } diff --git a/core/receipt_processor.go b/core/receipt_processor.go new file mode 100644 index 0000000000..57fb56a6dc --- /dev/null +++ b/core/receipt_processor.go @@ -0,0 +1,66 @@ +package core + +import ( + "bytes" + "sync" + + "github.com/ethereum/go-ethereum/core/types" +) + +type ReceiptProcessor interface { + Apply(receipt *types.Receipt) +} + +var ( + _ ReceiptProcessor = (*ReceiptBloomGenerator)(nil) + _ ReceiptProcessor = (*AsyncReceiptBloomGenerator)(nil) +) + +func NewReceiptBloomGenerator() *ReceiptBloomGenerator { + return &ReceiptBloomGenerator{} +} + +type ReceiptBloomGenerator struct { +} + +func (p *ReceiptBloomGenerator) Apply(receipt *types.Receipt) { + receipt.Bloom = types.CreateBloom(types.Receipts{receipt}) +} + +func NewAsyncReceiptBloomGenerator(txNums int) *AsyncReceiptBloomGenerator { + generator := &AsyncReceiptBloomGenerator{ + receipts: make(chan *types.Receipt, txNums), + } + generator.startWorker() + return generator +} + +type AsyncReceiptBloomGenerator struct { + receipts chan *types.Receipt + wg sync.WaitGroup + isClosed bool +} + +func (p *AsyncReceiptBloomGenerator) startWorker() { + p.wg.Add(1) + go func() { + defer p.wg.Done() + for receipt := range p.receipts { + if receipt != nil && bytes.Equal(receipt.Bloom[:], types.EmptyBloom[:]) { + receipt.Bloom = types.CreateBloom(types.Receipts{receipt}) + } + } + }() +} + +func (p *AsyncReceiptBloomGenerator) Apply(receipt *types.Receipt) { + if !p.isClosed { + p.receipts <- receipt + } +} + +func (p *AsyncReceiptBloomGenerator) Close() { + close(p.receipts) + p.isClosed = true + p.wg.Wait() +} \ No newline at end of file diff --git a/core/state_processor.go b/core/state_processor.go index 973fd27b54..14bd5d75b2 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -390,9 +390,14 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg blockContext := NewEVMBlockContext(header, p.bc, nil) vmenv := vm.NewEVM(blockContext, vm.TxContext{}, statedb, p.config, cfg) + txNum := len(block.Transactions()) // Iterate over and process the individual transactions posa, isPoSA := p.engine.(consensus.PoSA) - commonTxs := make([]*types.Transaction, 0, len(block.Transactions())) + commonTxs := make([]*types.Transaction, 0, txNum) + + // initilise bloom processors + bloomProcessors := NewAsyncReceiptBloomGenerator(txNum) + // usually do have two tx, one for validator set contract, another for system reward contract. systemTxs := make([]*types.Transaction, 0, 2) for i, tx := range block.Transactions() { @@ -410,7 +415,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return statedb, nil, nil, 0, err } statedb.Prepare(tx.Hash(), block.Hash(), i) - receipt, err := applyTransaction(msg, p.config, p.bc, nil, gp, statedb, header, tx, usedGas, vmenv) + receipt, err := applyTransaction(msg, p.config, p.bc, nil, gp, statedb, header, tx, usedGas, vmenv, bloomProcessors) if err != nil { return statedb, nil, nil, 0, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } @@ -418,6 +423,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg commonTxs = append(commonTxs, tx) receipts = append(receipts, receipt) } + bloomProcessors.Close() // Finalize the block, applying any consensus engine specific extras (e.g. block rewards) err := p.engine.Finalize(p.bc, header, statedb, &commonTxs, block.Uncles(), &receipts, &systemTxs, usedGas) @@ -431,7 +437,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return statedb, receipts, allLogs, *usedGas, nil } -func applyTransaction(msg types.Message, config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64, evm *vm.EVM) (*types.Receipt, error) { +func applyTransaction(msg types.Message, config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64, evm *vm.EVM, receiptProcessors ...ReceiptProcessor) (*types.Receipt, error) { // Create a new context to be used in the EVM environment. txContext := NewEVMTxContext(msg) evm.Reset(txContext, statedb) @@ -469,10 +475,12 @@ func applyTransaction(msg types.Message, config *params.ChainConfig, bc ChainCon // Set the receipt logs and create the bloom filter. receipt.Logs = statedb.GetLogs(tx.Hash()) - receipt.Bloom = types.CreateBloom(types.Receipts{receipt}) receipt.BlockHash = statedb.BlockHash() receipt.BlockNumber = header.Number receipt.TransactionIndex = uint(statedb.TxIndex()) + for _, receiptProcessor := range receiptProcessors { + receiptProcessor.Apply(receipt) + } return receipt, err } @@ -480,7 +488,7 @@ func applyTransaction(msg types.Message, config *params.ChainConfig, bc ChainCon // and uses the input parameters for its environment. It returns the receipt // for the transaction, gas used and an error if the transaction failed, // indicating the block was invalid. -func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64, cfg vm.Config) (*types.Receipt, error) { +func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64, cfg vm.Config, receiptProcessors ...ReceiptProcessor) (*types.Receipt, error) { msg, err := tx.AsMessage(types.MakeSigner(config, header.Number)) if err != nil { return nil, err @@ -493,5 +501,5 @@ func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *commo vm.EVMInterpreterPool.Put(ite) vm.EvmPool.Put(vmenv) }() - return applyTransaction(msg, config, bc, author, gp, statedb, header, tx, usedGas, vmenv) + return applyTransaction(msg, config, bc, author, gp, statedb, header, tx, usedGas, vmenv, receiptProcessors...) } diff --git a/core/types/bloom9.go b/core/types/bloom9.go index 1793c2adc7..342a0b0ec3 100644 --- a/core/types/bloom9.go +++ b/core/types/bloom9.go @@ -37,6 +37,8 @@ const ( BloomBitLength = 8 * BloomByteLength ) +var EmptyBloom = Bloom{} + // Bloom represents a 2048 bit bloom filter. type Bloom [BloomByteLength]byte diff --git a/core/types/transaction.go b/core/types/transaction.go index 1bb43d805f..b127cb2af6 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -479,6 +479,10 @@ func (t *TransactionsByPriceAndNonce) Pop() { heap.Pop(&t.heads) } +func (t *TransactionsByPriceAndNonce) CurrentSize() int { + return len(t.heads) +} + // Message is a fully derived transaction and implements core.Message // // NOTE: In a future PR this will be removed. diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index ccb0e6bb45..70de993b18 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -79,7 +79,7 @@ type blockExecutionEnv struct { func (env *blockExecutionEnv) commitTransaction(tx *types.Transaction, coinbase common.Address) error { vmconfig := *env.chain.GetVMConfig() - receipt, err := core.ApplyTransaction(env.chain.Config(), env.chain, &coinbase, env.gasPool, env.state, env.header, tx, &env.header.GasUsed, vmconfig) + receipt, err := core.ApplyTransaction(env.chain.Config(), env.chain, &coinbase, env.gasPool, env.state, env.header, tx, &env.header.GasUsed, vmconfig, core.NewReceiptBloomGenerator()) if err != nil { return err } diff --git a/miner/worker.go b/miner/worker.go index ef7a8c5630..2dcb75ac10 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -736,10 +736,10 @@ func (w *worker) updateSnapshot() { w.snapshotState = w.current.state.Copy() } -func (w *worker) commitTransaction(tx *types.Transaction, coinbase common.Address) ([]*types.Log, error) { +func (w *worker) commitTransaction(tx *types.Transaction, coinbase common.Address, receiptProcessors ...core.ReceiptProcessor) ([]*types.Log, error) { snap := w.current.state.Snapshot() - receipt, err := core.ApplyTransaction(w.chainConfig, w.chain, &coinbase, w.current.gasPool, w.current.state, w.current.header, tx, &w.current.header.GasUsed, *w.chain.GetVMConfig()) + receipt, err := core.ApplyTransaction(w.chainConfig, w.chain, &coinbase, w.current.gasPool, w.current.state, w.current.header, tx, &w.current.header.GasUsed, *w.chain.GetVMConfig(), receiptProcessors...) if err != nil { w.current.state.RevertToSnapshot(snap) return nil, err @@ -769,6 +769,14 @@ func (w *worker) commitTransactions(txs *types.TransactionsByPriceAndNonce, coin log.Debug("Time left for mining work", "left", (*delay - w.config.DelayLeftOver).String(), "leftover", w.config.DelayLeftOver) defer stopTimer.Stop() } + + // initilise bloom processors + processorCapacity := 100 + if txs.CurrentSize() < processorCapacity { + processorCapacity = txs.CurrentSize() + } + bloomProcessors := core.NewAsyncReceiptBloomGenerator(processorCapacity) + LOOP: for { // In the following three cases, we will interrupt the execution of the transaction. @@ -824,7 +832,7 @@ LOOP: // Start executing the transaction w.current.state.Prepare(tx.Hash(), common.Hash{}, w.current.tcount) - logs, err := w.commitTransaction(tx, coinbase) + logs, err := w.commitTransaction(tx, coinbase, bloomProcessors) switch { case errors.Is(err, core.ErrGasLimitReached): // Pop the current out-of-gas transaction without shifting in the next from the account @@ -859,6 +867,7 @@ LOOP: txs.Shift() } } + bloomProcessors.Close() if !w.isRunning() && len(coalescedLogs) > 0 { // We don't push the pendingLogsEvent while we are mining. The reason is that From 8f605f5904ae0aae9da0106b61dca73a650e9292 Mon Sep 17 00:00:00 2001 From: zjubfd <296179868@qq.com> Date: Fri, 15 Oct 2021 17:04:30 +0800 Subject: [PATCH 15/18] fix cache key (#454) --- core/state/database.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/state/database.go b/core/state/database.go index 24a1474d57..1c18c282e7 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -267,7 +267,7 @@ func (db *cachingDB) CopyTrie(t Trie) Trie { // ContractCode retrieves a particular contract's code. func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) { - if cached, ok := db.codeCache.Get(codeHash.Bytes()); ok { + if cached, ok := db.codeCache.Get(codeHash); ok { code := cached.([]byte) if len(code) > 0 { return code, nil @@ -276,7 +276,7 @@ func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error code := rawdb.ReadCode(db.db.DiskDB(), codeHash) if len(code) > 0 { - db.codeCache.Add(codeHash.Bytes(), code) + db.codeCache.Add(codeHash, code) db.codeSizeCache.Add(codeHash, len(code)) return code, nil } @@ -289,7 +289,7 @@ func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error func (db *cachingDB) ContractCodeWithPrefix(addrHash, codeHash common.Hash) ([]byte, error) { if cached, ok := db.codeCache.Get(codeHash.Bytes()); ok { code := cached.([]byte) - if len(code) > 0 { + if len(code) > 0 { return code, nil } } From aecb61296d928474b94511d614f8b5a47ae907de Mon Sep 17 00:00:00 2001 From: kyrie-yl <83150977+kyrie-yl@users.noreply.github.com> Date: Fri, 15 Oct 2021 18:10:58 +0800 Subject: [PATCH 16/18] fix cache key do not have hash func (#455) Signed-off-by: kyrie-yl --- core/state/database.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/state/database.go b/core/state/database.go index 1c18c282e7..b65dfca158 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -287,7 +287,7 @@ func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error // code can't be found in the cache, then check the existence with **new** // db scheme. func (db *cachingDB) ContractCodeWithPrefix(addrHash, codeHash common.Hash) ([]byte, error) { - if cached, ok := db.codeCache.Get(codeHash.Bytes()); ok { + if cached, ok := db.codeCache.Get(codeHash); ok { code := cached.([]byte) if len(code) > 0 { return code, nil @@ -295,7 +295,7 @@ func (db *cachingDB) ContractCodeWithPrefix(addrHash, codeHash common.Hash) ([]b } code := rawdb.ReadCodeWithPrefix(db.db.DiskDB(), codeHash) if len(code) > 0 { - db.codeCache.Add(codeHash.Bytes(), code) + db.codeCache.Add(codeHash, code) db.codeSizeCache.Add(codeHash, len(code)) return code, nil } From f2f68dba15e72728abd71b535d1a86173686ab06 Mon Sep 17 00:00:00 2001 From: dylanhuang Date: Mon, 18 Oct 2021 18:16:32 +0800 Subject: [PATCH 17/18] ci: unit test and truffle test (#456) * ci: add unit test fix: failed on test * ci: add truffle test * ci: update os version * ci: add cache for go build --- .github/workflows/build-test.yml | 50 ++++++ .github/workflows/integration-test.yml | 25 +++ .github/workflows/lint.yml | 48 ++++++ .github/workflows/release.yml | 26 +++- .github/workflows/unit-test.yml | 55 +++++++ .gitignore | 1 + Makefile | 14 +- build/ci.go | 4 + cmd/puppeth/testdata/stureby_aleth.json | 2 +- cmd/puppeth/testdata/stureby_parity.json | 2 +- core/blockchain.go | 12 +- core/blockchain_diff_test.go | 8 - core/receipt_processor.go | 2 +- core/state_processor.go | 9 +- docker/Dockerfile | 24 +++ docker/Dockerfile.truffle | 16 ++ docker/init_holders.template | 7 + docker/truffle-config.js | 70 +++++++++ eth/api_backend.go | 2 +- ethclient/ethclient_test.go | 21 --- go.mod | 2 +- go.sum | 147 +----------------- tests/state_test.go | 4 +- tests/truffle/.env | 5 + tests/truffle/config/config-bsc-rpc.toml | 67 ++++++++ tests/truffle/config/config-validator.toml | 49 ++++++ tests/truffle/docker-compose.yml | 62 ++++++++ ...0x59b02d4d2f94ea5c55230715a58ebb0b703bcd4b | 1 + ...0x7fd60c817837dcfefca6d0a52a44980d12f70c59 | 1 + ...0x8e1ad6fac6ea5871140594abef5b1d503385e936 | 1 + ...0xa2bc4cf857f3d7a22b29c71774b4d8f25cc7edd0 | 1 + ...0xb75573a04648535bddc52adf6fbc887149624253 | 1 + ...0xbb46abbcc95213754f549e0cfa2b13bef0abfab6 | 1 + ...0xc32ec0115bcb6693d4b4854531ca5e6a99217abf | 1 + ...0xc8d063a7e0a118432721dae5e059404b5598bd76 | 1 + tests/truffle/scripts/bootstrap.sh | 60 +++++++ tests/truffle/scripts/bsc-rpc.sh | 16 ++ tests/truffle/scripts/bsc-validator.sh | 18 +++ tests/truffle/scripts/truffle-test.sh | 5 + tests/truffle/scripts/utils.sh | 14 ++ 40 files changed, 660 insertions(+), 195 deletions(-) create mode 100644 .github/workflows/build-test.yml create mode 100644 .github/workflows/integration-test.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/unit-test.yml create mode 100644 docker/Dockerfile create mode 100644 docker/Dockerfile.truffle create mode 100644 docker/init_holders.template create mode 100644 docker/truffle-config.js create mode 100644 tests/truffle/.env create mode 100644 tests/truffle/config/config-bsc-rpc.toml create mode 100644 tests/truffle/config/config-validator.toml create mode 100644 tests/truffle/docker-compose.yml create mode 100644 tests/truffle/init-holders/0x59b02d4d2f94ea5c55230715a58ebb0b703bcd4b create mode 100644 tests/truffle/init-holders/0x7fd60c817837dcfefca6d0a52a44980d12f70c59 create mode 100644 tests/truffle/init-holders/0x8e1ad6fac6ea5871140594abef5b1d503385e936 create mode 100644 tests/truffle/init-holders/0xa2bc4cf857f3d7a22b29c71774b4d8f25cc7edd0 create mode 100644 tests/truffle/init-holders/0xb75573a04648535bddc52adf6fbc887149624253 create mode 100644 tests/truffle/init-holders/0xbb46abbcc95213754f549e0cfa2b13bef0abfab6 create mode 100644 tests/truffle/init-holders/0xc32ec0115bcb6693d4b4854531ca5e6a99217abf create mode 100644 tests/truffle/init-holders/0xc8d063a7e0a118432721dae5e059404b5598bd76 create mode 100755 tests/truffle/scripts/bootstrap.sh create mode 100755 tests/truffle/scripts/bsc-rpc.sh create mode 100755 tests/truffle/scripts/bsc-validator.sh create mode 100755 tests/truffle/scripts/truffle-test.sh create mode 100644 tests/truffle/scripts/utils.sh diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml new file mode 100644 index 0000000000..b0fb9833b3 --- /dev/null +++ b/.github/workflows/build-test.yml @@ -0,0 +1,50 @@ +name: Build Test + +on: + push: + branches: + - master + - develop + + pull_request: + branches: + - master + - develop + +jobs: + unit-test: + strategy: + matrix: + go-version: [1.16.x] + os: [ubuntu-18.04] + runs-on: ${{ matrix.os }} + steps: + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + + - name: Checkout code + uses: actions/checkout@v2 + + - uses: actions/cache@v2 + with: + # In order: + # * Module download cache + # * Build cache (Linux) + # * Build cache (Mac) + # * Build cache (Windows) + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Test Build + run: | + make geth + + diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml new file mode 100644 index 0000000000..a4774d856f --- /dev/null +++ b/.github/workflows/integration-test.yml @@ -0,0 +1,25 @@ +name: Integration Test + +on: + push: + branches: + - master + - develop + + pull_request: + branches: + - master + - develop + +jobs: + truffle-test: + strategy: + matrix: + os: [ubuntu-18.04] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Truffle test + run: make truffle-test diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000..288749ae41 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,48 @@ +name: Lint + +on: + push: + branches: + - master + - develop + + pull_request: + branches: + - master + - develop + +jobs: + unit-test: + strategy: + matrix: + go-version: [1.16.x] + os: [ubuntu-18.04] + runs-on: ${{ matrix.os }} + steps: + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + + - name: Checkout code + uses: actions/checkout@v2 + + - uses: actions/cache@v2 + with: + # In order: + # * Module download cache + # * Build cache (Linux) + # * Build cache (Mac) + # * Build cache (Windows) + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Lint + run: | + make lint diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 66e73dbc5b..1633752c23 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: go-version: [1.16.x] - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-18.04, macos-11, windows-2019] runs-on: ${{ matrix.os }} steps: - name: Install Go @@ -23,6 +23,22 @@ jobs: - name: Checkout Code uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + # In order: + # * Module download cache + # * Build cache (Linux) + # * Build cache (Mac) + # * Build cache (Windows) + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + # ============================== # Linux/Macos/Windows Build # ============================== @@ -36,21 +52,21 @@ jobs: - name: Upload Linux Build uses: actions/upload-artifact@v2 - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'ubuntu-18.04' with: name: linux path: ./build/bin/geth - name: Upload MacOS Build uses: actions/upload-artifact@v2 - if: matrix.os == 'macos-latest' + if: matrix.os == 'macos-11' with: name: macos path: ./build/bin/geth - name: Upload Windows Build uses: actions/upload-artifact@v2 - if: matrix.os == 'windows-latest' + if: matrix.os == 'windows-2019' with: name: windows path: ./build/bin/geth.exe @@ -58,7 +74,7 @@ jobs: release: name: Release needs: build - runs-on: ubuntu-latest + runs-on: ubuntu-18.04 steps: - name: Set Env run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml new file mode 100644 index 0000000000..737e4928a7 --- /dev/null +++ b/.github/workflows/unit-test.yml @@ -0,0 +1,55 @@ +name: Unit Test + +on: + push: + branches: + - master + - develop + + pull_request: + branches: + - master + - develop + +jobs: + unit-test: + strategy: + matrix: + go-version: [1.16.x] + os: [ubuntu-18.04] + runs-on: ${{ matrix.os }} + steps: + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + + - name: Checkout code + uses: actions/checkout@v2 + + - uses: actions/cache@v2 + with: + # In order: + # * Module download cache + # * Build cache (Linux) + # * Build cache (Mac) + # * Build cache (Windows) + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Test Build + run: | + make geth + + - name: Uint Test + env: + ANDROID_HOME: "" # Skip android test + run: | + make test + diff --git a/.gitignore b/.gitignore index 1ee8b83022..3d05da8816 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ */**/*tx_database* */**/*dapps* build/_vendor/pkg +/tests/truffle/storage #* .#* diff --git a/Makefile b/Makefile index cb5a87dad0..03f92e6f43 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ # with Go source code. If you know what GOPATH is then you probably # don't need to bother with make. -.PHONY: geth android ios geth-cross evm all test clean +.PHONY: geth android ios geth-cross evm all test truffle-test clean .PHONY: geth-linux geth-linux-386 geth-linux-amd64 geth-linux-mips64 geth-linux-mips64le .PHONY: geth-linux-arm geth-linux-arm-5 geth-linux-arm-6 geth-linux-arm-7 geth-linux-arm64 .PHONY: geth-darwin geth-darwin-386 geth-darwin-amd64 @@ -33,7 +33,17 @@ ios: @echo "Import \"$(GOBIN)/Geth.framework\" to use the library." test: all - $(GORUN) build/ci.go test + $(GORUN) build/ci.go test -timeout 1h + +truffle-test: + docker build . -f ./docker/Dockerfile --target bsc-genesis -t bsc-genesis + docker build . -f ./docker/Dockerfile --target bsc -t bsc + docker build . -f ./docker/Dockerfile.truffle -t truffle-test + docker-compose -f ./tests/truffle/docker-compose.yml up genesis + docker-compose -f ./tests/truffle/docker-compose.yml up -d bsc-rpc bsc-validator1 + sleep 30 + docker-compose -f ./tests/truffle/docker-compose.yml up --exit-code-from truffle-test truffle-test + docker-compose -f ./tests/truffle/docker-compose.yml down lint: ## Run linters. $(GORUN) build/ci.go lint diff --git a/build/ci.go b/build/ci.go index 18172c327a..df1cb50ab2 100644 --- a/build/ci.go +++ b/build/ci.go @@ -279,6 +279,7 @@ func doTest(cmdline []string) { cc = flag.String("cc", "", "Sets C compiler binary") coverage = flag.Bool("coverage", false, "Whether to record code coverage") verbose = flag.Bool("v", false, "Whether to log verbosely") + timeout = flag.String("timeout", "10m", `Timeout of runing tests`) ) flag.CommandLine.Parse(cmdline) @@ -299,6 +300,9 @@ func doTest(cmdline []string) { if *verbose { gotest.Args = append(gotest.Args, "-v") } + if *timeout != "" { + gotest.Args = append(gotest.Args, []string{"-timeout", *timeout}...) + } packages := []string{"./accounts/...", "./common/...", "./consensus/...", "./console/...", "./core/...", "./crypto/...", "./eth/...", "./ethclient/...", "./ethdb/...", "./event/...", "./graphql/...", "./les/...", diff --git a/cmd/puppeth/testdata/stureby_aleth.json b/cmd/puppeth/testdata/stureby_aleth.json index d18ba3854a..9f928f4889 100644 --- a/cmd/puppeth/testdata/stureby_aleth.json +++ b/cmd/puppeth/testdata/stureby_aleth.json @@ -14,7 +14,7 @@ "minGasLimit": "0x1388", "maxGasLimit": "0x7fffffffffffffff", "tieBreakingGas": false, - "gasLimitBoundDivisor": "0x400", + "gasLimitBoundDivisor": "0x100", "minimumDifficulty": "0x20000", "difficultyBoundDivisor": "0x800", "durationLimit": "0xd", diff --git a/cmd/puppeth/testdata/stureby_parity.json b/cmd/puppeth/testdata/stureby_parity.json index e9229f99b7..a9f27b8e41 100644 --- a/cmd/puppeth/testdata/stureby_parity.json +++ b/cmd/puppeth/testdata/stureby_parity.json @@ -25,7 +25,7 @@ "accountStartNonce": "0x0", "maximumExtraDataSize": "0x20", "minGasLimit": "0x1388", - "gasLimitBoundDivisor": "0x400", + "gasLimitBoundDivisor": "0x100", "networkID": "0x4cb2e", "chainID": "0x4cb2e", "maxCodeSize": "0x6000", diff --git a/core/blockchain.go b/core/blockchain.go index 2668ba601d..91371940cd 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -2161,8 +2161,16 @@ func (bc *BlockChain) updateHighestVerifiedHeader(header *types.Header) { return } - newTD := big.NewInt(0).Add(bc.GetTdByHash(header.ParentHash), header.Difficulty) - oldTD := big.NewInt(0).Add(bc.GetTdByHash(currentHeader.ParentHash), currentHeader.Difficulty) + newParentTD := bc.GetTdByHash(header.ParentHash) + if newParentTD == nil { + newParentTD = big.NewInt(0) + } + oldParentTD := bc.GetTdByHash(currentHeader.ParentHash) + if oldParentTD == nil { + oldParentTD = big.NewInt(0) + } + newTD := big.NewInt(0).Add(newParentTD, header.Difficulty) + oldTD := big.NewInt(0).Add(oldParentTD, currentHeader.Difficulty) if newTD.Cmp(oldTD) > 0 { bc.highestVerifiedHeader.Store(types.CopyHeader(header)) diff --git a/core/blockchain_diff_test.go b/core/blockchain_diff_test.go index bec463136e..0b289bdc1c 100644 --- a/core/blockchain_diff_test.go +++ b/core/blockchain_diff_test.go @@ -216,8 +216,6 @@ func rawDataToDiffLayer(data rlp.RawValue) (*types.DiffLayer, error) { } func TestProcessDiffLayer(t *testing.T) { - t.Parallel() - blockNum := 128 fullBackend := newTestBackend(blockNum, false) falseDiff := 5 @@ -279,8 +277,6 @@ func TestProcessDiffLayer(t *testing.T) { } func TestFreezeDiffLayer(t *testing.T) { - t.Parallel() - blockNum := 1024 fullBackend := newTestBackend(blockNum, true) defer fullBackend.close() @@ -302,8 +298,6 @@ func TestFreezeDiffLayer(t *testing.T) { } func TestPruneDiffLayer(t *testing.T) { - t.Parallel() - blockNum := 1024 fullBackend := newTestBackend(blockNum, true) defer fullBackend.close() @@ -363,8 +357,6 @@ func TestPruneDiffLayer(t *testing.T) { } func TestGetDiffAccounts(t *testing.T) { - t.Parallel() - blockNum := 128 fullBackend := newTestBackend(blockNum, false) defer fullBackend.close() diff --git a/core/receipt_processor.go b/core/receipt_processor.go index 57fb56a6dc..3b4c2579cc 100644 --- a/core/receipt_processor.go +++ b/core/receipt_processor.go @@ -63,4 +63,4 @@ func (p *AsyncReceiptBloomGenerator) Close() { close(p.receipts) p.isClosed = true p.wg.Wait() -} \ No newline at end of file +} diff --git a/core/state_processor.go b/core/state_processor.go index 14bd5d75b2..b529082063 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -42,11 +42,10 @@ import ( ) const ( - fullProcessCheck = 21 // On diff sync mode, will do full process every fullProcessCheck randomly - minNumberOfAccountPerTask = 5 - recentTime = 2048 * 3 - recentDiffLayerTimeout = 20 - farDiffLayerTimeout = 2 + fullProcessCheck = 21 // On diff sync mode, will do full process every fullProcessCheck randomly + recentTime = 2048 * 3 + recentDiffLayerTimeout = 20 + farDiffLayerTimeout = 2 ) // StateProcessor is a basic Processor, which takes care of transitioning diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000000..a00454c1ab --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,24 @@ +FROM golang:1.16-alpine as bsc + +RUN apk add --no-cache make gcc musl-dev linux-headers git bash + +ADD . /bsc +WORKDIR /bsc +RUN make geth +RUN mv /bsc/build/bin/geth /usr/local/bin/geth + +EXPOSE 8545 8547 30303 30303/udp +ENTRYPOINT [ "/usr/local/bin/geth" ] + +FROM ethereum/solc:0.6.4-alpine as bsc-genesis + +RUN apk add --d --no-cache ca-certificates npm nodejs bash alpine-sdk + +RUN git clone https://github.com/binance-chain/bsc-genesis-contract.git /root/genesis \ + && rm /root/genesis/package-lock.json && cd /root/genesis && npm install + +COPY docker/init_holders.template /root/genesis/init_holders.template + +COPY --from=bsc /usr/local/bin/geth /usr/local/bin/geth + +ENTRYPOINT [ "/bin/bash" ] diff --git a/docker/Dockerfile.truffle b/docker/Dockerfile.truffle new file mode 100644 index 0000000000..0ec7029377 --- /dev/null +++ b/docker/Dockerfile.truffle @@ -0,0 +1,16 @@ +FROM ethereum/solc:0.6.4-alpine + +RUN apk add --d --no-cache ca-certificates npm nodejs bash alpine-sdk git + +RUN git clone https://github.com/binance-chain/canonical-upgradeable-bep20.git /usr/app/canonical-upgradeable-bep20 + +WORKDIR /usr/app/canonical-upgradeable-bep20 +COPY docker/truffle-config.js /usr/app/canonical-upgradeable-bep20 + +RUN npm install -g n +RUN n 12.18.3 && node -v + +RUN npm install -g truffle@v5.1.14 +RUN npm install + +ENTRYPOINT [ "/bin/bash" ] diff --git a/docker/init_holders.template b/docker/init_holders.template new file mode 100644 index 0000000000..4012909164 --- /dev/null +++ b/docker/init_holders.template @@ -0,0 +1,7 @@ +const web3 = require("web3") + +const addresses = "{{INIT_HOLDER_ADDRESSES}}" +const balance = web3.utils.toBN("{{INIT_HOLDER_BALANCE}}").toString("hex") +const init_holders = addresses.split(",").map(address => ({ address, balance })); + +exports = module.exports = init_holders diff --git a/docker/truffle-config.js b/docker/truffle-config.js new file mode 100644 index 0000000000..91539a9ab6 --- /dev/null +++ b/docker/truffle-config.js @@ -0,0 +1,70 @@ +/** + * Use this file to configure your truffle project. It's seeded with some + * common settings for different networks and features like migrations, + * compilation and testing. Uncomment the ones you need or modify + * them to suit your project as necessary. + * + * More information about configuration can be found at: + * + * truffleframework.com/docs/advanced/configuration + * + * To deploy via Infura you'll need a wallet provider (like truffle-hdwallet-provider) + * to sign your transactions before they're sent to a remote public node. Infura accounts + * are available for free at: infura.io/register. + * + * You'll also need a mnemonic - the twelve word phrase the wallet uses to generate + * public/private key pairs. If you're publishing your code to GitHub make sure you load this + * phrase from a file you've .gitignored so it doesn't accidentally become public. + * + */ + +// const HDWalletProvider = require('truffle-hdwallet-provider'); +// const infuraKey = "fj4jll3k....."; +// +// const fs = require('fs'); +// const mnemonic = fs.readFileSync(".secret").toString().trim(); + +module.exports = { + /** + * Networks define how you connect to your ethereum client and let you set the + * defaults web3 uses to send transactions. If you don't specify one truffle + * will spin up a development blockchain for you on port 9545 when you + * run `develop` or `test`. You can ask a truffle command to use a specific + * network from the command line, e.g + * + * $ truffle test --network + */ + + networks: { + // Useful for testing. The `development` name is special - truffle uses it by default + // if it's defined here and no other network is specified at the command line. + // You should run a client (like ganache-cli, geth or parity) in a separate terminal + // tab if you use this network and you must also set the `host`, `port` and `network_id` + // options below to some value. + // + development: { + host: process.env.RPC_HOST || '127.0.0.1', // Localhost (default: none) + port: process.env.RPC_PORT || 8545, // Standard Ethereum port (default: none) + network_id: process.env.BSC_CHAIN_ID, // Any network (default: none) + }, + }, + + // Set default mocha options here, use special reporters etc. + mocha: { + // timeout: 100000 + }, + + // Configure your compilers + compilers: { + solc: { + version: "0.6.4", // Fetch exact version from solc-bin (default: truffle's version) + docker: false, // Use "0.5.1" you've installed locally with docker (default: false) + settings: { // See the solidity docs for advice about optimization and evmVersion + optimizer: { + enabled: true, + runs: 200 + } + } + } + } + } diff --git a/eth/api_backend.go b/eth/api_backend.go index 5c864a236b..18514b7cc3 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -279,7 +279,7 @@ func (b *EthAPIBackend) SuggestPrice(ctx context.Context) (*big.Int, error) { return b.gpo.SuggestPrice(ctx) } -func (b *EthAPIBackend) Chain() *core.BlockChain { +func (b *EthAPIBackend) Chain() *core.BlockChain { return b.eth.BlockChain() } diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index d2fd056041..380481a22d 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -599,27 +599,6 @@ func testCallContract(t *testing.T, client *rpc.Client) { } } -func sendTransaction(ec *Client) error { - // Retrieve chainID - chainID, err := ec.ChainID(context.Background()) - if err != nil { - return err - } - // Create transaction - tx := types.NewTransaction(0, common.Address{1}, big.NewInt(1), 23000, big.NewInt(100000), nil) - signer := types.LatestSignerForChainID(chainID) - signature, err := crypto.Sign(signer.Hash(tx).Bytes(), testKey) - if err != nil { - return err - } - signedTx, err := tx.WithSignature(signer, signature) - if err != nil { - return err - } - // Send transaction - return ec.SendTransaction(context.Background(), signedTx) -} - func testDiffAccounts(t *testing.T, client *rpc.Client) { ec := NewClient(client) ctx, cancel := context.WithTimeout(context.Background(), 1000*time.Millisecond) diff --git a/go.mod b/go.mod index fc5ec88fec..4ea9fc689a 100644 --- a/go.mod +++ b/go.mod @@ -55,8 +55,8 @@ require ( github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 github.com/olekukonko/tablewriter v0.0.5 - github.com/panjf2000/ants/v2 v2.4.5 github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c + github.com/panjf2000/ants/v2 v2.4.5 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 github.com/prometheus/tsdb v0.7.1 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect diff --git a/go.sum b/go.sum index 5fbb839b90..e94241a414 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -8,24 +7,16 @@ cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxK cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.51.0 h1:PvKAVQWCtlGUSlZkGW3QLelKaWq7KYv/MW1EboG8bfM= cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0 h1:sAbMqjY1PEQKZBWfbu6Y6bsupJ9c4QdHnzg/VvYTLcE= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigtable v1.2.0 h1:F4cCmA4nuV84V5zYQ3MKY+M1Cw1avHDuf3S/LcZPA9c= cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= -cloud.google.com/go/datastore v1.0.0 h1:Kt+gOPPp2LEPWp8CSfxhsM8ik9CcyE/gYu+0r+RnZvM= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0 h1:9/vpR43S4aJaROxqQHQ3nH9lfyKKV0dC3vOmnw8ebQQ= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0 h1:RPUcBvDeYgQFMfQu1eBMq6piD1SXmLH+vK3qjewZPus= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -collectd.org v0.3.0 h1:iNBHGw1VvPJxH2B6RiFWFZ+vsjo1lCdRszBeOuwGi00= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY= @@ -48,13 +39,9 @@ github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1Gn github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= @@ -62,19 +49,13 @@ github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQu github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/aead/siphash v1.0.1 h1:FwHfE/T45KPKYuuSAKyyvE+oPWcaQ+CUmFW0bPlM+kg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af h1:wVe6/Ea46ZMeNkQjjBW6xcqyQA/j5e0D6GytH95g0gQ= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db h1:nxAtV4VajJDhKysp2kdcJZsq8Ss1xSA0vZTkVHHJd0E= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= github.com/aws/aws-sdk-go-v2 v1.4.0 h1:Ryh4fNebT9SwLyCKPSk83dyEZj+KB6KzDyb1gXii7EI= github.com/aws/aws-sdk-go-v2 v1.4.0/go.mod h1:tI4KhsR5VkzlUa2DZAdwx7wCAYGwkZZ1H31PYrBFx1w= @@ -97,29 +78,19 @@ github.com/aws/smithy-go v1.4.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAm github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 h1:y4B3+GPxKlrigF1ha5FFErxK+sr6sWxQovRMzwMhejo= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= -github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= -github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= -github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd h1:qdGvebPBDuYDPGi1WCPjy1tGyMpmDK8IEapSsszn7HE= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= -github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723 h1:ZA/jbKoGcVAnER6pCHPEkGdZOV7U1oLUedErBHCUMs0= github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= -github.com/btcsuite/winsvc v1.0.0 h1:J9B4L7e3oqhXOcm+2IuNApwzQec85lE+QaikUcCs+dk= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/c-bata/go-prompt v0.2.2 h1:uyKRz6Z6DUyj49QVijyM339UJV9yhbr70gESwbNU3e0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= -github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= @@ -127,23 +98,16 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.14.0 h1:gFqGlGl/5f9UGXAaKapCGUfaTCgRKKnzu2VvzMZlOFA= github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= -github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572 h1:+R8G1+Ftumd0DaveLgMIjrFPcAS4G8MsVXWXiyZL5BY= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f h1:C43yEtQ6NIf4ftFXD/V55gnGFgPbMQobd//YlnLjUJ8= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/dave/jennifer v1.2.0 h1:S15ZkFMRoJ36mGAQgWL1tnr0NQJh9rZ8qatseX/VbBc= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -153,9 +117,7 @@ github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8 h1:akOQj8IVgoeFfBTzGOEQakCYshWD6RNo1M5pivFXt70= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954 h1:RMLoZVzv4GliuWafOuPuQDKSm1SJph7uCRnnS61JAn4= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dlclark/regexp2 v1.2.0 h1:8sAhBGEM0dRWogWqWyQeIJnxjWO6oIjl8FKqREDsGfk= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= @@ -163,13 +125,10 @@ github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf h1:sh8rkQZavChcmak github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 h1:Y9vTBSsV4hSwPSj4bacAU/eSnV3dAxVpepaghAdhGoQ= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= -github.com/eclipse/paho.mqtt.golang v1.2.0 h1:1F8mhG9+aO5/xpdtFkW4SxOJB67ukuDC3t2y2qayIX0= github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473 h1:4cmBvAEBNJaGARUEs3/suWRyfyBfhf7I60WBZq+bv2w= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3 h1:gSJmxrs37LgTqR/oyJBWok6k6SvXEUerFTbltIhXkBM= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= @@ -177,7 +136,6 @@ github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= -github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90 h1:WXb3TSNmHp2vHoCroCIB1foO/yQ36swABL8aOVeDpgg= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= @@ -186,13 +144,9 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= -github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd h1:r04MMPyLHj/QwZuMJ5+7tJcBr1AQjpiAK/rZWRrQT7o= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72 h1:b+9H1GAsx5RsjvDFLoS5zkNBzIQMuVKUYQDmxU3N5XE= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -203,27 +157,21 @@ github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-sourcemap/sourcemap v2.1.2+incompatible h1:0b/xya7BKGhXuqFESKM4oIiRo9WOt2ebz7KxfreD6ug= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= -github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= -github.com/golang/geo v0.0.0-20190916061304-5b978397cfec h1:lJwO/92dFXWeXOZdoGXgptLmNLwynMSHUmU6besqtiw= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -241,33 +189,25 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 h1:ur2rms48b3Ep1dxh7aUV2FZEQ8jEVO2F6ILKx8ofkAg= github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/flatbuffers v1.11.0 h1:O7CEyB8Cb3/DmtxODGtLHcEvpr81Jm5qLg/hsHnxA2A= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa h1:Q75Upo5UN4JbPFURXZ8nLKYUvF85dyFRop/vQ0Rv+64= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc h1:DLpL8pWq0v4JYoRpEhDfsJhhJyGKCcQM2WPW2TJs31c= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.5 h1:kxhtnfFVi+rYdOALN0B3k9UT86zVJKfBimRaciULW4I= github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -281,85 +221,55 @@ github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZ github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.1.1 h1:4JywC80b+/hSfljFlEBLHrrh+CIONLDz9NuFl0af4Mw= github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.1-0.20210310174557-0ca763054c88 h1:bcAj8KroPf552TScjFPIakjH2/tdIrIH8F+cc4v4SRo= github.com/huin/goupnp v1.0.1-0.20210310174557-0ca763054c88/go.mod h1:nNs7wvRfN1eKaMknBydLNQU6146XQim8t4h+q90biWo= -github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150 h1:vlNjIqmUZ9CMAWsbURYl3a6wZbw7q5RHVvlXTNS/Bs8= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68UUwekSh5iP2OMhRRZJiiBccgV7axzUG8vi56c= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/flux v0.65.1 h1:77BcVUCzvN5HMm8+j9PRBQ4iZcu98Dl4Y9rf+J5vhnc= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/influxdb v1.8.3 h1:WEypI1BQFTT4teLM+1qkEcvUi0dAvopAI/ir0vAiBg8= github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= -github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385 h1:ED4e5Cc3z5vSN2Tz2GkOHN7vs4Sxe2yds6CXvDnvZFE= github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= -github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e h1:/o3vQtpWJhvnIbXley4/jwzzqNeigJK9z+LZcJZ9zfM= github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE= -github.com/influxdata/promql/v2 v2.12.0 h1:kXn3p0D7zPw16rOtfDR+wo6aaiH8tSMfhPwONTxrlEc= github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8= -github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6 h1:UzJnB7VRL4PSkUJHwsyzseGOmrO/r4yA+AuxGJxiZmA= github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= -github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9 h1:MHTrDWmQpHq/hkq+7cw9oYAt2PqUw52TZazRA0N7PGE= github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= -github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368 h1:+TUUmaFa4YD1Q+7bH9o5NCHQGPMqZCYJiNW6lIIS9z4= github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e h1:UvSe12bq+Uj2hWd8aOlwPmoZ+CITRFrdit+sDGfAg8U= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= -github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89 h1:12K8AlpT0/6QUXSfV0yi4Q0jkbq8NDtIKFtF61AoqV0= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= -github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= -github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jsternberg/zap-logfmt v1.0.0 h1:0Dz2s/eturmdUS34GM82JwNEdQ9hPoJgqptcEKcbpzY= github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5 h1:PJr+ZMXIecYc1Ey2zucXdR73SMBtgjPgwa31099IMv0= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef h1:2jNeR4YUziVtswNP9sEFAI913cVrzH85T+8Q6LpYbT0= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 h1:I/yrLt2WilKxlQKCM52clh5rGzTKpVctGT1lH4Dc8Jw= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= -github.com/kisielk/errcheck v1.2.0 h1:reN85Pxc5larApoH1keMBiu2GWtPqXQ1nc9gx+jOU+E= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23 h1:FOOIBWrEkLgmlgGfMuZT83xIwfPDxEI2OHu6xUmJMFE= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.4.0 h1:8nsMz3tWa9SWWPL60G1V6CUsf4lLjWLTNEtibhe8gh8= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 h1:2U0HzY8BJ8hVwDKIzp7y4voR9CX/nvcfymLmg2UiOio= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 h1:KAZ1BW2TCmT6PRihDPpocIy1QTtsAsrx6TneU/4+CMg= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= -github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada h1:3L+neHp83cTjegPdCiOxVOJtRIy7/8RldvMTsyPYH10= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -367,7 +277,6 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= @@ -383,19 +292,13 @@ github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXT github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104 h1:d8RFOZ2IiFtFWBcKEHAFYJcPTf0wY5q0exFNJZVWa1U= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= @@ -403,7 +306,6 @@ github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcou github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= @@ -420,24 +322,20 @@ github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/panjf2000/ants/v2 v2.4.5 h1:kcGvjXB7ea0MrzzszpnlVFthhYKoFxLi75nRbsq01HY= -github.com/panjf2000/ants/v2 v2.4.5/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0= -github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs= +github.com/panjf2000/ants/v2 v2.4.5 h1:kcGvjXB7ea0MrzzszpnlVFthhYKoFxLi75nRbsq01HY= +github.com/panjf2000/ants/v2 v2.4.5/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= -github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= -github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5 h1:tFwafIEMf0B7NlcxV/zJ6leBIa81D3hgGSgsE5hCkOQ= github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -459,43 +357,29 @@ github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52 h1:RnWNS9Hlm8BIkjr6wx8li5abe0fr73jljLycdfemTp0= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= -github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= -github.com/segmentio/kafka-go v0.2.0 h1:HtCSf6B4gN/87yc5qTl7WsxPKQIIGXLPPM1bMCPOsoY= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= -github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -511,7 +395,6 @@ github.com/tendermint/iavl v0.12.0 h1:xcaFAr+ycqCj7WN1RzL2EfcBioRDOHcU1oWcg83K02 github.com/tendermint/iavl v0.12.0/go.mod h1:EoKMMv++tDOL5qKKVnoIqtVPshRrEPeJ0WsgDOLAauM= github.com/tendermint/tendermint v0.31.11 h1:TIs//4WfEAG4TOZc2eUfJPI3T8KrywXQCCPnGAaM1Wo= github.com/tendermint/tendermint v0.31.11/go.mod h1:ymcPyWblXCplCPQjbOYbrF1fWnpslATMVqiGgWbZrlc= -github.com/tinylib/msgp v1.0.2 h1:DfdQrzQa7Yh2es9SuLkixqxuXS2SxsdYn0KbdrOGWD8= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= @@ -519,25 +402,17 @@ github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefld github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= -github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/willf/bitset v1.1.3 h1:ekJIKh6+YbUIVt9DfNbkR5d6aFcFTLDRyJNAACURBg8= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6 h1:YdYsPAZ2pC6Tow/nPZOPQ96O3hm/ToAkGsPLzedXERk= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= -github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -558,11 +433,9 @@ golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxT golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299 h1:zQpM52jfKHG6II1ISZY1ZcpygvuSFZpLwfluuF89XOg= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -571,16 +444,13 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -608,7 +478,6 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -687,7 +556,6 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -696,12 +564,9 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1N golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.6.0 h1:DJy6UzXbahnGUf1ujUNkh/NEtK14qMo2nvlBPs4U5yw= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b h1:Qh4dB5D/WpoUUp3lSod7qgoyEHbDGPUWjIbnqdqqe1k= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -709,13 +574,11 @@ google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0 h1:yzlyyDW/J0w8yNFJIhiAJy4kq74S+1DOLdawELNxFMA= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -746,14 +609,11 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= @@ -780,9 +640,6 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.1.3 h1:qTakTkI6ni6LFD5sBwwsdSO+AQqbSIxOauHTTQKZ/7o= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= -rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/tests/state_test.go b/tests/state_test.go index b77a898c21..c1d9f6a0dc 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -19,8 +19,8 @@ package tests import ( "bufio" "bytes" + "errors" "fmt" - "reflect" "testing" "github.com/ethereum/go-ethereum/core/vm" @@ -108,7 +108,7 @@ func withTrace(t *testing.T, gasLimit uint64, test func(vm.Config) error) { tracer := vm.NewJSONLogger(&vm.LogConfig{DisableMemory: true}, w) config.Debug, config.Tracer = true, tracer err2 := test(config) - if !reflect.DeepEqual(err, err2) { + if !errors.Is(err, err2) { t.Errorf("different error for second run: %v", err2) } w.Flush() diff --git a/tests/truffle/.env b/tests/truffle/.env new file mode 100644 index 0000000000..d933e4cfef --- /dev/null +++ b/tests/truffle/.env @@ -0,0 +1,5 @@ +BSC_CHAIN_ID="99" +CLUSTER_CIDR=99.1.0.0/16 +BOOTSTRAP_PUB_KEY=177ae5db445a2f70db781b019aedd928f5b1528a7a43448840b022408f9a21509adcce0b37c87d59da68d47a16879cc1e95a62bbac9723f7b22f4365b2afabbe +BOOTSTRAP_TCP_PORT=30311 +VERBOSE=3 diff --git a/tests/truffle/config/config-bsc-rpc.toml b/tests/truffle/config/config-bsc-rpc.toml new file mode 100644 index 0000000000..66031308f8 --- /dev/null +++ b/tests/truffle/config/config-bsc-rpc.toml @@ -0,0 +1,67 @@ +[Eth] +NetworkId = {{NetworkId}} +LightPeers = 100 +TrieTimeout = 360000000000 +EnablePreimageRecording = false + +[Eth.Miner] +GasFloor = 30000000 +GasCeil = 40000000 +GasPrice = 5000000000 +Recommit = 10000000000 +Noverify = false + +[Eth.Ethash] +CacheDir = "" +CachesInMem = 0 +CachesOnDisk = 0 +CachesLockMmap = false +DatasetDir = "" +DatasetsInMem = 0 +DatasetsOnDisk = 0 +DatasetsLockMmap = false +PowMode = 0 + +[Eth.TxPool] +Locals = [] +NoLocals = true +Journal = "transactions.rlp" +Rejournal = 3600000000000 +PriceLimit = 5000000000 +PriceBump = 10 +AccountSlots = 128 +GlobalSlots = 10000 +AccountQueue = 64 +GlobalQueue = 5000 +Lifetime = 600000000000 + +[Eth.GPO] +Blocks = 20 +Percentile = 60 +OracleThreshold = 20 +Default = 5000000000 + +[Node] +DataDir = "node" +InsecureUnlockAllowed = true +NoUSB = true +IPCPath = "geth.ipc" +HTTPHost = "0.0.0.0" +HTTPPort = 8545 +HTTPVirtualHosts = ["*"] +HTTPModules = ["eth", "net", "web3", "txpool", "parlia"] +WSPort = 8547 +WSModules = ["net", "web3", "eth"] + +[Node.P2P] +MaxPeers = 30 +NoDiscovery = false +BootstrapNodes = [] +StaticNodes = [] +ListenAddr = ":30311" +EnableMsgEvents = false + +[Node.HTTPTimeouts] +ReadTimeout = 30000000000 +WriteTimeout = 30000000000 +IdleTimeout = 120000000000 diff --git a/tests/truffle/config/config-validator.toml b/tests/truffle/config/config-validator.toml new file mode 100644 index 0000000000..a96315a9db --- /dev/null +++ b/tests/truffle/config/config-validator.toml @@ -0,0 +1,49 @@ +[Eth] +NetworkId = {{NetworkId}} +SyncMode = "full" +NoPruning = false +NoPrefetch = false +LightPeers = 100 +DatabaseCache = 512 +DatabaseFreezer = "" +TrieCleanCache = 256 +TrieDirtyCache = 256 +TrieTimeout = 6000000000 +EnablePreimageRecording = false +EWASMInterpreter = "" +EVMInterpreter = "" + +[Eth.Ethash] +CacheDir = "" +CachesInMem = 0 +CachesOnDisk = 0 +CachesLockMmap = false +DatasetDir = "" +DatasetsInMem = 0 +DatasetsOnDisk = 0 +DatasetsLockMmap = false +PowMode = 0 + +[Eth.GPO] +Blocks = 20 +Percentile = 60 +OracleThreshold = 100 + +[Node] +DataDir = "" +InsecureUnlockAllowed = false +NoUSB = true +IPCPath = "geth.ipc" + +[Node.P2P] +MaxPeers = 50 +NoDiscovery = false +BootstrapNodes = [] +StaticNodes = [] +TrustedNodes = [] +ListenAddr = ":30311" +EnableMsgEvents = false + +[Node.HTTPTimeouts] +ReadTimeout = 30000000000 +WriteTimeout = 30000000000 diff --git a/tests/truffle/docker-compose.yml b/tests/truffle/docker-compose.yml new file mode 100644 index 0000000000..d5f007afee --- /dev/null +++ b/tests/truffle/docker-compose.yml @@ -0,0 +1,62 @@ +version: "3" + +services: + genesis: + image: bsc-genesis + env_file: .env + environment: + INIT_HOLDER_BALANCE: "500000000000000000000" + NUMS_OF_VALIDATOR: 1 + volumes: + - ./storage:/root/storage + - ./scripts:/root/scripts + - ./config:/root/config + - ./init-holders:/root/init-holders + command: /root/scripts/bootstrap.sh + + bsc-rpc: + image: bsc + healthcheck: + test: nc -z localhost 8545 + interval: 3s + timeout: 5s + retries: 15 + env_file: .env + environment: + NODE_ID: bsc-rpc + volumes: + - ./storage/bsc-rpc:/root/.ethereum + - ./scripts:/scripts + - ./config:/config + entrypoint: [ "sh", "-c", "/scripts/bsc-rpc.sh" ] + + bsc-validator1: + image: bsc + env_file: .env + environment: + NODE_ID: bsc-validator1 + BOOTSTRAP_HOST: bsc-rpc + volumes: + - ./storage/bsc-validator1:/root/.ethereum + - ./scripts:/scripts + entrypoint: [ "sh", "-c", "/scripts/bsc-validator.sh" ] + + truffle-test: + image: truffle-test + command: /scripts/truffle-test.sh + env_file: .env + environment: + RPC_HOST: bsc-rpc + RPC_PORT: 8545 + volumes: + - ./scripts:/scripts + depends_on: + bsc-rpc: + condition: service_healthy + +networks: + default: + ipam: + driver: default + config: + - subnet: 99.1.0.0/16 diff --git a/tests/truffle/init-holders/0x59b02d4d2f94ea5c55230715a58ebb0b703bcd4b b/tests/truffle/init-holders/0x59b02d4d2f94ea5c55230715a58ebb0b703bcd4b new file mode 100644 index 0000000000..f02e888f60 --- /dev/null +++ b/tests/truffle/init-holders/0x59b02d4d2f94ea5c55230715a58ebb0b703bcd4b @@ -0,0 +1 @@ +{"address":"59b02d4d2f94ea5c55230715a58ebb0b703bcd4b","crypto":{"cipher":"aes-128-ctr","ciphertext":"7d148d2d14ce811e6b6c407352b9dd9fa78012fb28cedfefd39c8c4821e6d495","cipherparams":{"iv":"4f896d1aaac704f23a5b0d982791af24"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"32ee8c617c9cf111efb1070aeaa6bdf1e23272e91826d7cf1c032a35e57647bf"},"mac":"9c46e2947e8a2184ab780dd8b99df83dc2ea7f555ba09edb39bac14e3cdd2ab1"},"id":"546d67c8-22aa-47f8-822c-d1338a56fde0","version":3} diff --git a/tests/truffle/init-holders/0x7fd60c817837dcfefca6d0a52a44980d12f70c59 b/tests/truffle/init-holders/0x7fd60c817837dcfefca6d0a52a44980d12f70c59 new file mode 100644 index 0000000000..7da28e4854 --- /dev/null +++ b/tests/truffle/init-holders/0x7fd60c817837dcfefca6d0a52a44980d12f70c59 @@ -0,0 +1 @@ +{"address":"7fd60c817837dcfefca6d0a52a44980d12f70c59","crypto":{"cipher":"aes-128-ctr","ciphertext":"de615dc06e491ffde4378effa527c307d5ad9b880e20e7246ed4c16768a99b66","cipherparams":{"iv":"daef85c2faea3d0bca64e5588bd10fb5"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"41a745c4fecdbe04139ca24cf203ff0c3c0c5890b918c8284fc32a285ec07f77"},"mac":"7ae357c9abcc3f0e41cf7f28edbc47089189699393f92dcd48420d9155063b2f"},"id":"dae91829-589f-46fd-bf83-2e05c99c2f76","version":3} diff --git a/tests/truffle/init-holders/0x8e1ad6fac6ea5871140594abef5b1d503385e936 b/tests/truffle/init-holders/0x8e1ad6fac6ea5871140594abef5b1d503385e936 new file mode 100644 index 0000000000..1e4048099c --- /dev/null +++ b/tests/truffle/init-holders/0x8e1ad6fac6ea5871140594abef5b1d503385e936 @@ -0,0 +1 @@ +{"address":"8e1ad6fac6ea5871140594abef5b1d503385e936","crypto":{"cipher":"aes-128-ctr","ciphertext":"2429bd13413d542e41033d12fa22fd7b848e6bd1fa73da42b146154b91f6bb42","cipherparams":{"iv":"ca266dd7bb8d42f440030c8c5057b720"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"3db50346ccd753740c3a997f406b2fff7ed382be16b5f2473e140971db098fe8"},"mac":"d08ca68beb30fa16f42553342d1c4c65c1ce198f490089588541c976892a105b"},"id":"44bf04b2-bb8b-4083-8a15-8ef0a7ebd2e4","version":3} diff --git a/tests/truffle/init-holders/0xa2bc4cf857f3d7a22b29c71774b4d8f25cc7edd0 b/tests/truffle/init-holders/0xa2bc4cf857f3d7a22b29c71774b4d8f25cc7edd0 new file mode 100644 index 0000000000..bd293be3e0 --- /dev/null +++ b/tests/truffle/init-holders/0xa2bc4cf857f3d7a22b29c71774b4d8f25cc7edd0 @@ -0,0 +1 @@ +{"address":"a2bc4cf857f3d7a22b29c71774b4d8f25cc7edd0","crypto":{"cipher":"aes-128-ctr","ciphertext":"2f294fe93b33feb559ff39180645259d05278be73940f763dfb94dec5a9eaaf5","cipherparams":{"iv":"6c8172356cea235580d451812bfb2939"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"ff0d64fcf4ee1b010c9e48addea6db74a27dd0b7f558a7ea39bd7879ccdc1d91"},"mac":"e500decff7ba7ad1acba4acabda507d147532dc59e15687c64e2e9cb286fbb1e"},"id":"54994cde-17cc-41e2-beba-8e5b67648229","version":3} diff --git a/tests/truffle/init-holders/0xb75573a04648535bddc52adf6fbc887149624253 b/tests/truffle/init-holders/0xb75573a04648535bddc52adf6fbc887149624253 new file mode 100644 index 0000000000..be31ac0c39 --- /dev/null +++ b/tests/truffle/init-holders/0xb75573a04648535bddc52adf6fbc887149624253 @@ -0,0 +1 @@ +{"address":"b75573a04648535bddc52adf6fbc887149624253","crypto":{"cipher":"aes-128-ctr","ciphertext":"968b6572ba689cc82aceacde5b075805a894f93f4d0a1a3eab2ec8426dd09864","cipherparams":{"iv":"f6dfc6e30df4de7637e5c22008ceab3b"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"e35e27d6040f86904eff8e1b2b93e96684dd542da51dc2f8f4700760d41861e6"},"mac":"a0dea8100a790961423d9138eddd2a95e8c291f0907673c121e9cee2ed988c1e"},"id":"ce7c3ea0-bb13-479a-be45-1b676ba4ea69","version":3} diff --git a/tests/truffle/init-holders/0xbb46abbcc95213754f549e0cfa2b13bef0abfab6 b/tests/truffle/init-holders/0xbb46abbcc95213754f549e0cfa2b13bef0abfab6 new file mode 100644 index 0000000000..a7fafc639c --- /dev/null +++ b/tests/truffle/init-holders/0xbb46abbcc95213754f549e0cfa2b13bef0abfab6 @@ -0,0 +1 @@ +{"address":"bb46abbcc95213754f549e0cfa2b13bef0abfab6","crypto":{"cipher":"aes-128-ctr","ciphertext":"b4e4bb220f060ddb88bb069bb779a86b2d4541df69dd074bbd5db20102c7bd9a","cipherparams":{"iv":"d8df230541cb6591436d13d8e921d748"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"1e250a04130e716bd14853bfb28539ceb2dd0bd199128e95e238878c5fe0f70d"},"mac":"e72ea48770406a43154a475657fb32b00de818f0f2c3722978aa1b3f4564f3b0"},"id":"a7eafd39-c349-437a-9734-bf41a88a9011","version":3} diff --git a/tests/truffle/init-holders/0xc32ec0115bcb6693d4b4854531ca5e6a99217abf b/tests/truffle/init-holders/0xc32ec0115bcb6693d4b4854531ca5e6a99217abf new file mode 100644 index 0000000000..78a7d24126 --- /dev/null +++ b/tests/truffle/init-holders/0xc32ec0115bcb6693d4b4854531ca5e6a99217abf @@ -0,0 +1 @@ +{"address":"c32ec0115bcb6693d4b4854531ca5e6a99217abf","crypto":{"cipher":"aes-128-ctr","ciphertext":"f274f8d49d2a7312be077da9fcc0d6be566952609a20720916ef8f0889b10ea9","cipherparams":{"iv":"b4e561354c859e32a05c71679c566e6d"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"a4dc8d1354590c5f15bf8fcdecfea60f3785b4496daab97caa48128c6d13a5c8"},"mac":"08600abf4f1d56a79bb410b500b60bf045f156eff858aab743aa1c0f5bae4fd3"},"id":"fab4ac2e-3714-498f-804a-d186e07e827c","version":3} diff --git a/tests/truffle/init-holders/0xc8d063a7e0a118432721dae5e059404b5598bd76 b/tests/truffle/init-holders/0xc8d063a7e0a118432721dae5e059404b5598bd76 new file mode 100644 index 0000000000..8968530b10 --- /dev/null +++ b/tests/truffle/init-holders/0xc8d063a7e0a118432721dae5e059404b5598bd76 @@ -0,0 +1 @@ +{"address":"c8d063a7e0a118432721dae5e059404b5598bd76","crypto":{"cipher":"aes-128-ctr","ciphertext":"9bf66fada0411b2a07fde861889f4fa5eb032e30bfbb74f4f3253d9ed9339330","cipherparams":{"iv":"b9e1045eff919d63f16e1182ee1b48ef"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"8efa2dbad843d7eeb3d26836e17b562cdb134c5ef7d875a5a34bd98d45bb28fb"},"mac":"b99064cfb5d043789a8684605ddd85efae6c87cab8a8a94924a2b49421d7f854"},"id":"eba9456c-e8eb-411f-ac75-33d0db977ffb","version":3} diff --git a/tests/truffle/scripts/bootstrap.sh b/tests/truffle/scripts/bootstrap.sh new file mode 100755 index 0000000000..0e5e00df99 --- /dev/null +++ b/tests/truffle/scripts/bootstrap.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +workspace=$(cd `dirname $0`; pwd)/.. + +function prepare() { + if ! [[ -f /usr/local/bin/geth ]];then + echo "geth do not exist!" + exit 1 + fi + rm -rf ${workspace}/storage/* + cd ${workspace}/genesis + rm -rf validators.conf +} + +function init_validator() { + node_id=$1 + mkdir -p ${workspace}/storage/${node_id} + geth --datadir ${workspace}/storage/${node_id} account new --password /dev/null > ${workspace}/storage/${node_id}Info + validatorAddr=`cat ${workspace}/storage/${node_id}Info|grep 'Public address of the key'|awk '{print $6}'` + echo "${validatorAddr},${validatorAddr},${validatorAddr},0x0000000010000000" >> ${workspace}/genesis/validators.conf + echo ${validatorAddr} > ${workspace}/storage/${node_id}/address +} + +function generate_genesis() { + INIT_HOLDER_ADDRESSES=$(ls ${workspace}/init-holders | tr '\n' ',') + INIT_HOLDER_ADDRESSES=${INIT_HOLDER_ADDRESSES/%,/} + sed "s/{{INIT_HOLDER_ADDRESSES}}/${INIT_HOLDER_ADDRESSES}/g" ${workspace}/genesis/init_holders.template | sed "s/{{INIT_HOLDER_BALANCE}}/${INIT_HOLDER_BALANCE}/g" > ${workspace}/genesis/init_holders.js + node generate-validator.js + chainIDHex=$(printf '%04x\n' ${BSC_CHAIN_ID}) + node generate-genesis.js --chainid ${BSC_CHAIN_ID} --bscChainId ${chainIDHex} +} + +function init_genesis_data() { + node_type=$1 + node_id=$2 + geth --datadir ${workspace}/storage/${node_id} init ${workspace}/genesis/genesis.json + cp ${workspace}/config/config-${node_type}.toml ${workspace}/storage/${node_id}/config.toml + sed -i -e "s/{{NetworkId}}/${BSC_CHAIN_ID}/g" ${workspace}/storage/${node_id}/config.toml + if [ "${node_id}" == "bsc-rpc" ]; then + cp ${workspace}/init-holders/* ${workspace}/storage/${node_id}/keystore + cp ${workspace}/genesis/genesis.json ${workspace}/storage/${node_id} + fi +} + +prepare + +# First, generate config for each validator +for((i=1;i<=${NUMS_OF_VALIDATOR};i++)); do + init_validator "bsc-validator${i}" +done + +# Then, use validator configs to generate genesis file +generate_genesis + +# Finally, use genesis file to init cluster data +init_genesis_data bsc-rpc bsc-rpc + +for((i=1;i<=${NUMS_OF_VALIDATOR};i++)); do + init_genesis_data validator "bsc-validator${i}" +done diff --git a/tests/truffle/scripts/bsc-rpc.sh b/tests/truffle/scripts/bsc-rpc.sh new file mode 100755 index 0000000000..33bcdc1fb2 --- /dev/null +++ b/tests/truffle/scripts/bsc-rpc.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +DATA_DIR=/root/.ethereum + +account_cnt=$(ls ${DATA_DIR}/keystore | wc -l) +i=1 +unlock_sequences="0" +while [ "$i" -lt ${account_cnt} ]; do + unlock_sequences="${unlock_sequences},${i}" + i=$(( i + 1 )) +done + +geth --config ${DATA_DIR}/config.toml --datadir ${DATA_DIR} --netrestrict ${CLUSTER_CIDR} \ + --verbosity ${VERBOSE} --nousb \ + --rpc --rpc.allow-unprotected-txs --txlookuplimit 15768000 \ + -unlock ${unlock_sequences} --password /dev/null diff --git a/tests/truffle/scripts/bsc-validator.sh b/tests/truffle/scripts/bsc-validator.sh new file mode 100755 index 0000000000..b5cb38465b --- /dev/null +++ b/tests/truffle/scripts/bsc-validator.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +source /scripts/utils.sh + +DATA_DIR=/root/.ethereum + +wait_for_host_port ${BOOTSTRAP_HOST} ${BOOTSTRAP_TCP_PORT} +BOOTSTRAP_IP=$(get_host_ip $BOOTSTRAP_HOST) +VALIDATOR_ADDR=$(cat ${DATA_DIR}/address) +HOST_IP=$(hostname -i) + +echo "validator id: ${HOST_IP}" + +geth --config ${DATA_DIR}/config.toml --datadir ${DATA_DIR} --netrestrict ${CLUSTER_CIDR} \ + --verbosity ${VERBOSE} --nousb \ + --bootnodes enode://${BOOTSTRAP_PUB_KEY}@${BOOTSTRAP_IP}:${BOOTSTRAP_TCP_PORT} \ + --mine -unlock ${VALIDATOR_ADDR} --password /dev/null \ + --light.serve 50 \ + --rpc.allow-unprotected-txs --txlookuplimit 15768000 diff --git a/tests/truffle/scripts/truffle-test.sh b/tests/truffle/scripts/truffle-test.sh new file mode 100755 index 0000000000..7b483e42ee --- /dev/null +++ b/tests/truffle/scripts/truffle-test.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +sed -i -e "s/localhost:8545/${RPC_HOST}:${RPC_PORT}/g" test/TestProxyBEP20.js + +npm run truffle:test diff --git a/tests/truffle/scripts/utils.sh b/tests/truffle/scripts/utils.sh new file mode 100644 index 0000000000..c13e54bc67 --- /dev/null +++ b/tests/truffle/scripts/utils.sh @@ -0,0 +1,14 @@ +wait_for_host_port() { + while ! nc -v -z -w 3 ${1} ${2} >/dev/null 2>&1 < /dev/null; do + echo "$(date) - waiting for ${1}:${2}..." + sleep 5 + done +} + +get_host_ip() { + local host_ip + while [ -z ${host_ip} ]; do + host_ip=$(getent hosts ${1}| awk '{ print $1 ; exit }') + done + echo $host_ip +} From eb581fd9497bdf5249e4b083f51b6d1bed8d8ae6 Mon Sep 17 00:00:00 2001 From: guagualvcha <296179868@qq.com> Date: Tue, 19 Oct 2021 10:55:19 +0800 Subject: [PATCH 18/18] prepare for release v1.1.3 --- CHANGELOG.md | 23 +++++++++++++++++++++++ params/version.go | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2eb20ff22..c0bd5774f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,27 @@ # Changelog +## v1.1.3 +Improvement +* [\#456](https://github.com/binance-chain/bsc/pull/456) git-flow support lint, unit test, and integration test +* [\#449](https://github.com/binance-chain/bsc/pull/449) cache bitmap and change the cache type of GetCode +* [\#454](https://github.com/binance-chain/bsc/pull/454) fix cache key do not have hash func +* [\#446](https://github.com/binance-chain/bsc/pull/446) parallel bloom calculation +* [\#442](https://github.com/binance-chain/bsc/pull/442) ignore empty tx in GetDiffAccountsWithScope +* [\#426](https://github.com/binance-chain/bsc/pull/426) add block proccess backoff time when validator is not in turn and received in turn block +* [\#398](https://github.com/binance-chain/bsc/pull/398) ci pipeline for release page + + +BUGFIX +* [\#446](https://github.com/binance-chain/bsc/pull/446) fix concurrent write of subfetcher +* [\#444](https://github.com/binance-chain/bsc/pull/444) fix blockhash not correct for the logs of system tx receipt +* [\#409](https://github.com/binance-chain/bsc/pull/409) fix nil point in downloader +* [\#408](https://github.com/binance-chain/bsc/pull/408) core/state/snapshot: fix typo + + +FEATURES +* [\#431](https://github.com/binance-chain/bsc/pull/431) Export get diff accounts in block api +* [\#412](https://github.com/binance-chain/bsc/pull/412) add extension in eth protocol handshake to disable tx broadcast +* [\#376](https://github.com/binance-chain/bsc/pull/376) implement diff sync + ## v1.1.2 Security * [\#379](https://github.com/binance-chain/bsc/pull/379) A pre-announced hotfix release to patch a vulnerability in the EVM (CVE-2021-39137). diff --git a/params/version.go b/params/version.go index c3d0ab20ee..fe8f2938e6 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 1 // Minor version component of the current release - VersionPatch = 2 // Patch version component of the current release + VersionPatch = 3 // Patch version component of the current release VersionMeta = "" // Version metadata to append to the version string )