diff --git a/cmd/gossamer/flags.go b/cmd/gossamer/flags.go index 41f36b99de..ba3559234b 100644 --- a/cmd/gossamer/flags.go +++ b/cmd/gossamer/flags.go @@ -264,6 +264,29 @@ var ( } ) +// State Prune flags +var ( + // BloomFilterSizeFlag size for bloom filter, valid for the use with prune-state subcommand + BloomFilterSizeFlag = cli.IntFlag{ + Name: "bloom-size", + Usage: "Megabytes of memory allocated to bloom-filter for pruning", + Value: 2048, + } + + // DBPathFlag data directory for pruned DB, valid for the use with prune-state subcommand + DBPathFlag = cli.StringFlag{ + Name: "pruned-db-path", + Usage: "Data directory for the output DB", + } + + // RetainBlockNumberFlag retain number of block from latest block while pruning, valid for the use with prune-state subcommand + RetainBlockNumberFlag = cli.IntFlag{ + Name: "retain-blocks", + Usage: "Retain number of block from latest block while pruning", + Value: 256, + } +) + // flag sets that are shared by multiple commands var ( // GlobalFlags are flags that are valid for use with the root command and all subcommands @@ -276,6 +299,9 @@ var ( CPUProfFlag, MemProfFlag, RewindFlag, + DBPathFlag, + BloomFilterSizeFlag, + RetainBlockNumberFlag, } // StartupFlags are flags that are valid for use with the root command and the export subcommand @@ -354,6 +380,14 @@ var ( HeaderFlag, FirstSlotFlag, } + + PruningFlags = []cli.Flag{ + ChainFlag, + ConfigFlag, + DBPathFlag, + BloomFilterSizeFlag, + RetainBlockNumberFlag, + } ) // FixFlagOrder allow us to use various flag order formats (ie, `gossamer init diff --git a/cmd/gossamer/main.go b/cmd/gossamer/main.go index b6d18f5a68..46fe0f7409 100644 --- a/cmd/gossamer/main.go +++ b/cmd/gossamer/main.go @@ -22,6 +22,7 @@ import ( "os" "github.com/ChainSafe/gossamer/dot" + "github.com/ChainSafe/gossamer/dot/state" "github.com/ChainSafe/gossamer/lib/keystore" "github.com/ChainSafe/gossamer/lib/utils" log "github.com/ChainSafe/log15" @@ -35,6 +36,7 @@ const ( buildSpecCommandName = "build-spec" importRuntimeCommandName = "import-runtime" importStateCommandName = "import-state" + pruningStateCommandName = "prune-state" ) // app is the cli application @@ -115,6 +117,18 @@ var ( "Input can be generated by using the RPC function state_getPairs.\n" + "\tUsage: gossamer import-state --state state.json --header header.json --first-slot \n", } + + pruningCommand = cli.Command{ + Action: FixFlagOrder(pruneState), + Name: pruningStateCommandName, + Usage: "Prune state will prune the state trie", + ArgsUsage: "", + Flags: PruningFlags, + Description: `prune-state will prune historical state data. + All trie nodes that do not belong to the specified version state will be deleted from the database. + + The default pruning target is the HEAD-256 state`, + } ) // init initialises the cli application @@ -132,6 +146,7 @@ func init() { buildSpecCommand, importRuntimeCommand, importStateCommand, + pruningCommand, } app.Flags = RootFlags } @@ -411,3 +426,39 @@ func buildSpecAction(ctx *cli.Context) error { return nil } + +func pruneState(ctx *cli.Context) error { + tomlCfg, _, err := setupConfigFromChain(ctx) + if err != nil { + logger.Error("failed to load chain configuration", "error", err) + return err + } + + inputDBPath := tomlCfg.Global.BasePath + prunedDBPath := ctx.GlobalString(DBPathFlag.Name) + if prunedDBPath == "" { + return fmt.Errorf("path not specified for badger db") + } + + bloomSize := ctx.GlobalUint64(BloomFilterSizeFlag.Name) + retainBlocks := ctx.GlobalInt64(RetainBlockNumberFlag.Name) + + pruner, err := state.NewPruner(inputDBPath, prunedDBPath, bloomSize, retainBlocks) + if err != nil { + return err + } + + logger.Info("Pruner initialised") + + err = pruner.SetBloomFilter() + if err != nil { + return fmt.Errorf("failed to set keys into bloom filter %w", err) + } + + err = pruner.Prune() + if err != nil { + return fmt.Errorf("failed to prune %w", err) + } + + return nil +} diff --git a/cmd/gossamer/prune_test.go b/cmd/gossamer/prune_test.go new file mode 100644 index 0000000000..e84e6c7c9e --- /dev/null +++ b/cmd/gossamer/prune_test.go @@ -0,0 +1,98 @@ +package main + +import ( + "fmt" + "strings" + "testing" + + "github.com/dgraph-io/badger/v2" + + "github.com/stretchr/testify/require" +) + +func iterateDB(db *badger.DB, cb func(*badger.Item)) { + txn := db.NewTransaction(false) + itr := txn.NewIterator(badger.DefaultIteratorOptions) + + for itr.Rewind(); itr.Valid(); itr.Next() { + cb(itr.Item()) + } +} +func runPruneCmd(t *testing.T, configFile, prunedDBPath string) { + ctx, err := newTestContext( + "Test state trie offline pruning --prune-state", + []string{"config", "pruned-db-path", "bloom-size", "retain-blocks"}, + []interface{}{configFile, prunedDBPath, "256", "5"}, + ) + if err != nil { + t.Fatal(err) + } + + command := pruningCommand + err = command.Run(ctx) + if err != nil { + t.Fatal(err) + } +} + +func TestPruneState(t *testing.T) { + var ( + inputDBPath = "../../tests/data/db" + configFile = "../../tests/data/db/config.toml" + prunedDBPath = fmt.Sprintf("%s/%s", t.TempDir(), "pruned") + storagePrefix = "storage" + ) + + inputDB, err := badger.Open(badger.DefaultOptions(inputDBPath).WithReadOnly(true)) + require.NoError(t, err) + + nonStorageKeys := make(map[string]interface{}) + var numStorageKeys int + + getKeysInputDB := func(item *badger.Item) { + key := string(item.Key()) + if strings.HasPrefix(key, storagePrefix) { + numStorageKeys++ + return + } + nonStorageKeys[key] = nil + } + iterateDB(inputDB, getKeysInputDB) + + err = inputDB.Close() + require.NoError(t, err) + + t.Log("Total keys in input DB", numStorageKeys+len(nonStorageKeys), "storage keys", numStorageKeys) + + t.Log("pruned DB path", prunedDBPath) + + runPruneCmd(t, configFile, prunedDBPath) + + prunedDB, err := badger.Open(badger.DefaultOptions(prunedDBPath)) + require.NoError(t, err) + + nonStorageKeysPruned := make(map[string]interface{}) + var numStorageKeysPruned int + + getKeysPrunedDB := func(item *badger.Item) { + key := string(item.Key()) + if strings.HasPrefix(key, storagePrefix) { + numStorageKeysPruned++ + return + } + nonStorageKeysPruned[key] = nil + } + iterateDB(prunedDB, getKeysPrunedDB) + + t.Log("Total keys in pruned DB", len(nonStorageKeysPruned)+numStorageKeysPruned, "storage keys", numStorageKeysPruned) + require.Equal(t, len(nonStorageKeysPruned), len(nonStorageKeys)) + + // Check all non storage keys are present. + for k := range nonStorageKeys { + _, ok := nonStorageKeysPruned[k] + require.True(t, ok) + } + + err = prunedDB.Close() + require.NoError(t, err) +} diff --git a/cmd/gossamer/utils.go b/cmd/gossamer/utils.go index b5feb371ab..08c34e0911 100644 --- a/cmd/gossamer/utils.go +++ b/cmd/gossamer/utils.go @@ -28,7 +28,6 @@ import ( "github.com/ChainSafe/gossamer/dot" "github.com/ChainSafe/gossamer/lib/utils" - log "github.com/ChainSafe/log15" "github.com/stretchr/testify/require" "github.com/urfave/cli" diff --git a/dot/network/message_cache_test.go b/dot/network/message_cache_test.go index 7acf50fce2..eb32d120dc 100644 --- a/dot/network/message_cache_test.go +++ b/dot/network/message_cache_test.go @@ -42,7 +42,8 @@ func TestMessageCache(t *testing.T) { ok = msgCache.exists(peerID, msg) require.True(t, ok) - time.Sleep(50 * time.Millisecond) + // TODO: Cache has issues with timeout. https://discuss.dgraph.io/t/setwithttl-doesnt-work/14192 + time.Sleep(3 * time.Second) ok = msgCache.exists(peerID, msg) require.False(t, ok) diff --git a/dot/state/bloom.go b/dot/state/bloom.go new file mode 100644 index 0000000000..bd8a70883b --- /dev/null +++ b/dot/state/bloom.go @@ -0,0 +1,59 @@ +package state + +import ( + "encoding/binary" + "errors" + + "github.com/ChainSafe/gossamer/lib/common" + log "github.com/ChainSafe/log15" + bloomfilter "github.com/holiman/bloomfilter/v2" +) + +// ErrKeySize is returned when key size does not fit +var ErrKeySize = errors.New("cannot have nil keystore") + +type bloomStateHasher []byte + +func (f bloomStateHasher) Write(p []byte) (n int, err error) { panic("not implemented") } +func (f bloomStateHasher) Sum(b []byte) []byte { panic("not implemented") } +func (f bloomStateHasher) Reset() { panic("not implemented") } +func (f bloomStateHasher) BlockSize() int { panic("not implemented") } +func (f bloomStateHasher) Size() int { return 8 } +func (f bloomStateHasher) Sum64() uint64 { return binary.BigEndian.Uint64(f) } + +// bloomState is a wrapper for bloom filter. +// The keys of all generated entries will be recorded here so that in the pruning +// stage the entries belong to the specific version can be avoided for deletion. +type bloomState struct { + bloom *bloomfilter.Filter +} + +// newBloomState creates a brand new state bloom for state generation +// The bloom filter will be created by the passing bloom filter size. the parameters +// are picked so that the false-positive rate for mainnet is low enough. +func newBloomState(size uint64) (*bloomState, error) { + bloom, err := bloomfilter.New(size*1024*1024*8, 4) + if err != nil { + return nil, err + } + log.Info("initialised state bloom", "size", float64(bloom.M()/8)) + return &bloomState{bloom: bloom}, nil +} + +// put writes key to bloom filter +func (sb *bloomState) put(key []byte) error { + if len(key) != common.HashLength { + return ErrKeySize + } + + sb.bloom.Add(bloomStateHasher(key)) + return nil +} + +// contain is the wrapper of the underlying contains function which +// reports whether the key is contained. +// - If it says yes, the key may be contained +// - If it says no, the key is definitely not contained. +func (sb *bloomState) contain(key []byte) bool { + return sb.bloom.Contains(bloomStateHasher(key)) +} diff --git a/dot/state/prune.go b/dot/state/prune.go new file mode 100644 index 0000000000..2df24d848a --- /dev/null +++ b/dot/state/prune.go @@ -0,0 +1,185 @@ +package state + +import ( + "context" + "fmt" + "strings" + + "github.com/ChainSafe/chaindb" + "github.com/ChainSafe/gossamer/lib/blocktree" + "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/lib/utils" + "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v2/pb" +) + +// Pruner is an offline tool to prune the stale state with the help of +// bloom filter, The workflow of pruner is very simple: +// - iterate the storage state, reconstruct the relevant state tries +// - iterate the database, stream all the targeted keys to new DB +type Pruner struct { + inputDB *chaindb.BadgerDB + storageState *StorageState + blockState *BlockState + bloom *bloomState + bestBlockHash common.Hash + retainBlockNum int64 + + inputDBPath string + prunedDBPath string +} + +// NewPruner creates an instance of Pruner. +func NewPruner(inputDBPath, prunedDBPath string, bloomSize uint64, retainBlockNum int64) (*Pruner, error) { + db, err := utils.LoadChainDB(inputDBPath) + if err != nil { + return nil, fmt.Errorf("failed to load DB %w", err) + } + + base := NewBaseState(db) + bestHash, err := base.LoadBestBlockHash() + if err != nil { + return nil, fmt.Errorf("failed to get best block hash: %w", err) + } + + // load blocktree + bt := blocktree.NewEmptyBlockTree(db) + if err = bt.Load(); err != nil { + return nil, fmt.Errorf("failed to load blocktree: %w", err) + } + + // create blockState state + blockState, err := NewBlockState(db, bt) + if err != nil { + return nil, fmt.Errorf("failed to create block state: %w", err) + } + + // create bloom filter + bloom, err := newBloomState(bloomSize) + if err != nil { + return nil, fmt.Errorf("failed to create new bloom filter of size %d %w", bloomSize, err) + } + + // load storage state + storageState, err := NewStorageState(db, blockState, trie.NewEmptyTrie()) + if err != nil { + return nil, fmt.Errorf("failed to create new storage state %w", err) + } + + return &Pruner{ + inputDB: db, + storageState: storageState, + blockState: blockState, + bloom: bloom, + bestBlockHash: bestHash, + retainBlockNum: retainBlockNum, + prunedDBPath: prunedDBPath, + inputDBPath: inputDBPath, + }, nil +} + +// SetBloomFilter loads keys with storage prefix of last `retainBlockNum` blocks into the bloom filter +func (p *Pruner) SetBloomFilter() error { + defer p.inputDB.Close() // nolint: errcheck + finalisedHash, err := p.blockState.GetFinalizedHash(0, 0) + if err != nil { + return err + } + + header, err := p.blockState.GetHeader(finalisedHash) + if err != nil { + return err + } + + latestBlockNum := header.Number.Int64() + keys := make(map[common.Hash]struct{}) + + logger.Info("Latest block number", "num", latestBlockNum) + + if latestBlockNum-p.retainBlockNum <= 0 { + return fmt.Errorf("not enough block to perform pruning") + } + + // loop from latest to last `retainBlockNum` blocks + for blockNum := header.Number.Int64(); blockNum > 0 && blockNum >= latestBlockNum-p.retainBlockNum; { + var tr *trie.Trie + tr, err = p.storageState.LoadFromDB(header.StateRoot) + if err != nil { + return err + } + + err = tr.GetNodeHashes(tr.RootNode(), keys) + if err != nil { + return err + } + + // get parent header of current block + header, err = p.blockState.GetHeader(header.ParentHash) + if err != nil { + return err + } + blockNum = header.Number.Int64() + } + + for key := range keys { + err = p.bloom.put(key.ToBytes()) + if err != nil { + return err + } + } + + logger.Info("Total keys added in bloom filter", "keysCount", len(keys)) + return nil +} + +// Prune starts streaming the data from input db to the pruned db. +func (p *Pruner) Prune() error { + inputDB, err := utils.LoadBadgerDB(p.inputDBPath) + if err != nil { + return fmt.Errorf("failed to load DB %w", err) + } + defer inputDB.Close() // nolint: errcheck + + prunedDB, err := utils.LoadBadgerDB(p.prunedDBPath) + if err != nil { + return fmt.Errorf("failed to load DB %w", err) + } + defer prunedDB.Close() // nolint: errcheck + + writer := prunedDB.NewStreamWriter() + if err = writer.Prepare(); err != nil { + return fmt.Errorf("cannot create stream writer in out DB at %s error %w", p.prunedDBPath, err) + } + + // Stream contents of DB to the output DB. + stream := inputDB.NewStream() + stream.LogPrefix = fmt.Sprintf("Streaming DB to new DB at %s ", p.prunedDBPath) + + stream.ChooseKey = func(item *badger.Item) bool { + key := string(item.Key()) + // All the non storage keys will be streamed to new db. + if !strings.HasPrefix(key, storagePrefix) { + return true + } + + // Only keys present in bloom filter will be streamed to new db + key = strings.TrimPrefix(key, storagePrefix) + exist := p.bloom.contain([]byte(key)) + return exist + } + + stream.Send = func(l *pb.KVList) error { + return writer.Write(l) + } + + if err = stream.Orchestrate(context.Background()); err != nil { + return fmt.Errorf("cannot stream DB to out DB at %s error %w", p.prunedDBPath, err) + } + + if err = writer.Flush(); err != nil { + return fmt.Errorf("cannot flush writer, error %w", err) + } + + return nil +} diff --git a/dot/state/storage.go b/dot/state/storage.go index 1aefb4f8ac..ea9d997808 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -29,6 +29,7 @@ import ( "github.com/ChainSafe/gossamer/lib/trie" ) +// storagePrefix storage key prefix. var storagePrefix = "storage" var codeKey = common.CodeKey diff --git a/go.mod b/go.mod index e5a2dcda3f..f9b8258164 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,8 @@ require ( github.com/centrifuge/go-substrate-rpc-client/v2 v2.0.1 github.com/cosmos/go-bip39 v1.0.0 github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f // indirect - github.com/dgraph-io/badger/v2 v2.2007.2 // indirect - github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de + github.com/dgraph-io/badger/v2 v2.2007.2 + github.com/dgraph-io/ristretto v0.0.4-0.20210122082011-bb5d392ed82d github.com/disiqueira/gotree v1.0.0 github.com/docker/docker v1.13.1 github.com/elastic/gosigar v0.14.0 // indirect @@ -26,6 +26,7 @@ require ( github.com/gorilla/websocket v1.4.2 github.com/gtank/merlin v0.1.1 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect + github.com/holiman/bloomfilter/v2 v2.0.3 github.com/huin/goupnp v1.0.1-0.20200620063722-49508fba0031 // indirect github.com/ipfs/go-ds-badger2 v0.1.0 github.com/jcelliott/lumber v0.0.0-20160324203708-dd349441af25 // indirect @@ -47,18 +48,15 @@ require ( github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 github.com/perlin-network/life v0.0.0-20191203030451-05c0e0f7eaea github.com/rs/cors v1.7.0 // indirect - github.com/sirupsen/logrus v1.2.0 + github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca // indirect github.com/urfave/cli v1.20.0 github.com/wasmerio/go-ext-wasm v0.3.2-0.20200326095750-0a32be6068ec + go.opencensus.io v0.22.5 // indirect golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect - golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd // indirect - golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect - golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect - golang.org/x/tools v0.0.0-20200221224223-e1da425f72fd // indirect google.golang.org/appengine v1.6.5 // indirect google.golang.org/protobuf v1.25.0 ) diff --git a/go.sum b/go.sum index bc840ce49e..0d4ff24d2c 100644 --- a/go.sum +++ b/go.sum @@ -49,11 +49,13 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY= github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -73,8 +75,9 @@ github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLI github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.0.4-0.20210122082011-bb5d392ed82d h1:eQYOG6A4td1tht0NdJB9Ls6DsXRGb2Ft6X9REU/MbbE= +github.com/dgraph-io/ristretto v0.0.4-0.20210122082011-bb5d392ed82d/go.mod h1:tv2ec8nA7vRpSYX7/MbP52ihrUMXIHit54CQMq8npXQ= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -101,6 +104,7 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-delve/delve v1.5.0/go.mod h1:c6b3a1Gry6x8a4LGCe/CWzrocrfaHvkUxCj3k4bvSUQ= github.com/go-interpreter/wagon v0.6.0 h1:BBxDxjiJiHgw9EdkYXAWs8NHhwnazZ5P2EWBW5hFNWw= github.com/go-interpreter/wagon v0.6.0/go.mod h1:5+b/MBYkclRZngKF5s6qrgWxSLgE9F5dFdO1hAueZLc= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= @@ -146,6 +150,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-dap v0.2.0/go.mod h1:5q8aYQFnHOAZEMP+6vmq25HKYAEwE+LF5yh7JKrrhSQ= github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY= github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= @@ -178,6 +183,8 @@ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= +github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goupnp v1.0.1-0.20200620063722-49508fba0031 h1:HarGZ5h9HD9LgEg1yRVMXyfiw4wlXiLiYM2oMjeA/SE= @@ -257,8 +264,9 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d h1:68u9r4wEvL3gYg2jvAOgROwZ3H+Y3hIDk4tbbmIjcYQ= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -460,11 +468,13 @@ github.com/libp2p/go-yamux v1.4.0 h1:7nqe0T95T2CWh40IdJ/tp8RMor4ubc9/wYZpB2a/Hx0 github.com/libp2p/go-yamux v1.4.0/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mattn/go-colorable v0.0.0-20170327083344-ded68f7a9561/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -487,6 +497,7 @@ github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKU github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mmcloughlin/avo v0.0.0-20201105074841-5d2f697d268f/go.mod h1:6aKT4zZIrpGqB3RpFU14ByCSSyKY6LfJz4J/JJChHfI= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -574,6 +585,7 @@ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYr github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/perlin-network/life v0.0.0-20191203030451-05c0e0f7eaea h1:okKoivlkNRRLqXraEtatHfEhW+D71QTwkaj+4n4M2Xc= github.com/perlin-network/life v0.0.0-20191203030451-05c0e0f7eaea/go.mod h1:3KEU5Dm8MAYWZqity880wOFJ9PhQjyKVZGwAEfc5Q4E= +github.com/peterh/liner v0.0.0-20170317030525-88609521dc4b/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/phuslu/iploc v1.0.20200807 h1:LIBm2Y9l5zmUvnJhQgMcLZ0iVwuG+5/L6AgbMwSOpE4= github.com/phuslu/iploc v1.0.20200807/go.mod h1:Q/0VX0txvbxekt4NhWIi3Q3eyZ139lHhnlzvDxyXhuc= github.com/pierrec/xxHash v0.1.5 h1:n/jBpwTHiER4xYvK3/CdPVnLDPchj8eTJFFLUb4QHBo= @@ -591,8 +603,9 @@ github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= @@ -602,8 +615,10 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.0-20170417170307-b6cb39589372/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v0.0.0-20170417173400-9e4c21054fa1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= @@ -621,8 +636,9 @@ github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca h1:Ld/zXl5t4+D6 github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc= github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4= -github.com/twitchyliquid64/golang-asm v0.0.0-20190126203739-365674df15fc h1:RTUQlKzoZZVG3umWNzOYeFecQLIh+dbxXvJp1zPQJTI= github.com/twitchyliquid64/golang-asm v0.0.0-20190126203739-365674df15fc/go.mod h1:NoCfSFWosfqMqmmD7hApkirIK9ozpHjxRnRxs1l413A= +github.com/twitchyliquid64/golang-asm v0.15.0 h1:WYZ15YKpC5xM8PwpBTDsAgemoLB/lyhRkzJSEw9eAew= +github.com/twitchyliquid64/golang-asm v0.15.0/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -642,12 +658,15 @@ github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1: github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.starlark.net v0.0.0-20190702223751-32f345186213/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -663,6 +682,8 @@ go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +golang.org/x/arch v0.0.0-20190927153633-4e8777c89be4/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= +golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -692,8 +713,8 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNT golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd h1:ePuNC7PZ6O5BzgPn9bZayERXBdfZjUYoXEf5BTfDfh8= -golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -707,8 +728,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -729,6 +750,7 @@ golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190306220234-b354f8bf4d9e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -743,6 +765,7 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM= @@ -765,9 +788,10 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191127201027-ecd32218bd7f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200221224223-e1da425f72fd h1:hHkvGJK23seRCflePJnVa9IMv8fsuavSCWKd11kDQFs= -golang.org/x/tools v0.0.0-20200221224223-e1da425f72fd/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20201105001634-bc3cf281b174 h1:0rx0F4EjJNbxTuzWe0KjKcIzs+3VEb/Mrs/d1ciNz1c= +golang.org/x/tools v0.0.0-20201105001634-bc3cf281b174/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -826,3 +850,4 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/lib/trie/database.go b/lib/trie/database.go index 7067ca7187..447c53171e 100644 --- a/lib/trie/database.go +++ b/lib/trie/database.go @@ -128,6 +128,26 @@ func (t *Trie) load(db chaindb.Database, curr node) error { return nil } +// GetNodeHashes return hash of each key of the trie. +func (t *Trie) GetNodeHashes(curr node, keys map[common.Hash]struct{}) error { + if c, ok := curr.(*branch); ok { + for _, child := range c.children { + if child == nil { + continue + } + + hash := child.getHash() + keys[common.BytesToHash(hash)] = struct{}{} + + err := t.GetNodeHashes(child, keys) + if err != nil { + return err + } + } + } + return nil +} + // PutInDB puts a value into the trie and writes the updates nodes the database. Since it needs to write all the nodes from the changed node up to the root, it writes these in a batch operation. func (t *Trie) PutInDB(db chaindb.Database, key, value []byte) error { t.Put(key, value) diff --git a/lib/utils/utils.go b/lib/utils/utils.go index e76de1da21..f2e22e9f06 100644 --- a/lib/utils/utils.go +++ b/lib/utils/utils.go @@ -25,6 +25,9 @@ import ( "path/filepath" "runtime" "strings" + + "github.com/ChainSafe/chaindb" + "github.com/dgraph-io/badger/v2" ) // PathExists returns true if the named file or directory exists, otherwise false @@ -193,18 +196,31 @@ func GetKusamaGenesisPath() string { return fp } -// GetDevGenesisPath gets the dev genesis path -func GetDevGenesisPath() string { - path1 := "../chain/dev/genesis.json" - path2 := "../../chain/dev/genesis.json" +// LoadChainDB load the db at the given path. +func LoadChainDB(basePath string) (*chaindb.BadgerDB, error) { + cfg := &chaindb.Config{ + DataDir: basePath, + } - var fp string + // TODO: Open the db in readonly mode. - if PathExists(path1) { - fp, _ = filepath.Abs(path1) - } else if PathExists(path2) { - fp, _ = filepath.Abs(path2) + // Open already existing DB + db, err := chaindb.NewBadgerDB(cfg) + if err != nil { + return nil, err } - return fp + return db, nil +} + +// LoadBadgerDB load the db at the given path. +func LoadBadgerDB(basePath string) (*badger.DB, error) { + opts := badger.DefaultOptions(basePath) + // Open already existing DB + db, err := badger.Open(opts) + if err != nil { + return nil, err + } + + return db, nil } diff --git a/tests/data/db/000000.vlog b/tests/data/db/000000.vlog new file mode 100644 index 0000000000..1e7e304124 Binary files /dev/null and b/tests/data/db/000000.vlog differ diff --git a/tests/data/db/000006.sst b/tests/data/db/000006.sst new file mode 100644 index 0000000000..bc81160b5b Binary files /dev/null and b/tests/data/db/000006.sst differ diff --git a/tests/data/db/KEYREGISTRY b/tests/data/db/KEYREGISTRY new file mode 100644 index 0000000000..4bfc7e5c9a --- /dev/null +++ b/tests/data/db/KEYREGISTRY @@ -0,0 +1 @@ +=ØÒ½\¡ÝéŸ{ÉDä'œHello Badger \ No newline at end of file diff --git a/tests/data/db/MANIFEST b/tests/data/db/MANIFEST new file mode 100644 index 0000000000..40204a43ad Binary files /dev/null and b/tests/data/db/MANIFEST differ diff --git a/tests/data/db/config.toml b/tests/data/db/config.toml new file mode 100644 index 0000000000..7592d17e54 --- /dev/null +++ b/tests/data/db/config.toml @@ -0,0 +1,38 @@ +[global] +basepath = "../../tests/data/db" +log = "info" +metrics-port = 9876 + +[log] +core = "" +network = "" +rpc = "" +state = "" +runtime = "" +babe = "" +grandpa = "" +sync = "" + +[init] +genesis = "./chain/gssmr/genesis.json" + +[account] +key = "" +unlock = "" + +[core] +roles = 4 +babe-authority = true +grandpa-authority = true + +[network] +port = 7001 +nobootstrap = false +nomdns = false + +[rpc] +enabled = false +port = 8545 +host = "localhost" +modules = ["system", "author", "chain", "state", "rpc", "grandpa"] +ws-port = 8546 diff --git a/tests/stress/stress_test.go b/tests/stress/stress_test.go index d1567b7763..9a2105ad4a 100644 --- a/tests/stress/stress_test.go +++ b/tests/stress/stress_test.go @@ -29,11 +29,10 @@ import ( gosstypes "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/tests/utils" + log "github.com/ChainSafe/log15" gsrpc "github.com/centrifuge/go-substrate-rpc-client/v2" "github.com/centrifuge/go-substrate-rpc-client/v2/signature" "github.com/centrifuge/go-substrate-rpc-client/v2/types" - - log "github.com/ChainSafe/log15" "github.com/stretchr/testify/require" )