diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f060f462f9a..d95527efcd0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,7 +35,50 @@ Ref: https://keepachangelog.com/en/1.0.0/ # Changelog -## [Unreleased] +## [v0.38.5] - TBD + +The pruning changes introduced in 0.38.0 has bugs that may cause data loss, even after upgrading +to 0.38.5. When upgrading from 0.38.x it is important to follow the instructions below, to prevent +data loss and database corruption. + +Do not modify pruning settings with <=0.38.4 as that may cause data corruption - the following +assumes pruning settings have not been modified since the node started using 0.38.x. The default +pruning setting `syncable` used `KeepEvery:100`. + +* If using `KeepEvery:1` (pruning settings `nothing` or `everything`), upgrading to 0.38.5 is safe. + +* Otherwise, halt block processing with `--halt-height` after committing a height divisible by + `KeepEvery` - e.g. at block 147600 with `KeepEvery:100`. The node must _never_ have processed a + height beyond that at any time in its past. Upgrading to 0.38.5 is then safe. + +* Otherwise, set the 0.38.5 `KeepEvery` setting to the same as the previous `KeepEvery` setting + (<=0.38.4 and 0.38.5 both default to `KeepEvery:100`). Upgrading to 0.38.5 is then safe as long + as you wait one `KeepEvery` interval plus one `KeepRecent` interval plus one pruning `Interval` + before changing pruning settings or deleting the last <=0.38.4 height (so wait 210 heights with + the default configuration). + +* Otherwise, make sure the last version persisted with <=0.38.4 is never deleted after upgrading to + 0.38.5, as doing so may cause data loss and data corruption. + +* Otherwise, consider syncing the node from scratch with 0.38.5. + +### Improvements + +* (deps) Bump Tendermint version to [v0.33.5](https://github.com/tendermint/tendermint/releases/tag/v0.33.5) +* (deps) Bump IAVL version to [v0.14.0](https://github.com/cosmos/iavl/releases/tag/v0.14.0) + +### Bug Fixes + +* (store) Revert IAVL pruning functionality introduced in [v0.13.0](https://github.com/cosmos/iavl/releases/tag/v0.13.0), +where the IAVL no longer keeps states in-memory in which it flushes periodically. IAVL now commits and +flushes every state to disk as it did pre-v0.13.0. The SDK's multi-store will track and ensure the proper +heights are pruned. The operator can set the pruning options via a `pruning` config via the CLI or +through `app.toml`. The `pruning` flag exposes `default|everything|nothing|custom` as options -- +see docs for further details. If the operator chooses `custom`, they may provide granular pruning +options `pruning-keep-recent`, `pruning-keep-every`, and `pruning-interval`. The former two options +dictate how many recent versions are kept on disk and the offset of what versions are kept after that +respectively, and the latter defines the height interval in which versions are deleted in a batch. +**Note, there are some client-facing API breaking changes with regard to IAVL, stores, and pruning settings.** ## [v0.38.4] - 2020-05-21 diff --git a/baseapp/baseapp_test.go b/baseapp/baseapp_test.go index 6ee5aabbb0a8..60b434530fef 100644 --- a/baseapp/baseapp_test.go +++ b/baseapp/baseapp_test.go @@ -168,7 +168,7 @@ func initStore(t *testing.T, db dbm.DB, storeKey string, k, v []byte) { func checkStore(t *testing.T, db dbm.DB, ver int64, storeKey string, k, v []byte) { rs := rootmulti.NewStore(db) - rs.SetPruning(store.PruneSyncable) + rs.SetPruning(store.PruneDefault) key := sdk.NewKVStoreKey(storeKey) rs.MountStoreWithDB(key, store.StoreTypeIAVL, nil) err := rs.LoadLatestVersion() @@ -273,7 +273,7 @@ func TestSetLoader(t *testing.T) { func TestAppVersionSetterGetter(t *testing.T) { logger := defaultLogger() - pruningOpt := SetPruning(store.PruneSyncable) + pruningOpt := SetPruning(store.PruneDefault) db := dbm.NewMemDB() name := t.Name() app := NewBaseApp(name, logger, db, nil, pruningOpt) @@ -329,8 +329,9 @@ func TestLoadVersionInvalid(t *testing.T) { func TestLoadVersionPruning(t *testing.T) { logger := log.NewNopLogger() pruningOptions := store.PruningOptions{ - KeepEvery: 2, - SnapshotEvery: 6, + KeepRecent: 2, + KeepEvery: 3, + Interval: 1, } pruningOpt := SetPruning(pruningOptions) db := dbm.NewMemDB() @@ -351,61 +352,33 @@ func TestLoadVersionPruning(t *testing.T) { require.Equal(t, int64(0), lastHeight) require.Equal(t, emptyCommitID, lastID) - // execute a block - header := abci.Header{Height: 1} - app.BeginBlock(abci.RequestBeginBlock{Header: header}) - res := app.Commit() - - // execute a block, collect commit ID - header = abci.Header{Height: 2} - app.BeginBlock(abci.RequestBeginBlock{Header: header}) - res = app.Commit() - commitID2 := sdk.CommitID{Version: 2, Hash: res.Data} - - // execute a block - header = abci.Header{Height: 3} - app.BeginBlock(abci.RequestBeginBlock{Header: header}) - res = app.Commit() - commitID3 := sdk.CommitID{Version: 3, Hash: res.Data} - - // reload with LoadLatestVersion, check it loads last flushed version - app = NewBaseApp(name, logger, db, nil, pruningOpt) - app.MountStores(capKey) - err = app.LoadLatestVersion(capKey) - require.Nil(t, err) - testLoadVersionHelper(t, app, int64(2), commitID2) + var lastCommitID sdk.CommitID - // re-execute block 3 and check it is same CommitID - header = abci.Header{Height: 3} - app.BeginBlock(abci.RequestBeginBlock{Header: header}) - res = app.Commit() - recommitID3 := sdk.CommitID{Version: 3, Hash: res.Data} - require.Equal(t, commitID3, recommitID3, "Commits of identical blocks not equal after reload") + // Commit seven blocks, of which 7 (latest) is kept in addition to 6, 5 + // (keep recent) and 3 (keep every). + for i := int64(1); i <= 7; i++ { + app.BeginBlock(abci.RequestBeginBlock{Header: abci.Header{Height: i}}) + res := app.Commit() + lastCommitID = sdk.CommitID{Version: i, Hash: res.Data} + } - // execute a block, collect commit ID - header = abci.Header{Height: 4} - app.BeginBlock(abci.RequestBeginBlock{Header: header}) - res = app.Commit() - commitID4 := sdk.CommitID{Version: 4, Hash: res.Data} + for _, v := range []int64{1, 2, 4} { + _, err = app.cms.CacheMultiStoreWithVersion(v) + require.Error(t, err) + } - // execute a block - header = abci.Header{Height: 5} - app.BeginBlock(abci.RequestBeginBlock{Header: header}) - res = app.Commit() + for _, v := range []int64{3, 5, 6, 7} { + _, err = app.cms.CacheMultiStoreWithVersion(v) + require.NoError(t, err) + } - // reload with LoadLatestVersion, check it loads last flushed version + // reload with LoadLatestVersion, check it loads last version app = NewBaseApp(name, logger, db, nil, pruningOpt) app.MountStores(capKey) + err = app.LoadLatestVersion(capKey) require.Nil(t, err) - testLoadVersionHelper(t, app, int64(4), commitID4) - - // reload with LoadVersion of previous flushed version - // and check it fails since previous flush should be pruned - app = NewBaseApp(name, logger, db, nil, pruningOpt) - app.MountStores(capKey) - err = app.LoadVersion(2, capKey) - require.NotNil(t, err) + testLoadVersionHelper(t, app, int64(7), lastCommitID) } func testLoadVersionHelper(t *testing.T, app *BaseApp, expectedHeight int64, expectedID sdk.CommitID) { diff --git a/client/context/context.go b/client/context/context.go index 28bfe8e830b0..ec01b50f3caa 100644 --- a/client/context/context.go +++ b/client/context/context.go @@ -12,6 +12,7 @@ import ( "github.com/tendermint/tendermint/libs/cli" tmlite "github.com/tendermint/tendermint/lite" rpcclient "github.com/tendermint/tendermint/rpc/client" + rpchttp "github.com/tendermint/tendermint/rpc/client/http" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/codec" @@ -65,7 +66,7 @@ func NewCLIContextWithInputAndFrom(input io.Reader, from string) CLIContext { if !genOnly { nodeURI = viper.GetString(flags.FlagNode) if nodeURI != "" { - rpc, err = rpcclient.NewHTTP(nodeURI, "/websocket") + rpc, err = rpchttp.New(nodeURI, "/websocket") if err != nil { fmt.Printf("failted to get client: %v\n", err) os.Exit(1) @@ -157,7 +158,7 @@ func (ctx CLIContext) WithTrustNode(trustNode bool) CLIContext { // WithNodeURI returns a copy of the context with an updated node URI. func (ctx CLIContext) WithNodeURI(nodeURI string) CLIContext { ctx.NodeURI = nodeURI - client, err := rpcclient.NewHTTP(nodeURI, "/websocket") + client, err := rpchttp.New(nodeURI, "/websocket") if err != nil { panic(err) } diff --git a/client/context/verifier.go b/client/context/verifier.go index 856c585b5a28..b849d3e397d1 100644 --- a/client/context/verifier.go +++ b/client/context/verifier.go @@ -7,7 +7,7 @@ import ( "github.com/tendermint/tendermint/libs/log" tmlite "github.com/tendermint/tendermint/lite" tmliteproxy "github.com/tendermint/tendermint/lite/proxy" - rpcclient "github.com/tendermint/tendermint/rpc/client" + rpchttp "github.com/tendermint/tendermint/rpc/client/http" ) const ( @@ -43,7 +43,7 @@ func CreateVerifier(ctx CLIContext, cacheSize int) (tmlite.Verifier, error) { // create an RPC client based off of the RPC URI if no RPC client exists client := ctx.Client if client == nil { - client, err = rpcclient.NewHTTP(ctx.NodeURI, "/websocket") + client, err = rpchttp.New(ctx.NodeURI, "/websocket") if err != nil { return nil, err } diff --git a/client/lcd/root.go b/client/lcd/root.go index af0e74b48557..a99f99fe9faf 100644 --- a/client/lcd/root.go +++ b/client/lcd/root.go @@ -12,7 +12,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/tendermint/tendermint/libs/log" - rpcserver "github.com/tendermint/tendermint/rpc/lib/server" + tmrpcserver "github.com/tendermint/tendermint/rpc/jsonrpc/server" "github.com/cosmos/cosmos-sdk/client/context" "github.com/cosmos/cosmos-sdk/client/flags" @@ -52,12 +52,12 @@ func (rs *RestServer) Start(listenAddr string, maxOpen int, readTimeout, writeTi rs.log.Error("error closing listener", "err", err) }) - cfg := rpcserver.DefaultConfig() + cfg := tmrpcserver.DefaultConfig() cfg.MaxOpenConnections = maxOpen cfg.ReadTimeout = time.Duration(readTimeout) * time.Second cfg.WriteTimeout = time.Duration(writeTimeout) * time.Second - rs.listener, err = rpcserver.Listen(listenAddr, cfg) + rs.listener, err = tmrpcserver.Listen(listenAddr, cfg) if err != nil { return } @@ -68,7 +68,7 @@ func (rs *RestServer) Start(listenAddr string, maxOpen int, readTimeout, writeTi ), ) - return rpcserver.StartHTTPServer(rs.listener, rs.Mux, rs.log, cfg) + return tmrpcserver.Serve(rs.listener, rs.Mux, rs.log, cfg) } // ServeCommand will start the application REST service as a blocking process. It diff --git a/go.mod b/go.mod index 32d549a01dc8..33c31036d1a1 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,12 @@ +go 1.13 + module github.com/cosmos/cosmos-sdk require ( github.com/99designs/keyring v1.1.3 github.com/bartekn/go-bip39 v0.0.0-20171116152956-a05967ea095d github.com/bgentry/speakeasy v0.1.0 - github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d + github.com/btcsuite/btcd v0.20.1-beta github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d github.com/cosmos/ledger-cosmos-go v0.11.1 github.com/gogo/protobuf v1.3.1 @@ -16,20 +18,19 @@ require ( github.com/pkg/errors v0.9.1 github.com/rakyll/statik v0.1.6 github.com/spf13/afero v1.2.1 // indirect - github.com/spf13/cobra v0.0.6 + github.com/spf13/cobra v1.0.0 github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.6.2 - github.com/stretchr/testify v1.5.1 + github.com/spf13/viper v1.6.3 + github.com/stretchr/testify v1.6.1 github.com/tendermint/btcd v0.1.1 github.com/tendermint/crypto v0.0.0-20191022145703-50d29ede1e15 github.com/tendermint/go-amino v0.15.1 - github.com/tendermint/iavl v0.13.2 - github.com/tendermint/tendermint v0.33.3 - github.com/tendermint/tm-db v0.5.0 - gopkg.in/yaml.v2 v2.2.8 + github.com/tendermint/iavl v0.14.0-rc2 + github.com/tendermint/tendermint v0.33.5 + github.com/tendermint/tm-db v0.5.1 + google.golang.org/grpc v1.30.0 // indirect + gopkg.in/yaml.v2 v2.3.0 ) -go 1.13 - replace github.com/keybase/go-keychain => github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 diff --git a/go.sum b/go.sum index 90cb15b46352..fee75af0e78d 100644 --- a/go.sum +++ b/go.sum @@ -6,14 +6,15 @@ github.com/99designs/keyring v1.1.3 h1:mEV3iyZWjkxQ7R8ia8GcG97vCX5zQQ7n4o8R2Bylw github.com/99designs/keyring v1.1.3/go.mod h1:657DQuMrBZRtuL/voxVyiyb6zpMehlm5vLB9Qwrv904= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/ChainSafe/go-schnorrkel v0.0.0-20200102211924-4bcbc698314f h1:4O1om+UVU+Hfcihr1timk8YNXHxzZWgCo7ofnrZRApw= -github.com/ChainSafe/go-schnorrkel v0.0.0-20200102211924-4bcbc698314f/go.mod h1:URdX5+vg25ts3aCh8H5IFZybJYKWhJHYMTnf+ULtoC4= +github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d h1:nalkkPQcITbvhmL4+C4cKA87NW0tfm3Kl9VXRoPywFg= +github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d/go.mod h1:URdX5+vg25ts3aCh8H5IFZybJYKWhJHYMTnf+ULtoC4= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/Workiva/go-datastructures v1.0.52 h1:PLSK6pwn8mYdaoaCZEMsXBpBotr4HHn9abU0yMQt0NI= github.com/Workiva/go-datastructures v1.0.52/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= @@ -43,9 +44,14 @@ github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d h1:xG8Pj6Y6J760xwETNmMzmlt38QSwz0BLp1cZ09g27uw= github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d/go.mod h1:d3C0AkH6BRcvO8T0UEPu53cnw4IbV63x1bEjildYhO0= +github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20180706230648-ab6388e0c60a h1:RQMUrEILyYJEoAT34XS/kLu40vC0+po/UfxrBBA4qZE= github.com/btcsuite/btcutil v0.0.0-20180706230648-ab6388e0c60a/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/btcutil v1.0.2 h1:9iZ1Terx9fMIOtq1VrwdqfsATL9MC2l8ZrUY6YZ2uts= +github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= @@ -65,6 +71,7 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -99,8 +106,6 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/etcd-io/bbolt v1.3.3 h1:gSJmxrs37LgTqR/oyJBWok6k6SvXEUerFTbltIhXkBM= -github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ= github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= @@ -152,8 +157,12 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -179,8 +188,8 @@ github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -188,8 +197,9 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= -github.com/gtank/merlin v0.1.1-0.20191105220539-8318aed1a79f h1:8N8XWLZelZNibkhM1FuF+3Ad3YIbgirjdMiVA0eUkaM= github.com/gtank/merlin v0.1.1-0.20191105220539-8318aed1a79f/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s= +github.com/gtank/merlin v0.1.1 h1:eQ90iG7K9pOhtereWsmyRJ6RAwcP4tHTDBHXNg+u5is= +github.com/gtank/merlin v0.1.1/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s= github.com/gtank/ristretto255 v0.1.2 h1:JEqUCPA1NvLq5DwYtuzigd7ss8fwbYay9fi4/5uMzcc= github.com/gtank/ristretto255 v0.1.2/go.mod h1:Ph5OpO6c7xKUGROZfWVLiJf9icMDwUeIvY4OmlYW69o= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= @@ -240,7 +250,6 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -252,7 +261,6 @@ github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoR github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -267,6 +275,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643 h1:hLDRPB66XQT/8+wG9WsDpiCvZf1yKO7sz7scAjSlBa0= github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM= +github.com/minio/highwayhash v1.0.0 h1:iMSDhgUILCr0TNm8LWlSjF8N0ZIj2qbO8WHp6Q/J2BA= +github.com/minio/highwayhash v1.0.0/go.mod h1:xQboMTeM9nY9v/LlAOxFctujiv5+Aq2hR5dxBpaMbdc= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -317,7 +327,6 @@ github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9 github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -327,24 +336,20 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= -github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= -github.com/prometheus/client_golang v1.5.0 h1:Ctq0iGpCmr3jeP77kbF2UxgvRwzWWz+4Bh9/vJTyg1A= -github.com/prometheus/client_golang v1.5.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= +github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= @@ -352,7 +357,6 @@ github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= @@ -360,8 +364,9 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rakyll/statik v0.1.6 h1:uICcfUXpgqtw2VopbIncslhAmE5hwc4g20TEyEENBNs= github.com/rakyll/statik v0.1.6/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= @@ -387,19 +392,18 @@ github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTd github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.6 h1:breEStsVwemnKh2/s6gMvSdMEkwW0sK8vGStnlVBMCs= -github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= -github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= +github.com/spf13/viper v1.6.3 h1:pDDu1OyEDTKzpJwdq4TiuLyMsUgRa/BT5cn5O62NoHs= +github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= @@ -409,10 +413,10 @@ github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d h1:gZZadD8H+fF+n9CmNhYL1Y0dJB+kLOmKd7FbPJLeGHs= @@ -426,15 +430,12 @@ github.com/tendermint/crypto v0.0.0-20191022145703-50d29ede1e15/go.mod h1:z4YtwM github.com/tendermint/go-amino v0.14.1/go.mod h1:i/UKE5Uocn+argJJBb12qTZsCDBcAYMbR92AaJVmKso= github.com/tendermint/go-amino v0.15.1 h1:D2uk35eT4iTsvJd9jWIetzthE5C0/k2QmMFkCN+4JgQ= github.com/tendermint/go-amino v0.15.1/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME= -github.com/tendermint/iavl v0.13.2 h1:O1m08/Ciy53l9IYmf75uIRVvrNsfjEbre8u/yCu/oqk= -github.com/tendermint/iavl v0.13.2/go.mod h1:vE1u0XAGXYjHykd4BLp8p/yivrw2PF1TuoljBcsQoGA= -github.com/tendermint/tendermint v0.33.2 h1:NzvRMTuXJxqSsFed2J7uHmMU5N1CVzSpfi3nCc882KY= -github.com/tendermint/tendermint v0.33.2/go.mod h1:25DqB7YvV1tN3tHsjWoc2vFtlwICfrub9XO6UBO+4xk= -github.com/tendermint/tendermint v0.33.3 h1:6lMqjEoCGejCzAghbvfQgmw87snGSqEhDTo/jw+W8CI= -github.com/tendermint/tendermint v0.33.3/go.mod h1:25DqB7YvV1tN3tHsjWoc2vFtlwICfrub9XO6UBO+4xk= -github.com/tendermint/tm-db v0.4.1/go.mod h1:JsJ6qzYkCGiGwm5GHl/H5GLI9XLb6qZX7PRe425dHAY= -github.com/tendermint/tm-db v0.5.0 h1:qtM5UTr1dlRnHtDY6y7MZO5Di8XAE2j3lc/pCnKJ5hQ= -github.com/tendermint/tm-db v0.5.0/go.mod h1:lSq7q5WRR/njf1LnhiZ/lIJHk2S8Y1Zyq5oP/3o9C2U= +github.com/tendermint/iavl v0.14.0-rc2 h1:S/r/XSfKWpnmh85nR319j6LeGgLfKBgCJpPFi4V/Bj4= +github.com/tendermint/iavl v0.14.0-rc2/go.mod h1:QmfViflFiXzxKLQE4tAUuWQHq+RSuQFxablW5oJZ6sE= +github.com/tendermint/tendermint v0.33.5 h1:jYgRd9ImkzA9iOyhpmgreYsqSB6tpDa6/rXYPb8HKE8= +github.com/tendermint/tendermint v0.33.5/go.mod h1:0yUs9eIuuDq07nQql9BmI30FtYGcEC60Tu5JzB5IezM= +github.com/tendermint/tm-db v0.5.1 h1:H9HDq8UEA7Eeg13kdYckkgwwkQLBnJGgX4PgLJRhieY= +github.com/tendermint/tm-db v0.5.1/go.mod h1:g92zWjHpCYlEvQXvy9M168Su8V1IBEeawpXVVBaK4f4= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= @@ -464,11 +465,13 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200406173513-056763e48d71/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 h1:IaQbIIB2X/Mp/DKctl6ROxz1KyMlKp4uyvL6+kQ7C88= +golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -493,10 +496,10 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -513,6 +516,7 @@ golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -521,11 +525,10 @@ golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -571,15 +574,19 @@ google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.30.0 h1:M5a8xTlYTxwMn5ZFkwhRabsygDY5G8TYLyQDBxJNAxE= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -596,13 +603,13 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/server/config/config.go b/server/config/config.go index 9752ae85273a..f046bbe56535 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "github.com/cosmos/cosmos-sdk/store" + storetypes "github.com/cosmos/cosmos-sdk/store/types" sdk "github.com/cosmos/cosmos-sdk/types" ) @@ -19,6 +19,11 @@ type BaseConfig struct { // specified in this config (e.g. 0.25token1;0.0001token2). MinGasPrices string `mapstructure:"minimum-gas-prices"` + Pruning string `mapstructure:"pruning"` + PruningKeepRecent string `mapstructure:"pruning-keep-recent"` + PruningKeepEvery string `mapstructure:"pruning-keep-every"` + PruningInterval string `mapstructure:"pruning-interval"` + // HaltHeight contains a non-zero block height at which a node will gracefully // halt and shutdown that can be used to assist upgrades and testing. // @@ -34,8 +39,6 @@ type BaseConfig struct { // InterBlockCache enables inter-block caching. InterBlockCache bool `mapstructure:"inter-block-cache"` - - Pruning string `mapstructure:"pruning"` } // Config defines the server's top level configuration @@ -73,10 +76,13 @@ func (c *Config) GetMinGasPrices() sdk.DecCoins { // DefaultConfig returns server's default configuration. func DefaultConfig() *Config { return &Config{ - BaseConfig{ - MinGasPrices: defaultMinGasPrices, - InterBlockCache: true, - Pruning: store.PruningStrategySyncable, + BaseConfig: BaseConfig{ + MinGasPrices: defaultMinGasPrices, + InterBlockCache: true, + Pruning: storetypes.PruningOptionDefault, + PruningKeepRecent: "0", + PruningKeepEvery: "0", + PruningInterval: "0", }, } } diff --git a/server/config/toml.go b/server/config/toml.go index e556ed57656b..6fb3f166f976 100644 --- a/server/config/toml.go +++ b/server/config/toml.go @@ -18,6 +18,17 @@ const defaultConfigTemplate = `# This is a TOML config file. # specified in this config (e.g. 0.25token1;0.0001token2). minimum-gas-prices = "{{ .BaseConfig.MinGasPrices }}" +# default: the last 100 states are kept in addition to every 500th state; pruning at 10 block intervals +# nothing: all historic states will be saved, nothing will be deleted (i.e. archiving node) +# everything: all saved states will be deleted, storing only the current state; pruning at 10 block intervals +# custom: allow pruning options to be manually specified through 'pruning-keep-recent', 'pruning-keep-every', and 'pruning-interval' +pruning = "{{ .BaseConfig.Pruning }}" + +# These are applied if and only if the pruning strategy is custom. +pruning-keep-recent = "{{ .BaseConfig.PruningKeepRecent }}" +pruning-keep-every = "{{ .BaseConfig.PruningKeepEvery }}" +pruning-interval = "{{ .BaseConfig.PruningInterval }}" + # HaltHeight contains a non-zero block height at which a node will gracefully # halt and shutdown that can be used to assist upgrades and testing. # @@ -33,12 +44,6 @@ halt-time = {{ .BaseConfig.HaltTime }} # InterBlockCache enables inter-block caching. inter-block-cache = {{ .BaseConfig.InterBlockCache }} - -# Pruning sets the pruning strategy: syncable, nothing, everything -# syncable: only those states not needed for state syncing will be deleted (keeps last 100 + every 10000th) -# nothing: all historic states will be saved, nothing will be deleted (i.e. archiving node) -# everything: all saved states will be deleted, storing only the current state -pruning = "{{ .BaseConfig.Pruning }}" ` var configTemplate *template.Template diff --git a/server/pruning.go b/server/pruning.go new file mode 100644 index 000000000000..26497f4b8863 --- /dev/null +++ b/server/pruning.go @@ -0,0 +1,38 @@ +package server + +import ( + "fmt" + "strings" + + "github.com/spf13/viper" + + "github.com/cosmos/cosmos-sdk/store" + "github.com/cosmos/cosmos-sdk/store/types" +) + +// GetPruningOptionsFromFlags parses command flags and returns the correct +// PruningOptions. If a pruning strategy is provided, that will be parsed and +// returned, otherwise, it is assumed custom pruning options are provided. +func GetPruningOptionsFromFlags() (types.PruningOptions, error) { + strategy := strings.ToLower(viper.GetString(FlagPruning)) + + switch strategy { + case types.PruningOptionDefault, types.PruningOptionNothing, types.PruningOptionEverything: + return types.NewPruningOptionsFromString(strategy), nil + + case types.PruningOptionCustom: + opts := types.NewPruningOptions( + viper.GetUint64(FlagPruningKeepRecent), + viper.GetUint64(FlagPruningKeepEvery), viper.GetUint64(FlagPruningInterval), + ) + + if err := opts.Validate(); err != nil { + return opts, fmt.Errorf("invalid custom pruning options: %w", err) + } + + return opts, nil + + default: + return store.PruningOptions{}, fmt.Errorf("unknown pruning strategy %s", strategy) + } +} diff --git a/server/pruning_test.go b/server/pruning_test.go new file mode 100644 index 000000000000..d05f2d2c2902 --- /dev/null +++ b/server/pruning_test.go @@ -0,0 +1,64 @@ +package server + +import ( + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/require" + + "github.com/cosmos/cosmos-sdk/store/types" +) + +func TestGetPruningOptionsFromFlags(t *testing.T) { + tests := []struct { + name string + initParams func() + expectedOptions types.PruningOptions + wantErr bool + }{ + { + name: FlagPruning, + initParams: func() { + viper.Set(FlagPruning, types.PruningOptionNothing) + }, + expectedOptions: types.PruneNothing, + }, + { + name: "custom pruning options", + initParams: func() { + viper.Set(FlagPruning, types.PruningOptionCustom) + viper.Set(FlagPruningKeepRecent, 1234) + viper.Set(FlagPruningKeepEvery, 4321) + viper.Set(FlagPruningInterval, 10) + }, + expectedOptions: types.PruningOptions{ + KeepRecent: 1234, + KeepEvery: 4321, + Interval: 10, + }, + }, + { + name: types.PruningOptionDefault, + initParams: func() {}, + expectedOptions: types.PruneDefault, + }, + } + + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(j *testing.T) { + viper.Reset() + viper.SetDefault(FlagPruning, types.PruningOptionDefault) + tt.initParams() + + opts, err := GetPruningOptionsFromFlags() + if tt.wantErr { + require.Error(t, err) + return + } + + require.Equal(t, tt.expectedOptions, opts) + }) + } +} diff --git a/server/start.go b/server/start.go index 5a9cd0500884..c96c1e71c84f 100644 --- a/server/start.go +++ b/server/start.go @@ -16,6 +16,8 @@ import ( "github.com/tendermint/tendermint/p2p" pvm "github.com/tendermint/tendermint/privval" "github.com/tendermint/tendermint/proxy" + + storetypes "github.com/cosmos/cosmos-sdk/store/types" ) // Tendermint full-node start flags @@ -23,13 +25,17 @@ const ( flagWithTendermint = "with-tendermint" flagAddress = "address" flagTraceStore = "trace-store" - flagPruning = "pruning" flagCPUProfile = "cpu-profile" FlagMinGasPrices = "minimum-gas-prices" FlagHaltHeight = "halt-height" FlagHaltTime = "halt-time" FlagInterBlockCache = "inter-block-cache" FlagUnsafeSkipUpgrades = "unsafe-skip-upgrades" + + FlagPruning = "pruning" + FlagPruningKeepRecent = "pruning-keep-recent" + FlagPruningKeepEvery = "pruning-keep-every" + FlagPruningInterval = "pruning-interval" ) // StartCmd runs the service passed in, either stand-alone or in-process with @@ -41,11 +47,15 @@ func StartCmd(ctx *Context, appCreator AppCreator) *cobra.Command { Long: `Run the full node application with Tendermint in or out of process. By default, the application will run with Tendermint in process. -Pruning options can be provided via the '--pruning' flag. The options are as follows: +Pruning options can be provided via the '--pruning' flag or alternatively with '--pruning-keep-recent', +'pruning-keep-every', and 'pruning-interval' together. -syncable: only those states not needed for state syncing will be deleted (flushes every 100th to disk and keeps every 10000th) +For '--pruning' the options are as follows: + +default: the last 100 states are kept in addition to every 500th state; pruning at 10 block intervals nothing: all historic states will be saved, nothing will be deleted (i.e. archiving node) -everything: all saved states will be deleted, storing only the current state +everything: all saved states will be deleted, storing only the current state; pruning at 10 block intervals +custom: allow pruning options to be manually specified through 'pruning-keep-recent', 'pruning-keep-every', and 'pruning-interval' Node halting configurations exist in the form of two flags: '--halt-height' and '--halt-time'. During the ABCI Commit phase, the node will check if the current block height is greater than or equal to @@ -56,6 +66,10 @@ will not be able to commit subsequent blocks. For profiling and benchmarking purposes, CPU profiling can be enabled via the '--cpu-profile' flag which accepts a path for the resulting pprof file. `, + PreRunE: func(cmd *cobra.Command, args []string) error { + _, err := GetPruningOptionsFromFlags() + return err + }, RunE: func(cmd *cobra.Command, args []string) error { if !viper.GetBool(flagWithTendermint) { ctx.Logger.Info("starting ABCI without Tendermint") @@ -73,7 +87,6 @@ which accepts a path for the resulting pprof file. cmd.Flags().Bool(flagWithTendermint, true, "Run abci app embedded in-process with tendermint") cmd.Flags().String(flagAddress, "tcp://0.0.0.0:26658", "Listen address") cmd.Flags().String(flagTraceStore, "", "Enable KVStore tracing to an output file") - cmd.Flags().String(flagPruning, "syncable", "Pruning strategy: syncable, nothing, everything") cmd.Flags().String( FlagMinGasPrices, "", "Minimum gas prices to accept for transactions; Any fee in a tx must meet this minimum (e.g. 0.01photino;0.0001stake)", @@ -84,6 +97,15 @@ which accepts a path for the resulting pprof file. cmd.Flags().Bool(FlagInterBlockCache, true, "Enable inter-block caching") cmd.Flags().String(flagCPUProfile, "", "Enable CPU profiling and write to the provided file") + cmd.Flags().String(FlagPruning, storetypes.PruningOptionDefault, "Pruning strategy (default|nothing|everything|custom)") + cmd.Flags().Uint64(FlagPruningKeepRecent, 0, "Number of recent heights to keep on disk (ignored if pruning is not 'custom')") + cmd.Flags().Uint64(FlagPruningKeepEvery, 0, "Offset heights to keep on disk after 'keep-every' (ignored if pruning is not 'custom')") + cmd.Flags().Uint64(FlagPruningInterval, 0, "Height interval at which pruned heights are removed from disk (ignored if pruning is not 'custom')") + viper.BindPFlag(FlagPruning, cmd.Flags().Lookup(FlagPruning)) + viper.BindPFlag(FlagPruningKeepRecent, cmd.Flags().Lookup(FlagPruningKeepRecent)) + viper.BindPFlag(FlagPruningKeepEvery, cmd.Flags().Lookup(FlagPruningKeepEvery)) + viper.BindPFlag(FlagPruningInterval, cmd.Flags().Lookup(FlagPruningInterval)) + // add support for all Tendermint-specific command line options tcmd.AddNodeFlags(cmd) return cmd @@ -151,8 +173,6 @@ func startInProcess(ctx *Context, appCreator AppCreator) (*node.Node, error) { return nil, err } - UpgradeOldPrivValFile(cfg) - // create & start tendermint node tmNode, err := node.NewNode( cfg, diff --git a/server/tm_cmds.go b/server/tm_cmds.go index 2667e55a7df5..1f325c716547 100644 --- a/server/tm_cmds.go +++ b/server/tm_cmds.go @@ -44,10 +44,12 @@ func ShowValidatorCmd(ctx *Context) *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { cfg := ctx.Config - UpgradeOldPrivValFile(cfg) - privValidator := pvm.LoadOrGenFilePV( - cfg.PrivValidatorKeyFile(), cfg.PrivValidatorStateFile()) - valPubKey := privValidator.GetPubKey() + privValidator := pvm.LoadOrGenFilePV(cfg.PrivValidatorKeyFile(), cfg.PrivValidatorStateFile()) + + valPubKey, err := privValidator.GetPubKey() + if err != nil { + return err + } if viper.GetString(cli.OutputFlag) == "json" { return printlnJSON(valPubKey) @@ -75,9 +77,7 @@ func ShowAddressCmd(ctx *Context) *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { cfg := ctx.Config - UpgradeOldPrivValFile(cfg) - privValidator := pvm.LoadOrGenFilePV( - cfg.PrivValidatorKeyFile(), cfg.PrivValidatorStateFile()) + privValidator := pvm.LoadOrGenFilePV(cfg.PrivValidatorKeyFile(), cfg.PrivValidatorStateFile()) valConsAddr := (sdk.ConsAddress)(privValidator.GetAddress()) if viper.GetString(cli.OutputFlag) == "json" { diff --git a/server/util.go b/server/util.go index eb22c230268e..08f31fc664e0 100644 --- a/server/util.go +++ b/server/util.go @@ -19,7 +19,6 @@ import ( "github.com/tendermint/tendermint/libs/cli" tmflags "github.com/tendermint/tendermint/libs/cli/flags" "github.com/tendermint/tendermint/libs/log" - pvm "github.com/tendermint/tendermint/privval" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/codec" @@ -217,16 +216,6 @@ func TrapSignal(cleanupFunc func()) { }() } -// UpgradeOldPrivValFile converts old priv_validator.json file (prior to Tendermint 0.28) -// to the new priv_validator_key.json and priv_validator_state.json files. -func UpgradeOldPrivValFile(config *cfg.Config) { - if _, err := os.Stat(config.OldPrivValidatorFile()); !os.IsNotExist(err) { - if oldFilePV, err := pvm.LoadOldFilePV(config.OldPrivValidatorFile()); err == nil { - oldFilePV.Upgrade(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) - } - } -} - func skipInterface(iface net.Interface) bool { if iface.Flags&net.FlagUp == 0 { return true // interface down diff --git a/store/cache/cache_test.go b/store/cache/cache_test.go index 3fc5f9157dbc..93ece75083ed 100644 --- a/store/cache/cache_test.go +++ b/store/cache/cache_test.go @@ -20,7 +20,7 @@ func TestGetOrSetStoreCache(t *testing.T) { sKey := types.NewKVStoreKey("test") tree, err := iavl.NewMutableTree(db, 100) require.NoError(t, err) - store := iavlstore.UnsafeNewStore(tree, types.PruneNothing) + store := iavlstore.UnsafeNewStore(tree) store2 := mngr.GetStoreCache(sKey, store) require.NotNil(t, store2) @@ -34,7 +34,7 @@ func TestUnwrap(t *testing.T) { sKey := types.NewKVStoreKey("test") tree, err := iavl.NewMutableTree(db, 100) require.NoError(t, err) - store := iavlstore.UnsafeNewStore(tree, types.PruneNothing) + store := iavlstore.UnsafeNewStore(tree) _ = mngr.GetStoreCache(sKey, store) require.Equal(t, store, mngr.Unwrap(sKey)) @@ -48,7 +48,7 @@ func TestStoreCache(t *testing.T) { sKey := types.NewKVStoreKey("test") tree, err := iavl.NewMutableTree(db, 100) require.NoError(t, err) - store := iavlstore.UnsafeNewStore(tree, types.PruneNothing) + store := iavlstore.UnsafeNewStore(tree) kvStore := mngr.GetStoreCache(sKey, store) for i := uint(0); i < cache.DefaultCommitKVStoreCacheSize*2; i++ { diff --git a/store/iavl/store.go b/store/iavl/store.go index d1e1d9c4cae0..f5af4099bc0b 100644 --- a/store/iavl/store.go +++ b/store/iavl/store.go @@ -1,11 +1,9 @@ package iavl import ( - "fmt" "io" "sync" - "github.com/pkg/errors" "github.com/tendermint/iavl" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/crypto/merkle" @@ -31,39 +29,14 @@ var ( // Store Implements types.KVStore and CommitKVStore. type Store struct { - tree Tree - pruning types.PruningOptions + tree Tree } // LoadStore returns an IAVL Store as a CommitKVStore. Internally, it will load the // store's version (id) from the provided DB. An error is returned if the version // fails to load. -func LoadStore(db dbm.DB, id types.CommitID, pruning types.PruningOptions, lazyLoading bool) (types.CommitKVStore, error) { - if !pruning.IsValid() { - return nil, fmt.Errorf("pruning options are invalid: %v", pruning) - } - - var keepRecent int64 - - // Determine the value of keepRecent based on the following: - // - // If KeepEvery = 1, keepRecent should be 0 since there is no need to keep - // latest version in a in-memory cache. - // - // If KeepEvery > 1, keepRecent should be 1 so that state changes in between - // flushed states can be saved in the in-memory latest tree. - if pruning.KeepEvery == 1 { - keepRecent = 0 - } else { - keepRecent = 1 - } - - tree, err := iavl.NewMutableTreeWithOpts( - db, - dbm.NewMemDB(), - defaultIAVLCacheSize, - iavl.PruningOptions(pruning.KeepEvery, keepRecent), - ) +func LoadStore(db dbm.DB, id types.CommitID, lazyLoading bool) (types.CommitKVStore, error) { + tree, err := iavl.NewMutableTree(db, defaultIAVLCacheSize) if err != nil { return nil, err } @@ -79,8 +52,7 @@ func LoadStore(db dbm.DB, id types.CommitID, pruning types.PruningOptions, lazyL } return &Store{ - tree: tree, - pruning: pruning, + tree: tree, }, nil } @@ -90,10 +62,9 @@ func LoadStore(db dbm.DB, id types.CommitID, pruning types.PruningOptions, lazyL // CONTRACT: The IAVL tree should be fully loaded. // CONTRACT: PruningOptions passed in as argument must be the same as pruning options // passed into iavl.MutableTree -func UnsafeNewStore(tree *iavl.MutableTree, po types.PruningOptions) *Store { +func UnsafeNewStore(tree *iavl.MutableTree) *Store { return &Store{ - tree: tree, - pruning: po, + tree: tree, } } @@ -113,8 +84,7 @@ func (st *Store) GetImmutable(version int64) (*Store, error) { } return &Store{ - tree: &immutableTree{iTree}, - pruning: st.pruning, + tree: &immutableTree{iTree}, }, nil } @@ -123,25 +93,9 @@ func (st *Store) GetImmutable(version int64) (*Store, error) { func (st *Store) Commit() types.CommitID { hash, version, err := st.tree.SaveVersion() if err != nil { - // TODO: Do we want to extend Commit to allow returning errors? panic(err) } - // If the version we saved got flushed to disk, check if previous flushed - // version should be deleted. - if st.pruning.FlushVersion(version) { - previous := version - st.pruning.KeepEvery - - // Previous flushed version should only be pruned if the previous version is - // not a snapshot version OR if snapshotting is disabled (SnapshotEvery == 0). - if previous != 0 && !st.pruning.SnapshotVersion(previous) { - err := st.tree.DeleteVersion(previous) - if errCause := errors.Cause(err); errCause != nil && errCause != iavl.ErrVersionDoesNotExist { - panic(err) - } - } - } - return types.CommitID{ Version: version, Hash: hash, @@ -204,6 +158,13 @@ func (st *Store) Delete(key []byte) { st.tree.Remove(key) } +// DeleteVersions deletes a series of versions from the MutableTree. An error +// is returned if any single version is invalid or the delete fails. All writes +// happen in a single batch with a single commit. +func (st *Store) DeleteVersions(versions ...int64) error { + return st.tree.DeleteVersions(versions...) +} + // Implements types.KVStore. func (st *Store) Iterator(start, end []byte) types.Iterator { var iTree *iavl.ImmutableTree diff --git a/store/iavl/store_test.go b/store/iavl/store_test.go index ed138c84964f..7a65c1880c64 100644 --- a/store/iavl/store_test.go +++ b/store/iavl/store_test.go @@ -52,7 +52,7 @@ func newAlohaTree(t *testing.T, db dbm.DB) (*iavl.MutableTree, types.CommitID) { func TestGetImmutable(t *testing.T) { db := dbm.NewMemDB() tree, cID := newAlohaTree(t, db) - store := UnsafeNewStore(tree, types.PruneNothing) + store := UnsafeNewStore(tree) require.True(t, tree.Set([]byte("hello"), []byte("adios"))) hash, ver, err := tree.SaveVersion() @@ -82,7 +82,7 @@ func TestGetImmutable(t *testing.T) { func TestTestGetImmutableIterator(t *testing.T) { db := dbm.NewMemDB() tree, cID := newAlohaTree(t, db) - store := UnsafeNewStore(tree, types.PruneNothing) + store := UnsafeNewStore(tree) newStore, err := store.GetImmutable(cID.Version) require.NoError(t, err) @@ -105,7 +105,7 @@ func TestTestGetImmutableIterator(t *testing.T) { func TestIAVLStoreGetSetHasDelete(t *testing.T) { db := dbm.NewMemDB() tree, _ := newAlohaTree(t, db) - iavlStore := UnsafeNewStore(tree, types.PruneNothing) + iavlStore := UnsafeNewStore(tree) key := "hello" @@ -130,14 +130,14 @@ func TestIAVLStoreGetSetHasDelete(t *testing.T) { func TestIAVLStoreNoNilSet(t *testing.T) { db := dbm.NewMemDB() tree, _ := newAlohaTree(t, db) - iavlStore := UnsafeNewStore(tree, types.PruneNothing) + iavlStore := UnsafeNewStore(tree) require.Panics(t, func() { iavlStore.Set([]byte("key"), nil) }, "setting a nil value should panic") } func TestIAVLIterator(t *testing.T) { db := dbm.NewMemDB() tree, _ := newAlohaTree(t, db) - iavlStore := UnsafeNewStore(tree, types.PruneNothing) + iavlStore := UnsafeNewStore(tree) iter := iavlStore.Iterator([]byte("aloha"), []byte("hellz")) expected := []string{"aloha", "hello"} var i int @@ -213,7 +213,7 @@ func TestIAVLReverseIterator(t *testing.T) { tree, err := iavl.NewMutableTree(db, cacheSize) require.NoError(t, err) - iavlStore := UnsafeNewStore(tree, types.PruneNothing) + iavlStore := UnsafeNewStore(tree) iavlStore.Set([]byte{0x00}, []byte("0")) iavlStore.Set([]byte{0x00, 0x00}, []byte("0 0")) @@ -246,7 +246,7 @@ func TestIAVLPrefixIterator(t *testing.T) { tree, err := iavl.NewMutableTree(db, cacheSize) require.NoError(t, err) - iavlStore := UnsafeNewStore(tree, types.PruneNothing) + iavlStore := UnsafeNewStore(tree) iavlStore.Set([]byte("test1"), []byte("test1")) iavlStore.Set([]byte("test2"), []byte("test2")) @@ -310,7 +310,7 @@ func TestIAVLReversePrefixIterator(t *testing.T) { tree, err := iavl.NewMutableTree(db, cacheSize) require.NoError(t, err) - iavlStore := UnsafeNewStore(tree, types.PruneNothing) + iavlStore := UnsafeNewStore(tree) iavlStore.Set([]byte("test1"), []byte("test1")) iavlStore.Set([]byte("test2"), []byte("test2")) @@ -373,95 +373,12 @@ func nextVersion(iavl *Store) { iavl.Commit() } -func TestIAVLDefaultPruning(t *testing.T) { - //Expected stored / deleted version numbers for: - //numRecent = 5, storeEvery = 3, snapshotEvery = 5 - var states = []pruneState{ - {[]int64{}, []int64{}}, - {[]int64{1}, []int64{}}, - {[]int64{1, 2}, []int64{}}, - {[]int64{1, 2, 3}, []int64{}}, - {[]int64{1, 2, 3, 4}, []int64{}}, - {[]int64{1, 2, 3, 4, 5}, []int64{}}, - {[]int64{2, 4, 5, 6}, []int64{1, 3}}, - {[]int64{4, 5, 6, 7}, []int64{1, 2, 3}}, - {[]int64{4, 5, 6, 7, 8}, []int64{1, 2, 3}}, - {[]int64{5, 6, 7, 8, 9}, []int64{1, 2, 3, 4}}, - {[]int64{6, 7, 8, 9, 10}, []int64{1, 2, 3, 4, 5}}, - {[]int64{6, 7, 8, 9, 10, 11}, []int64{1, 2, 3, 4, 5}}, - {[]int64{6, 8, 10, 11, 12}, []int64{1, 2, 3, 4, 5, 7, 9}}, - {[]int64{6, 10, 11, 12, 13}, []int64{1, 2, 3, 4, 5, 7, 8, 9}}, - {[]int64{6, 10, 11, 12, 13, 14}, []int64{1, 2, 3, 4, 5, 7, 8, 9}}, - {[]int64{6, 11, 12, 13, 14, 15}, []int64{1, 2, 3, 4, 5, 7, 8, 9, 10}}, - } - testPruning(t, int64(5), int64(3), int64(6), states) -} - -func TestIAVLAlternativePruning(t *testing.T) { - //Expected stored / deleted version numbers for: - //numRecent = 3, storeEvery = 5, snapshotEvery = 10 - var states = []pruneState{ - {[]int64{}, []int64{}}, - {[]int64{1}, []int64{}}, - {[]int64{1, 2}, []int64{}}, - {[]int64{1, 2, 3}, []int64{}}, - {[]int64{2, 3, 4}, []int64{1}}, - {[]int64{3, 4, 5}, []int64{1, 2}}, - {[]int64{4, 5, 6}, []int64{1, 2, 3}}, - {[]int64{5, 6, 7}, []int64{1, 2, 3, 4}}, - {[]int64{5, 6, 7, 8}, []int64{1, 2, 3, 4}}, - {[]int64{5, 7, 8, 9}, []int64{1, 2, 3, 4, 6}}, - {[]int64{8, 9, 10}, []int64{1, 2, 3, 4, 6, 7}}, - {[]int64{9, 10, 11}, []int64{1, 2, 3, 4, 6, 7, 8}}, - {[]int64{10, 11, 12}, []int64{1, 2, 3, 4, 5, 6, 7, 8, 9}}, - {[]int64{10, 11, 12, 13}, []int64{1, 2, 3, 4, 5, 6, 7, 8, 9}}, - {[]int64{10, 12, 13, 14}, []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 11}}, - {[]int64{10, 13, 14, 15}, []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12}}, - } - testPruning(t, int64(3), int64(5), int64(10), states) -} - -type pruneState struct { - stored []int64 - deleted []int64 -} - -func testPruning(t *testing.T, numRecent int64, storeEvery int64, snapshotEvery int64, states []pruneState) { - db := dbm.NewMemDB() - pruningOpts := types.PruningOptions{ - KeepEvery: storeEvery, - SnapshotEvery: snapshotEvery, - } - iavlOpts := iavl.PruningOptions(storeEvery, numRecent) - - tree, err := iavl.NewMutableTreeWithOpts(db, dbm.NewMemDB(), cacheSize, iavlOpts) - require.NoError(t, err) - - iavlStore := UnsafeNewStore(tree, pruningOpts) - - for step, state := range states { - for _, ver := range state.stored { - require.True(t, iavlStore.VersionExists(ver), - "missing version %d with latest version %d; should save last %d, store every %d, and snapshot every %d", - ver, step, numRecent, storeEvery, snapshotEvery) - } - - for _, ver := range state.deleted { - require.False(t, iavlStore.VersionExists(ver), - "not pruned version %d with latest version %d; should prune all but last %d and every %d with intermediate flush interval %d", - ver, step, numRecent, snapshotEvery, storeEvery) - } - - nextVersion(iavlStore) - } -} - func TestIAVLNoPrune(t *testing.T) { db := dbm.NewMemDB() tree, err := iavl.NewMutableTree(db, cacheSize) require.NoError(t, err) - iavlStore := UnsafeNewStore(tree, types.PruneNothing) + iavlStore := UnsafeNewStore(tree) nextVersion(iavlStore) for i := 1; i < 100; i++ { @@ -475,37 +392,12 @@ func TestIAVLNoPrune(t *testing.T) { } } -func TestIAVLPruneEverything(t *testing.T) { - db := dbm.NewMemDB() - iavlOpts := iavl.PruningOptions(0, 1) // only store latest version in memory - - tree, err := iavl.NewMutableTreeWithOpts(db, dbm.NewMemDB(), cacheSize, iavlOpts) - require.NoError(t, err) - - iavlStore := UnsafeNewStore(tree, types.PruneEverything) - nextVersion(iavlStore) - - for i := 1; i < 100; i++ { - for j := 1; j < i; j++ { - require.False(t, iavlStore.VersionExists(int64(j)), - "not pruned version %d with latest version %d; should prune all old versions", - j, i) - } - - require.True(t, iavlStore.VersionExists(int64(i)), - "missing current version on step %d; should not prune current state tree", - i) - - nextVersion(iavlStore) - } -} - func TestIAVLStoreQuery(t *testing.T) { db := dbm.NewMemDB() tree, err := iavl.NewMutableTree(db, cacheSize) require.NoError(t, err) - iavlStore := UnsafeNewStore(tree, types.PruneNothing) + iavlStore := UnsafeNewStore(tree) k1, v1 := []byte("key1"), []byte("val1") k2, v2 := []byte("key2"), []byte("val2") @@ -604,7 +496,7 @@ func BenchmarkIAVLIteratorNext(b *testing.B) { tree.Set(key, value) } - iavlStore := UnsafeNewStore(tree, types.PruneNothing) + iavlStore := UnsafeNewStore(tree) iterators := make([]types.Iterator, b.N/treeSize) for i := 0; i < len(iterators); i++ { diff --git a/store/iavl/tree.go b/store/iavl/tree.go index deae294b2c5d..9873034fa550 100644 --- a/store/iavl/tree.go +++ b/store/iavl/tree.go @@ -23,6 +23,7 @@ type ( Remove(key []byte) ([]byte, bool) SaveVersion() ([]byte, int64, error) DeleteVersion(version int64) error + DeleteVersions(versions ...int64) error Version() int64 Hash() []byte VersionExists(version int64) bool @@ -55,6 +56,10 @@ func (it *immutableTree) DeleteVersion(_ int64) error { panic("cannot call 'DeleteVersion' on an immutable IAVL tree") } +func (it *immutableTree) DeleteVersions(_ ...int64) error { + panic("cannot call 'DeleteVersions' on an immutable IAVL tree") +} + func (it *immutableTree) VersionExists(version int64) bool { return it.Version() == version } diff --git a/store/prefix/store_test.go b/store/prefix/store_test.go index eb1f0a4d4433..a4f8e6bbc0a5 100644 --- a/store/prefix/store_test.go +++ b/store/prefix/store_test.go @@ -86,7 +86,7 @@ func TestIAVLStorePrefix(t *testing.T) { db := dbm.NewMemDB() tree, err := tiavl.NewMutableTree(db, cacheSize) require.NoError(t, err) - iavlStore := iavl.UnsafeNewStore(tree, types.PruneNothing) + iavlStore := iavl.UnsafeNewStore(tree) testPrefixStore(t, iavlStore, []byte("test")) } diff --git a/store/reexport.go b/store/reexport.go index 9e4dba0ed5f6..61a3d9f795a3 100644 --- a/store/reexport.go +++ b/store/reexport.go @@ -30,10 +30,3 @@ type ( GasMeter = types.GasMeter GasConfig = types.GasConfig ) - -// nolint - reexport -var ( - PruneNothing = types.PruneNothing - PruneEverything = types.PruneEverything - PruneSyncable = types.PruneSyncable -) diff --git a/store/rootmulti/proof_test.go b/store/rootmulti/proof_test.go index 4d316f2c85aa..676fc777cd7e 100644 --- a/store/rootmulti/proof_test.go +++ b/store/rootmulti/proof_test.go @@ -14,7 +14,7 @@ import ( func TestVerifyIAVLStoreQueryProof(t *testing.T) { // Create main tree for testing. db := dbm.NewMemDB() - iStore, err := iavl.LoadStore(db, types.CommitID{}, types.PruneNothing, false) + iStore, err := iavl.LoadStore(db, types.CommitID{}, false) store := iStore.(*iavl.Store) require.Nil(t, err) store.Set([]byte("MYKEY"), []byte("MYVALUE")) diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index 42c7b05ab589..b0dc4c269657 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -5,6 +5,8 @@ import ( "io" "strings" + "github.com/pkg/errors" + iavltree "github.com/tendermint/iavl" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/crypto/merkle" "github.com/tendermint/tendermint/crypto/tmhash" @@ -21,6 +23,7 @@ import ( const ( latestVersionKey = "s/latest" + pruneHeightsKey = "s/pruneheights" commitInfoKeyFmt = "s/%d" // s/ ) @@ -35,6 +38,7 @@ type Store struct { stores map[types.StoreKey]types.CommitKVStore keysByName map[string]types.StoreKey lazyLoading bool + pruneHeights []int64 traceWriter io.Writer traceContext types.TraceContext @@ -42,8 +46,10 @@ type Store struct { interBlockCache types.MultiStorePersistentCache } -var _ types.CommitMultiStore = (*Store)(nil) -var _ types.Queryable = (*Store)(nil) +var ( + _ types.CommitMultiStore = (*Store)(nil) + _ types.Queryable = (*Store)(nil) +) // NewStore returns a reference to a new Store object with the provided DB. The // store will be created with a PruneNothing pruning strategy by default. After @@ -56,20 +62,15 @@ func NewStore(db dbm.DB) *Store { storesParams: make(map[types.StoreKey]storeParams), stores: make(map[types.StoreKey]types.CommitKVStore), keysByName: make(map[string]types.StoreKey), + pruneHeights: make([]int64, 0), } } // SetPruning sets the pruning strategy on the root store and all the sub-stores. // Note, calling SetPruning on the root store prior to LoadVersion or // LoadLatestVersion performs a no-op as the stores aren't mounted yet. -// -// TODO: Consider removing this API altogether on sub-stores as a pruning -// strategy should only be provided on initialization. func (rs *Store) SetPruning(pruningOpts types.PruningOptions) { rs.pruningOpts = pruningOpts - for _, substore := range rs.stores { - substore.SetPruning(pruningOpts) - } } // SetLazyLoading sets if the iavl store should be loaded lazily or not @@ -200,6 +201,12 @@ func (rs *Store) loadVersion(ver int64, upgrades *types.StoreUpgrades) error { rs.lastCommitInfo = cInfo rs.stores = newStores + // load any pruned heights we missed from disk to be pruned on the next run + ph, err := getPruningHeights(rs.db) + if err == nil && len(ph) > 0 { + rs.pruneHeights = ph + } + return nil } @@ -286,22 +293,59 @@ func (rs *Store) LastCommitID() types.CommitID { // Implements Committer/CommitStore. func (rs *Store) Commit() types.CommitID { - - // Commit stores. - version := rs.lastCommitInfo.Version + 1 + previousHeight := rs.lastCommitInfo.Version + version := previousHeight + 1 rs.lastCommitInfo = commitStores(version, rs.stores) - // write CommitInfo to disk only if this version was flushed to disk - if rs.pruningOpts.FlushVersion(version) { - flushCommitInfo(rs.db, version, rs.lastCommitInfo) + // Determine if pruneHeight height needs to be added to the list of heights to + // be pruned, where pruneHeight = (commitHeight - 1) - KeepRecent. + if int64(rs.pruningOpts.KeepRecent) < previousHeight { + pruneHeight := previousHeight - int64(rs.pruningOpts.KeepRecent) + // We consider this height to be pruned iff: + // + // - KeepEvery is zero as that means that all heights should be pruned. + // - KeepEvery % (height - KeepRecent) != 0 as that means the height is not + // a 'snapshot' height. + if rs.pruningOpts.KeepEvery == 0 || pruneHeight%int64(rs.pruningOpts.KeepEvery) != 0 { + rs.pruneHeights = append(rs.pruneHeights, pruneHeight) + } + } + + // batch prune if the current height is a pruning interval height + if rs.pruningOpts.Interval > 0 && version%int64(rs.pruningOpts.Interval) == 0 { + rs.pruneStores() } - // Prepare for next version. - commitID := types.CommitID{ + flushMetadata(rs.db, version, rs.lastCommitInfo, rs.pruneHeights) + + return types.CommitID{ Version: version, Hash: rs.lastCommitInfo.Hash(), } - return commitID +} + +// pruneStores will batch delete a list of heights from each mounted sub-store. +// Afterwards, pruneHeights is reset. +func (rs *Store) pruneStores() { + if len(rs.pruneHeights) == 0 { + return + } + + for key, store := range rs.stores { + if store.GetStoreType() == types.StoreTypeIAVL { + // If the store is wrapped with an inter-block cache, we must first unwrap + // it to get the underlying IAVL store. + store = rs.GetCommitKVStore(key) + + if err := store.(*iavl.Store).DeleteVersions(rs.pruneHeights...); err != nil { + if errCause := errors.Cause(err); errCause != nil && errCause != iavltree.ErrVersionDoesNotExist { + panic(err) + } + } + } + } + + rs.pruneHeights = make([]int64, 0) } // Implements CacheWrapper/Store/CommitStore. @@ -480,8 +524,6 @@ func parsePath(path string) (storeName string, subpath string, err error) { return storeName, subpath, nil } -//---------------------------------------- -// Note: why do we use key and params.key in different places. Seems like there should be only one key used. func (rs *Store) loadCommitStoreFromParams(key types.StoreKey, id types.CommitID, params storeParams) (types.CommitKVStore, error) { var db dbm.DB @@ -497,7 +539,7 @@ func (rs *Store) loadCommitStoreFromParams(key types.StoreKey, id types.CommitID panic("recursive MultiStores not yet supported") case types.StoreTypeIAVL: - store, err := iavl.LoadStore(db, id, rs.pruningOpts, rs.lazyLoading) + store, err := iavl.LoadStore(db, id, rs.lazyLoading) if err != nil { return nil, err } @@ -620,12 +662,6 @@ func getLatestVersion(db dbm.DB) int64 { return latest } -// Set the latest version. -func setLatestVersion(batch dbm.Batch, version int64) { - latestBytes, _ := cdc.MarshalBinaryLengthPrefixed(version) - batch.Set([]byte(latestVersionKey), latestBytes) -} - // Commits each store and returns a new commitInfo. func commitStores(version int64, storeMap map[types.StoreKey]types.CommitKVStore) commitInfo { storeInfos := make([]storeInfo, 0, len(storeMap)) @@ -651,9 +687,8 @@ func commitStores(version int64, storeMap map[types.StoreKey]types.CommitKVStore // Gets commitInfo from disk. func getCommitInfo(db dbm.DB, ver int64) (commitInfo, error) { - - // Get from DB. cInfoKey := fmt.Sprintf(commitInfoKeyFmt, ver) + cInfoBytes, err := db.Get([]byte(cInfoKey)) if err != nil { return commitInfo{}, fmt.Errorf("failed to get commit info: %v", err) @@ -671,20 +706,48 @@ func getCommitInfo(db dbm.DB, ver int64) (commitInfo, error) { return cInfo, nil } -// Set a commitInfo for given version. func setCommitInfo(batch dbm.Batch, version int64, cInfo commitInfo) { cInfoBytes := cdc.MustMarshalBinaryLengthPrefixed(cInfo) cInfoKey := fmt.Sprintf(commitInfoKeyFmt, version) batch.Set([]byte(cInfoKey), cInfoBytes) } -// flushCommitInfo flushes a commitInfo for given version to the DB. Note, this -// needs to happen atomically. -func flushCommitInfo(db dbm.DB, version int64, cInfo commitInfo) { +func setLatestVersion(batch dbm.Batch, version int64) { + latestBytes := cdc.MustMarshalBinaryLengthPrefixed(version) + batch.Set([]byte(latestVersionKey), latestBytes) +} + +func setPruningHeights(batch dbm.Batch, pruneHeights []int64) { + bz := cdc.MustMarshalBinaryBare(pruneHeights) + batch.Set([]byte(pruneHeightsKey), bz) +} + +func getPruningHeights(db dbm.DB) ([]int64, error) { + bz, err := db.Get([]byte(pruneHeightsKey)) + if err != nil { + return nil, fmt.Errorf("failed to get pruned heights: %w", err) + } + if len(bz) == 0 { + return nil, errors.New("no pruned heights found") + } + + var prunedHeights []int64 + if err := cdc.UnmarshalBinaryBare(bz, &prunedHeights); err != nil { + return nil, fmt.Errorf("failed to unmarshal pruned heights: %w", err) + } + + return prunedHeights, nil +} + +func flushMetadata(db dbm.DB, version int64, cInfo commitInfo, pruneHeights []int64) { batch := db.NewBatch() defer batch.Close() setCommitInfo(batch, version, cInfo) setLatestVersion(batch, version) - batch.Write() + setPruningHeights(batch, pruneHeights) + + if err := batch.Write(); err != nil { + panic(fmt.Errorf("error on batch write %w", err)) + } } diff --git a/store/rootmulti/store_test.go b/store/rootmulti/store_test.go index 164f231a2ac2..ddf876f45fdb 100644 --- a/store/rootmulti/store_test.go +++ b/store/rootmulti/store_test.go @@ -22,7 +22,7 @@ func TestStoreType(t *testing.T) { func TestGetCommitKVStore(t *testing.T) { var db dbm.DB = dbm.NewMemDB() - ms := newMultiStoreWithMounts(db, types.PruneSyncable) + ms := newMultiStoreWithMounts(db, types.PruneDefault) err := ms.LoadLatestVersion() require.Nil(t, err) @@ -166,7 +166,7 @@ func TestMultistoreCommitLoad(t *testing.T) { // XXX: confirm old commit is overwritten and we have rolled back // LatestVersion - store = newMultiStoreWithMounts(db, types.PruneSyncable) + store = newMultiStoreWithMounts(db, types.PruneDefault) err = store.LoadLatestVersion() require.Nil(t, err) commitID = getExpectedCommitID(store, ver+1) @@ -293,8 +293,9 @@ func TestParsePath(t *testing.T) { func TestMultiStoreRestart(t *testing.T) { db := dbm.NewMemDB() pruning := types.PruningOptions{ - KeepEvery: 3, - SnapshotEvery: 6, + KeepRecent: 2, + KeepEvery: 3, + Interval: 1, } multi := newMultiStoreWithMounts(db, pruning) err := multi.LoadLatestVersion() @@ -322,8 +323,8 @@ func TestMultiStoreRestart(t *testing.T) { multi.Commit() cinfo, err := getCommitInfo(multi.db, int64(i)) - require.NotNil(t, err) - require.Equal(t, commitInfo{}, cinfo) + require.NoError(t, err) + require.Equal(t, int64(i), cinfo.Version) } // Set and commit data in one store. @@ -347,15 +348,15 @@ func TestMultiStoreRestart(t *testing.T) { multi.Commit() postFlushCinfo, err := getCommitInfo(multi.db, 4) - require.NotNil(t, err) - require.Equal(t, commitInfo{}, postFlushCinfo, "Commit changed after in-memory commit") + require.NoError(t, err) + require.Equal(t, int64(4), postFlushCinfo.Version, "Commit changed after in-memory commit") multi = newMultiStoreWithMounts(db, pruning) err = multi.LoadLatestVersion() require.Nil(t, err) reloadedCid := multi.LastCommitID() - require.Equal(t, flushedCinfo.CommitID(), reloadedCid, "Reloaded CID is not the same as last flushed CID") + require.Equal(t, int64(4), reloadedCid.Version, "Reloaded CID is not the same as last flushed CID") // Check that store1 and store2 retained date from 3rd commit store1 = multi.getStoreByName("store1").(types.KVStore) @@ -369,7 +370,7 @@ func TestMultiStoreRestart(t *testing.T) { // Check that store3 still has data from last commit even though update happened on 2nd commit store3 = multi.getStoreByName("store3").(types.KVStore) val3 := store3.Get([]byte(k3)) - require.Equal(t, []byte(fmt.Sprintf("%s:%d", v3, 2)), val3, "Reloaded value not the same as last flushed value") + require.Equal(t, []byte(fmt.Sprintf("%s:%d", v3, 3)), val3, "Reloaded value not the same as last flushed value") } func TestMultiStoreQuery(t *testing.T) { @@ -442,6 +443,80 @@ func TestMultiStoreQuery(t *testing.T) { require.Equal(t, v2, qres.Value) } +func TestMultiStore_Pruning(t *testing.T) { + testCases := []struct { + name string + numVersions int64 + po types.PruningOptions + deleted []int64 + saved []int64 + }{ + {"prune nothing", 10, types.PruneNothing, nil, []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}}, + {"prune everything", 10, types.PruneEverything, []int64{1, 2, 3, 4, 5, 6, 7, 8, 9}, []int64{10}}, + {"prune some; no batch", 10, types.NewPruningOptions(2, 3, 1), []int64{1, 2, 4, 5, 7}, []int64{3, 6, 8, 9, 10}}, + {"prune some; small batch", 10, types.NewPruningOptions(2, 3, 3), []int64{1, 2, 4, 5}, []int64{3, 6, 7, 8, 9, 10}}, + {"prune some; large batch", 10, types.NewPruningOptions(2, 3, 11), nil, []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}}, + } + + for _, tc := range testCases { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + db := dbm.NewMemDB() + ms := newMultiStoreWithMounts(db, tc.po) + require.NoError(t, ms.LoadLatestVersion()) + + for i := int64(0); i < tc.numVersions; i++ { + ms.Commit() + } + + for _, v := range tc.saved { + _, err := ms.CacheMultiStoreWithVersion(v) + require.NoError(t, err, "expected error when loading height: %d", v) + } + + for _, v := range tc.deleted { + _, err := ms.CacheMultiStoreWithVersion(v) + require.Error(t, err, "expected error when loading height: %d", v) + } + }) + } +} + +func TestMultiStore_PruningRestart(t *testing.T) { + db := dbm.NewMemDB() + ms := newMultiStoreWithMounts(db, types.NewPruningOptions(2, 3, 11)) + require.NoError(t, ms.LoadLatestVersion()) + + // Commit enough to build up heights to prune, where on the next block we should + // batch delete. + for i := int64(0); i < 10; i++ { + ms.Commit() + } + + pruneHeights := []int64{1, 2, 4, 5, 7} + + // ensure we've persisted the current batch of heights to prune to the store's DB + ph, err := getPruningHeights(ms.db) + require.NoError(t, err) + require.Equal(t, pruneHeights, ph) + + // "restart" + ms = newMultiStoreWithMounts(db, types.NewPruningOptions(2, 3, 11)) + err = ms.LoadLatestVersion() + require.NoError(t, err) + require.Equal(t, pruneHeights, ms.pruneHeights) + + // commit one more block and ensure the heights have been pruned + ms.Commit() + require.Empty(t, ms.pruneHeights) + + for _, v := range pruneHeights { + _, err := ms.CacheMultiStoreWithVersion(v) + require.Error(t, err, "expected error when loading height: %d", v) + } +} + //----------------------------------------------------------------------- // utils diff --git a/store/store.go b/store/store.go index ba0311763814..2c068c413f47 100644 --- a/store/store.go +++ b/store/store.go @@ -8,13 +8,6 @@ import ( "github.com/cosmos/cosmos-sdk/store/types" ) -// Pruning strategies that may be provided to a KVStore to enable pruning. -const ( - PruningStrategyNothing = "nothing" - PruningStrategyEverything = "everything" - PruningStrategySyncable = "syncable" -) - func NewCommitMultiStore(db dbm.DB) types.CommitMultiStore { return rootmulti.NewStore(db) } @@ -22,17 +15,3 @@ func NewCommitMultiStore(db dbm.DB) types.CommitMultiStore { func NewCommitKVStoreCacheManager() types.MultiStorePersistentCache { return cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize) } - -func NewPruningOptionsFromString(strategy string) (opt PruningOptions) { - switch strategy { - case PruningStrategyNothing: - opt = PruneNothing - case PruningStrategyEverything: - opt = PruneEverything - case PruningStrategySyncable: - opt = PruneSyncable - default: - opt = PruneSyncable - } - return -} diff --git a/store/types/iterator_test.go b/store/types/iterator_test.go index e081de80a3d5..3086917b605b 100644 --- a/store/types/iterator_test.go +++ b/store/types/iterator_test.go @@ -12,7 +12,7 @@ import ( func newMemTestKVStore(t *testing.T) types.KVStore { db := dbm.NewMemDB() - store, err := iavl.LoadStore(db, types.CommitID{}, types.PruneNothing, false) + store, err := iavl.LoadStore(db, types.CommitID{}, false) require.NoError(t, err) return store } diff --git a/store/types/pruning.go b/store/types/pruning.go index c540443521ff..823da76c6585 100644 --- a/store/types/pruning.go +++ b/store/types/pruning.go @@ -1,66 +1,77 @@ package types +import "fmt" + +// Pruning option string constants +const ( + PruningOptionDefault = "default" + PruningOptionEverything = "everything" + PruningOptionNothing = "nothing" + PruningOptionCustom = "custom" +) + var ( - // PruneEverything defines a pruning strategy where all committed states will - // be deleted, persisting only the current state. - PruneEverything = PruningOptions{ - KeepEvery: 1, - SnapshotEvery: 0, - } + // PruneDefault defines a pruning strategy where the last 100 heights are kept + // in addition to every 100th and where to-be pruned heights are pruned at + // every 10th height. + PruneDefault = NewPruningOptions(100, 100, 10) - // PruneNothing defines a pruning strategy where all committed states will be - // kept on disk, i.e. no states will be pruned. - PruneNothing = PruningOptions{ - KeepEvery: 1, - SnapshotEvery: 1, - } + // PruneEverything defines a pruning strategy where all committed heights are + // deleted, storing only the current height and where to-be pruned heights are + // pruned at every 10th height. + PruneEverything = NewPruningOptions(0, 0, 10) - // PruneSyncable defines a pruning strategy where only those states not needed - // for state syncing will be pruned. It flushes every 100th state to disk and - // keeps every 10000th. - PruneSyncable = PruningOptions{ - KeepEvery: 100, - SnapshotEvery: 10000, - } + // PruneNothing defines a pruning strategy where all heights are kept on disk. + PruneNothing = NewPruningOptions(0, 1, 0) ) -// PruningOptions defines the specific pruning strategy every store in a multi-store -// will use when committing state, where keepEvery determines which committed -// heights are flushed to disk and snapshotEvery determines which of these heights -// are kept after pruning. +// PruningOptions defines the pruning strategy used when determining which +// heights are removed from disk when committing state. type PruningOptions struct { - KeepEvery int64 - SnapshotEvery int64 + // KeepRecent defines how many recent heights to keep on disk. + KeepRecent uint64 + + // KeepEvery defines how many offset heights are kept on disk past KeepRecent. + KeepEvery uint64 + + // Interval defines when the pruned heights are removed from disk. + Interval uint64 } -// IsValid verifies if the pruning options are valid. It returns false if invalid -// and true otherwise. Pruning options are considered valid iff: -// -// - KeepEvery > 0 -// - SnapshotEvery >= 0 -// - SnapshotEvery % KeepEvery = 0 -func (po PruningOptions) IsValid() bool { - // must flush at positive block interval - if po.KeepEvery <= 0 { - return false +func NewPruningOptions(keepRecent, keepEvery, interval uint64) PruningOptions { + return PruningOptions{ + KeepRecent: keepRecent, + KeepEvery: keepEvery, + Interval: interval, } +} - // cannot snapshot negative intervals - if po.SnapshotEvery < 0 { - return false +func (po PruningOptions) Validate() error { + if po.KeepEvery == 0 && po.Interval == 0 { + return fmt.Errorf("invalid 'Interval' when pruning everything: %d", po.Interval) + } + if po.KeepEvery == 1 && po.Interval != 0 { // prune nothing + return fmt.Errorf("invalid 'Interval' when pruning nothing: %d", po.Interval) + } + if po.KeepEvery > 1 && po.Interval == 0 { + return fmt.Errorf("invalid 'Interval' when pruning: %d", po.Interval) } - return po.SnapshotEvery%po.KeepEvery == 0 + return nil } -// FlushVersion returns a boolean signaling if the provided version/height should -// be flushed to disk. -func (po PruningOptions) FlushVersion(ver int64) bool { - return po.KeepEvery != 0 && ver%po.KeepEvery == 0 -} +func NewPruningOptionsFromString(strategy string) PruningOptions { + switch strategy { + case PruningOptionEverything: + return PruneEverything -// SnapshotVersion returns a boolean signaling if the provided version/height -// should be snapshotted (kept on disk). -func (po PruningOptions) SnapshotVersion(ver int64) bool { - return po.SnapshotEvery != 0 && ver%po.SnapshotEvery == 0 + case PruningOptionNothing: + return PruneNothing + + case PruningOptionDefault: + return PruneDefault + + default: + return PruneDefault + } } diff --git a/store/types/pruning_test.go b/store/types/pruning_test.go new file mode 100644 index 000000000000..2b88905ee8d2 --- /dev/null +++ b/store/types/pruning_test.go @@ -0,0 +1,29 @@ +package types + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPruningOptions_Validate(t *testing.T) { + testCases := []struct { + keepRecent uint64 + keepEvery uint64 + interval uint64 + expectErr bool + }{ + {100, 500, 10, false}, // default + {0, 0, 10, false}, // everything + {0, 1, 0, false}, // nothing + {0, 10, 10, false}, + {100, 0, 0, true}, // invalid interval + {0, 1, 5, true}, // invalid interval + } + + for _, tc := range testCases { + po := NewPruningOptions(tc.keepRecent, tc.keepEvery, tc.interval) + err := po.Validate() + require.Equal(t, tc.expectErr, err != nil, "options: %v, err: %s", po, err) + } +} diff --git a/store/types/store.go b/store/types/store.go index 377b43909e11..400b3f64b080 100644 --- a/store/types/store.go +++ b/store/types/store.go @@ -18,6 +18,8 @@ type Store interface { //nolint type Committer interface { Commit() CommitID LastCommitID() CommitID + + // TODO: Deprecate after 0.38.5 SetPruning(PruningOptions) } diff --git a/tests/util.go b/tests/util.go index 9b37d12c6988..c68595e7e3a3 100644 --- a/tests/util.go +++ b/tests/util.go @@ -10,10 +10,9 @@ import ( "time" "github.com/stretchr/testify/require" - - tmclient "github.com/tendermint/tendermint/rpc/client" + rpchttp "github.com/tendermint/tendermint/rpc/client/http" ctypes "github.com/tendermint/tendermint/rpc/core/types" - rpcclient "github.com/tendermint/tendermint/rpc/lib/client" + tmjsonrpc "github.com/tendermint/tendermint/rpc/jsonrpc/client" "github.com/cosmos/cosmos-sdk/codec" ) @@ -29,7 +28,7 @@ func WaitForNextHeightTM(port string) { func WaitForNextNBlocksTM(n int64, port string) { // get the latest block and wait for n more url := fmt.Sprintf("http://localhost:%v", port) - cl, err := tmclient.NewHTTP(url, "/websocket") + cl, err := rpchttp.New(url, "/websocket") if err != nil { panic(fmt.Sprintf("failed to create Tendermint HTTP client: %s", err)) } @@ -56,7 +55,7 @@ func WaitForHeightTM(height int64, port string) { } func waitForHeightTM(height int64, url string) { - cl, err := tmclient.NewHTTP(url, "/websocket") + cl, err := rpchttp.New(url, "/websocket") if err != nil { panic(fmt.Sprintf("failed to create Tendermint HTTP client: %s", err)) } @@ -186,7 +185,7 @@ func WaitForStart(url string) { // Wait for the RPC server to respond to /status func WaitForRPC(laddr string) { fmt.Println("LADDR", laddr) - client, err := rpcclient.NewJSONRPCClient(laddr) + client, err := tmjsonrpc.New(laddr) if err != nil { panic(fmt.Sprintf("failed to create Tendermint RPC client: %s", err)) } diff --git a/x/genutil/utils.go b/x/genutil/utils.go index 7d1a761ef562..4aea9d9fd500 100644 --- a/x/genutil/utils.go +++ b/x/genutil/utils.go @@ -11,8 +11,6 @@ import ( "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/privval" tmtypes "github.com/tendermint/tendermint/types" - - "github.com/cosmos/cosmos-sdk/server" ) // ExportGenesisFile creates and writes the genesis configuration to disk. An @@ -47,28 +45,28 @@ func ExportGenesisFileWithTime( } // InitializeNodeValidatorFiles creates private validator and p2p configuration files. -func InitializeNodeValidatorFiles(config *cfg.Config, -) (nodeID string, valPubKey crypto.PubKey, err error) { - +func InitializeNodeValidatorFiles(config *cfg.Config) (nodeID string, valPubKey crypto.PubKey, err error) { nodeKey, err := p2p.LoadOrGenNodeKey(config.NodeKeyFile()) if err != nil { - return nodeID, valPubKey, err + return "", nil, err } nodeID = string(nodeKey.ID()) - server.UpgradeOldPrivValFile(config) pvKeyFile := config.PrivValidatorKeyFile() if err := tmos.EnsureDir(filepath.Dir(pvKeyFile), 0777); err != nil { - return nodeID, valPubKey, nil + return "", nil, err } pvStateFile := config.PrivValidatorStateFile() if err := tmos.EnsureDir(filepath.Dir(pvStateFile), 0777); err != nil { - return nodeID, valPubKey, nil + return "", nil, err } - valPubKey = privval.LoadOrGenFilePV(pvKeyFile, pvStateFile).GetPubKey() + valPubKey, err = privval.LoadOrGenFilePV(pvKeyFile, pvStateFile).GetPubKey() + if err != nil { + return "", nil, err + } return nodeID, valPubKey, nil }