feat(dot/state): Implement online pruning of state tries.

cmd/gossamer/config.go

@@ -25,6 +25,7 @@ import (
 	"github.com/ChainSafe/gossamer/dot"
 	ctoml "github.com/ChainSafe/gossamer/dot/config/toml"
 	"github.com/ChainSafe/gossamer/dot/state"
+	"github.com/ChainSafe/gossamer/dot/state/pruner"
 	"github.com/ChainSafe/gossamer/dot/types"
 	"github.com/ChainSafe/gossamer/lib/common"
 	"github.com/ChainSafe/gossamer/lib/genesis"
@@ -443,6 +444,9 @@ func setDotGlobalConfigFromToml(tomlCfg *ctoml.Config, cfg *dot.GlobalConfig) {
 		}
 
 		cfg.MetricsPort = tomlCfg.Global.MetricsPort
+
+		cfg.RetainBlocks = tomlCfg.Global.RetainBlocks
+		cfg.Pruning = pruner.Mode(tomlCfg.Global.Pruning)
 	}
 }
 
@@ -472,6 +476,8 @@ func setDotGlobalConfigFromFlags(ctx *cli.Context, cfg *dot.GlobalConfig) {
 		cfg.MetricsPort = uint32(metricsPort)
 	}
 
+	cfg.RetainBlocks = ctx.Int64(RetainBlockNumberFlag.Name)
+	cfg.Pruning = pruner.Mode(ctx.String(PruningFlag.Name))
 	cfg.NoTelemetry = ctx.Bool("no-telemetry")
 }
 

cmd/gossamer/export.go

@@ -79,11 +79,13 @@ func dotConfigToToml(dcfg *dot.Config) *ctoml.Config {
 	cfg := &ctoml.Config{}
 
 	cfg.Global = ctoml.GlobalConfig{
-		Name:        dcfg.Global.Name,
-		ID:          dcfg.Global.ID,
-		BasePath:    dcfg.Global.BasePath,
-		LogLvl:      dcfg.Global.LogLvl.String(),
-		MetricsPort: dcfg.Global.MetricsPort,
+		Name:         dcfg.Global.Name,
+		ID:           dcfg.Global.ID,
+		BasePath:     dcfg.Global.BasePath,
+		LogLvl:       dcfg.Global.LogLvl.String(),
+		MetricsPort:  dcfg.Global.MetricsPort,
+		RetainBlocks: dcfg.Global.RetainBlocks,
+		Pruning:      string(dcfg.Global.Pruning),
 	}
 
 	cfg.Log = ctoml.LogConfig{

cmd/gossamer/flags.go

@@ -283,7 +283,15 @@ var (
 	RetainBlockNumberFlag = cli.IntFlag{
 		Name:  "retain-blocks",
 		Usage: "Retain number of block from latest block while pruning",
-		Value: 256,
+		Value: 512,
+	}
+
+	// PruningFlag triggers the online pruning of historical state tries. It's either full or archive. To enable pruning the value
+	// should be set to `full`.
+	PruningFlag = cli.StringFlag{
+		Name:  "pruning",
+		Usage: `State trie online pruning ("full", "archive")`,
+		Value: "archive",
 	}
 )
 
@@ -301,7 +309,6 @@ var (
 		RewindFlag,
 		DBPathFlag,
 		BloomFilterSizeFlag,
-		RetainBlockNumberFlag,
 	}
 
 	// StartupFlags are flags that are valid for use with the root command and the export subcommand
@@ -346,6 +353,8 @@ var (
 	InitFlags = append([]cli.Flag{
 		ForceFlag,
 		GenesisFlag,
+		PruningFlag,
+		RetainBlockNumberFlag,
 	}, GlobalFlags...)
 
 	BuildSpecFlags = append([]cli.Flag{

cmd/gossamer/main.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/ChainSafe/gossamer/dot"
 	"github.com/ChainSafe/gossamer/dot/state"
+	"github.com/ChainSafe/gossamer/dot/state/pruner"
 	"github.com/ChainSafe/gossamer/lib/keystore"
 	"github.com/ChainSafe/gossamer/lib/utils"
 	log "github.com/ChainSafe/log15"
@@ -37,6 +38,7 @@ const (
 	importRuntimeCommandName = "import-runtime"
 	importStateCommandName   = "import-state"
 	pruningStateCommandName  = "prune-state"
+	defaultRetainBlocks      = 512
 )
 
 // app is the cli application
@@ -334,6 +336,18 @@ func initAction(ctx *cli.Context) error {
 	// from createDotConfig because dot config should not include expanded path)
 	cfg.Global.BasePath = utils.ExpandDir(cfg.Global.BasePath)
 
+	mode := pruner.Mode(ctx.String(PruningFlag.Name))
+	if !mode.IsValid() {
+		return fmt.Errorf("--%s must be either %s or %s", PruningFlag.Name, pruner.Full, pruner.Archive)
+	}
+	cfg.Global.Pruning = mode
+
+	rb := ctx.Int64(RetainBlockNumberFlag.Name)
+	if rb < defaultRetainBlocks {
+		return fmt.Errorf("--%s cannot be less than %d", RetainBlockNumberFlag.Name, defaultRetainBlocks)
+	}
+	cfg.Global.RetainBlocks = rb
+
 	// check if node has been initialised (expected false - no warning log)
 	if dot.NodeInitialized(cfg.Global.BasePath, false) {
 
@@ -441,14 +455,14 @@ func pruneState(ctx *cli.Context) error {
 	}
 
 	bloomSize := ctx.GlobalUint64(BloomFilterSizeFlag.Name)
-	retainBlocks := ctx.GlobalInt64(RetainBlockNumberFlag.Name)
+	retainBlocks := ctx.Int64(RetainBlockNumberFlag.Name)
 
-	pruner, err := state.NewPruner(inputDBPath, prunedDBPath, bloomSize, retainBlocks)
+	pruner, err := state.NewOfflinePruner(inputDBPath, prunedDBPath, bloomSize, retainBlocks)
 	if err != nil {
 		return err
 	}
 
-	logger.Info("Pruner initialised")
+	logger.Info("Offline pruner initialised")
 
 	err = pruner.SetBloomFilter()
 	if err != nil {

dot/build_spec.go

@@ -107,7 +107,11 @@ func BuildFromDB(path string) (*BuildSpec, error) {
 	tmpGen.Genesis.Raw = make(map[string]map[string]string)
 	tmpGen.Genesis.Runtime = make(map[string]map[string]interface{})
 
-	stateSrvc := state.NewService(path, log.LvlCrit)
+	config := state.Config{
+		Path:     path,
+		LogLevel: log.LvlInfo,
+	}
+	stateSrvc := state.NewService(config)
 
 	// start state service (initialise state database)
 	err := stateSrvc.Start()

dot/config.go

@@ -23,6 +23,7 @@ import (
 	"github.com/ChainSafe/gossamer/chain/gssmr"
 	"github.com/ChainSafe/gossamer/chain/kusama"
 	"github.com/ChainSafe/gossamer/chain/polkadot"
+	"github.com/ChainSafe/gossamer/dot/state/pruner"
 	"github.com/ChainSafe/gossamer/dot/types"
 	log "github.com/ChainSafe/log15"
 )
@@ -52,6 +53,8 @@ type GlobalConfig struct {
 	PublishMetrics bool
 	MetricsPort    uint32
 	NoTelemetry    bool
+	RetainBlocks   int64
+	Pruning        pruner.Mode
 }
 
 // LogConfig represents the log levels for individual packages

dot/config/toml/config.go

@@ -29,11 +29,13 @@ type Config struct {
 
 // GlobalConfig is to marshal/unmarshal toml global config vars
 type GlobalConfig struct {
-	Name        string `toml:"name,omitempty"`
-	ID          string `toml:"id,omitempty"`
-	BasePath    string `toml:"basepath,omitempty"`
-	LogLvl      string `toml:"log,omitempty"`
-	MetricsPort uint32 `toml:"metrics-port,omitempty"`
+	Name         string `toml:"name,omitempty"`
+	ID           string `toml:"id,omitempty"`
+	BasePath     string `toml:"basepath,omitempty"`
+	LogLvl       string `toml:"log,omitempty"`
+	MetricsPort  uint32 `toml:"metrics-port,omitempty"`
+	RetainBlocks int64  `toml:"retain-blocks,omitempty"`
+	Pruning      string `toml:"pruning,omitempty"`
 }
 
 // LogConfig represents the log levels for individual packages

dot/core/digest_test.go

@@ -35,7 +35,12 @@ import (
 func newTestDigestHandler(t *testing.T, withBABE, withGrandpa bool) *DigestHandler { //nolint
 	testDatadirPath, err := ioutil.TempDir("/tmp", "test-datadir-*")
 	require.NoError(t, err)
-	stateSrvc := state.NewService(testDatadirPath, log.LvlInfo)
+
+	config := state.Config{
+		Path:     testDatadirPath,
+		LogLevel: log.LvlInfo,
+	}
+	stateSrvc := state.NewService(config)
 	stateSrvc.UseMemDB()
 
 	gen, genTrie, genHeader := newTestGenesisWithTrieAndHeader(t)

dot/core/test_helpers.go

@@ -130,7 +130,11 @@ func NewTestService(t *testing.T, cfg *Config) *Service {
 	gen, genTrie, genHeader := newTestGenesisWithTrieAndHeader(t)
 
 	if cfg.BlockState == nil || cfg.StorageState == nil || cfg.TransactionState == nil || cfg.EpochState == nil {
-		stateSrvc = state.NewService(testDatadirPath, log.LvlInfo)
+		config := state.Config{
+			Path:     testDatadirPath,
+			LogLevel: log.LvlInfo,
+		}
+		stateSrvc = state.NewService(config)
 		stateSrvc.UseMemDB()
 
 		err = stateSrvc.Initialise(gen, genHeader, genTrie)

dot/import.go

@@ -45,7 +45,11 @@ func ImportState(basepath, stateFP, headerFP string, firstSlot uint64) error {
 
 	log.Info("ImportState", "header", header)
 
-	srv := state.NewService(basepath, log.LvlInfo)
+	config := state.Config{
+		Path:     basepath,
+		LogLevel: log.LvlInfo,
+	}
+	srv := state.NewService(config)
 	return srv.Import(header, tr, firstSlot)
 }
 

dot/node.go

@@ -31,6 +31,7 @@ import (
 	gssmrmetrics "github.com/ChainSafe/gossamer/dot/metrics"
 	"github.com/ChainSafe/gossamer/dot/network"
 	"github.com/ChainSafe/gossamer/dot/state"
+	"github.com/ChainSafe/gossamer/dot/state/pruner"
 	"github.com/ChainSafe/gossamer/dot/telemetry"
 	"github.com/ChainSafe/gossamer/dot/types"
 	"github.com/ChainSafe/gossamer/lib/genesis"
@@ -90,8 +91,20 @@ func InitNode(cfg *Config) error {
 		return fmt.Errorf("failed to create genesis block from trie: %w", err)
 	}
 
+	config := state.Config{
+		Path:     cfg.Global.BasePath,
+		LogLevel: cfg.Global.LogLvl,
+		PrunerCfg: struct {
+			Mode           pruner.Mode
+			RetainedBlocks int64
+		}{
+			Mode:           cfg.Global.Pruning,
+			RetainedBlocks: cfg.Global.RetainBlocks,
+		},
+	}
+
 	// create new state service
-	stateSrvc := state.NewService(cfg.Global.BasePath, cfg.Global.LogLvl)
+	stateSrvc := state.NewService(config)
 
 	// initialise state service with genesis data, block, and trie
 	err = stateSrvc.Initialise(gen, header, t)

dot/node_test.go

@@ -212,7 +212,11 @@ func TestInitNode_LoadGenesisData(t *testing.T) {
 	err := InitNode(cfg)
 	require.NoError(t, err)
 
-	stateSrvc := state.NewService(cfg.Global.BasePath, log.LvlTrace)
+	config := state.Config{
+		Path:     cfg.Global.BasePath,
+		LogLevel: log.LvlInfo,
+	}
+	stateSrvc := state.NewService(config)
 
 	gen, err := genesis.NewGenesisFromJSONRaw(genPath)
 	require.NoError(t, err)

dot/rpc/modules/chain_test.go

@@ -280,7 +280,12 @@ var gen, genTrie, genesisHeader = newTestGenesisWithTrieAndHeader()
 func newTestStateService(t *testing.T) *state.Service {
 	testDatadirPath, err := ioutil.TempDir("/tmp", "test-datadir-*")
 	require.NoError(t, err)
-	stateSrvc := state.NewService(testDatadirPath, log.LvlInfo)
+
+	config := state.Config{
+		Path:     testDatadirPath,
+		LogLevel: log.LvlInfo,
+	}
+	stateSrvc := state.NewService(config)
 	stateSrvc.UseMemDB()
 
 	err = stateSrvc.Initialise(gen, genesisHeader, genTrie)

dot/rpc/modules/state_test.go

@@ -429,7 +429,7 @@ func setupStateModule(t *testing.T) (*StateModule, *common.Hash, *common.Hash) {
 
 	sr1, err := ts.Root()
 	require.NoError(t, err)
-	err = chain.Storage.StoreTrie(ts)
+	err = chain.Storage.StoreTrie(ts, nil)
 	require.NoError(t, err)
 
 	err = chain.Block.AddBlock(&types.Block{

dot/rpc/modules/system_test.go

@@ -366,7 +366,7 @@ func setupSystemModule(t *testing.T) *SystemModule {
 	require.NoError(t, err)
 	ts.Set(aliceAcctStoKey, aliceAcctEncoded)
 
-	err = chain.Storage.StoreTrie(ts)
+	err = chain.Storage.StoreTrie(ts, nil)
 	require.NoError(t, err)
 	err = chain.Block.AddBlock(&types.Block{
 		Header: &types.Header{

dot/services.go

@@ -22,7 +22,6 @@ import (
 	"path/filepath"
 
 	"github.com/ChainSafe/chaindb"
-
 	"github.com/ChainSafe/gossamer/dot/core"
 	"github.com/ChainSafe/gossamer/dot/network"
 	"github.com/ChainSafe/gossamer/dot/rpc"
@@ -53,7 +52,13 @@ func newInMemoryDB(path string) (chaindb.Database, error) {
 // createStateService creates the state service and initialise state database
 func createStateService(cfg *Config) (*state.Service, error) {
 	logger.Debug("creating state service...")
-	stateSrvc := state.NewService(cfg.Global.BasePath, cfg.Log.StateLvl)
+
+	config := state.Config{
+		Path:     cfg.Global.BasePath,
+		LogLevel: cfg.Log.StateLvl,
+	}
+
+	stateSrvc := state.NewService(config)
 
 	// start state service (initialise state database)
 	err := stateSrvc.Start()

dot/state/base.go

@@ -21,10 +21,10 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/ChainSafe/chaindb"
+	"github.com/ChainSafe/gossamer/dot/state/pruner"
 	"github.com/ChainSafe/gossamer/lib/common"
 	"github.com/ChainSafe/gossamer/lib/genesis"
-
-	"github.com/ChainSafe/chaindb"
 )
 
 // BaseState is a wrapper for the chaindb.Database, without any prefixes
@@ -139,3 +139,26 @@ func (s *BaseState) loadFirstSlot() (uint64, error) {
 
 	return binary.LittleEndian.Uint64(data), nil
 }
+
+// storePruningData stores the pruner configuration.
+func (s *BaseState) storePruningData(mode pruner.Config) error {
+	encMode, err := json.Marshal(mode)
+	if err != nil {
+		return fmt.Errorf("cannot scale encode pruning mode: %s", err)
+	}
+
+	return s.db.Put(common.PruningKey, encMode)
+}
+
+// loadPruningData retrieves pruner configuration from db.
+func (s *BaseState) loadPruningData() (pruner.Config, error) {
+	data, err := s.db.Get(common.PruningKey)
+	if err != nil {
+		return pruner.Config{}, err
+	}
+
+	var mode pruner.Config
+	err = json.Unmarshal(data, &mode)
+
+	return mode, err
+}

dot/state/initialize.go

@@ -21,6 +21,8 @@ import (
 	"fmt"
 	"path/filepath"
 
+	"github.com/ChainSafe/chaindb"
+	"github.com/ChainSafe/gossamer/dot/state/pruner"
 	"github.com/ChainSafe/gossamer/dot/types"
 	"github.com/ChainSafe/gossamer/lib/blocktree"
 	"github.com/ChainSafe/gossamer/lib/genesis"
@@ -29,8 +31,6 @@ import (
 	"github.com/ChainSafe/gossamer/lib/runtime/wasmer"
 	"github.com/ChainSafe/gossamer/lib/trie"
 	"github.com/ChainSafe/gossamer/lib/utils"
-
-	"github.com/ChainSafe/chaindb"
 )
 
 // Initialise initialises the genesis state of the DB using the given storage trie. The trie should be loaded with the genesis storage state.
@@ -89,7 +89,7 @@ func (s *Service) Initialise(gen *genesis.Genesis, header *types.Header, t *trie
 	}
 
 	// create storage state from genesis trie
-	storageState, err := NewStorageState(db, blockState, t)
+	storageState, err := NewStorageState(db, blockState, t, pruner.Config{})
 	if err != nil {
 		return fmt.Errorf("failed to create storage state from trie: %s", err)
 	}
@@ -174,6 +174,10 @@ func (s *Service) storeInitialValues(data *genesis.Data, header *types.Header, t
 		return fmt.Errorf("failed to write genesis data to database: %s", err)
 	}
 
+	if err := s.Base.storePruningData(s.PrunerCfg); err != nil {
+		return fmt.Errorf("failed to write pruning data to database: %s", err)
+	}
+
 	return nil
 }