go/worker/storage: Remove separate storage sync status store

.changelog/4565.internal.md

@@ -0,0 +1,6 @@
+go/worker/storage: Remove separate storage sync status store
+
+Previously the worker maintaned a separate store that kept information about
+the progress of storage sync. Since it was a separate store this could cause
+problems if it got out of sync (e.g. due to partial manual copies). This
+should make the process more robust as there is only one source of truth.

go/consensus/tendermint/abci/prune.go

@@ -122,11 +122,7 @@ type genericPruner struct {
 
 func (p *genericPruner) Initialize() error {
 	// Figure out the eldest version currently present in the tree.
-	var err error
-	if p.earliestVersion, err = p.ndb.GetEarliestVersion(context.Background()); err != nil {
-		return fmt.Errorf("failed to get earliest version: %w", err)
-	}
-
+	p.earliestVersion = p.ndb.GetEarliestVersion()
 	// Initially, the earliest version is the last retained version.
 	p.lastRetainedVersion = p.earliestVersion
 

go/consensus/tendermint/abci/prune_test.go

@@ -46,38 +46,35 @@ func TestPruneKeepN(t *testing.T) {
 		require.NoError(err, "Finalize")
 	}
 
-	earliestVersion, err := ndb.GetEarliestVersion(ctx)
-	require.NoError(err, "GetEarliestVersion")
+	earliestVersion := ndb.GetEarliestVersion()
 	require.EqualValues(1, earliestVersion, "earliest version should be correct")
-	latestVersion, err := ndb.GetLatestVersion(ctx)
-	require.NoError(err, "GetLatestVersion")
+	latestVersion, exists := ndb.GetLatestVersion()
 	require.EqualValues(11, latestVersion, "latest version should be correct")
+	require.True(exists, "latest version should exist")
 
 	pruner, err := newStatePruner(&PruneConfig{
 		Strategy: PruneKeepN,
 		NumKept:  2,
 	}, ndb)
 	require.NoError(err, "newStatePruner failed")
 
-	earliestVersion, err = ndb.GetEarliestVersion(ctx)
-	require.NoError(err, "GetEarliestVersion")
+	earliestVersion = ndb.GetEarliestVersion()
 	require.EqualValues(1, earliestVersion, "earliest version should be correct")
-	latestVersion, err = ndb.GetLatestVersion(ctx)
-	require.NoError(err, "GetLatestVersion")
+	latestVersion, exists = ndb.GetLatestVersion()
 	require.EqualValues(11, latestVersion, "latest version should be correct")
+	require.True(exists, "latest version should exist")
 
 	lastRetainedVersion := pruner.GetLastRetainedVersion()
 	require.EqualValues(1, lastRetainedVersion, "last retained version should be correct")
 
 	err = pruner.Prune(ctx, 11)
 	require.NoError(err, "Prune")
 
-	earliestVersion, err = ndb.GetEarliestVersion(ctx)
-	require.NoError(err, "GetEarliestVersion")
+	earliestVersion = ndb.GetEarliestVersion()
 	require.EqualValues(9, earliestVersion, "earliest version should be correct")
-	latestVersion, err = ndb.GetLatestVersion(ctx)
-	require.NoError(err, "GetLatestVersion")
+	latestVersion, exists = ndb.GetLatestVersion()
 	require.EqualValues(11, latestVersion, "latest version should be correct")
+	require.True(exists, "latest version should exist")
 
 	lastRetainedVersion = pruner.GetLastRetainedVersion()
 	require.EqualValues(9, lastRetainedVersion, "last retained version should be correct")

go/consensus/tendermint/abci/state.go

@@ -498,10 +498,7 @@ func InitStateStorage(ctx context.Context, cfg *ApplicationConfig) (storage.Loca
 	}()
 
 	// Figure out the latest version/hash if any, and use that as the block height/hash.
-	latestVersion, err := ndb.GetLatestVersion(ctx)
-	if err != nil {
-		return nil, nil, nil, err
-	}
+	latestVersion, _ := ndb.GetLatestVersion()
 	roots, err := ndb.GetRootsForVersion(ctx, latestVersion)
 	if err != nil {
 		return nil, nil, nil, err

go/oasis-node/cmd/node/node.go

@@ -351,7 +351,6 @@ func (n *Node) initRuntimeWorkers() error {
 		n.CommonWorker,
 		n.RegistrationWorker,
 		n.Genesis,
-		n.commonStore,
 	)
 	if err != nil {
 		return err

go/storage/mkvs/checkpoint/checkpointer.go

@@ -218,10 +218,7 @@ func (c *checkpointer) maybeCheckpoint(ctx context.Context, version uint64, para
 	sort.Slice(cpVersions, func(i, j int) bool { return cpVersions[i] < cpVersions[j] })
 
 	// Make sure to not start earlier than the earliest version.
-	earlyVersion, err := c.ndb.GetEarliestVersion(ctx)
-	if err != nil {
-		return fmt.Errorf("checkpointer: failed to get earliest version: %w", err)
-	}
+	earlyVersion := c.ndb.GetEarliestVersion()
 	firstCheckpointVersion := lastCheckpointVersion + 1 // We can checkpoint the next version.
 	if firstCheckpointVersion < earlyVersion {
 		firstCheckpointVersion = earlyVersion

go/storage/mkvs/db/api/api.go

@@ -88,10 +88,12 @@ type NodeDB interface {
 	GetWriteLog(ctx context.Context, startRoot, endRoot node.Root) (writelog.Iterator, error)
 
 	// GetLatestVersion returns the most recent version in the node database.
-	GetLatestVersion(ctx context.Context) (uint64, error)
+	//
+	// The boolean flag signifies whether any version exists to disambiguate version zero.
+	GetLatestVersion() (uint64, bool)
 
 	// GetEarliestVersion returns the earliest version in the node database.
-	GetEarliestVersion(ctx context.Context) (uint64, error)
+	GetEarliestVersion() uint64
 
 	// GetRootsForVersion returns a list of roots stored under the given version.
 	GetRootsForVersion(ctx context.Context, version uint64) ([]node.Root, error)
@@ -216,12 +218,12 @@ func (d *nopNodeDB) GetWriteLog(ctx context.Context, startRoot, endRoot node.Roo
 	return nil, ErrWriteLogNotFound
 }
 
-func (d *nopNodeDB) GetLatestVersion(ctx context.Context) (uint64, error) {
-	return 0, nil
+func (d *nopNodeDB) GetLatestVersion() (uint64, bool) {
+	return 0, false
 }
 
-func (d *nopNodeDB) GetEarliestVersion(ctx context.Context) (uint64, error) {
-	return 0, nil
+func (d *nopNodeDB) GetEarliestVersion() uint64 {
+	return 0
 }
 
 func (d *nopNodeDB) GetRootsForVersion(ctx context.Context, version uint64) ([]node.Root, error) {

go/storage/mkvs/db/badger/badger.go

@@ -468,13 +468,12 @@ func (d *badgerNodeDB) GetWriteLog(ctx context.Context, startRoot, endRoot node.
 	return nil, api.ErrWriteLogNotFound
 }
 
-func (d *badgerNodeDB) GetLatestVersion(ctx context.Context) (uint64, error) {
-	version, _ := d.meta.getLastFinalizedVersion()
-	return version, nil
+func (d *badgerNodeDB) GetLatestVersion() (uint64, bool) {
+	return d.meta.getLastFinalizedVersion()
 }
 
-func (d *badgerNodeDB) GetEarliestVersion(ctx context.Context) (uint64, error) {
-	return d.meta.getEarliestVersion(), nil
+func (d *badgerNodeDB) GetEarliestVersion() uint64 {
+	return d.meta.getEarliestVersion()
 }
 
 func (d *badgerNodeDB) GetRootsForVersion(ctx context.Context, version uint64) (roots []node.Root, err error) {

go/storage/mkvs/tree_test.go

@@ -1318,12 +1318,11 @@ func testPruneBasic(t *testing.T, ndb db.NodeDB, factory NodeDBFactory) {
 	err = ndb.Finalize(ctx, []node.Root{root3})
 	require.NoError(t, err, "Finalize")
 
-	earliestVersion, err := ndb.GetEarliestVersion(ctx)
-	require.NoError(t, err, "GetEarliestVersion")
+	earliestVersion := ndb.GetEarliestVersion()
 	require.EqualValues(t, 0, earliestVersion, "earliest version should be correct")
-	latestVersion, err := ndb.GetLatestVersion(ctx)
-	require.NoError(t, err, "GetLatestVersion")
+	latestVersion, exists := ndb.GetLatestVersion()
 	require.EqualValues(t, 2, latestVersion, "latest version should be correct")
+	require.True(t, exists, "latest version should exist")
 
 	// Prune version 0.
 	err = ndb.Prune(ctx, 0)
@@ -1335,12 +1334,11 @@ func testPruneBasic(t *testing.T, ndb db.NodeDB, factory NodeDBFactory) {
 	require.NoError(t, err, "ndb.New")
 	defer ndb.Close()
 
-	earliestVersion, err = ndb.GetEarliestVersion(ctx)
-	require.NoError(t, err, "GetEarliestVersion")
+	earliestVersion = ndb.GetEarliestVersion()
 	require.EqualValues(t, 1, earliestVersion, "earliest version should be correct")
-	latestVersion, err = ndb.GetLatestVersion(ctx)
-	require.NoError(t, err, "GetLatestVersion")
+	latestVersion, exists = ndb.GetLatestVersion()
 	require.EqualValues(t, 2, latestVersion, "latest version should be correct")
+	require.True(t, exists, "latest version should exist")
 
 	// Keys must still be available in version 2.
 	tree = NewWithRoot(nil, ndb, node.Root{Namespace: testNs, Version: 2, Type: node.RootTypeState, Hash: rootHash3})

go/worker/client/service.go

@@ -139,10 +139,7 @@ func (s *service) GetLastRetainedBlock(ctx context.Context, runtimeID common.Nam
 	// we don't actually have state available. This may be because there is only a later checkpoint
 	// available.
 	if lsb, ok := rt.Storage().(storage.LocalBackend); ok {
-		version, err := lsb.NodeDB().GetEarliestVersion(ctx)
-		if err != nil {
-			return nil, err
-		}
+		version := lsb.NodeDB().GetEarliestVersion()
 
 		if version > blk.Header.Round {
 			blk, err = rt.History().GetBlock(ctx, version)