feat: merkle_root bundles migration

app/app.go

@@ -409,6 +409,7 @@ func New(
 			app.StakersKeeper,
 			app.StakingKeeper,
 			app.BankKeeper,
+			app.BundlesKeeper,
 		),
 	)
 

app/upgrades/v2_0/upgrade.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 
+	bundleskeeper "github.com/KYVENetwork/chain/x/bundles/keeper"
+
 	"github.com/KYVENetwork/chain/x/stakers/types"
 	bankkeeper "github.com/cosmos/cosmos-sdk/x/bank/keeper"
 
@@ -33,6 +35,7 @@ func CreateUpgradeHandler(
 	stakersKeeper *stakerskeeper.Keeper,
 	stakingKeeper *stakingkeeper.Keeper,
 	bankKeeper bankkeeper.Keeper,
+	bundlesKeeper bundleskeeper.Keeper,
 ) upgradetypes.UpgradeHandler {
 	return func(ctx context.Context, plan upgradetypes.Plan, fromVM module.VersionMap) (module.VersionMap, error) {
 		sdkCtx := sdk.UnwrapSDKContext(ctx)
@@ -45,6 +48,9 @@ func CreateUpgradeHandler(
 		// Run KYVE migrations
 		migrateProtocolStakers(sdkCtx, delegationKeeper, stakersKeeper, stakingKeeper, bankKeeper)
 
+		// Run Bundles Merkle Roots migrations
+		bundlesKeeper.SetBundlesMigrationUpgradeHeight(sdkCtx, uint64(sdkCtx.BlockHeight()))
+
 		logger.Info(fmt.Sprintf("finished upgrade %v", UpgradeName))
 
 		return migratedVersionMap, err

x/bundles/keeper/getters_migration.go

@@ -0,0 +1,30 @@
+package keeper
+
+import (
+	"encoding/binary"
+	"errors"
+
+	"github.com/KYVENetwork/chain/x/bundles/types"
+	"github.com/cosmos/cosmos-sdk/runtime"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+)
+
+func (k Keeper) GetBundlesMigrationUpgradeHeight(ctx sdk.Context) (uint64, error) {
+	storeAdapter := runtime.KVStoreAdapter(k.storeService.OpenKVStore(ctx))
+
+	if storeAdapter.Has(types.BundlesMigrationHeightKey) {
+		return binary.BigEndian.Uint64(storeAdapter.Get(types.BundlesMigrationHeightKey)), nil
+	}
+	return 0, errors.New("upgrade height can't be zero")
+}
+
+// SetBundlesMigrationUpgradeHeight stores the upgrade height of the v2.0 bundles migration
+// upgrade in the KV-Store.
+func (k Keeper) SetBundlesMigrationUpgradeHeight(ctx sdk.Context, upgradeHeight uint64) {
+	storeAdapter := runtime.KVStoreAdapter(k.storeService.OpenKVStore(ctx))
+
+	bz := make([]byte, 8)
+	binary.BigEndian.PutUint64(bz, upgradeHeight)
+
+	storeAdapter.Set(types.BundlesMigrationHeightKey, bz)
+}

x/bundles/keeper/keeper.go

@@ -6,8 +6,8 @@ import (
 	"cosmossdk.io/collections"
 	"cosmossdk.io/core/store"
 	"cosmossdk.io/log"
-
 	storetypes "cosmossdk.io/store/types"
+
 	"github.com/KYVENetwork/chain/util"
 	"github.com/KYVENetwork/chain/x/bundles/types"
 	"github.com/cosmos/cosmos-sdk/codec"
@@ -92,3 +92,13 @@ func (k Keeper) InitMemStore(gasCtx sdk.Context) {
 		memStoreInitialized = true
 	}
 }
+
+// TODO: remove after v2 migration
+func (k Keeper) Migration_GetStoreService() store.KVStoreService {
+	return k.storeService
+}
+
+// TODO: remove after v2 migration
+func (k Keeper) Migration_GetCodec() codec.BinaryCodec {
+	return k.cdc
+}

x/bundles/migration/README.md

@@ -0,0 +1,33 @@
+## Bundles Migration
+
+This migration is used to update the bundle summaries by adding the created Merkle roots that are
+required by the [Trustless API](https://docs.kyve.network/access-data-sets/trustless-api/overview).
+To create the correct Merkle proofs for the archived bundles, an approach was used that downloads
+the bundle and computes the correct Merkle proof. To validate the correct proof calculation, it was
+implemented in [Go and Python](https://github.com/KYVENetwork/merkle-script) , whereas it was tested with the TypeScript runtime implementation.
+
+These scripts created the `files/merkle_roots_pool_X` binary files, that are used for the v2 migration.
+Both Python and Go implementations were used to compute the hashes, which provide identical results:
+
+```
+merkle_roots_pool_0
+da4bb9bf0a60c5c79e399d8bb54ae4cf916f6c1dbdd5cdae45cb991f4e56158f
+
+merkle_roots_pool_1
+3c4eeb915cd01c6adea3241ea3536dfce5cec87017557b7e43d92c6ceec3096e  
+
+merkle_roots_pool_2
+754eb4680fe550cd3a7277ab0fc12c8f7ce794d18ca71d247561e40b05629c39  
+
+merkle_roots_pool_3
+df26b886928dbec03e84eca9b41c02b15ae7c5e7cf39ab540fcf381d3e1d27cc  
+
+merkle_roots_pool_5
+051efd6e44d7ac5bca41abb20aaf79d34dd095b5d6797d536bf13face7e397f9  
+
+merkle_roots_pool_7
+303d5ccaa18cc9e23298d599e3ba4c5bcf46f44d0fb5dd2cfdebcd02dcd8dc95 
+
+merkle_roots_pool_9
+e2f1c174350e5925d3f61b7adfb077f38507aec1562900b79c645099809ae617 
+```

x/bundles/migration/migration.go

@@ -0,0 +1,120 @@
+package migration
+
+import (
+	"embed"
+	_ "embed"
+	"encoding/hex"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"cosmossdk.io/log"
+	"cosmossdk.io/store/prefix"
+	"github.com/KYVENetwork/chain/util"
+	bundleskeeper "github.com/KYVENetwork/chain/x/bundles/keeper"
+	"github.com/KYVENetwork/chain/x/bundles/types"
+	"github.com/cosmos/cosmos-sdk/runtime"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+)
+
+var logger log.Logger
+
+const (
+	BundlesMigrationStepSizePerPool uint64 = 100
+	WaitingBlockPeriod              int64  = 1
+)
+
+//go:embed files/*
+var merkelRoots embed.FS
+
+type BundlesMigrationEntry struct {
+	merkleRoots []byte
+	poolId      uint64
+	maxBundleId uint64
+}
+
+// bundlesMigration includes the poolId and maxBundleId (exclusive) to determine which bundles are migrated
+var bundlesMigration []BundlesMigrationEntry
+
+func init() {
+	dir, err := merkelRoots.ReadDir("files")
+	if err != nil {
+		panic(err)
+	}
+
+	for _, file := range dir {
+		readFile, err := merkelRoots.ReadFile(fmt.Sprintf("files/%s", file.Name()))
+		if err != nil {
+			panic(err)
+		}
+
+		poolId, err := strconv.ParseUint(strings.ReplaceAll(file.Name(), "merkle_roots_pool_", ""), 10, 64)
+		if err != nil {
+			panic(err)
+		}
+
+		bundlesMigration = append(bundlesMigration, BundlesMigrationEntry{
+			merkleRoots: readFile,
+			poolId:      poolId,
+			maxBundleId: uint64(len(readFile)) / 32,
+		})
+	}
+}
+
+// MigrateBundlesModule migrates the bundles by adding the missing Merkle Roots to the bundle summary.
+func MigrateBundlesModule(sdkCtx sdk.Context, bundlesKeeper bundleskeeper.Keeper, upgradeHeight int64) {
+	logger = sdkCtx.Logger().With("upgrade", "bundles-migration")
+
+	if sdkCtx.BlockHeight()-upgradeHeight < WaitingBlockPeriod {
+		logger.Info("sdkCtx.BlockHeight()-upgradeHeight < WaitingBlockPeriod > return")
+		return
+	}
+
+	for _, bundlesMigrationEntry := range bundlesMigration {
+		step := sdkCtx.BlockHeight() - upgradeHeight - WaitingBlockPeriod
+		offset := uint64(step) * BundlesMigrationStepSizePerPool
+
+		// Skip if all bundles have already been migrated
+		if offset > bundlesMigrationEntry.maxBundleId+BundlesMigrationStepSizePerPool {
+			continue
+		}
+
+		if err := migrateFinalizedBundles(sdkCtx, bundlesKeeper, offset, bundlesMigrationEntry); err != nil {
+			panic(err)
+		}
+	}
+}
+
+// migrateFinalizedBundles sets the updated bundles for a certain range.
+func migrateFinalizedBundles(ctx sdk.Context, bundlesKeeper bundleskeeper.Keeper, offset uint64, bundlesMigrationEntry BundlesMigrationEntry) error {
+	// Init Bundles Store
+	storeAdapter := runtime.KVStoreAdapter(bundlesKeeper.Migration_GetStoreService().OpenKVStore(ctx))
+	store := prefix.NewStore(storeAdapter, util.GetByteKey(types.FinalizedBundlePrefix, bundlesMigrationEntry.poolId))
+
+	iterator := store.Iterator(util.GetByteKey(offset), util.GetByteKey(offset+BundlesMigrationStepSizePerPool))
+
+	var migratedBundles []types.FinalizedBundle
+
+	for ; iterator.Valid(); iterator.Next() {
+		var rawFinalizedBundle types.FinalizedBundle
+		if err := bundlesKeeper.Migration_GetCodec().Unmarshal(iterator.Value(), &rawFinalizedBundle); err != nil {
+			return err
+		}
+
+		if rawFinalizedBundle.Id >= bundlesMigrationEntry.maxBundleId {
+			break
+		}
+
+		merkleRoot := bundlesMigrationEntry.merkleRoots[rawFinalizedBundle.Id*32 : rawFinalizedBundle.Id*32+32]
+
+		rawFinalizedBundle.BundleSummary = fmt.Sprintf("{\"merkle_root\":\"%v\"}", hex.EncodeToString(merkleRoot))
+
+		migratedBundles = append(migratedBundles, rawFinalizedBundle)
+	}
+	iterator.Close()
+
+	for _, migratedBundle := range migratedBundles {
+		bundlesKeeper.SetFinalizedBundle(ctx, migratedBundle)
+	}
+	return nil
+}

x/bundles/module.go

@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/KYVENetwork/chain/x/bundles/migration"
+
 	"cosmossdk.io/core/appmodule"
 	"cosmossdk.io/core/store"
 	"cosmossdk.io/depinject"
@@ -175,6 +177,12 @@ func (am AppModule) BeginBlock(ctx context.Context) error {
 	sdkCtx := sdk.UnwrapSDKContext(ctx)
 	am.keeper.InitMemStore(sdkCtx)
 	SplitInflation(sdkCtx, am.keeper, am.bankKeeper, am.mintKeeper, am.poolKeeper, am.teamKeeper, am.upgradeKeeper)
+
+	upgradeHeight, err := am.keeper.GetBundlesMigrationUpgradeHeight(sdkCtx)
+	if err == nil {
+		migration.MigrateBundlesModule(sdkCtx, am.keeper, int64(upgradeHeight))
+	}
+
 	return nil
 }
 

x/bundles/types/keys.go

@@ -27,6 +27,8 @@ var (
 	FinalizedBundleVersionMapKey = []byte{3}
 	// RoundRobinProgressPrefix ...
 	RoundRobinProgressPrefix = []byte{4}
+	// BundlesMigrationHeightKey ...
+	BundlesMigrationHeightKey = []byte{5}
 
 	FinalizedBundleByIndexPrefix = []byte{11}
 )