From 1821bd24475266c5caeb5f282e9cd5a3ac64a793 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Thu, 9 Jan 2025 15:47:54 +0100 Subject: [PATCH] More sims tests --- scripts/build/simulations.mk | 61 +++---- simapp/v2/sim_runner.go | 290 ++++++++++++++++++------------- simapp/v2/sim_test.go | 102 ++++++++++- simsx/environment.go | 3 + simsx/runner.go | 43 +++-- simsx/v2/valset_history.go | 17 +- types/simulation/config.go | 4 +- x/simulation/client/cli/flags.go | 10 +- 8 files changed, 338 insertions(+), 192 deletions(-) diff --git a/scripts/build/simulations.mk b/scripts/build/simulations.mk index 26b4661aa404..6a5a2d6cfe48 100644 --- a/scripts/build/simulations.mk +++ b/scripts/build/simulations.mk @@ -2,9 +2,10 @@ #? test-sim-nondeterminism: Run non-determinism test for simapp test-sim-nondeterminism: - # @echo "Running non-determinism test..." - # @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout=30m -tags='sims' -run TestAppStateDeterminism \ - # -NumBlocks=100 -BlockSize=200 -Period=0 + @echo "Running non-determinism test..." + @cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -timeout=30m -tags='sims' -run TestAppStateDeterminism \ + -NumBlocks=100 -BlockSize=200 + # Requires an exported plugin. See store/streaming/README.md for documentation. # @@ -18,51 +19,45 @@ test-sim-nondeterminism: test-sim-nondeterminism-streaming: # @echo "Running non-determinism-streaming test..." # @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout=30m -tags='sims' -run TestAppStateDeterminism \ - # -NumBlocks=100 -BlockSize=200 -Period=0 -EnableStreaming=true + # -NumBlocks=100 -BlockSize=200 -EnableStreaming=true test-sim-custom-genesis-fast: # @echo "Running custom genesis simulation..." # @echo "By default, ${HOME}/.simapp/config/genesis.json will be used." # @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout=30m -tags='sims' -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \ - # -NumBlocks=100 -BlockSize=200 -Seed=99 -Period=5 -SigverifyTx=false + # -NumBlocks=100 -BlockSize=200 -Seed=99 -SigverifyTx=false test-sim-import-export: - # @echo "Running application import/export simulation. This may take several minutes..." - # @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout 20m -tags='sims' -run TestAppImportExport \ - # -NumBlocks=50 -Period=5 +# @echo "Running application import/export simulation. This may take several minutes..." +# @cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -timeout 20m -tags='sims' -run TestAppImportExport \ + # -NumBlocks=50 test-sim-after-import: - # @echo "Running application simulation-after-import. This may take several minutes..." - # @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestAppSimulationAfterImport \ - # -NumBlocks=50 -Period=5 + @echo "Running application simulation-after-import. This may take several minutes..." + @cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestAppSimulationAfterImport \ + -NumBlocks=50 test-sim-custom-genesis-multi-seed: # @echo "Running multi-seed custom genesis simulation..." # @echo "By default, ${HOME}/.simapp/config/genesis.json will be used." - # @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \ - # -NumBlocks=400 -Period=5 + # @cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \ + # -NumBlocks=400 test-sim-multi-seed-long: - # @echo "Running long multi-seed application simulation. This may take awhile!" - # @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout=2h -tags='sims' -run TestFullAppSimulation \ - # -NumBlocks=150 -Period=50 - -test-sim-multi-seed-short: test-v2-sim - # @echo "Running short multi-seed application simulation. This may take awhile!" - # @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation \ - # -NumBlocks=50 -Period=10 -FauxMerkle=true + @echo "Running long multi-seed application simulation. This may take awhile!" + @cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -timeout=2h -tags='sims' -run TestFullAppSimulation \ + -NumBlocks=150 -.Phony: test-v2-sim -test-v2-sim: - @echo "Running short multi-seed application simulation. This may take awhile!" - @cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestSimsAppV2 \ -# -NumBlocks=50 -Period=10 -FauxMerkle=true +test-sim-multi-seed-short: + @echo "Running short multi-seed application simulation. This may take awhile!" + @cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation \ + -NumBlocks=50 test-sim-benchmark-invariants: # @echo "Running simulation invariant benchmarks..." # cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -benchmem -bench=BenchmarkInvariants -tags='sims' -run=^$ \ # -Enabled=true -NumBlocks=1000 -BlockSize=200 \ - # -Period=1 -Commit=true -Seed=57 -v -timeout 24h + # -Commit=true -Seed=57 -v -timeout 24h .PHONY: \ test-sim-nondeterminism \ @@ -81,20 +76,20 @@ SIM_COMMIT ?= true #? test-sim-fuzz: Run fuzz test for simapp test-sim-fuzz: - @echo "Running application fuzz for numBlocks=2, blockSize=20. This may take awhile!" +# @echo "Running application fuzz for numBlocks=2, blockSize=20. This may take awhile!" #ld flags are a quick fix to make it work on current osx - @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -json -tags='sims' -ldflags="-extldflags=-Wl,-ld_classic" -timeout=60m -fuzztime=60m -run=^$$ -fuzz=FuzzFullAppSimulation -GenesisTime=1714720615 -NumBlocks=2 -BlockSize=20 +# @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -json -tags='sims' -ldflags="-extldflags=-Wl,-ld_classic" -timeout=60m -fuzztime=60m -run=^$$ -fuzz=FuzzFullAppSimulation -GenesisTime=1714720615 -NumBlocks=2 -BlockSize=20 #? test-sim-benchmark: Run benchmark test for simapp test-sim-benchmark: - @echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!" - @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -tags='sims' -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \ +# @echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!" +# @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -tags='sims' -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \ -Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h test-sim-profile: - @echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!" - @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -tags='sims' -benchmem -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \ +# @echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!" +# @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -tags='sims' -benchmem -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \ -Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out .PHONY: test-sim-profile test-sim-benchmark test-sim-fuzz diff --git a/simapp/v2/sim_runner.go b/simapp/v2/sim_runner.go index 3ae56ad77878..38b0e021def9 100644 --- a/simapp/v2/sim_runner.go +++ b/simapp/v2/sim_runner.go @@ -27,7 +27,6 @@ import ( "cosmossdk.io/log" "cosmossdk.io/runtime/v2" "cosmossdk.io/server/v2/appmanager" - cometbfttypes "cosmossdk.io/server/v2/cometbft/types" storev2 "cosmossdk.io/store/v2" consensustypes "cosmossdk.io/x/consensus/types" @@ -89,6 +88,14 @@ type ( // SimulationApp abstract blockchain app SimulationApp[T Tx] interface { + appmanager.TransactionFuzzer[T] + InitGenesis( + ctx context.Context, + blockRequest *server.BlockRequest[T], + initGenesisJSON []byte, + txDecoder transaction.Codec[T], + ) (*server.BlockResponse, store.WriterMap, error) + GetApp() *runtime.App[T] TxConfig() client.TxConfig AppCodec() codec.Codec @@ -99,6 +106,7 @@ type ( // TestInstance system under test TestInstance[T Tx] struct { + Seed int64 App SimulationApp[T] TxDecoder transaction.Codec[T] BankKeeper BankKeeper @@ -110,18 +118,24 @@ type ( } AppFactory[T Tx, V SimulationApp[T]] func(config depinject.Config, outputs ...any) (V, error) + AppConfigFactory func() depinject.Config ) // SetupTestInstance initializes and returns the system under test. -func SetupTestInstance[T Tx, V SimulationApp[T]](t *testing.T, factory AppFactory[T, V], appConfig depinject.Config) TestInstance[T] { - t.Helper() +func SetupTestInstance[T Tx, V SimulationApp[T]]( + tb testing.TB, + appFactory AppFactory[T, V], + appConfigFactory AppConfigFactory, + seed int64, +) TestInstance[T] { + tb.Helper() vp := viper.New() vp.Set("store.app-db-backend", "memdb") - vp.Set("home", t.TempDir()) + vp.Set("home", tb.TempDir()) depInjCfg := depinject.Configs( depinject.Supply(log.NewNopLogger(), runtime.GlobalConfig(vp.AllSettings())), - appConfig, + appConfigFactory(), ) var ( bankKeeper BankKeeper @@ -134,11 +148,12 @@ func SetupTestInstance[T Tx, V SimulationApp[T]](t *testing.T, factory AppFactor &bankKeeper, &stKeeper, ) - require.NoError(t, err) + require.NoError(tb, err) - xapp, err := factory(depinject.Configs(depinject.Supply(log.NewNopLogger(), runtime.GlobalConfig(vp.AllSettings())))) - require.NoError(t, err) + xapp, err := appFactory(depinject.Configs(depinject.Supply(log.NewNopLogger(), runtime.GlobalConfig(vp.AllSettings())))) + require.NoError(tb, err) return TestInstance[T]{ + Seed: seed, App: xapp, BankKeeper: bankKeeper, AuthKeeper: authKeeper, @@ -150,80 +165,128 @@ func SetupTestInstance[T Tx, V SimulationApp[T]](t *testing.T, factory AppFactor } } +// InitializeChain sets up the blockchain with an initial state, validator set, and history using the provided genesis data. +func (ti TestInstance[T]) InitializeChain(tb testing.TB, + ctx context.Context, + chainID string, + genesisTimestamp time.Time, + initialHeight uint64, + genesisAppState json.RawMessage, +) ChainState[T] { + tb.Helper() + initRsp, stateRoot := doChainInitWithGenesis( + tb, + ctx, + chainID, + genesisTimestamp, + initialHeight, + genesisAppState, + ti, + ) + activeValidatorSet := simsxv2.NewValSet().Update(initRsp.ValidatorUpdates) + valsetHistory := simsxv2.NewValSetHistory(initialHeight) + valsetHistory.Add(genesisTimestamp, activeValidatorSet) + return ChainState[T]{ + ChainID: chainID, + BlockTime: genesisTimestamp, + BlockHeight: initialHeight, + ActiveValidatorSet: activeValidatorSet, + ValsetHistory: valsetHistory, + AppHash: stateRoot, + } +} + // RunWithSeeds runs a series of subtests using the default set of random seeds for deterministic simulation testing. -func RunWithSeeds[T Tx]( +func RunWithSeeds[T Tx, V SimulationApp[T]]( t *testing.T, + appFactory AppFactory[T, V], + appConfigFactory AppConfigFactory, seeds []int64, - postRunActions ...func(t testing.TB, app TestInstance[T], accs []simtypes.Account), + postRunActions ...func(t testing.TB, cs ChainState[T], app TestInstance[T], accs []simtypes.Account), ) { t.Helper() cfg := cli.NewConfigFromFlags() cfg.ChainID = SimAppChainID - for i := range seeds { - seed := seeds[i] + for _, seed := range seeds { t.Run(fmt.Sprintf("seed: %d", seed), func(t *testing.T) { t.Parallel() - RunWithSeed(t, NewSimApp[T], AppConfig(), cfg, seed, postRunActions...) + RunWithSeed(t, appFactory, appConfigFactory, cfg, seed, postRunActions...) }) } } // RunWithSeed initializes and executes a simulation run with the given seed, generating blocks and transactions. func RunWithSeed[T Tx, V SimulationApp[T]]( - t *testing.T, + tb testing.TB, appFactory AppFactory[T, V], - appConfig depinject.Config, + appConfigFactory AppConfigFactory, tCfg simtypes.Config, seed int64, - postRunActions ...func(t testing.TB, app TestInstance[T], accs []simtypes.Account), + postRunActions ...func(t testing.TB, cs ChainState[T], app TestInstance[T], accs []simtypes.Account), ) { - t.Helper() - r := rand.New(rand.NewSource(seed)) - testInstance := SetupTestInstance[T, V](t, appFactory, appConfig) - accounts, genesisAppState, chainID, genesisTimestamp := prepareInitialGenesisState(testInstance.App, r, testInstance.BankKeeper, tCfg, testInstance.ModuleManager) + tb.Helper() + initialBlockHeight := tCfg.InitialBlockHeight + require.NotEmpty(tb, initialBlockHeight, "initial block height must not be 0") + + setupFn := func(ctx context.Context, r *rand.Rand) (TestInstance[T], ChainState[T], []simtypes.Account) { + testInstance := SetupTestInstance[T, V](tb, appFactory, appConfigFactory, seed) + accounts, genesisAppState, chainID, genesisTimestamp := prepareInitialGenesisState( + testInstance.App, + r, + testInstance.BankKeeper, + tCfg, + testInstance.ModuleManager, + ) + cs := testInstance.InitializeChain( + tb, + ctx, + chainID, + genesisTimestamp, + initialBlockHeight, + genesisAppState, + ) + + return testInstance, cs, accounts + } + RunWithSeedX(tb, tCfg, setupFn, seed, postRunActions...) +} - appManager := testInstance.AppManager - appStore := testInstance.App.Store() - txConfig := testInstance.App.TxConfig() +// RunWithSeedX entrypoint for custom chain setups. +// The function runs the full simulation test circle for the specified seed and setup function, followed by optional post-run actions. +func RunWithSeedX[T Tx]( + tb testing.TB, + tCfg simtypes.Config, + setupChainStateFn func(ctx context.Context, r *rand.Rand) (TestInstance[T], ChainState[T], []simtypes.Account), + seed int64, + postRunActions ...func(t testing.TB, cs ChainState[T], app TestInstance[T], accs []simtypes.Account), +) { + tb.Helper() + r := rand.New(rand.NewSource(seed)) rootCtx, done := context.WithCancel(context.Background()) defer done() - initRsp, stateRoot := doChainInitWithGenesis(t, rootCtx, chainID, genesisTimestamp, appManager, testInstance.TxDecoder, genesisAppState, appStore) - activeValidatorSet := simsxv2.NewValSet().Update(initRsp.ValidatorUpdates) - valsetHistory := simsxv2.NewValSetHistory(1) - valsetHistory.Add(genesisTimestamp, activeValidatorSet) + + testInstance, chainState, accounts := setupChainStateFn(rootCtx, r) emptySimParams := make(map[string]json.RawMessage) // todo read sims params from disk as before modules := testInstance.ModuleManager.Modules() msgFactoriesFn := prepareSimsMsgFactories(r, modules, simsx.ParamWeightSource(emptySimParams)) - cs := chainState[T]{ - chainID: chainID, - blockTime: genesisTimestamp, - activeValidatorSet: activeValidatorSet, - valsetHistory: valsetHistory, - stateRoot: stateRoot, - app: appManager, - appStore: appStore, - txConfig: txConfig, - } doMainLoop( - t, + tb, rootCtx, - cs, + testInstance, + &chainState, msgFactoriesFn, r, - testInstance.AuthKeeper, - testInstance.BankKeeper, + tCfg, accounts, - testInstance.TXBuilder, - testInstance.StakingKeeper, ) - require.NoError(t, testInstance.App.Close(), "closing app") for _, step := range postRunActions { - step(t, testInstance, accounts) + step(tb, chainState, testInstance, accounts) } + require.NoError(tb, testInstance.App.Close(), "closing app") } // prepareInitialGenesisState initializes the genesis state for simulation by generating accounts, app state, chain ID, and timestamp. @@ -257,18 +320,20 @@ func prepareInitialGenesisState[T Tx]( // doChainInitWithGenesis initializes the blockchain state with the provided genesis data and returns the initial block response and state root. func doChainInitWithGenesis[T Tx]( - t *testing.T, + tb testing.TB, ctx context.Context, chainID string, genesisTimestamp time.Time, - app appmanager.AppManager[T], - txDecoder transaction.Codec[T], + initialHeight uint64, genesisAppState json.RawMessage, - appStore cometbfttypes.Store, + testInstance TestInstance[T], ) (*server.BlockResponse, store.Hash) { - t.Helper() + tb.Helper() + app := testInstance.App + txDecoder := testInstance.TxDecoder + appStore := testInstance.App.Store() genesisReq := &server.BlockRequest[T]{ - Height: 0, + Height: initialHeight, Time: genesisTimestamp, Hash: make([]byte, 32), ChainId: chainID, @@ -290,27 +355,25 @@ func doChainInitWithGenesis[T Tx]( } genesisCtx := context.WithValue(ctx, corecontext.CometParamsInitInfoKey, initialConsensusParams) initRsp, genesisStateChanges, err := app.InitGenesis(genesisCtx, genesisReq, genesisAppState, txDecoder) - require.NoError(t, err) + require.NoError(tb, err) - require.NoError(t, appStore.SetInitialVersion(0)) + require.NoError(tb, appStore.SetInitialVersion(initialHeight-1)) changeSet, err := genesisStateChanges.GetStateChanges() - require.NoError(t, err) + require.NoError(tb, err) - stateRoot, err := appStore.Commit(&store.Changeset{Changes: changeSet}) - require.NoError(t, err) + stateRoot, err := appStore.Commit(&store.Changeset{Changes: changeSet, Version: initialHeight - 1}) + require.NoError(tb, err) return initRsp, stateRoot } -// chainState represents the state of a blockchain during a simulation run. -type chainState[T Tx] struct { - chainID string - blockTime time.Time - activeValidatorSet simsxv2.WeightedValidators - valsetHistory *simsxv2.ValSetHistory - stateRoot store.Hash - app appmanager.TransactionFuzzer[T] - appStore storev2.RootStore - txConfig client.TxConfig +// ChainState represents the state of a blockchain during a simulation run. +type ChainState[T Tx] struct { + ChainID string + BlockTime time.Time + BlockHeight uint64 + ActiveValidatorSet simsxv2.WeightedValidators + ValsetHistory *simsxv2.ValSetHistory + AppHash store.Hash } // doMainLoop executes the main simulation loop after chain setup with genesis block. @@ -318,34 +381,23 @@ type chainState[T Tx] struct { // and executed. Events like validators missing votes or double signing are included in this // process. The runtime tracks the validator's state and history. func doMainLoop[T Tx]( - t *testing.T, + tb testing.TB, rootCtx context.Context, - cs chainState[T], + testInstance TestInstance[T], + cs *ChainState[T], nextMsgFactory func() simsx.SimMsgFactoryX, r *rand.Rand, - authKeeper AuthKeeper, - bankKeeper simsx.BalanceSource, + tCfg simtypes.Config, accounts []simtypes.Account, - txBuilder simsxv2.TXBuilder[T], - stakingKeeper StakingKeeper, ) { - t.Helper() - blockTime := cs.blockTime - activeValidatorSet := cs.activeValidatorSet - if len(activeValidatorSet) == 0 { - t.Fatal("no active validators in chain setup") + tb.Helper() + if len(cs.ActiveValidatorSet) == 0 { + tb.Fatal("no active validators in chain setup") return } - valsetHistory := cs.valsetHistory - stateRoot := cs.stateRoot - chainID := cs.chainID - app := cs.app - appStore := cs.appStore - - const ( // todo: read from CLI instead - numBlocks = 100 // 500 default - maxTXPerBlock = 200 // 200 default - ) + + numBlocks := tCfg.NumBlocks + maxTXPerBlock := tCfg.BlockSize var ( txSkippedCounter int @@ -354,39 +406,39 @@ func doMainLoop[T Tx]( rootReporter := simsx.NewBasicSimulationReporter() futureOpsReg := simsxv2.NewFutureOpsRegistry() - for i := 0; i < numBlocks; i++ { - if len(activeValidatorSet) == 0 { - t.Skipf("run out of validators in block: %d\n", i+1) + for end := cs.BlockHeight + numBlocks; cs.BlockHeight < end; cs.BlockHeight++ { + if len(cs.ActiveValidatorSet) == 0 { + tb.Skipf("run out of validators in block: %d\n", cs.BlockHeight) return } - blockTime = blockTime.Add(minTimePerBlock) - blockTime = blockTime.Add(time.Duration(int64(r.Intn(int(timeRangePerBlock/time.Second)))) * time.Second) - valsetHistory.Add(blockTime, activeValidatorSet) + cs.BlockTime = cs.BlockTime.Add(minTimePerBlock). + Add(time.Duration(int64(r.Intn(int(timeRangePerBlock/time.Second)))) * time.Second) + cs.ValsetHistory.Add(cs.BlockTime, cs.ActiveValidatorSet) blockReqN := &server.BlockRequest[T]{ - Height: uint64(1 + i), - Time: blockTime, - Hash: stateRoot, - AppHash: stateRoot, - ChainId: chainID, + Height: cs.BlockHeight, + Time: cs.BlockTime, + Hash: cs.AppHash, + AppHash: cs.AppHash, + ChainId: cs.ChainID, } cometInfo := comet.Info{ ValidatorsHash: nil, - Evidence: valsetHistory.MissBehaviour(r), - ProposerAddress: activeValidatorSet[0].Address, - LastCommit: activeValidatorSet.NewCommitInfo(r), + Evidence: cs.ValsetHistory.MissBehaviour(r), + ProposerAddress: cs.ActiveValidatorSet[0].Address, // todo: pick random one + LastCommit: cs.ActiveValidatorSet.NewCommitInfo(r), } - fOps, pos := futureOpsReg.PopScheduledFor(blockTime), 0 - addressCodec := cs.txConfig.SigningContext().AddressCodec() + fOps, pos := futureOpsReg.PopScheduledFor(cs.BlockTime), 0 + addressCodec := testInstance.App.TxConfig().SigningContext().AddressCodec() simsCtx := context.WithValue(rootCtx, corecontext.CometInfoKey, cometInfo) // required for ContextAwareCometInfoService resultHandlers := make([]simsx.SimDeliveryResultHandler, 0, maxTXPerBlock) var txPerBlockCounter int - blockRsp, updates, err := app.DeliverSims(simsCtx, blockReqN, func(ctx context.Context) iter.Seq[T] { + blockRsp, updates, err := testInstance.App.DeliverSims(simsCtx, blockReqN, func(ctx context.Context) iter.Seq[T] { return func(yield func(T) bool) { - unbondingTime, err := stakingKeeper.UnbondingTime(ctx) - require.NoError(t, err) - valsetHistory.SetMaxHistory(minBlocksInUnbondingPeriod(unbondingTime)) - testData := simsx.NewChainDataSource(ctx, r, authKeeper, bankKeeper, addressCodec, accounts...) + unbondingTime, err := testInstance.StakingKeeper.UnbondingTime(ctx) + require.NoError(tb, err) + cs.ValsetHistory.SetMaxHistory(minBlocksInUnbondingPeriod(unbondingTime)) + testData := simsx.NewChainDataSource(ctx, r, testInstance.AuthKeeper, testInstance.BankKeeper, addressCodec, accounts...) for txPerBlockCounter < maxTXPerBlock { txPerBlockCounter++ @@ -407,36 +459,36 @@ func doMainLoop[T Tx]( signers, msg := mergedMsgFactory.Create()(ctx, testData, reporter) if reporter.IsSkipped() { txSkippedCounter++ - require.NoError(t, reporter.Close()) + require.NoError(tb, reporter.Close()) continue } resultHandlers = append(resultHandlers, mergedMsgFactory.DeliveryResultHandler()) reporter.Success(msg) - require.NoError(t, reporter.Close()) + require.NoError(tb, reporter.Close()) - tx, err := txBuilder.Build(ctx, authKeeper, signers, msg, r, chainID) - require.NoError(t, err) + tx, err := testInstance.TXBuilder.Build(ctx, testInstance.AuthKeeper, signers, msg, r, cs.ChainID) + require.NoError(tb, err) if !yield(tx) { return } } } }) - require.NoError(t, err, "%d, %s", blockReqN.Height, blockReqN.Time) + require.NoError(tb, err, "%d, %s", blockReqN.Height, blockReqN.Time) changeSet, err := updates.GetStateChanges() - require.NoError(t, err) - stateRoot, err = appStore.Commit(&store.Changeset{ + require.NoError(tb, err) + cs.AppHash, err = testInstance.App.Store().Commit(&store.Changeset{ Version: blockReqN.Height, Changes: changeSet, }) - require.NoError(t, err) - require.Equal(t, len(resultHandlers), len(blockRsp.TxResults), "txPerBlockCounter: %d, totalSkipped: %d", txPerBlockCounter, txSkippedCounter) + require.NoError(tb, err) + require.Equal(tb, len(resultHandlers), len(blockRsp.TxResults), "txPerBlockCounter: %d, totalSkipped: %d", txPerBlockCounter, txSkippedCounter) for i, v := range blockRsp.TxResults { - require.NoError(t, resultHandlers[i](v.Error)) + require.NoError(tb, resultHandlers[i](v.Error)) } txTotalCounter += txPerBlockCounter - activeValidatorSet = activeValidatorSet.Update(blockRsp.ValidatorUpdates) + cs.ActiveValidatorSet = cs.ActiveValidatorSet.Update(blockRsp.ValidatorUpdates) } fmt.Println("+++ reporter:\n" + rootReporter.Summary().String()) fmt.Printf("Tx total: %d skipped: %d\n", txTotalCounter, txSkippedCounter) diff --git a/simapp/v2/sim_test.go b/simapp/v2/sim_test.go index c92aeaedefc3..ade890ad1de4 100644 --- a/simapp/v2/sim_test.go +++ b/simapp/v2/sim_test.go @@ -1,7 +1,103 @@ +//go:build sims + package simapp -import "testing" +import ( + "bytes" + "context" + "math/rand" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + simtypes "github.com/cosmos/cosmos-sdk/types/simulation" + v2 "github.com/cosmos/cosmos-sdk/x/genutil/v2" + simcli "github.com/cosmos/cosmos-sdk/x/simulation/client/cli" +) + +func init() { + simcli.GetSimulatorFlags() +} + +func TestFullAppSimulation(t *testing.T) { + RunWithSeeds[Tx](t, NewSimApp[Tx], AppConfig, defaultSeeds) +} + +func TestAppStateDeterminism(t *testing.T) { + var seeds []int64 + if s := simcli.NewConfigFromFlags().Seed; s != simcli.DefaultSeedValue { + // override defaults with user data + seeds = []int64{s, s, s} // run same simulation 3 times + } else { + seeds = []int64{ // some random seeds, tripled to ensure same app-hash on all runs + 1, 1, 1, + 3, 3, 3, + 5, 5, 5, + } + } + + var mx sync.Mutex + appHashResults := make(map[int64][]byte) + captureAndCheckHash := func(tb testing.TB, cs ChainState[Tx], ti TestInstance[Tx], _ []simtypes.Account) { + tb.Helper() + mx.Lock() + defer mx.Unlock() + otherHashes, ok := appHashResults[ti.Seed] + if !ok { + appHashResults[ti.Seed] = cs.AppHash + return + } + if !bytes.Equal(otherHashes, cs.AppHash) { + tb.Fatalf("non-determinism in seed %d", ti.Seed) + } + } + // run simulations + RunWithSeeds(t, NewSimApp[Tx], AppConfig, seeds, captureAndCheckHash) +} + +// ExportableApp defines an interface for exporting application state and validator set. +type ExportableApp interface { + ExportAppStateAndValidators(forZeroHeight bool, jailAllowedAddrs []string) (v2.ExportedApp, error) +} + +// Scenario: +// +// Start a fresh node and run n blocks, export state +// set up a new node instance, Init chain from exported genesis +// run new instance for n blocks +func TestAppSimulationAfterImport(t *testing.T) { + appFactory := NewSimApp[Tx] + cfg := simcli.NewConfigFromFlags() + cfg.ChainID = SimAppChainID + + exportAndStartChainFromGenesisPostAction := func(tb testing.TB, cs ChainState[Tx], ti TestInstance[Tx], accs []simtypes.Account) { + tb.Helper() + tb.Log("exporting genesis...\n") + app, ok := ti.App.(ExportableApp) + require.True(tb, ok) + exported, err := app.ExportAppStateAndValidators(false, []string{}) + require.NoError(tb, err) + + genesisTimestamp := cs.BlockTime.Add(24 * time.Hour) + startHeight := uint64(exported.Height + 1) + chainID := SimAppChainID + "_2" -func TestSimsAppV2(t *testing.T) { - RunWithSeeds[Tx](t, defaultSeeds) + importGenesisChainStateFactory := func(ctx context.Context, r *rand.Rand) (TestInstance[Tx], ChainState[Tx], []simtypes.Account) { + testInstance := SetupTestInstance(tb, appFactory, AppConfig, ti.Seed) + newCs := testInstance.InitializeChain( + tb, + ctx, + chainID, + genesisTimestamp, + startHeight, + exported.AppState, + ) + return testInstance, newCs, accs + } + // run sims with new app setup from exported genesis + RunWithSeedX[Tx](tb, cfg, importGenesisChainStateFactory, ti.Seed) + } + RunWithSeeds[Tx, *SimApp[Tx]](t, appFactory, AppConfig, defaultSeeds, exportAndStartChainFromGenesisPostAction) } diff --git a/simsx/environment.go b/simsx/environment.go index e5302802cce4..b01ad401be72 100644 --- a/simsx/environment.go +++ b/simsx/environment.go @@ -263,6 +263,9 @@ func NewChainDataSource( codec address.Codec, oldSimAcc ...simtypes.Account, ) *ChainDataSource { + if len(oldSimAcc) == 0 { + panic("empty accounts") + } acc := make([]SimAccount, len(oldSimAcc)) index := make(map[string]int, len(oldSimAcc)) bank := contextAwareBalanceSource{ctx: ctx, bank: bk} diff --git a/simsx/runner.go b/simsx/runner.go index 86992617eac0..cb0820527f2f 100644 --- a/simsx/runner.go +++ b/simsx/runner.go @@ -72,13 +72,13 @@ type SimulationApp interface { func Run[T SimulationApp]( t *testing.T, appFactory func( - logger log.Logger, - db corestore.KVStoreWithBatch, - traceStore io.Writer, - loadLatest bool, - appOpts server.DynamicConfig, - baseAppOptions ...func(*baseapp.BaseApp), - ) T, + logger log.Logger, + db corestore.KVStoreWithBatch, + traceStore io.Writer, + loadLatest bool, + appOpts server.DynamicConfig, + baseAppOptions ...func(*baseapp.BaseApp), +) T, setupStateFactory func(app T) SimStateFactory, postRunActions ...func(t testing.TB, app TestInstance[T], accs []simtypes.Account), ) { @@ -98,13 +98,13 @@ func Run[T SimulationApp]( func RunWithSeeds[T SimulationApp]( t *testing.T, appFactory func( - logger log.Logger, - db corestore.KVStoreWithBatch, - traceStore io.Writer, - loadLatest bool, - appOpts server.DynamicConfig, - baseAppOptions ...func(*baseapp.BaseApp), - ) T, + logger log.Logger, + db corestore.KVStoreWithBatch, + traceStore io.Writer, + loadLatest bool, + appOpts server.DynamicConfig, + baseAppOptions ...func(*baseapp.BaseApp), +) T, setupStateFactory func(app T) SimStateFactory, seeds []int64, fuzzSeed []byte, @@ -118,13 +118,13 @@ func RunWithSeeds[T SimulationApp]( func RunWithSeedsAndRandAcc[T SimulationApp]( t *testing.T, appFactory func( - logger log.Logger, - db corestore.KVStoreWithBatch, - traceStore io.Writer, - loadLatest bool, - appOpts server.DynamicConfig, - baseAppOptions ...func(*baseapp.BaseApp), - ) T, + logger log.Logger, + db corestore.KVStoreWithBatch, + traceStore io.Writer, + loadLatest bool, + appOpts server.DynamicConfig, + baseAppOptions ...func(*baseapp.BaseApp), +) T, setupStateFactory func(app T) SimStateFactory, seeds []int64, fuzzSeed []byte, @@ -349,7 +349,6 @@ func NewSimulationAppInstance[T SimulationApp]( }) appOptions := make(simtestutil.AppOptionsMap) appOptions[flags.FlagHome] = workDir - appOptions[flags.FlagInvCheckPeriod] = cli.FlagPeriodValue opts := []func(*baseapp.BaseApp){baseapp.SetChainID(tCfg.ChainID)} if tCfg.FauxMerkle { opts = append(opts, FauxMerkleModeOpt) diff --git a/simsx/v2/valset_history.go b/simsx/v2/valset_history.go index 9c05978ef900..e390f7c83ebd 100644 --- a/simsx/v2/valset_history.go +++ b/simsx/v2/valset_history.go @@ -16,15 +16,17 @@ type historicValSet struct { } type ValSetHistory struct { maxElements int - blockOffset int + blockOffset uint64 vals []historicValSet } -func NewValSetHistory(maxElements int) *ValSetHistory { +// NewValSetHistory constructor. The maximum of historic valsets must not exceed the block or time limit for +// valid evidence. +func NewValSetHistory(initialHeight uint64) *ValSetHistory { return &ValSetHistory{ - maxElements: maxElements, - blockOffset: 1, // start at height 1 - vals: make([]historicValSet, 0, maxElements), + maxElements: 1, + blockOffset: initialHeight, + vals: make([]historicValSet, 0, 1), } } @@ -58,7 +60,7 @@ func (h *ValSetHistory) MissBehaviour(r *rand.Rand) []comet.Evidence { evidence := comet.Evidence{ Type: comet.DuplicateVote, Validator: comet.Validator{Address: badVal.Address, Power: badVal.Power}, - Height: int64(h.blockOffset + n), + Height: int64(h.blockOffset) + int64(n), Time: h.vals[n].blockTime, TotalVotingPower: h.vals[n].vals.TotalPower(), } @@ -68,11 +70,12 @@ func (h *ValSetHistory) MissBehaviour(r *rand.Rand) []comet.Evidence { return []comet.Evidence{evidence} } +// SetMaxHistory sets the maximum number of historical validator sets to retain. Reduces retained history if it exceeds the limit. func (h *ValSetHistory) SetMaxHistory(v int) { h.maxElements = v if len(h.vals) > h.maxElements { diff := len(h.vals) - h.maxElements h.vals = h.vals[diff:] - h.blockOffset += diff + h.blockOffset += uint64(diff) } } diff --git a/types/simulation/config.go b/types/simulation/config.go index 6f93684d0822..c5efa35e0d01 100644 --- a/types/simulation/config.go +++ b/types/simulation/config.go @@ -13,9 +13,9 @@ type Config struct { ExportStatsPath string // custom file path to save the exported simulation statistics JSON Seed int64 // simulation random seed - InitialBlockHeight int // initial block to start the simulation + InitialBlockHeight uint64 // initial block to start the simulation GenesisTime int64 // genesis time to start the simulation - NumBlocks int // number of new blocks to simulate from the initial block height + NumBlocks uint64 // number of new blocks to simulate from the initial block height BlockSize int // operations per block ChainID string // chain-id used on the simulation diff --git a/x/simulation/client/cli/flags.go b/x/simulation/client/cli/flags.go index 1dc05fa47f29..4ede979e956c 100644 --- a/x/simulation/client/cli/flags.go +++ b/x/simulation/client/cli/flags.go @@ -18,8 +18,8 @@ var ( FlagExportStatePathValue string FlagExportStatsPathValue string FlagSeedValue int64 - FlagInitialBlockHeightValue int - FlagNumBlocksValue int + FlagInitialBlockHeightValue uint64 + FlagNumBlocksValue uint64 FlagBlockSizeValue int FlagLeanValue bool FlagCommitValue bool @@ -27,7 +27,6 @@ var ( FlagEnabledValue bool FlagVerboseValue bool - FlagPeriodValue uint FlagGenesisTimeValue int64 FlagSigverifyTxValue bool FlagFauxMerkle bool @@ -42,8 +41,8 @@ func GetSimulatorFlags() { flag.IntVar(&FlagExportParamsHeightValue, "ExportParamsHeight", 0, "height to which export the randomly generated params") flag.StringVar(&FlagExportStatePathValue, "ExportStatePath", "", "custom file path to save the exported app state JSON") flag.Int64Var(&FlagSeedValue, "Seed", DefaultSeedValue, "simulation random seed") - flag.IntVar(&FlagInitialBlockHeightValue, "InitialBlockHeight", 1, "initial block to start the simulation") - flag.IntVar(&FlagNumBlocksValue, "NumBlocks", 500, "number of new blocks to simulate from the initial block height") + flag.Uint64Var(&FlagInitialBlockHeightValue, "InitialBlockHeight", 1, "initial block to start the simulation") + flag.Uint64Var(&FlagNumBlocksValue, "NumBlocks", 500, "number of new blocks to simulate from the initial block height") flag.IntVar(&FlagBlockSizeValue, "BlockSize", 200, "operations per block") flag.BoolVar(&FlagLeanValue, "Lean", false, "lean simulation log output") flag.BoolVar(&FlagCommitValue, "Commit", true, "have the simulation commit") @@ -52,7 +51,6 @@ func GetSimulatorFlags() { // simulation flags flag.BoolVar(&FlagEnabledValue, "Enabled", false, "enable the simulation") flag.BoolVar(&FlagVerboseValue, "Verbose", false, "verbose log output") - flag.UintVar(&FlagPeriodValue, "Period", 0, "run slow invariants only once every period assertions") flag.Int64Var(&FlagGenesisTimeValue, "GenesisTime", time.Now().Unix(), "use current time as genesis UNIX time for default") flag.BoolVar(&FlagSigverifyTxValue, "SigverifyTx", true, "whether to sigverify check for transaction ") flag.BoolVar(&FlagFauxMerkle, "FauxMerkle", false, "use faux merkle instead of iavl")