Skip to content

Commit

Permalink
Refactor to sims2 msg factories
Browse files Browse the repository at this point in the history
(cherry picked from commit 4f4db31)
  • Loading branch information
alpe committed Sep 6, 2024
1 parent 889becd commit b3cf35c
Show file tree
Hide file tree
Showing 133 changed files with 6,141 additions and 9,629 deletions.
280 changes: 280 additions & 0 deletions go.sum

Large diffs are not rendered by default.

59 changes: 18 additions & 41 deletions scripts/build/simulations.mk
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
#? test-sim-nondeterminism: Run non-determinism test for simapp
test-sim-nondeterminism:
@echo "Running non-determinism test..."
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -tags='sims' -run TestAppStateDeterminism \
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout=30m -tags='sims' -run TestAppStateDeterminism \
-NumBlocks=100 -BlockSize=200 -Period=0

# Requires an exported plugin. See store/streaming/README.md for documentation.
Expand All @@ -16,45 +16,49 @@ test-sim-nondeterminism:
# make test-sim-nondeterminism-streaming
test-sim-nondeterminism-streaming:
@echo "Running non-determinism-streaming test..."
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -tags='sims' -run TestAppStateDeterminism \
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout=30m -tags='sims' -run TestAppStateDeterminism \
-NumBlocks=100 -BlockSize=200 -Period=0 -EnableStreaming=true

test-sim-custom-genesis-fast:
@echo "Running custom genesis simulation..."
@echo "By default, ${HOME}/.simapp/config/genesis.json will be used."
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -tags='sims' -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout=30m -tags='sims' -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \
-NumBlocks=100 -BlockSize=200 -Seed=99 -Period=5 -SigverifyTx=false

test-sim-import-export:
@echo "Running application import/export simulation. This may take several minutes..."
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 20m -tags='sims' -run TestAppImportExport \
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout 20m -tags='sims' -run TestAppImportExport \
-NumBlocks=50 -Period=5

test-sim-after-import:
@echo "Running application simulation-after-import. This may take several minutes..."
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -tags='sims' -run TestAppSimulationAfterImport \
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestAppSimulationAfterImport \
-NumBlocks=50 -Period=5


test-sim-custom-genesis-multi-seed:
@echo "Running multi-seed custom genesis simulation..."
@echo "By default, ${HOME}/.simapp/config/genesis.json will be used."
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \
-NumBlocks=400 -Period=5

test-sim-multi-seed-long:
@echo "Running long multi-seed application simulation. This may take awhile!"
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=2h -tags='sims' -run TestFullAppSimulation \
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout=2h -tags='sims' -run TestFullAppSimulation \
-NumBlocks=150 -Period=50

test-sim-multi-seed-short:
@echo "Running short multi-seed application simulation. This may take awhile!"
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation \
-NumBlocks=50 -Period=10
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation \
-NumBlocks=50 -Period=10 -FauxMerkle=true

test-v2-sim-wip:
@echo "Running v2 simapp. This may take awhile!"
@cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestSimsAppV2

test-sim-benchmark-invariants:
@echo "Running simulation invariant benchmarks..."
cd ${CURRENT_DIR}/simapp && go test -mod=readonly -benchmem -bench=BenchmarkInvariants -tags='sims' -run=^$ \
cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -benchmem -bench=BenchmarkInvariants -tags='sims' -run=^$ \
-Enabled=true -NumBlocks=1000 -BlockSize=200 \
-Period=1 -Commit=true -Seed=57 -v -timeout 24h

Expand All @@ -77,50 +81,23 @@ SIM_COMMIT ?= true
test-sim-fuzz:
@echo "Running application fuzz for numBlocks=2, blockSize=20. This may take awhile!"
#ld flags are a quick fix to make it work on current osx
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -json -tags='sims' -ldflags="-extldflags=-Wl,-ld_classic" -timeout=60m -fuzztime=60m -run=^$$ -fuzz=FuzzFullAppSimulation -GenesisTime=1714720615 -NumBlocks=2 -BlockSize=20
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -json -tags='sims' -ldflags="-extldflags=-Wl,-ld_classic" -timeout=60m -fuzztime=60m -run=^$$ -fuzz=FuzzFullAppSimulation -GenesisTime=1714720615 -NumBlocks=2 -BlockSize=20

#? test-sim-benchmark: Run benchmark test for simapp
test-sim-benchmark:
@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -tags='sims' -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -tags='sims' -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h

# Requires an exported plugin. See store/streaming/README.md for documentation.
#
# example:
# export COSMOS_SDK_ABCI_V1=<path-to-plugin-binary>
# make test-sim-benchmark-streaming
#
# Using the built-in examples:
# export COSMOS_SDK_ABCI_V1=<path-to-sdk>/store/streaming/abci/examples/file/file
# make test-sim-benchmark-streaming
test-sim-benchmark-streaming:
@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -tags='sims' -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -EnableStreaming=true

test-sim-profile:
@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -tags='sims' -benchmem -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -tags='sims' -benchmem -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out

# Requires an exported plugin. See store/streaming/README.md for documentation.
#
# example:
# export COSMOS_SDK_ABCI_V1=<path-to-plugin-binary>
# make test-sim-profile-streaming
#
# Using the built-in examples:
# export COSMOS_SDK_ABCI_V1=<path-to-sdk>/store/streaming/abci/examples/file/file
# make test-sim-profile-streaming
test-sim-profile-streaming:
@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -tags='sims' -benchmem -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out -EnableStreaming=true

.PHONY: test-sim-profile test-sim-benchmark test-sim-fuzz

#? benchmark: Run benchmark tests
benchmark:
@go test -mod=readonly -bench=. $(PACKAGES_NOSIMULATION)
@go test -failfast -mod=readonly -bench=. $(PACKAGES_NOSIMULATION)
.PHONY: benchmark
8 changes: 4 additions & 4 deletions simapp/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -457,15 +457,15 @@ func NewSimApp(
auth.NewAppModule(appCodec, app.AuthKeeper, app.AccountsKeeper, authsims.RandomGenesisAccounts, nil),
vesting.NewAppModule(app.AuthKeeper, app.BankKeeper),
bank.NewAppModule(appCodec, app.BankKeeper, app.AuthKeeper),
feegrantmodule.NewAppModule(appCodec, app.AuthKeeper, app.BankKeeper, app.FeeGrantKeeper, app.interfaceRegistry),
feegrantmodule.NewAppModule(appCodec, app.FeeGrantKeeper, app.interfaceRegistry),
gov.NewAppModule(appCodec, &app.GovKeeper, app.AuthKeeper, app.BankKeeper, app.PoolKeeper),
mint.NewAppModule(appCodec, app.MintKeeper, app.AuthKeeper, nil),
slashing.NewAppModule(appCodec, app.SlashingKeeper, app.AuthKeeper, app.BankKeeper, app.StakingKeeper, app.interfaceRegistry, cometService),
distr.NewAppModule(appCodec, app.DistrKeeper, app.AuthKeeper, app.BankKeeper, app.StakingKeeper),
staking.NewAppModule(appCodec, app.StakingKeeper, app.AuthKeeper, app.BankKeeper),
distr.NewAppModule(appCodec, app.DistrKeeper, app.StakingKeeper),
staking.NewAppModule(appCodec, app.StakingKeeper),
upgrade.NewAppModule(app.UpgradeKeeper),
evidence.NewAppModule(appCodec, app.EvidenceKeeper, cometService),
authzmodule.NewAppModule(appCodec, app.AuthzKeeper, app.AuthKeeper, app.BankKeeper, app.interfaceRegistry),
authzmodule.NewAppModule(appCodec, app.AuthzKeeper, app.interfaceRegistry),
groupmodule.NewAppModule(appCodec, app.GroupKeeper, app.AuthKeeper, app.BankKeeper, app.interfaceRegistry),
nftmodule.NewAppModule(appCodec, app.NFTKeeper, app.AuthKeeper, app.BankKeeper, app.interfaceRegistry),
consensus.NewAppModule(appCodec, app.ConsensusParamsKeeper),
Expand Down
76 changes: 1 addition & 75 deletions simapp/sim_bench_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,33 +3,12 @@
package simapp

import (
"os"
"testing"

"cosmossdk.io/log"
dbm "github.com/cosmos/cosmos-db"
"github.com/cosmos/cosmos-sdk/testutils/sims"

flag "github.com/spf13/pflag"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"

"github.com/cosmos/cosmos-sdk/baseapp"
"github.com/cosmos/cosmos-sdk/client/flags"
"github.com/cosmos/cosmos-sdk/server"
simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims"
simtypes "github.com/cosmos/cosmos-sdk/types/simulation"
"github.com/cosmos/cosmos-sdk/x/simulation"
simcli "github.com/cosmos/cosmos-sdk/x/simulation/client/cli"
)

var FlagEnableBenchStreamingValue bool

// Get flags every time the simulator is run
func init() {
flag.BoolVar(&FlagEnableBenchStreamingValue, "EnableStreaming", false, "Enable streaming service")
}

// Profile with:
// /usr/local/go/bin/go test -benchmem -run=^$ cosmossdk.io/simapp -bench ^BenchmarkFullAppSimulation$ -Commit=true -cpuprofile cpu.out
func BenchmarkFullAppSimulation(b *testing.B) {
Expand All @@ -38,58 +17,5 @@ func BenchmarkFullAppSimulation(b *testing.B) {
config := simcli.NewConfigFromFlags()
config.ChainID = sims.SimAppChainID

db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "goleveldb-app-sim", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue)
if err != nil {
b.Fatalf("simulation setup failed: %s", err.Error())
}

if skip {
b.Skip("skipping benchmark application simulation")
}

defer func() {
require.NoError(b, db.Close())
require.NoError(b, os.RemoveAll(dir))
}()

appOptions := viper.New()
appOptions.SetDefault(flags.FlagHome, DefaultNodeHome)
appOptions.SetDefault(server.FlagInvCheckPeriod, simcli.FlagPeriodValue)

app := NewSimApp(logger, db, nil, true, appOptions, interBlockCacheOpt(), baseapp.SetChainID(sims.SimAppChainID))

blockedAddrs, err := BlockedAddresses(app.InterfaceRegistry().SigningContext().AddressCodec())
require.NoError(b, err)

// run randomized simulation
simParams, simErr := simulation.SimulateFromSeedX(
b,
log.NewNopLogger(),
os.Stdout,
app.BaseApp,
simtestutil.AppStateFn(app.AppCodec(), app.AuthKeeper.AddressCodec(), app.StakingKeeper.ValidatorAddressCodec(), app.SimulationManager(), app.DefaultGenesis()),
simtypes.RandomAccounts,
simtestutil.SimulationOperations(app, app.AppCodec(), config, app.txConfig),
blockedAddrs,
config,
app.AppCodec(),
app.txConfig.SigningContext().AddressCodec(),
&simulation.DummyLogWriter{},
)

// export state and simParams before the simulation error is checked
if err = simtestutil.CheckExportSimulation(app, config, simParams); err != nil {
b.Fatal(err)
}

if simErr != nil {
b.Fatal(simErr)
}

if config.Commit {
db, ok := db.(dbm.DB)
if ok {
simtestutil.PrintStats(db)
}
}
sims.RunWithSeed(b, config, NewSimApp, setupStateFactory, 1, nil)
}
81 changes: 44 additions & 37 deletions simapp/sim_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"strings"
"sync"
"testing"
"time"

corestore "cosmossdk.io/core/store"
"cosmossdk.io/log"
Expand All @@ -22,6 +23,9 @@ import (
stakingtypes "cosmossdk.io/x/staking/types"
abci "github.com/cometbft/cometbft/api/cometbft/abci/v1"
cmtproto "github.com/cometbft/cometbft/api/cometbft/types/v1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/cosmos/cosmos-sdk/baseapp"
servertypes "github.com/cosmos/cosmos-sdk/server/types"
simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims"
Expand All @@ -30,8 +34,6 @@ import (
simtypes "github.com/cosmos/cosmos-sdk/types/simulation"
"github.com/cosmos/cosmos-sdk/x/simulation"
simcli "github.com/cosmos/cosmos-sdk/x/simulation/client/cli"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

// SimAppChainID hardcoded chainID for simulation
Expand All @@ -57,19 +59,21 @@ func TestFullAppSimulation(t *testing.T) {
func setupStateFactory(app *SimApp) sims.SimStateFactory {
blockedAddre, _ := BlockedAddresses(app.interfaceRegistry.SigningContext().AddressCodec())
return sims.SimStateFactory{
Codec: app.AppCodec(),
AppStateFn: simtestutil.AppStateFn(app.AppCodec(), app.AuthKeeper.AddressCodec(), app.StakingKeeper.ValidatorAddressCodec(), app.SimulationManager(), app.DefaultGenesis()),
BlockedAddr: blockedAddre,
Codec: app.AppCodec(),
AppStateFn: simtestutil.AppStateFn(app.AppCodec(), app.AuthKeeper.AddressCodec(), app.StakingKeeper.ValidatorAddressCodec(), app.SimulationManager().Modules, app.DefaultGenesis()),
BlockedAddr: blockedAddre,
AccountSource: app.AuthKeeper,
BalanceSource: app.BankKeeper,
}
}

var (
exportAllModules = []string{}
exportWithValidatorSet = []string{}
exportAllModules []string
exportWithValidatorSet []string
)

func TestAppImportExport(t *testing.T) {
sims.Run(t, NewSimApp, setupStateFactory, func(t *testing.T, ti sims.TestInstance[*SimApp]) {
sims.Run(t, NewSimApp, setupStateFactory, func(t testing.TB, ti sims.TestInstance[*SimApp]) {
app := ti.App
t.Log("exporting genesis...\n")
exported, err := app.ExportAppStateAndValidators(false, exportWithValidatorSet, exportAllModules)
Expand Down Expand Up @@ -111,40 +115,36 @@ func TestAppImportExport(t *testing.T) {
// set up a new node instance, Init chain from exported genesis
// run new instance for n blocks
func TestAppSimulationAfterImport(t *testing.T) {
sims.Run(t, NewSimApp, setupStateFactory, func(t *testing.T, ti sims.TestInstance[*SimApp]) {
sims.Run(t, NewSimApp, setupStateFactory, func(t testing.TB, ti sims.TestInstance[*SimApp]) {
app := ti.App
t.Log("exporting genesis...\n")
exported, err := app.ExportAppStateAndValidators(false, exportWithValidatorSet, exportAllModules)
require.NoError(t, err)

t.Log("importing genesis...\n")
newTestInstance := sims.NewSimulationAppInstance(t, ti.Cfg, NewSimApp)
newApp := newTestInstance.App
_, err = newApp.InitChain(&abci.InitChainRequest{
AppStateBytes: exported.AppState,
ChainId: sims.SimAppChainID,
})
if IsEmptyValidatorSetErr(err) {
t.Skip("Skipping simulation as all validators have been unbonded")
return
importGenesisStateFactory := func(app *SimApp) sims.SimStateFactory {
return sims.SimStateFactory{
Codec: app.AppCodec(),
AppStateFn: func(r *rand.Rand, accs []simtypes.Account, config simtypes.Config) (json.RawMessage, []simtypes.Account, string, time.Time) {
_, err = app.InitChain(&abci.InitChainRequest{
AppStateBytes: exported.AppState,
ChainId: sims.SimAppChainID,
})
if IsEmptyValidatorSetErr(err) {
t.Skip("Skipping simulation as all validators have been unbonded")
return nil, nil, "", time.Time{}
}
acc, err := simtestutil.AccountsFromAppState(app.AppCodec(), exported.AppState)
require.NoError(t, err)
genesisTimestamp := time.Unix(config.GenesisTime, 0)
return exported.AppState, acc, config.ChainID, genesisTimestamp
},
BlockedAddr: must(BlockedAddresses(app.AuthKeeper.AddressCodec())),
AccountSource: app.AuthKeeper,
BalanceSource: app.BankKeeper,
}
}
require.NoError(t, err)
newStateFactory := setupStateFactory(newApp)
_, err = simulation.SimulateFromSeedX(
t,
newTestInstance.AppLogger,
sims.WriteToDebugLog(newTestInstance.AppLogger),
newApp.BaseApp,
newStateFactory.AppStateFn,
simtypes.RandomAccounts,
simtestutil.SimulationOperations(newApp, newApp.AppCodec(), newTestInstance.Cfg, newApp.TxConfig()),
newStateFactory.BlockedAddr,
newTestInstance.Cfg,
newStateFactory.Codec,
newApp.TxConfig().SigningContext().AddressCodec(),
ti.ExecLogWriter,
)
require.NoError(t, err)
sims.RunWithSeed(t, ti.Cfg, NewSimApp, importGenesisStateFactory, ti.Cfg.Seed, ti.Cfg.FuzzSeed)
})
}

Expand Down Expand Up @@ -190,7 +190,7 @@ func TestAppStateDeterminism(t *testing.T) {
var mx sync.Mutex
appHashResults := make(map[int64][][]byte)
appSimLogger := make(map[int64][]simulation.LogWriter)
captureAndCheckHash := func(t *testing.T, ti sims.TestInstance[*SimApp]) {
captureAndCheckHash := func(t testing.TB, ti sims.TestInstance[*SimApp]) {
seed, appHash := ti.Cfg.Seed, ti.App.LastCommitID().Hash
mx.Lock()
otherHashes, execWriters := appHashResults[seed], appSimLogger[seed]
Expand Down Expand Up @@ -226,7 +226,7 @@ type ComparableStoreApp interface {
GetStoreKeys() []storetypes.StoreKey
}

func AssertEqualStores(t *testing.T, app ComparableStoreApp, newApp ComparableStoreApp, storeDecoders simtypes.StoreDecoderRegistry, skipPrefixes map[string][][]byte) {
func AssertEqualStores(t testing.TB, app, newApp ComparableStoreApp, storeDecoders simtypes.StoreDecoderRegistry, skipPrefixes map[string][][]byte) {
ctxA := app.NewContextLegacy(true, cmtproto.Header{Height: app.LastBlockHeight()})
ctxB := newApp.NewContextLegacy(true, cmtproto.Header{Height: app.LastBlockHeight()})

Expand Down Expand Up @@ -273,3 +273,10 @@ func FuzzFullAppSimulation(f *testing.F) {
)
})
}

func must[T any](r T, err error) T {
if err != nil {
panic(err)
}
return r
}
1 change: 1 addition & 0 deletions simapp/v2/app_di.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
slashingkeeper "cosmossdk.io/x/slashing/keeper"
stakingkeeper "cosmossdk.io/x/staking/keeper"
upgradekeeper "cosmossdk.io/x/upgrade/keeper"
_ "github.com/cosmos/cosmos-sdk/x/genutil"

"github.com/cosmos/cosmos-sdk/client"
"github.com/cosmos/cosmos-sdk/codec"
Expand Down
Loading

0 comments on commit b3cf35c

Please sign in to comment.