From 53e82b12eb1657b61cfd8694b4fa3295f98a545b Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Thu, 30 May 2024 15:16:29 +0200 Subject: [PATCH 01/15] Replace runsim with Go stdlib testing --- .github/workflows/sims.yml | 146 +++++---- Makefile | 51 +-- baseapp/baseapp.go | 5 + contrib/devtools/Makefile | 69 ----- simapp/runner.go | 229 ++++++++++++++ simapp/sim_bench_test.go | 12 +- simapp/sim_test.go | 461 ++++++++++------------------ testutil/sims/simulation_helpers.go | 10 +- testutil/sims/state_helpers.go | 33 +- types/simulation/config.go | 28 +- types/simulation/rand_util.go | 25 +- types/simulation/rand_util_test.go | 24 +- x/simulation/client/cli/flags.go | 9 +- x/simulation/params.go | 8 - x/simulation/simulate.go | 136 ++++---- 15 files changed, 653 insertions(+), 593 deletions(-) delete mode 100644 contrib/devtools/Makefile create mode 100644 simapp/runner.go diff --git a/.github/workflows/sims.yml b/.github/workflows/sims.yml index 6427977d9098..e6089e3b048f 100644 --- a/.github/workflows/sims.yml +++ b/.github/workflows/sims.yml @@ -2,10 +2,13 @@ name: Sims # Sims workflow runs multiple types of simulations (nondeterminism, import-export, after-import, multi-seed-short) # This workflow will run on all Pull Requests, if a .go, .mod or .sum file have been changed on: - schedule: - - cron: "0 */2 * * *" - release: - types: [published] + push: + branches: + - main + - release/** + pull_request: +permissions: + contents: read concurrency: group: ci-${{ github.ref }}-sims @@ -15,7 +18,7 @@ jobs: build: permissions: contents: read # for actions/checkout to fetch code - runs-on: ubuntu-latest + runs-on: large-sdk-runner if: "!contains(github.event.head_commit.message, 'skip-sims')" steps: - uses: actions/checkout@v4 @@ -24,15 +27,9 @@ jobs: go-version: "1.22" check-latest: true - run: make build - - name: Install runsim - run: go install github.com/cosmos/tools/cmd/runsim@v1.0.0 - - uses: actions/cache@v4 - with: - path: ~/go/bin - key: ${{ runner.os }}-go-runsim-binary test-sim-import-export: - runs-on: ubuntu-latest + runs-on: large-sdk-runner needs: [build] timeout-minutes: 60 steps: @@ -41,16 +38,12 @@ jobs: with: go-version: "1.22" check-latest: true - - uses: actions/cache@v4 - with: - path: ~/go/bin - key: ${{ runner.os }}-go-runsim-binary - name: test-sim-import-export run: | make test-sim-import-export test-sim-after-import: - runs-on: ubuntu-latest + runs-on: large-sdk-runner needs: [build] steps: - uses: actions/checkout@v4 @@ -58,16 +51,25 @@ jobs: with: go-version: "1.22" check-latest: true - - uses: actions/cache@v4 - with: - path: ~/go/bin - key: ${{ runner.os }}-go-runsim-binary - name: test-sim-after-import run: | make test-sim-after-import + test-sim-deterministic: + runs-on: large-sdk-runner + needs: [build] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.22" + check-latest: true + - name: test-sim-nondeterminism-streaming + run: | + make test-sim-nondeterminism-streaming + test-sim-multi-seed-short: - runs-on: ubuntu-latest + runs-on: large-sdk-runner needs: [build] timeout-minutes: 60 steps: @@ -76,55 +78,65 @@ jobs: with: go-version: "1.22" check-latest: true - - uses: actions/cache@v4 - with: - path: ~/go/bin - key: ${{ runner.os }}-go-runsim-binary - name: test-sim-multi-seed-short run: | make test-sim-multi-seed-short - sims-notify-success: - needs: - [test-sim-multi-seed-short, test-sim-after-import, test-sim-import-export] - runs-on: ubuntu-latest - if: ${{ success() }} + test-sim-fuzz: + runs-on: large-sdk-runner + needs: [build] + timeout-minutes: 60 steps: - - name: Check out repository - uses: actions/checkout@v4 - - name: Get previous workflow status - uses: ./.github/actions/last-workflow-status - id: last_status + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: - github_token: ${{ secrets.GITHUB_TOKEN }} - - - name: Notify Slack on success - if: ${{ steps.last_status.outputs.last_status == 'failure' }} - uses: rtCamp/action-slack-notify@v2.3.0 - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - SLACK_CHANNEL: sdk-sims - SLACK_USERNAME: Sim Tests - SLACK_ICON_EMOJI: ":white_check_mark:" - SLACK_COLOR: good - SLACK_MESSAGE: Sims are passing - SLACK_FOOTER: "" + go-version: "1.22" + check-latest: true + - name: test-sim-fuzz + run: | + make test-sim-fuzz - sims-notify-failure: - permissions: - contents: none - needs: - [test-sim-multi-seed-short, test-sim-after-import, test-sim-import-export] - runs-on: ubuntu-latest - if: ${{ failure() }} - steps: - - name: Notify Slack on failure - uses: rtCamp/action-slack-notify@v2.3.0 - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - SLACK_CHANNEL: sdk-sims - SLACK_USERNAME: Sim Tests - SLACK_ICON_EMOJI: ":skull:" - SLACK_COLOR: danger - SLACK_MESSAGE: Sims are failing - SLACK_FOOTER: "" +# sims-notify-success: +# needs: +# [test-sim-multi-seed-short, test-sim-after-import, test-sim-import-export] +# runs-on: large-sdk-runner +# if: ${{ success() }} +# steps: +# - name: Check out repository +# uses: actions/checkout@v4 +# - name: Get previous workflow status +# uses: ./.github/actions/last-workflow-status +# id: last_status +# with: +# github_token: ${{ secrets.GITHUB_TOKEN }} +# +# - name: Notify Slack on success +# if: ${{ steps.last_status.outputs.last_status == 'failure' }} +# uses: rtCamp/action-slack-notify@v2.3.0 +# env: +# SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} +# SLACK_CHANNEL: sdk-sims +# SLACK_USERNAME: Sim Tests +# SLACK_ICON_EMOJI: ":white_check_mark:" +# SLACK_COLOR: good +# SLACK_MESSAGE: Sims are passing +# SLACK_FOOTER: "" +# +# sims-notify-failure: +# permissions: +# contents: none +# needs: +# [test-sim-multi-seed-short, test-sim-after-import, test-sim-import-export] +# runs-on: large-sdk-runner +# if: ${{ failure() }} +# steps: +# - name: Notify Slack on failure +# uses: rtCamp/action-slack-notify@v2.3.0 +# env: +# SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} +# SLACK_CHANNEL: sdk-sims +# SLACK_USERNAME: Sim Tests +# SLACK_ICON_EMOJI: ":skull:" +# SLACK_COLOR: danger +# SLACK_MESSAGE: Sims are failing +# SLACK_FOOTER: "" diff --git a/Makefile b/Makefile index 2131edb3025e..a480d6ff9b53 100644 --- a/Makefile +++ b/Makefile @@ -103,9 +103,6 @@ endif #? all: Run tools build all: build -# The below include contains the tools and runsim targets. -include contrib/devtools/Makefile - ############################################################################### ### Build ### ############################################################################### @@ -158,7 +155,7 @@ $(MOCKS_DIR): mkdir -p $(MOCKS_DIR) #? distclean: Run `make clean` and `make tools-clean` -distclean: clean tools-clean +distclean: clean #? clean: Clean some auto generated directory clean: @@ -280,8 +277,8 @@ endif #? test-sim-nondeterminism: Run non-determinism test for simapp test-sim-nondeterminism: @echo "Running non-determinism test..." - @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -run TestAppStateDeterminism -Enabled=true \ - -NumBlocks=100 -BlockSize=200 -Commit=true -Period=0 -v -timeout 24h + @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -run TestAppStateDeterminism \ + -NumBlocks=100 -BlockSize=200 -Period=0 # Requires an exported plugin. See store/streaming/README.md for documentation. # @@ -294,35 +291,41 @@ test-sim-nondeterminism: # make test-sim-nondeterminism-streaming test-sim-nondeterminism-streaming: @echo "Running non-determinism-streaming test..." - @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -run TestAppStateDeterminism -Enabled=true \ - -NumBlocks=100 -BlockSize=200 -Commit=true -Period=0 -v -timeout 24h -EnableStreaming=true + @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -run TestAppStateDeterminism \ + -NumBlocks=100 -BlockSize=200 -Period=0 -EnableStreaming=true test-sim-custom-genesis-fast: @echo "Running custom genesis simulation..." @echo "By default, ${HOME}/.simapp/config/genesis.json will be used." - @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \ - -Enabled=true -NumBlocks=100 -BlockSize=200 -Commit=true -Seed=99 -Period=5 -SigverifyTx=false -v -timeout 24h + @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \ + -NumBlocks=100 -BlockSize=200 -Seed=99 -Period=5 -SigverifyTx=false -test-sim-import-export: runsim +test-sim-import-export: @echo "Running application import/export simulation. This may take several minutes..." - @cd ${CURRENT_DIR}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 50 5 TestAppImportExport + @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 20m -run TestAppImportExport \ + -NumBlocks=50 -Period=5 -test-sim-after-import: runsim +test-sim-after-import: @echo "Running application simulation-after-import. This may take several minutes..." - @cd ${CURRENT_DIR}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 50 5 TestAppSimulationAfterImport + @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -run TestAppSimulationAfterImport \ + -NumBlocks=50 -Period=5 + -test-sim-custom-genesis-multi-seed: runsim +test-sim-custom-genesis-multi-seed: @echo "Running multi-seed custom genesis simulation..." @echo "By default, ${HOME}/.simapp/config/genesis.json will be used." - @cd ${CURRENT_DIR}/simapp && $(BINDIR)/runsim -Genesis=${HOME}/.simapp/config/genesis.json -SigverifyTx=false -SimAppPkg=. -ExitOnFail 400 5 TestFullAppSimulation + @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \ + -NumBlocks=400 -Period=5 -test-sim-multi-seed-long: runsim +test-sim-multi-seed-long: @echo "Running long multi-seed application simulation. This may take awhile!" - @cd ${CURRENT_DIR}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 500 50 TestFullAppSimulation + @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=1h -run TestFullAppSimulation \ + -NumBlocks=500 -Period=50 -test-sim-multi-seed-short: runsim +test-sim-multi-seed-short: @echo "Running short multi-seed application simulation. This may take awhile!" - @cd ${CURRENT_DIR}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 50 10 TestFullAppSimulation + @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -run TestFullAppSimulation \ + -NumBlocks=50 -Period=10 test-sim-benchmark-invariants: @echo "Running simulation invariant benchmarks..." @@ -345,6 +348,12 @@ SIM_NUM_BLOCKS ?= 500 SIM_BLOCK_SIZE ?= 200 SIM_COMMIT ?= true +#? test-sim-fuzz: Run fuzz test for simapp +test-sim-fuzz: + @echo "Running application fuzz for numBlocks=2, blockSize=20. This may take awhile!" +#ld flags are a quick fix to make it work on current osx + @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -json -ldflags="-extldflags=-Wl,-ld_classic" -timeout=60m -fuzztime=60m -run=^$$ -fuzz=FuzzFullAppSimulation -GenesisTime=1714720615 -NumBlocks=2 -BlockSize=20 + #? test-sim-benchmark: Run benchmark test for simapp test-sim-benchmark: @echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!" @@ -384,7 +393,7 @@ test-sim-profile-streaming: @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -benchmem -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \ -Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out -EnableStreaming=true -.PHONY: test-sim-profile test-sim-benchmark +.PHONY: test-sim-profile test-sim-benchmark test-sim-fuzz #? benchmark: Run benchmark tests benchmark: diff --git a/baseapp/baseapp.go b/baseapp/baseapp.go index 164c215d9545..36121b7f221d 100644 --- a/baseapp/baseapp.go +++ b/baseapp/baseapp.go @@ -1165,3 +1165,8 @@ func (app *BaseApp) Close() error { return errors.Join(errs...) } + +// GetBaseApp returns the pointer to itself. +func (app *BaseApp) GetBaseApp() *BaseApp { + return app +} diff --git a/contrib/devtools/Makefile b/contrib/devtools/Makefile deleted file mode 100644 index e302695bdb18..000000000000 --- a/contrib/devtools/Makefile +++ /dev/null @@ -1,69 +0,0 @@ -### -# Find OS and Go environment -# GO contains the Go binary -# FS contains the OS file separator -### -ifeq ($(OS),Windows_NT) - GO := $(shell where go.exe 2> NUL) - FS := "\\" -else - GO := $(shell command -v go 2> /dev/null) - FS := "/" -endif - -ifeq ($(GO),) - $(error could not find go. Is it in PATH? $(GO)) -endif - -############################################################################### -### Functions ### -############################################################################### - -go_get = $(if $(findstring Windows_NT,$(OS)),\ -IF NOT EXIST $(GITHUBDIR)$(FS)$(1)$(FS) ( mkdir $(GITHUBDIR)$(FS)$(1) ) else (cd .) &\ -IF NOT EXIST $(GITHUBDIR)$(FS)$(1)$(FS)$(2)$(FS) ( cd $(GITHUBDIR)$(FS)$(1) && git clone https://github.com/$(1)/$(2) ) else (cd .) &\ -,\ -mkdir -p $(GITHUBDIR)$(FS)$(1) &&\ -(test ! -d $(GITHUBDIR)$(FS)$(1)$(FS)$(2) && cd $(GITHUBDIR)$(FS)$(1) && git clone https://github.com/$(1)/$(2)) || true &&\ -)\ -cd $(GITHUBDIR)$(FS)$(1)$(FS)$(2) && git fetch origin && git checkout -q $(3) - -mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) -mkfile_dir := $(shell cd $(shell dirname $(mkfile_path)); pwd) - - -############################################################################### -### Tools ### -############################################################################### - -PREFIX ?= /usr/local -BIN ?= $(PREFIX)/bin -UNAME_S ?= $(shell uname -s) -UNAME_M ?= $(shell uname -m) - -GOPATH ?= $(shell $(GO) env GOPATH) -GITHUBDIR := $(GOPATH)$(FS)src$(FS)github.com - -BUF_VERSION ?= 0.11.0 - -TOOLS_DESTDIR ?= $(GOPATH)/bin -RUNSIM = $(TOOLS_DESTDIR)/runsim - -tools: tools-stamp -tools-stamp: runsim - # Create dummy file to satisfy dependency and avoid - # rebuilding when this Makefile target is hit twice - # in a row. - touch $@ - -# Install the runsim binary -runsim: $(RUNSIM) -$(RUNSIM): - @echo "Installing runsim..." - @go install github.com/cosmos/tools/cmd/runsim@v1.0.0 - -tools-clean: - rm -f $(GOLANGCI_LINT) $(RUNSIM) - rm -f tools-stamp - -.PHONY: tools-clean runsim \ No newline at end of file diff --git a/simapp/runner.go b/simapp/runner.go new file mode 100644 index 000000000000..b7babd4bc6fe --- /dev/null +++ b/simapp/runner.go @@ -0,0 +1,229 @@ +package simapp + +import ( + "fmt" + "io" + "path/filepath" + "testing" + + "github.com/cosmos/cosmos-sdk/client" + + "cosmossdk.io/log" + + dbm "github.com/cosmos/cosmos-db" + "github.com/cosmos/cosmos-sdk/baseapp" + "github.com/cosmos/cosmos-sdk/client/flags" + "github.com/cosmos/cosmos-sdk/codec" + "github.com/cosmos/cosmos-sdk/runtime" + "github.com/cosmos/cosmos-sdk/server" + servertypes "github.com/cosmos/cosmos-sdk/server/types" + simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims" + simtypes "github.com/cosmos/cosmos-sdk/types/simulation" + "github.com/cosmos/cosmos-sdk/x/simulation" + "github.com/cosmos/cosmos-sdk/x/simulation/client/cli" + "github.com/stretchr/testify/require" +) + +const SimAppChainID = "simulation-app" + +// this list of seeds was imported from the original simulation runner: https://github.com/cosmos/tools/blob/v1.0.0/cmd/runsim/main.go#L32 +var defaultSeeds = []int64{ + 1, 2, 4, 7, + 32, 123, 124, 582, 1893, 2989, + 3012, 4728, 37827, 981928, 87821, 891823782, + 989182, 89182391, 11, 22, 44, 77, 99, 2020, + 3232, 123123, 124124, 582582, 18931893, + 29892989, 30123012, 47284728, 7601778, 8090485, + 977367484, 491163361, 424254581, 673398983, +} + +type SimStateFactory struct { + Codec codec.Codec + AppStateFn simtypes.AppStateFn + BlockedAddr map[string]bool +} + +// SimulationApp abstract app that is used by sims +type SimulationApp interface { + runtime.AppSimI + SetNotSigverifyTx() + GetBaseApp() *baseapp.BaseApp + TxConfig() client.TxConfig +} + +// Run is a helper function that runs a simulation test with the given parameters. +// It calls the RunWithSeeds function with the default seeds and parameters. +// +// This is the entrypoint to run simulation tests that used to run with the runsim binary. +func Run[T SimulationApp]( + t *testing.T, + appFactory func( + logger log.Logger, + db dbm.DB, + traceStore io.Writer, + loadLatest bool, + appOpts servertypes.AppOptions, + baseAppOptions ...func(*baseapp.BaseApp), + ) T, + setupStateFactory func(app T) SimStateFactory, + postRunActions ...func(t *testing.T, app TestInstance[T]), +) { + RunWithSeeds(t, appFactory, setupStateFactory, defaultSeeds, nil, postRunActions...) +} + +// RunWithSeeds is a helper function that runs a simulation test with the given parameters. +// It iterates over the provided seeds and runs the simulation test for each seed in parallel. +// +// It sets up the environment, creates an instance of the simulation app, +// calls the simulation.SimulateFromSeed function to run the simulation, and performs post-run actions for each seed. +// The execution is deterministic and can be used for fuzz tests as well. +// +// The system under test is isolated for each run but unlike the old runsim command, there is no Process separation. +// This means, global caches may be reused for example. This implementation build upon the vanialla Go stdlib test framework. +func RunWithSeeds[T SimulationApp]( + t *testing.T, + appFactory func( + logger log.Logger, + db dbm.DB, + traceStore io.Writer, + loadLatest bool, + appOpts servertypes.AppOptions, + baseAppOptions ...func(*baseapp.BaseApp), + ) T, + setupStateFactory func(app T) SimStateFactory, + seeds []int64, + fuzzSeed []byte, + postRunActions ...func(t *testing.T, app TestInstance[T]), +) { + cfg := cli.NewConfigFromFlags() + cfg.ChainID = SimAppChainID + for i := range seeds { + seed := seeds[i] + t.Run(fmt.Sprintf("seed: %d", seed), func(t *testing.T) { + t.Parallel() + // setup environment + tCfg := cfg.Clone() + tCfg.Seed = seed + tCfg.FuzzSeed = fuzzSeed + tCfg.T = t + testInstance := NewSimulationAppInstance(t, tCfg, appFactory) + var runLogger log.Logger + if cli.FlagVerboseValue { + runLogger = log.NewTestLogger(t) + } else { + runLogger = log.NewTestLoggerInfo(t) + } + runLogger = runLogger.With("seed", tCfg.Seed) + + app := testInstance.App + stateFactory := setupStateFactory(app) + simParams, err := simulation.SimulateFromSeed( + t, + runLogger, + WriteToDebugLog(runLogger), + app.GetBaseApp(), + stateFactory.AppStateFn, + simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 + simtestutil.SimulationOperations(app, stateFactory.Codec, tCfg, testInstance.App.TxConfig()), + stateFactory.BlockedAddr, + tCfg, + stateFactory.Codec, + app.TxConfig().SigningContext().AddressCodec(), + ) + require.NoError(t, err) + err = simtestutil.CheckExportSimulation(app, tCfg, simParams) + require.NoError(t, err) + if tCfg.Commit { + simtestutil.PrintStats(testInstance.DB) + } + for _, step := range postRunActions { + step(t, testInstance) + } + }) + } +} + +// TestInstance is a generic type that represents an instance of a SimulationApp used for testing simulations. +// It contains the following fields: +// - App: The instance of the SimulationApp under test. +// - DB: The LevelDB database for the simulation app. +// - WorkDir: The temporary working directory for the simulation app. +// - Cfg: The configuration flags for the simulator. +// - Logger: The logger used for logging in the app during the simulation, with seed value attached. +type TestInstance[T SimulationApp] struct { + App T + DB dbm.DB + WorkDir string + Cfg simtypes.Config + Logger log.Logger +} + +// NewSimulationAppInstance initializes and returns a TestInstance of a SimulationApp. +// The function takes a testing.T instance, a simtypes.Config instance, and an appFactory function as parameters. +// It creates a temporary working directory and a LevelDB database for the simulation app. +// The function then initializes a logger based on the verbosity flag and sets the logger's seed to the test configuration's seed. +// The database is closed and cleaned up on test completion. +func NewSimulationAppInstance[T SimulationApp]( + t *testing.T, + tCfg simtypes.Config, + appFactory func(logger log.Logger, db dbm.DB, traceStore io.Writer, loadLatest bool, appOpts servertypes.AppOptions, baseAppOptions ...func(*baseapp.BaseApp)) T, +) TestInstance[T] { + workDir := t.TempDir() + dbDir := filepath.Join(workDir, "leveldb-app-sim") + var logger log.Logger + if cli.FlagVerboseValue { + logger = log.NewTestLogger(t) + } else { + logger = log.NewTestLoggerError(t) + } + logger = logger.With("seed", tCfg.Seed) + + db, err := dbm.NewDB("Simulation", dbm.BackendType(tCfg.DBBackend), dbDir) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, db.Close()) + }) + appOptions := make(simtestutil.AppOptionsMap) + appOptions[flags.FlagHome] = workDir + appOptions[server.FlagInvCheckPeriod] = cli.FlagPeriodValue + app := appFactory(logger, db, nil, true, appOptions, baseapp.SetChainID(SimAppChainID)) + if !cli.FlagSigverifyTxValue { + app.SetNotSigverifyTx() + } + return TestInstance[T]{ + App: app, + DB: db, + WorkDir: workDir, + Cfg: tCfg, + Logger: logger, + } +} + +var _ io.Writer = writerFn(nil) + +type writerFn func(p []byte) (n int, err error) + +func (w writerFn) Write(p []byte) (n int, err error) { + return w(p) +} + +// WriteToDebugLog is an adapter to io.Writer interface +func WriteToDebugLog(logger log.Logger) io.Writer { + return writerFn(func(p []byte) (n int, err error) { + logger.Debug(string(p)) + return len(p), nil + }) +} + +// AppOptionsFn is an adapter to the single method AppOptions interface +type AppOptionsFn func(string) any + +func (f AppOptionsFn) Get(k string) any { + return f(k) +} + +// FauxMerkleModeOpt returns a BaseApp option to use a dbStoreAdapter instead of +// an IAVLStore for faster simulation speed. +func FauxMerkleModeOpt(bapp *baseapp.BaseApp) { + bapp.SetFauxMerkleMode() +} diff --git a/simapp/sim_bench_test.go b/simapp/sim_bench_test.go index e4d01fcd44f4..c0ddd7df1edb 100644 --- a/simapp/sim_bench_test.go +++ b/simapp/sim_bench_test.go @@ -4,13 +4,14 @@ import ( "os" "testing" + "cosmossdk.io/log" + flag "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/stretchr/testify/require" "github.com/cosmos/cosmos-sdk/baseapp" "github.com/cosmos/cosmos-sdk/client/flags" - codectestutil "github.com/cosmos/cosmos-sdk/codec/testutil" "github.com/cosmos/cosmos-sdk/server" simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims" simtypes "github.com/cosmos/cosmos-sdk/types/simulation" @@ -63,17 +64,18 @@ func BenchmarkFullAppSimulation(b *testing.B) { app := NewSimApp(logger, db, nil, true, appOptions, interBlockCacheOpt(), baseapp.SetChainID(SimAppChainID)) // run randomized simulation - _, simParams, simErr := simulation.SimulateFromSeed( + simParams, simErr := simulation.SimulateFromSeed( b, + log.NewNopLogger(), os.Stdout, app.BaseApp, simtestutil.AppStateFn(app.AppCodec(), app.AuthKeeper.AddressCodec(), app.StakingKeeper.ValidatorAddressCodec(), app.SimulationManager(), app.DefaultGenesis()), - simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 - simtestutil.SimulationOperations(app, app.AppCodec(), config), + simtypes.RandomAccounts, + simtestutil.SimulationOperations(app, app.AppCodec(), config, app.txConfig), BlockedAddresses(), config, app.AppCodec(), - codectestutil.CodecOptions{}.GetAddressCodec(), + app.txConfig.SigningContext().AddressCodec(), ) // export state and simParams before the simulation error is checked diff --git a/simapp/sim_test.go b/simapp/sim_test.go index 17b530bba2a9..846556d2f5ca 100644 --- a/simapp/sim_test.go +++ b/simapp/sim_test.go @@ -1,18 +1,22 @@ package simapp import ( + "encoding/binary" "encoding/json" "flag" + "io" "math/rand" - "os" - "runtime/debug" "strings" + "sync" "testing" + servertypes "github.com/cosmos/cosmos-sdk/server/types" + + sdk "github.com/cosmos/cosmos-sdk/types" + abci "github.com/cometbft/cometbft/abci/types" cmtproto "github.com/cometbft/cometbft/api/cometbft/types/v1" dbm "github.com/cosmos/cosmos-db" - "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -25,9 +29,6 @@ import ( stakingtypes "cosmossdk.io/x/staking/types" "github.com/cosmos/cosmos-sdk/baseapp" - "github.com/cosmos/cosmos-sdk/client/flags" - codectestutil "github.com/cosmos/cosmos-sdk/codec/testutil" - "github.com/cosmos/cosmos-sdk/server" simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims" simtypes "github.com/cosmos/cosmos-sdk/types/simulation" "github.com/cosmos/cosmos-sdk/x/simulation" @@ -35,7 +36,6 @@ import ( ) // SimAppChainID hardcoded chainID for simulation -const SimAppChainID = "simulation-app" var FlagEnableStreamingValue bool @@ -45,12 +45,6 @@ func init() { flag.BoolVar(&FlagEnableStreamingValue, "EnableStreaming", false, "Enable streaming service") } -// fauxMerkleModeOpt returns a BaseApp option to use a dbStoreAdapter instead of -// an IAVLStore for faster simulation speed. -func fauxMerkleModeOpt(bapp *baseapp.BaseApp) { - bapp.SetFauxMerkleMode() -} - // interBlockCacheOpt returns a BaseApp option function that sets the persistent // inter-block write-through cache. func interBlockCacheOpt() func(*baseapp.BaseApp) { @@ -58,107 +52,164 @@ func interBlockCacheOpt() func(*baseapp.BaseApp) { } func TestFullAppSimulation(t *testing.T) { - config, db, _, app := setupSimulationApp(t, "skipping application simulation") - // run randomized simulation - _, simParams, simErr := simulation.SimulateFromSeed( - t, - os.Stdout, - app.BaseApp, - simtestutil.AppStateFn(app.AppCodec(), app.AuthKeeper.AddressCodec(), app.StakingKeeper.ValidatorAddressCodec(), app.SimulationManager(), app.DefaultGenesis()), - simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 - simtestutil.SimulationOperations(app, app.AppCodec(), config), - BlockedAddresses(), - config, - app.AppCodec(), - codectestutil.CodecOptions{}.GetAddressCodec(), - ) - - // export state and simParams before the simulation error is checked - err := simtestutil.CheckExportSimulation(app, config, simParams) - require.NoError(t, err) - require.NoError(t, simErr) + Run(t, NewSimApp, setupStateFactory) +} - if config.Commit { - simtestutil.PrintStats(db) +func setupStateFactory(app *SimApp) SimStateFactory { + return SimStateFactory{ + Codec: app.AppCodec(), + AppStateFn: simtestutil.AppStateFn(app.AppCodec(), app.AuthKeeper.AddressCodec(), app.StakingKeeper.ValidatorAddressCodec(), app.SimulationManager(), app.DefaultGenesis()), + BlockedAddr: BlockedAddresses(), } } func TestAppImportExport(t *testing.T) { - config, db, appOptions, app := setupSimulationApp(t, "skipping application import/export simulation") + Run(t, NewSimApp, setupStateFactory, func(t *testing.T, ti TestInstance[*SimApp]) { + app := ti.App + t.Log("exporting genesis...\n") + exported, err := app.ExportAppStateAndValidators(false, []string{}, []string{}) + require.NoError(t, err) + + t.Log("importing genesis...\n") + newTestInstance := NewSimulationAppInstance(t, ti.Cfg, NewSimApp) + newApp := newTestInstance.App + var genesisState GenesisState + require.NoError(t, json.Unmarshal(exported.AppState, &genesisState)) + ctxB := newApp.NewContextLegacy(true, cmtproto.Header{Height: app.LastBlockHeight()}) + _, err = newApp.ModuleManager.InitGenesis(ctxB, genesisState) + if IsEmptyValidatorSetErr(err) { + t.Skip("Skipping simulation as all validators have been unbonded") + return + } + require.NoError(t, err) + err = newApp.StoreConsensusParams(ctxB, exported.ConsensusParams) + require.NoError(t, err) + + t.Log("comparing stores...") + // skip certain prefixes + skipPrefixes := map[string][][]byte{ + stakingtypes.StoreKey: { + stakingtypes.UnbondingQueueKey, stakingtypes.RedelegationQueueKey, stakingtypes.ValidatorQueueKey, + stakingtypes.HistoricalInfoKey, stakingtypes.UnbondingIDKey, stakingtypes.UnbondingIndexKey, + stakingtypes.UnbondingTypeKey, + }, + authzkeeper.StoreKey: {authzkeeper.GrantQueuePrefix}, + feegrant.StoreKey: {feegrant.FeeAllowanceQueueKeyPrefix}, + slashingtypes.StoreKey: {slashingtypes.ValidatorMissedBlockBitmapKeyPrefix}, + } + AssertEqualStores(t, app, newApp, app.SimulationManager().StoreDecoders, skipPrefixes) + }) +} - // Run randomized simulation - _, simParams, simErr := simulation.SimulateFromSeed( - t, - os.Stdout, - app.BaseApp, - simtestutil.AppStateFn(app.AppCodec(), app.AuthKeeper.AddressCodec(), app.StakingKeeper.ValidatorAddressCodec(), app.SimulationManager(), app.DefaultGenesis()), - simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 - simtestutil.SimulationOperations(app, app.AppCodec(), config), - BlockedAddresses(), - config, - app.AppCodec(), - codectestutil.CodecOptions{}.GetAddressCodec(), - ) +// Scenario: +// +// Start a fresh node and run n blocks, export state +// set up a new node instance, Init chain from exported genesis +// run new instance for n blocks +func TestAppSimulationAfterImport(t *testing.T) { + Run(t, NewSimApp, setupStateFactory, func(t *testing.T, ti TestInstance[*SimApp]) { + app := ti.App + t.Log("exporting genesis...\n") + exported, err := app.ExportAppStateAndValidators(false, []string{}, []string{}) + require.NoError(t, err) + + t.Log("importing genesis...\n") + newTestInstance := NewSimulationAppInstance(t, ti.Cfg, NewSimApp) + newApp := newTestInstance.App + _, err = newApp.InitChain(&abci.InitChainRequest{ + AppStateBytes: exported.AppState, + ChainId: SimAppChainID, + }) + if IsEmptyValidatorSetErr(err) { + t.Skip("Skipping simulation as all validators have been unbonded") + return + } + require.NoError(t, err) + newStateFactory := setupStateFactory(newApp) + _, err = simulation.SimulateFromSeed( + t, + newTestInstance.Logger, + WriteToDebugLog(newTestInstance.Logger), + newApp.BaseApp, + newStateFactory.AppStateFn, + simtypes.RandomAccounts, + simtestutil.SimulationOperations(newApp, newApp.AppCodec(), newTestInstance.Cfg, newApp.TxConfig()), + newStateFactory.BlockedAddr, + newTestInstance.Cfg, + newStateFactory.Codec, + newApp.TxConfig().SigningContext().AddressCodec(), + ) + require.NoError(t, err) + }) +} - // export state and simParams before the simulation error is checked - err := simtestutil.CheckExportSimulation(app, config, simParams) - require.NoError(t, err) - require.NoError(t, simErr) +func IsEmptyValidatorSetErr(err error) bool { + return err != nil && strings.Contains(err.Error(), "validator set is empty after InitGenesis") +} - if config.Commit { - simtestutil.PrintStats(db) +func TestAppStateDeterminism(t *testing.T) { + const numTimesToRunPerSeed = 3 + var seeds []int64 + if s := simcli.NewConfigFromFlags().Seed; s != simcli.DefaultSeedValue { + // We will be overriding the random seed and just run a single simulation on the provided seed value + for j := 0; j < numTimesToRunPerSeed; j++ { // multiple rounds + seeds = append(seeds, s) + } + } else { + // setup with 3 random seeds + for i := 0; i < 3; i++ { + seed := rand.Int63() + for j := 0; j < numTimesToRunPerSeed; j++ { // multiple rounds + seeds = append(seeds, seed) + } + } } - - t.Log("exporting genesis...\n") - - exported, err := app.ExportAppStateAndValidators(false, []string{}, []string{}) - require.NoError(t, err) - - t.Log("importing genesis...\n") - - newDB, newDir, _, _, err := simtestutil.SetupSimulation(config, "leveldb-app-sim-2", "Simulation-2", simcli.FlagVerboseValue, simcli.FlagEnabledValue) - require.NoError(t, err, "simulation setup failed") - - defer func() { - require.NoError(t, newDB.Close()) - require.NoError(t, os.RemoveAll(newDir)) - }() - - appOptions[flags.FlagHome] = newDir // ensure a unique folder for the new app - newApp := NewSimApp(log.NewNopLogger(), newDB, nil, true, appOptions, fauxMerkleModeOpt, baseapp.SetChainID(SimAppChainID)) - require.Equal(t, "SimApp", newApp.Name()) - - var genesisState GenesisState - err = json.Unmarshal(exported.AppState, &genesisState) - require.NoError(t, err) - - ctxA := app.NewContextLegacy(true, cmtproto.Header{Height: app.LastBlockHeight()}) - ctxB := newApp.NewContextLegacy(true, cmtproto.Header{Height: app.LastBlockHeight()}) - _, err = newApp.ModuleManager.InitGenesis(ctxB, genesisState) - if err != nil { - if strings.Contains(err.Error(), "validator set is empty after InitGenesis") { - t.Log("Skipping simulation as all validators have been unbonded") - t.Logf("err: %s stacktrace: %s\n", err, string(debug.Stack())) - return + // overwrite default app config + interBlockCachingAppFactory := func(logger log.Logger, db dbm.DB, traceStore io.Writer, loadLatest bool, appOpts servertypes.AppOptions, baseAppOptions ...func(*baseapp.BaseApp)) *SimApp { + if FlagEnableStreamingValue { + m := map[string]any{ + "streaming.abci.keys": []string{"*"}, + "streaming.abci.plugin": "abci_v1", + "streaming.abci.stop-node-on-err": true, + } + others := appOpts + appOpts = AppOptionsFn(func(k string) any { + if v, ok := m[k]; ok { + return v + } + return others.Get(k) + }) } + return NewSimApp(logger, db, nil, true, appOpts, append(baseAppOptions, interBlockCacheOpt())...) } + var mx sync.Mutex + appHashResults := make(map[int64][][]byte) + captureAndCheckHash := func(t *testing.T, ti TestInstance[*SimApp]) { + seed, appHash := ti.Cfg.Seed, ti.App.LastCommitID().Hash + mx.Lock() + otherHashes := appHashResults[seed] + appHashResults[seed] = append(otherHashes, appHash) + t.Logf("+++ hashes: %#X\n", appHashResults) + mx.Unlock() + // and check that all app hashes per seed are equal for each iteration + for i := 0; i < len(otherHashes); i++ { + require.Equal(t, otherHashes[i], appHash, "non-determinism in seed %d: %v\n", seed, otherHashes) + } + } + // run simulations + RunWithSeeds(t, interBlockCachingAppFactory, setupStateFactory, seeds, []byte{}, captureAndCheckHash) +} - require.NoError(t, err) - err = newApp.StoreConsensusParams(ctxB, exported.ConsensusParams) - require.NoError(t, err) +type ComparableStoreApp interface { + LastBlockHeight() int64 + NewContextLegacy(isCheckTx bool, header cmtproto.Header) sdk.Context + GetKey(storeKey string) *storetypes.KVStoreKey + GetStoreKeys() []storetypes.StoreKey +} - t.Log("comparing stores...") - // skip certain prefixes - skipPrefixes := map[string][][]byte{ - stakingtypes.StoreKey: { - stakingtypes.UnbondingQueueKey, stakingtypes.RedelegationQueueKey, stakingtypes.ValidatorQueueKey, - stakingtypes.HistoricalInfoKey, stakingtypes.UnbondingIDKey, stakingtypes.UnbondingIndexKey, - stakingtypes.UnbondingTypeKey, - }, - authzkeeper.StoreKey: {authzkeeper.GrantQueuePrefix}, - feegrant.StoreKey: {feegrant.FeeAllowanceQueueKeyPrefix}, - slashingtypes.StoreKey: {slashingtypes.ValidatorMissedBlockBitmapKeyPrefix}, - } +func AssertEqualStores(t *testing.T, app ComparableStoreApp, newApp ComparableStoreApp, storeDecoders simtypes.StoreDecoderRegistry, skipPrefixes map[string][][]byte) { + ctxA := app.NewContextLegacy(true, cmtproto.Header{Height: app.LastBlockHeight()}) + ctxB := newApp.NewContextLegacy(true, cmtproto.Header{Height: app.LastBlockHeight()}) storeKeys := app.GetStoreKeys() require.NotEmpty(t, storeKeys) @@ -179,7 +230,7 @@ func TestAppImportExport(t *testing.T) { require.Equal(t, len(failedKVAs), len(failedKVBs), "unequal sets of key-values to compare %s, key stores %s and %s", keyName, appKeyA, appKeyB) t.Logf("compared %d different key/value pairs between %s and %s\n", len(failedKVAs), appKeyA, appKeyB) - if !assert.Equal(t, 0, len(failedKVAs), simtestutil.GetSimulationLog(keyName, app.SimulationManager().StoreDecoders, failedKVAs, failedKVBs)) { + if !assert.Equal(t, 0, len(failedKVAs), simtestutil.GetSimulationLog(keyName, storeDecoders, failedKVAs, failedKVBs)) { for _, v := range failedKVAs { t.Logf("store mismatch: %q\n", v) } @@ -188,198 +239,18 @@ func TestAppImportExport(t *testing.T) { } } -func TestAppSimulationAfterImport(t *testing.T) { - config, db, appOptions, app := setupSimulationApp(t, "skipping application simulation after import") - - // Run randomized simulation - stopEarly, simParams, simErr := simulation.SimulateFromSeed( - t, - os.Stdout, - app.BaseApp, - simtestutil.AppStateFn(app.AppCodec(), app.AuthKeeper.AddressCodec(), app.StakingKeeper.ValidatorAddressCodec(), app.SimulationManager(), app.DefaultGenesis()), - simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 - simtestutil.SimulationOperations(app, app.AppCodec(), config), - BlockedAddresses(), - config, - app.AppCodec(), - codectestutil.CodecOptions{}.GetAddressCodec(), - ) - require.NoError(t, simErr) - - // export state and simParams before the simulation error is checked - err := simtestutil.CheckExportSimulation(app, config, simParams) - require.NoError(t, err) - - if config.Commit { - simtestutil.PrintStats(db) - } - - if stopEarly { - t.Log("can't export or import a zero-validator genesis, exiting test...") - return - } - - t.Logf("exporting genesis...\n") - - exported, err := app.ExportAppStateAndValidators(true, []string{}, []string{}) - require.NoError(t, err) - - t.Logf("importing genesis...\n") - - newDB, newDir, _, _, err := simtestutil.SetupSimulation(config, "leveldb-app-sim-2", "Simulation-2", simcli.FlagVerboseValue, simcli.FlagEnabledValue) - require.NoError(t, err, "simulation setup failed") - - defer func() { - require.NoError(t, newDB.Close()) - require.NoError(t, os.RemoveAll(newDir)) - }() - - newApp := NewSimApp(log.NewNopLogger(), newDB, nil, true, appOptions, fauxMerkleModeOpt, baseapp.SetChainID(SimAppChainID)) - if !simcli.FlagSigverifyTxValue { - newApp.SetNotSigverifyTx() - } - require.Equal(t, "SimApp", newApp.Name()) - - _, err = newApp.InitChain(&abci.InitChainRequest{ - AppStateBytes: exported.AppState, - ChainId: SimAppChainID, - }) - require.NoError(t, err) - _, _, err = simulation.SimulateFromSeed( - t, - os.Stdout, - newApp.BaseApp, - simtestutil.AppStateFn(app.AppCodec(), app.AuthKeeper.AddressCodec(), app.StakingKeeper.ValidatorAddressCodec(), app.SimulationManager(), app.DefaultGenesis()), - simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 - simtestutil.SimulationOperations(newApp, newApp.AppCodec(), config), - BlockedAddresses(), - config, - app.AppCodec(), - codectestutil.CodecOptions{}.GetAddressCodec(), - ) - require.NoError(t, err) -} - -func setupSimulationApp(t *testing.T, msg string) (simtypes.Config, dbm.DB, simtestutil.AppOptionsMap, *SimApp) { - t.Helper() - config := simcli.NewConfigFromFlags() - config.ChainID = SimAppChainID - - db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "leveldb-app-sim", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue) - if skip { - t.Skip(msg) - } - require.NoError(t, err, "simulation setup failed") - - t.Cleanup(func() { - require.NoError(t, db.Close()) - require.NoError(t, os.RemoveAll(dir)) - }) - - appOptions := make(simtestutil.AppOptionsMap, 0) - appOptions[flags.FlagHome] = dir // ensure a unique folder - appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue - - app := NewSimApp(logger, db, nil, true, appOptions, fauxMerkleModeOpt, baseapp.SetChainID(SimAppChainID)) - if !simcli.FlagSigverifyTxValue { - app.SetNotSigverifyTx() - } - require.Equal(t, "SimApp", app.Name()) - return config, db, appOptions, app -} - -// TODO: Make another test for the fuzzer itself, which just has noOp txs -// and doesn't depend on the application. -func TestAppStateDeterminism(t *testing.T) { - if !simcli.FlagEnabledValue { - t.Skip("skipping application simulation") - } - - config := simcli.NewConfigFromFlags() - config.InitialBlockHeight = 1 - config.ExportParamsPath = "" - config.OnOperation = false - config.AllInvariants = false - config.ChainID = SimAppChainID - - numSeeds := 3 - numTimesToRunPerSeed := 3 // This used to be set to 5, but we've temporarily reduced it to 3 for the sake of faster CI. - appHashList := make([]json.RawMessage, numTimesToRunPerSeed) - - // We will be overriding the random seed and just run a single simulation on the provided seed value - if config.Seed != simcli.DefaultSeedValue { - numSeeds = 1 - } - - appOptions := viper.New() - if FlagEnableStreamingValue { - m := make(map[string]interface{}) - m["streaming.abci.keys"] = []string{"*"} - m["streaming.abci.plugin"] = "abci_v1" - m["streaming.abci.stop-node-on-err"] = true - for key, value := range m { - appOptions.SetDefault(key, value) - } - } - appOptions.SetDefault(server.FlagInvCheckPeriod, simcli.FlagPeriodValue) - if simcli.FlagVerboseValue { - appOptions.SetDefault(flags.FlagLogLevel, "debug") - } - - for i := 0; i < numSeeds; i++ { - if config.Seed == simcli.DefaultSeedValue { - config.Seed = rand.Int63() - } - - t.Log("config.Seed: ", config.Seed) - - for j := 0; j < numTimesToRunPerSeed; j++ { - var logger log.Logger - if simcli.FlagVerboseValue { - logger = log.NewTestLogger(t) - } else { - logger = log.NewNopLogger() - } - - appOptions.SetDefault(flags.FlagHome, t.TempDir()) - db := dbm.NewMemDB() - app := NewSimApp(logger, db, nil, true, appOptions, interBlockCacheOpt(), baseapp.SetChainID(SimAppChainID)) - if !simcli.FlagSigverifyTxValue { - app.SetNotSigverifyTx() - } - - t.Logf( - "running non-determinism simulation; seed %d: %d/%d, attempt: %d/%d\n", - config.Seed, i+1, numSeeds, j+1, numTimesToRunPerSeed, - ) - - _, _, err := simulation.SimulateFromSeed( - t, - os.Stdout, - app.BaseApp, - simtestutil.AppStateFn(app.AppCodec(), app.AuthKeeper.AddressCodec(), app.StakingKeeper.ValidatorAddressCodec(), app.SimulationManager(), app.DefaultGenesis()), - simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 - simtestutil.SimulationOperations(app, app.AppCodec(), config), - BlockedAddresses(), - config, - app.AppCodec(), - codectestutil.CodecOptions{}.GetAddressCodec(), - ) - require.NoError(t, err) - - if config.Commit { - simtestutil.PrintStats(db) - } - - appHash := app.LastCommitID().Hash - appHashList[j] = appHash - - if j != 0 { - require.Equal( - t, string(appHashList[0]), string(appHashList[j]), - "non-determinism in seed %d: %d/%d, attempt: %d/%d\n", config.Seed, i+1, numSeeds, j+1, numTimesToRunPerSeed, - ) - } +func FuzzFullAppSimulation(f *testing.F) { + f.Fuzz(func(t *testing.T, rawSeed []byte) { + if len(rawSeed) < 8 { + t.Skip() + return } - } + RunWithSeeds( + t, + NewSimApp, + setupStateFactory, + []int64{int64(binary.BigEndian.Uint64(rawSeed))}, + rawSeed[8:], + ) + }) } diff --git a/testutil/sims/simulation_helpers.go b/testutil/sims/simulation_helpers.go index c06ab9e3e64e..e2d93e86bd47 100644 --- a/testutil/sims/simulation_helpers.go +++ b/testutil/sims/simulation_helpers.go @@ -7,12 +7,12 @@ import ( "os" "sync" + "github.com/cosmos/cosmos-sdk/client" + dbm "github.com/cosmos/cosmos-db" "cosmossdk.io/log" storetypes "cosmossdk.io/store/types" - authtx "cosmossdk.io/x/auth/tx" - "github.com/cosmos/cosmos-sdk/codec" "github.com/cosmos/cosmos-sdk/runtime" sdk "github.com/cosmos/cosmos-sdk/types" @@ -31,7 +31,7 @@ func SetupSimulation(config simtypes.Config, dirPrefix, dbName string, verbose, var logger log.Logger if verbose { - logger = log.NewLogger(os.Stdout) // TODO(mr): enable selection of log destination. + logger = log.NewLogger(os.Stdout) } else { logger = log.NewNopLogger() } @@ -51,14 +51,14 @@ func SetupSimulation(config simtypes.Config, dirPrefix, dbName string, verbose, // SimulationOperations retrieves the simulation params from the provided file path // and returns all the modules weighted operations -func SimulationOperations(app runtime.AppSimI, cdc codec.Codec, config simtypes.Config) []simtypes.WeightedOperation { +func SimulationOperations(app runtime.AppSimI, cdc codec.Codec, config simtypes.Config, txConfig client.TxConfig) []simtypes.WeightedOperation { signingCtx := cdc.InterfaceRegistry().SigningContext() simState := module.SimulationState{ AppParams: make(simtypes.AppParams), Cdc: cdc, AddressCodec: signingCtx.AddressCodec(), ValidatorCodec: signingCtx.ValidatorAddressCodec(), - TxConfig: authtx.NewTxConfig(cdc, signingCtx.AddressCodec(), signingCtx.ValidatorAddressCodec(), authtx.DefaultSignModes), // TODO(tip): we should extract this from app + TxConfig: txConfig, BondDenom: sdk.DefaultBondDenom, } diff --git a/testutil/sims/state_helpers.go b/testutil/sims/state_helpers.go index 68885c52568d..8a3ce8a488a3 100644 --- a/testutil/sims/state_helpers.go +++ b/testutil/sims/state_helpers.go @@ -36,30 +36,35 @@ const ( ) // AppStateFn returns the initial application state using a genesis or the simulation parameters. -// It calls AppStateFnWithExtendedCb with nil rawStateCb. -func AppStateFn(cdc codec.JSONCodec, addresCodec, validatorCodec address.Codec, simManager *module.SimulationManager, genesisState map[string]json.RawMessage) simtypes.AppStateFn { - return AppStateFnWithExtendedCb(cdc, addresCodec, validatorCodec, simManager, genesisState, nil) +// It calls appStateFnWithExtendedCb with nil rawStateCb. +func AppStateFn( + cdc codec.JSONCodec, + addresCodec, validatorCodec address.Codec, + simManager *module.SimulationManager, + genesisState map[string]json.RawMessage, +) simtypes.AppStateFn { + return appStateFnWithExtendedCb(cdc, addresCodec, validatorCodec, simManager, genesisState, nil) } -// AppStateFnWithExtendedCb returns the initial application state using a genesis or the simulation parameters. -// It calls AppStateFnWithExtendedCbs with nil moduleStateCb. -func AppStateFnWithExtendedCb( +// appStateFnWithExtendedCb returns the initial application state using a genesis or the simulation parameters. +// It calls appStateFnWithExtendedCbs with nil moduleStateCb. +func appStateFnWithExtendedCb( cdc codec.JSONCodec, addresCodec, validatorCodec address.Codec, simManager *module.SimulationManager, genesisState map[string]json.RawMessage, rawStateCb func(rawState map[string]json.RawMessage), ) simtypes.AppStateFn { - return AppStateFnWithExtendedCbs(cdc, addresCodec, validatorCodec, simManager, genesisState, nil, rawStateCb) + return appStateFnWithExtendedCbs(cdc, addresCodec, validatorCodec, simManager, genesisState, nil, rawStateCb) } -// AppStateFnWithExtendedCbs returns the initial application state using a genesis or the simulation parameters. +// appStateFnWithExtendedCbs returns the initial application state using a genesis or the simulation parameters. // It panics if the user provides files for both of them. // If a file is not given for the genesis or the sim params, it creates a randomized one. // genesisState is the default genesis state of the whole app. // moduleStateCb is the callback function to access moduleState. // rawStateCb is the callback function to extend rawState. -func AppStateFnWithExtendedCbs( +func appStateFnWithExtendedCbs( cdc codec.JSONCodec, addressCodec, validatorCodec address.Codec, simManager *module.SimulationManager, @@ -139,7 +144,6 @@ func AppStateFnWithExtendedCbs( notBondedCoins := sdk.NewCoin(stakingState.Params.BondDenom, notBondedTokens) // edit bank state to make it have the not bonded pool tokens bankStateBz, ok := rawState[testutil.BankModuleName] - // TODO(fdymylja/jonathan): should we panic in this case if !ok { panic("bank genesis state is missing") } @@ -220,15 +224,6 @@ func AppStateRandomizedFn( numInitiallyBonded = numAccs } - fmt.Printf( - `Selected randomly generated parameters for simulated genesis: -{ - stake_per_account: "%d", - initially_bonded_validators: "%d" -} -`, initialStake.Uint64(), numInitiallyBonded, - ) - simState := &module.SimulationState{ AppParams: appParams, Cdc: cdc, diff --git a/types/simulation/config.go b/types/simulation/config.go index d2bb114133e0..59feff6dee70 100644 --- a/types/simulation/config.go +++ b/types/simulation/config.go @@ -1,5 +1,7 @@ package simulation +import "testing" + // Config contains the necessary configuration flags for the simulator type Config struct { GenesisFile string // custom simulation genesis file; cannot be used with params file @@ -20,9 +22,29 @@ type Config struct { Lean bool // lean simulation log output Commit bool // have the simulation commit - OnOperation bool // run slow invariants every operation - AllInvariants bool // print all failed invariants if a broken invariant is found - DBBackend string // custom db backend type BlockMaxGas int64 // custom max gas for block + FuzzSeed []byte + T testing.TB +} + +func (c Config) Clone() Config { + return Config{ + GenesisFile: c.GenesisFile, + ParamsFile: c.ParamsFile, + ExportParamsPath: c.ExportParamsPath, + ExportParamsHeight: c.ExportParamsHeight, + ExportStatePath: c.ExportStatePath, + ExportStatsPath: c.ExportStatsPath, + Seed: c.Seed, + InitialBlockHeight: c.InitialBlockHeight, + GenesisTime: c.GenesisTime, + NumBlocks: c.NumBlocks, + BlockSize: c.BlockSize, + ChainID: c.ChainID, + Lean: c.Lean, + Commit: c.Commit, + DBBackend: c.DBBackend, + BlockMaxGas: c.BlockMaxGas, + } } diff --git a/types/simulation/rand_util.go b/types/simulation/rand_util.go index adacd90ad436..2dd39ead89bd 100644 --- a/types/simulation/rand_util.go +++ b/types/simulation/rand_util.go @@ -147,31 +147,8 @@ func RandSubsetCoins(r *rand.Rand, coins sdk.Coins) sdk.Coins { } // DeriveRand derives a new Rand deterministically from another random source. -// Unlike rand.New(rand.NewSource(seed)), the result is "more random" -// depending on the source and state of r. // // NOTE: not crypto safe. func DeriveRand(r *rand.Rand) *rand.Rand { - const num = 8 // TODO what's a good number? Too large is too slow. - ms := multiSource(make([]rand.Source, num)) - - for i := 0; i < num; i++ { - ms[i] = rand.NewSource(r.Int63()) - } - - return rand.New(ms) -} - -type multiSource []rand.Source - -func (ms multiSource) Int63() (r int64) { - for _, source := range ms { - r ^= source.Int63() - } - - return r -} - -func (ms multiSource) Seed(_ int64) { - panic("multiSource Seed should not be called") + return rand.New(rand.NewSource(r.Int63())) } diff --git a/types/simulation/rand_util_test.go b/types/simulation/rand_util_test.go index 0bfbd24423dd..f88fdfecfd63 100644 --- a/types/simulation/rand_util_test.go +++ b/types/simulation/rand_util_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "cosmossdk.io/math" @@ -176,23 +178,11 @@ func TestRandomIntBetween(t *testing.T) { } func TestDeriveRand(t *testing.T) { - t.Parallel() - tests := []struct { - name string - r *rand.Rand - exp int64 - }{ - {"seed equal to zero", rand.New(rand.NewSource(0)), 8759604767892952359}, - {"seed positive number", rand.New(rand.NewSource(1)), 4609759310771376844}, - {"seed negative number", rand.New(rand.NewSource(-2)), 7949885017563827825}, - } - for _, tt := range tests { - tc := tt - t.Run(tc.name, func(t *testing.T) { - got := simulation.DeriveRand(tc.r) - require.Equal(t, got.Int63(), tc.exp) - }) - } + src := rand.New(rand.NewSource(0)) + derived := simulation.DeriveRand(src) + got := derived.Int() + assert.NotEqual(t, got, src) + assert.NotEqual(t, got, rand.New(rand.NewSource(0)).Int()) } func mustParseCoins(s string) sdk.Coins { diff --git a/x/simulation/client/cli/flags.go b/x/simulation/client/cli/flags.go index f57c44a21a2e..409e2aabf80a 100644 --- a/x/simulation/client/cli/flags.go +++ b/x/simulation/client/cli/flags.go @@ -23,8 +23,6 @@ var ( FlagBlockSizeValue int FlagLeanValue bool FlagCommitValue bool - FlagOnOperationValue bool // TODO: Remove in favor of binary search for invariant violation - FlagAllInvariantsValue bool FlagDBBackendValue string FlagEnabledValue bool @@ -42,15 +40,12 @@ func GetSimulatorFlags() { flag.StringVar(&FlagExportParamsPathValue, "ExportParamsPath", "", "custom file path to save the exported params JSON") flag.IntVar(&FlagExportParamsHeightValue, "ExportParamsHeight", 0, "height to which export the randomly generated params") flag.StringVar(&FlagExportStatePathValue, "ExportStatePath", "", "custom file path to save the exported app state JSON") - flag.StringVar(&FlagExportStatsPathValue, "ExportStatsPath", "", "custom file path to save the exported simulation statistics JSON") flag.Int64Var(&FlagSeedValue, "Seed", DefaultSeedValue, "simulation random seed") flag.IntVar(&FlagInitialBlockHeightValue, "InitialBlockHeight", 1, "initial block to start the simulation") flag.IntVar(&FlagNumBlocksValue, "NumBlocks", 500, "number of new blocks to simulate from the initial block height") flag.IntVar(&FlagBlockSizeValue, "BlockSize", 200, "operations per block") flag.BoolVar(&FlagLeanValue, "Lean", false, "lean simulation log output") - flag.BoolVar(&FlagCommitValue, "Commit", false, "have the simulation commit") - flag.BoolVar(&FlagOnOperationValue, "SimulateEveryOperation", false, "run slow invariants every operation") - flag.BoolVar(&FlagAllInvariantsValue, "PrintAllInvariants", false, "print all invariants if a broken invariant is found") + flag.BoolVar(&FlagCommitValue, "Commit", true, "have the simulation commit") flag.StringVar(&FlagDBBackendValue, "DBBackend", "goleveldb", "custom db backend type") // simulation flags @@ -77,8 +72,6 @@ func NewConfigFromFlags() simulation.Config { BlockSize: FlagBlockSizeValue, Lean: FlagLeanValue, Commit: FlagCommitValue, - OnOperation: FlagOnOperationValue, - AllInvariants: FlagAllInvariantsValue, DBBackend: FlagDBBackendValue, } } diff --git a/x/simulation/params.go b/x/simulation/params.go index eee74ac20100..da62ffe1e87a 100644 --- a/x/simulation/params.go +++ b/x/simulation/params.go @@ -2,7 +2,6 @@ package simulation import ( "encoding/json" - "fmt" "math/rand" cmtproto "github.com/cometbft/cometbft/api/cometbft/types/v1" @@ -198,12 +197,5 @@ func randomConsensusParams(r *rand.Rand, appState json.RawMessage, cdc codec.JSO MaxAgeDuration: stakingGenesisState.Params.UnbondingTime, }, } - - bz, err := json.MarshalIndent(&consensusParams, "", " ") - if err != nil { - panic(err) - } - fmt.Printf("Selected randomly generated consensus parameters:\n%s\n", bz) - return consensusParams } diff --git a/x/simulation/simulate.go b/x/simulation/simulate.go index ddfe004a7ed5..ad1f60ab1bad 100644 --- a/x/simulation/simulate.go +++ b/x/simulation/simulate.go @@ -1,15 +1,17 @@ package simulation import ( + "bytes" + "encoding/binary" + "encoding/hex" "fmt" "io" "math/rand" - "os" - "os/signal" - "syscall" "testing" "time" + "cosmossdk.io/log" + abci "github.com/cometbft/cometbft/abci/types" cmtproto "github.com/cometbft/cometbft/api/cometbft/types/v1" @@ -59,6 +61,7 @@ func initChain( // operations, testing the provided invariants, but using the provided config.Seed. func SimulateFromSeed( tb testing.TB, + logger log.Logger, w io.Writer, app *baseapp.BaseApp, appStateFn simulation.AppStateFn, @@ -67,17 +70,18 @@ func SimulateFromSeed( blockedAddrs map[string]bool, config simulation.Config, cdc codec.JSONCodec, - addresscodec address.Codec, -) (stopEarly bool, exportedParams Params, err error) { + addressCodec address.Codec, +) (exportedParams Params, err error) { tb.Helper() // in case we have to end early, don't os.Exit so that we can run cleanup code. testingMode, _, b := getTestingMode(tb) - r := rand.New(rand.NewSource(config.Seed)) + r := rand.New(NewByteSource(config.FuzzSeed, config.Seed)) params := RandomParams(r) - fmt.Fprintf(w, "Starting SimulateFromSeed with randomness created with seed %d\n", int(config.Seed)) - fmt.Fprintf(w, "Randomized simulation params: \n%s\n", mustMarshalJSONIndent(params)) + startTime := time.Now() + logger.Info("Starting SimulateFromSeed with randomness", "time", startTime) + logger.Debug("Randomized simulation setup", "params", mustMarshalJSONIndent(params)) timeDiff := maxTimePerBlock - minTimePerBlock accs := randAccFn(r, params.NumKeys()) @@ -89,23 +93,18 @@ func SimulateFromSeed( // At least 2 accounts must be added here, otherwise when executing SimulateMsgSend // two accounts will be selected to meet the conditions from != to and it will fall into an infinite loop. if len(accs) <= 1 { - return true, params, fmt.Errorf("at least two genesis accounts are required") + return params, fmt.Errorf("at least two genesis accounts are required") } config.ChainID = chainID - fmt.Printf( - "Starting the simulation from time %v (unixtime %v)\n", - blockTime.UTC().Format(time.UnixDate), blockTime.Unix(), - ) - // remove module account address if they exist in accs var tmpAccs []simulation.Account for _, acc := range accs { - accAddr, err := addresscodec.BytesToString(acc.Address) + accAddr, err := addressCodec.BytesToString(acc.Address) if err != nil { - return true, params, err + return params, err } if !blockedAddrs[accAddr] { tmpAccs = append(tmpAccs, acc) @@ -125,17 +124,6 @@ func SimulateFromSeed( opCount = 0 ) - // Setup code to catch SIGTERM's - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGINT) - - go func() { - receivedSignal := <-c - fmt.Fprintf(w, "\nExiting early due to %s, on block %d, operation %d\n", receivedSignal, blockHeight, opCount) - err = fmt.Errorf("exited due to %s", receivedSignal) - stopEarly = true - }() - finalizeBlockReq := RandomRequestFinalizeBlock( r, params, @@ -171,7 +159,7 @@ func SimulateFromSeed( // recover logs in case of panic defer func() { if r := recover(); r != nil { - _, _ = fmt.Fprintf(w, "simulation halted due to panic on block %d\n", blockHeight) + logger.Error("simulation halted due to panic", "height", blockHeight) logWriter.PrintLogs() panic(r) } @@ -183,7 +171,7 @@ func SimulateFromSeed( exportedParams = params } - for blockHeight < int64(config.NumBlocks+config.InitialBlockHeight) && !stopEarly { + for blockHeight < int64(config.NumBlocks+config.InitialBlockHeight) { pastTimes = append(pastTimes, blockTime) pastVoteInfos = append(pastVoteInfos, finalizeBlockReq.DecidedLastCommit.Votes) @@ -192,7 +180,7 @@ func SimulateFromSeed( res, err := app.FinalizeBlock(finalizeBlockReq) if err != nil { - return true, params, err + return params, err } ctx := app.NewContextLegacy(false, cmtproto.Header{ @@ -241,14 +229,13 @@ func SimulateFromSeed( if config.Commit { _, err := app.Commit() if err != nil { - return true, params, err + return params, err } } if proposerAddress == nil { - fmt.Fprintf(w, "\nSimulation stopped early as all validators have been unbonded; nobody left to propose a block!\n") - stopEarly = true + logger.Info("Simulation stopped early as all validators have been unbonded; nobody left to propose a block", "height", blockHeight) break } @@ -267,22 +254,8 @@ func SimulateFromSeed( } } - if stopEarly { - if config.ExportStatsPath != "" { - fmt.Println("Exporting simulation statistics...") - eventStats.ExportJSON(config.ExportStatsPath) - } else { - eventStats.Print(w) - } - - return true, exportedParams, err - } - - fmt.Fprintf( - w, - "\nSimulation complete; Final height (blocks): %d, final time (seconds): %v, operations ran: %d\n", - blockHeight, blockTime, opCount, - ) + logger.Info("Simulation complete", "height", blockHeight, "block-time", blockTime, "opsCount", opCount, + "run-time", time.Since(startTime), "app-hash", hex.EncodeToString(app.LastCommitID().Hash)) if config.ExportStatsPath != "" { fmt.Println("Exporting simulation statistics...") @@ -290,8 +263,7 @@ func SimulateFromSeed( } else { eventStats.Print(w) } - - return false, exportedParams, nil + return exportedParams, err } type blockSimFn func( @@ -361,7 +333,7 @@ Comment: %s`, queueOperations(operationQueue, timeOperationQueue, futureOps) if testingMode && opCount%50 == 0 { - fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ", + _, _ = fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ", header.Height, config.NumBlocks, opCount, blocksize) } @@ -444,3 +416,63 @@ func runQueuedTimeOperations(tb testing.TB, queueOps []simulation.FutureOperatio return numOpsRan, allFutureOps } + +const ( + rngMax = 1 << 63 + rngMask = rngMax - 1 +) + +type ByteSource struct { + seed *bytes.Reader + fallback *rand.Rand +} + +func NewByteSource(fuzzSeed []byte, seed int64) *ByteSource { + return &ByteSource{ + seed: bytes.NewReader(fuzzSeed), + fallback: rand.New(rand.NewSource(seed)), + } +} + +func (s *ByteSource) Uint64() uint64 { + if s.seed.Len() < 8 { + return s.fallback.Uint64() + } + var b [8]byte + if _, err := s.seed.Read(b[:]); err != nil && err != io.EOF { + panic(err) // Should not happen. + } + return binary.BigEndian.Uint64(b[:]) +} + +func (s *ByteSource) Int63() int64 { + return int64(s.Uint64() & rngMask) +} +func (s *ByteSource) Seed(seed int64) {} + +type arraySource struct { + pos int + arr []int64 + src *rand.Rand +} + +// Int63 returns a non-negative pseudo-random 63-bit integer as an int64. +func (rng *arraySource) Int63() int64 { + return int64(rng.Uint64() & rngMask) +} + +// Uint64 returns a non-negative pseudo-random 64-bit integer as an uint64. +func (rng *arraySource) Uint64() uint64 { + if rng.pos >= len(rng.arr) { + return rng.src.Uint64() + } + val := rng.arr[rng.pos] + rng.pos = rng.pos + 1 + if val < 0 { + return uint64(-val) + } + + return uint64(val) +} + +func (rng *arraySource) Seed(seed int64) {} From 40879670e839e095c2194ebdeccc3d506d550a77 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Thu, 30 May 2024 17:52:46 +0200 Subject: [PATCH 02/15] Add tag to exclude sims from normal test runs --- Makefile | 18 +++++++++--------- simapp/sim_bench_test.go | 11 ++--------- simapp/sim_test.go | 2 ++ 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index a480d6ff9b53..26a29c05173b 100644 --- a/Makefile +++ b/Makefile @@ -277,7 +277,7 @@ endif #? test-sim-nondeterminism: Run non-determinism test for simapp test-sim-nondeterminism: @echo "Running non-determinism test..." - @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -run TestAppStateDeterminism \ + @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -tags='sims' -run TestAppStateDeterminism \ -NumBlocks=100 -BlockSize=200 -Period=0 # Requires an exported plugin. See store/streaming/README.md for documentation. @@ -291,45 +291,45 @@ test-sim-nondeterminism: # make test-sim-nondeterminism-streaming test-sim-nondeterminism-streaming: @echo "Running non-determinism-streaming test..." - @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -run TestAppStateDeterminism \ + @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -tags='sims' -run TestAppStateDeterminism \ -NumBlocks=100 -BlockSize=200 -Period=0 -EnableStreaming=true test-sim-custom-genesis-fast: @echo "Running custom genesis simulation..." @echo "By default, ${HOME}/.simapp/config/genesis.json will be used." - @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \ + @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -tags='sims' -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \ -NumBlocks=100 -BlockSize=200 -Seed=99 -Period=5 -SigverifyTx=false test-sim-import-export: @echo "Running application import/export simulation. This may take several minutes..." - @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 20m -run TestAppImportExport \ + @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 20m -tags='sims' -run TestAppImportExport \ -NumBlocks=50 -Period=5 test-sim-after-import: @echo "Running application simulation-after-import. This may take several minutes..." - @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -run TestAppSimulationAfterImport \ + @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -tags='sims' -run TestAppSimulationAfterImport \ -NumBlocks=50 -Period=5 test-sim-custom-genesis-multi-seed: @echo "Running multi-seed custom genesis simulation..." @echo "By default, ${HOME}/.simapp/config/genesis.json will be used." - @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \ + @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \ -NumBlocks=400 -Period=5 test-sim-multi-seed-long: @echo "Running long multi-seed application simulation. This may take awhile!" - @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=1h -run TestFullAppSimulation \ + @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=1h -tags='sims' -run TestFullAppSimulation \ -NumBlocks=500 -Period=50 test-sim-multi-seed-short: @echo "Running short multi-seed application simulation. This may take awhile!" - @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -run TestFullAppSimulation \ + @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation \ -NumBlocks=50 -Period=10 test-sim-benchmark-invariants: @echo "Running simulation invariant benchmarks..." - cd ${CURRENT_DIR}/simapp && @go test -mod=readonly -benchmem -bench=BenchmarkInvariants -run=^$ \ + cd ${CURRENT_DIR}/simapp && go test -mod=readonly -benchmem -bench=BenchmarkInvariants -tags='sims' -run=^$ \ -Enabled=true -NumBlocks=1000 -BlockSize=200 \ -Period=1 -Commit=true -Seed=57 -v -timeout 24h diff --git a/simapp/sim_bench_test.go b/simapp/sim_bench_test.go index c0ddd7df1edb..0348ca999982 100644 --- a/simapp/sim_bench_test.go +++ b/simapp/sim_bench_test.go @@ -1,3 +1,5 @@ +//go:build sims + package simapp import ( @@ -49,15 +51,6 @@ func BenchmarkFullAppSimulation(b *testing.B) { }() appOptions := viper.New() - if FlagEnableStreamingValue { - m := make(map[string]interface{}) - m["streaming.abci.keys"] = []string{"*"} - m["streaming.abci.plugin"] = "abci_v1" - m["streaming.abci.stop-node-on-err"] = true - for key, value := range m { - appOptions.SetDefault(key, value) - } - } appOptions.SetDefault(flags.FlagHome, DefaultNodeHome) appOptions.SetDefault(server.FlagInvCheckPeriod, simcli.FlagPeriodValue) diff --git a/simapp/sim_test.go b/simapp/sim_test.go index 846556d2f5ca..072f40a3bb1b 100644 --- a/simapp/sim_test.go +++ b/simapp/sim_test.go @@ -1,3 +1,5 @@ +//go:build sims + package simapp import ( From 0f54fddb5365f1e5e2e77bebce238be8a8c456b3 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Fri, 31 May 2024 17:17:37 +0200 Subject: [PATCH 03/15] Add build tag to fuzz test target --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c165b6b7d6e1..425b012be7f4 100644 --- a/Makefile +++ b/Makefile @@ -352,7 +352,7 @@ SIM_COMMIT ?= true test-sim-fuzz: @echo "Running application fuzz for numBlocks=2, blockSize=20. This may take awhile!" #ld flags are a quick fix to make it work on current osx - @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -json -ldflags="-extldflags=-Wl,-ld_classic" -timeout=60m -fuzztime=60m -run=^$$ -fuzz=FuzzFullAppSimulation -GenesisTime=1714720615 -NumBlocks=2 -BlockSize=20 + @cd ${CURRENT_DIR}/simapp && go test -mod=readonly -json -tags='sims' -ldflags="-extldflags=-Wl,-ld_classic" -timeout=60m -fuzztime=60m -run=^$$ -fuzz=FuzzFullAppSimulation -GenesisTime=1714720615 -NumBlocks=2 -BlockSize=20 #? test-sim-benchmark: Run benchmark test for simapp test-sim-benchmark: From c988e19f4d67439eb80a4f52324aa23d81cfd958 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Fri, 31 May 2024 17:20:13 +0200 Subject: [PATCH 04/15] Add changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3773d6e78ab1..45ece71b4a89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ Every module contains its own CHANGELOG.md. Please refer to the module you are interested in. ### Features - +* (perf)[#20490](https://github.com/cosmos/cosmos-sdk/pull/20490) Sims: Replace runsim command with Go stdlib testing * (tests) [#20013](https://github.com/cosmos/cosmos-sdk/pull/20013) Introduce system tests to run multi node local testnet in CI * (runtime) [#19953](https://github.com/cosmos/cosmos-sdk/pull/19953) Implement `core/transaction.Service` in runtime. * (client) [#19905](https://github.com/cosmos/cosmos-sdk/pull/19905) Add grpc client config to `client.toml`. From 711e147f79c09bc28728723e67db3cb5dfb47b60 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Fri, 31 May 2024 17:31:19 +0200 Subject: [PATCH 05/15] Restore cron based sims action --- .github/workflows/sims.yml | 105 ++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 61 deletions(-) diff --git a/.github/workflows/sims.yml b/.github/workflows/sims.yml index e6089e3b048f..fcac31a88096 100644 --- a/.github/workflows/sims.yml +++ b/.github/workflows/sims.yml @@ -2,13 +2,10 @@ name: Sims # Sims workflow runs multiple types of simulations (nondeterminism, import-export, after-import, multi-seed-short) # This workflow will run on all Pull Requests, if a .go, .mod or .sum file have been changed on: - push: - branches: - - main - - release/** - pull_request: -permissions: - contents: read + schedule: + - cron: "0 */2 * * *" + release: + types: [published] concurrency: group: ci-${{ github.ref }}-sims @@ -82,61 +79,47 @@ jobs: run: | make test-sim-multi-seed-short - test-sim-fuzz: + sims-notify-success: + needs: + [test-sim-multi-seed-short, test-sim-after-import, test-sim-import-export] runs-on: large-sdk-runner - needs: [build] - timeout-minutes: 60 + if: ${{ success() }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - name: Check out repository + uses: actions/checkout@v4 + - name: Get previous workflow status + uses: ./.github/actions/last-workflow-status + id: last_status with: - go-version: "1.22" - check-latest: true - - name: test-sim-fuzz - run: | - make test-sim-fuzz + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Notify Slack on success + if: ${{ steps.last_status.outputs.last_status == 'failure' }} + uses: rtCamp/action-slack-notify@v2.3.0 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACK_CHANNEL: sdk-sims + SLACK_USERNAME: Sim Tests + SLACK_ICON_EMOJI: ":white_check_mark:" + SLACK_COLOR: good + SLACK_MESSAGE: Sims are passing + SLACK_FOOTER: "" -# sims-notify-success: -# needs: -# [test-sim-multi-seed-short, test-sim-after-import, test-sim-import-export] -# runs-on: large-sdk-runner -# if: ${{ success() }} -# steps: -# - name: Check out repository -# uses: actions/checkout@v4 -# - name: Get previous workflow status -# uses: ./.github/actions/last-workflow-status -# id: last_status -# with: -# github_token: ${{ secrets.GITHUB_TOKEN }} -# -# - name: Notify Slack on success -# if: ${{ steps.last_status.outputs.last_status == 'failure' }} -# uses: rtCamp/action-slack-notify@v2.3.0 -# env: -# SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} -# SLACK_CHANNEL: sdk-sims -# SLACK_USERNAME: Sim Tests -# SLACK_ICON_EMOJI: ":white_check_mark:" -# SLACK_COLOR: good -# SLACK_MESSAGE: Sims are passing -# SLACK_FOOTER: "" -# -# sims-notify-failure: -# permissions: -# contents: none -# needs: -# [test-sim-multi-seed-short, test-sim-after-import, test-sim-import-export] -# runs-on: large-sdk-runner -# if: ${{ failure() }} -# steps: -# - name: Notify Slack on failure -# uses: rtCamp/action-slack-notify@v2.3.0 -# env: -# SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} -# SLACK_CHANNEL: sdk-sims -# SLACK_USERNAME: Sim Tests -# SLACK_ICON_EMOJI: ":skull:" -# SLACK_COLOR: danger -# SLACK_MESSAGE: Sims are failing -# SLACK_FOOTER: "" + sims-notify-failure: + permissions: + contents: none + needs: + [test-sim-multi-seed-short, test-sim-after-import, test-sim-import-export] + runs-on: large-sdk-runner + if: ${{ failure() }} + steps: + - name: Notify Slack on failure + uses: rtCamp/action-slack-notify@v2.3.0 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACK_CHANNEL: sdk-sims + SLACK_USERNAME: Sim Tests + SLACK_ICON_EMOJI: ":skull:" + SLACK_COLOR: danger + SLACK_MESSAGE: Sims are failing + SLACK_FOOTER: "" From 1008ebd394e280aabe1d6ff663ebc302facdc120 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Mon, 3 Jun 2024 15:35:38 +0200 Subject: [PATCH 06/15] Review feedback --- .github/workflows/sims.yml | 2 ++ simapp/sim_bench_test.go | 8 +++++--- simapp/sim_test.go | 28 +++++++++++++++------------- {simapp => testutils/sims}/runner.go | 2 +- x/simulation/simulate.go | 7 ++++++- 5 files changed, 29 insertions(+), 18 deletions(-) rename {simapp => testutils/sims}/runner.go (99%) diff --git a/.github/workflows/sims.yml b/.github/workflows/sims.yml index fcac31a88096..4b695d79ab4b 100644 --- a/.github/workflows/sims.yml +++ b/.github/workflows/sims.yml @@ -42,6 +42,7 @@ jobs: test-sim-after-import: runs-on: large-sdk-runner needs: [build] + timeout-minutes: 60 steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 @@ -55,6 +56,7 @@ jobs: test-sim-deterministic: runs-on: large-sdk-runner needs: [build] + timeout-minutes: 60 steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 diff --git a/simapp/sim_bench_test.go b/simapp/sim_bench_test.go index 0348ca999982..be0286c3000f 100644 --- a/simapp/sim_bench_test.go +++ b/simapp/sim_bench_test.go @@ -6,7 +6,9 @@ import ( "os" "testing" - "cosmossdk.io/log" + "github.com/cosmos/cosmos-sdk/testutils/sims" + + "cosmossdk.io/core/log" flag "github.com/spf13/pflag" "github.com/spf13/viper" @@ -34,7 +36,7 @@ func BenchmarkFullAppSimulation(b *testing.B) { b.ReportAllocs() config := simcli.NewConfigFromFlags() - config.ChainID = SimAppChainID + config.ChainID = sims.SimAppChainID db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "goleveldb-app-sim", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue) if err != nil { @@ -54,7 +56,7 @@ func BenchmarkFullAppSimulation(b *testing.B) { appOptions.SetDefault(flags.FlagHome, DefaultNodeHome) appOptions.SetDefault(server.FlagInvCheckPeriod, simcli.FlagPeriodValue) - app := NewSimApp(logger, db, nil, true, appOptions, interBlockCacheOpt(), baseapp.SetChainID(SimAppChainID)) + app := NewSimApp(logger, db, nil, true, appOptions, interBlockCacheOpt(), baseapp.SetChainID(sims.SimAppChainID)) // run randomized simulation simParams, simErr := simulation.SimulateFromSeed( diff --git a/simapp/sim_test.go b/simapp/sim_test.go index 072f40a3bb1b..a703f608079c 100644 --- a/simapp/sim_test.go +++ b/simapp/sim_test.go @@ -12,6 +12,8 @@ import ( "sync" "testing" + "github.com/cosmos/cosmos-sdk/testutils/sims" + servertypes "github.com/cosmos/cosmos-sdk/server/types" sdk "github.com/cosmos/cosmos-sdk/types" @@ -54,11 +56,11 @@ func interBlockCacheOpt() func(*baseapp.BaseApp) { } func TestFullAppSimulation(t *testing.T) { - Run(t, NewSimApp, setupStateFactory) + sims.Run(t, NewSimApp, setupStateFactory) } -func setupStateFactory(app *SimApp) SimStateFactory { - return SimStateFactory{ +func setupStateFactory(app *SimApp) sims.SimStateFactory { + return sims.SimStateFactory{ Codec: app.AppCodec(), AppStateFn: simtestutil.AppStateFn(app.AppCodec(), app.AuthKeeper.AddressCodec(), app.StakingKeeper.ValidatorAddressCodec(), app.SimulationManager(), app.DefaultGenesis()), BlockedAddr: BlockedAddresses(), @@ -66,14 +68,14 @@ func setupStateFactory(app *SimApp) SimStateFactory { } func TestAppImportExport(t *testing.T) { - Run(t, NewSimApp, setupStateFactory, func(t *testing.T, ti TestInstance[*SimApp]) { + sims.Run(t, NewSimApp, setupStateFactory, func(t *testing.T, ti sims.TestInstance[*SimApp]) { app := ti.App t.Log("exporting genesis...\n") exported, err := app.ExportAppStateAndValidators(false, []string{}, []string{}) require.NoError(t, err) t.Log("importing genesis...\n") - newTestInstance := NewSimulationAppInstance(t, ti.Cfg, NewSimApp) + newTestInstance := sims.NewSimulationAppInstance(t, ti.Cfg, NewSimApp) newApp := newTestInstance.App var genesisState GenesisState require.NoError(t, json.Unmarshal(exported.AppState, &genesisState)) @@ -109,18 +111,18 @@ func TestAppImportExport(t *testing.T) { // set up a new node instance, Init chain from exported genesis // run new instance for n blocks func TestAppSimulationAfterImport(t *testing.T) { - Run(t, NewSimApp, setupStateFactory, func(t *testing.T, ti TestInstance[*SimApp]) { + sims.Run(t, NewSimApp, setupStateFactory, func(t *testing.T, ti sims.TestInstance[*SimApp]) { app := ti.App t.Log("exporting genesis...\n") exported, err := app.ExportAppStateAndValidators(false, []string{}, []string{}) require.NoError(t, err) t.Log("importing genesis...\n") - newTestInstance := NewSimulationAppInstance(t, ti.Cfg, NewSimApp) + newTestInstance := sims.NewSimulationAppInstance(t, ti.Cfg, NewSimApp) newApp := newTestInstance.App _, err = newApp.InitChain(&abci.InitChainRequest{ AppStateBytes: exported.AppState, - ChainId: SimAppChainID, + ChainId: sims.SimAppChainID, }) if IsEmptyValidatorSetErr(err) { t.Skip("Skipping simulation as all validators have been unbonded") @@ -131,7 +133,7 @@ func TestAppSimulationAfterImport(t *testing.T) { _, err = simulation.SimulateFromSeed( t, newTestInstance.Logger, - WriteToDebugLog(newTestInstance.Logger), + sims.WriteToDebugLog(newTestInstance.Logger), newApp.BaseApp, newStateFactory.AppStateFn, simtypes.RandomAccounts, @@ -175,7 +177,7 @@ func TestAppStateDeterminism(t *testing.T) { "streaming.abci.stop-node-on-err": true, } others := appOpts - appOpts = AppOptionsFn(func(k string) any { + appOpts = sims.AppOptionsFn(func(k string) any { if v, ok := m[k]; ok { return v } @@ -186,7 +188,7 @@ func TestAppStateDeterminism(t *testing.T) { } var mx sync.Mutex appHashResults := make(map[int64][][]byte) - captureAndCheckHash := func(t *testing.T, ti TestInstance[*SimApp]) { + captureAndCheckHash := func(t *testing.T, ti sims.TestInstance[*SimApp]) { seed, appHash := ti.Cfg.Seed, ti.App.LastCommitID().Hash mx.Lock() otherHashes := appHashResults[seed] @@ -199,7 +201,7 @@ func TestAppStateDeterminism(t *testing.T) { } } // run simulations - RunWithSeeds(t, interBlockCachingAppFactory, setupStateFactory, seeds, []byte{}, captureAndCheckHash) + sims.RunWithSeeds(t, interBlockCachingAppFactory, setupStateFactory, seeds, []byte{}, captureAndCheckHash) } type ComparableStoreApp interface { @@ -247,7 +249,7 @@ func FuzzFullAppSimulation(f *testing.F) { t.Skip() return } - RunWithSeeds( + sims.RunWithSeeds( t, NewSimApp, setupStateFactory, diff --git a/simapp/runner.go b/testutils/sims/runner.go similarity index 99% rename from simapp/runner.go rename to testutils/sims/runner.go index b7babd4bc6fe..cbb24002afd0 100644 --- a/simapp/runner.go +++ b/testutils/sims/runner.go @@ -1,4 +1,4 @@ -package simapp +package sims import ( "fmt" diff --git a/x/simulation/simulate.go b/x/simulation/simulate.go index ad1f60ab1bad..d5f7aff72c05 100644 --- a/x/simulation/simulate.go +++ b/x/simulation/simulate.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "cosmossdk.io/log" + "cosmossdk.io/core/log" abci "github.com/cometbft/cometbft/abci/types" cmtproto "github.com/cometbft/cometbft/api/cometbft/types/v1" @@ -422,11 +422,16 @@ const ( rngMask = rngMax - 1 ) +// ByteSource offers deterministic pseudo-random numbers for math.Rand with fuzzer support. +// The 'seed' data is read in big endian to uint64. When exhausted, +// it falls back to a standard random number generator initialized with a specific 'seed' value. type ByteSource struct { seed *bytes.Reader fallback *rand.Rand } +// NewByteSource creates a new ByteSource with a specified byte slice and seed. This gives a fixed sequence of pseudo-random numbers. +// Initially, it utilizes the byte slice. Once that's exhausted, it continues generating numbers using the provided seed. func NewByteSource(fuzzSeed []byte, seed int64) *ByteSource { return &ByteSource{ seed: bytes.NewReader(fuzzSeed), From de7373324112935630f8e24b31dee057872a7912 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Mon, 3 Jun 2024 15:56:32 +0200 Subject: [PATCH 07/15] Update changelog --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45ece71b4a89..f6a08ff5b370 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,6 @@ Ref: https://keepachangelog.com/en/1.0.0/ Every module contains its own CHANGELOG.md. Please refer to the module you are interested in. ### Features -* (perf)[#20490](https://github.com/cosmos/cosmos-sdk/pull/20490) Sims: Replace runsim command with Go stdlib testing * (tests) [#20013](https://github.com/cosmos/cosmos-sdk/pull/20013) Introduce system tests to run multi node local testnet in CI * (runtime) [#19953](https://github.com/cosmos/cosmos-sdk/pull/19953) Implement `core/transaction.Service` in runtime. * (client) [#19905](https://github.com/cosmos/cosmos-sdk/pull/19905) Add grpc client config to `client.toml`. @@ -194,7 +193,7 @@ Every module contains its own CHANGELOG.md. Please refer to the module you are i * (runtime) [#19040](https://github.com/cosmos/cosmos-sdk/pull/19040) Simplify app config implementation and deprecate `/cosmos/app/v1alpha1/config` query. ### CLI Breaking Changes - +* (perf)[#20490](https://github.com/cosmos/cosmos-sdk/pull/20490) Sims: Replace runsim command with Go stdlib testing. CLI: `Commit` default true, `Lean`, `SimulateEveryOperation`, `PrintAllInvariants`, `DBBackend` params removed * (server) [#18303](https://github.com/cosmos/cosmos-sdk/pull/18303) `appd export` has moved with other genesis commands, use `appd genesis export` instead. ### Deprecated From 99bbf1866820f6fac7452073ed546568fd896739 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Tue, 4 Jun 2024 10:32:11 +0200 Subject: [PATCH 08/15] Review feedback --- CHANGELOG.md | 2 ++ simapp/sim_test.go | 21 ++++++++------------- testutil/sims/simulation_helpers.go | 4 ++-- testutils/sims/runner.go | 20 ++++++++++++-------- 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6a08ff5b370..e5e7b852c3d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ Every module contains its own CHANGELOG.md. Please refer to the module you are interested in. ### Features + * (tests) [#20013](https://github.com/cosmos/cosmos-sdk/pull/20013) Introduce system tests to run multi node local testnet in CI * (runtime) [#19953](https://github.com/cosmos/cosmos-sdk/pull/19953) Implement `core/transaction.Service` in runtime. * (client) [#19905](https://github.com/cosmos/cosmos-sdk/pull/19905) Add grpc client config to `client.toml`. @@ -193,6 +194,7 @@ Every module contains its own CHANGELOG.md. Please refer to the module you are i * (runtime) [#19040](https://github.com/cosmos/cosmos-sdk/pull/19040) Simplify app config implementation and deprecate `/cosmos/app/v1alpha1/config` query. ### CLI Breaking Changes + * (perf)[#20490](https://github.com/cosmos/cosmos-sdk/pull/20490) Sims: Replace runsim command with Go stdlib testing. CLI: `Commit` default true, `Lean`, `SimulateEveryOperation`, `PrintAllInvariants`, `DBBackend` params removed * (server) [#18303](https://github.com/cosmos/cosmos-sdk/pull/18303) `appd export` has moved with other genesis commands, use `appd genesis export` instead. diff --git a/simapp/sim_test.go b/simapp/sim_test.go index a703f608079c..c3edd4228e5a 100644 --- a/simapp/sim_test.go +++ b/simapp/sim_test.go @@ -12,18 +12,6 @@ import ( "sync" "testing" - "github.com/cosmos/cosmos-sdk/testutils/sims" - - servertypes "github.com/cosmos/cosmos-sdk/server/types" - - sdk "github.com/cosmos/cosmos-sdk/types" - - abci "github.com/cometbft/cometbft/abci/types" - cmtproto "github.com/cometbft/cometbft/api/cometbft/types/v1" - dbm "github.com/cosmos/cosmos-db" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "cosmossdk.io/log" "cosmossdk.io/store" storetypes "cosmossdk.io/store/types" @@ -31,12 +19,19 @@ import ( "cosmossdk.io/x/feegrant" slashingtypes "cosmossdk.io/x/slashing/types" stakingtypes "cosmossdk.io/x/staking/types" - + abci "github.com/cometbft/cometbft/abci/types" + cmtproto "github.com/cometbft/cometbft/api/cometbft/types/v1" + dbm "github.com/cosmos/cosmos-db" "github.com/cosmos/cosmos-sdk/baseapp" + servertypes "github.com/cosmos/cosmos-sdk/server/types" simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims" + "github.com/cosmos/cosmos-sdk/testutils/sims" + sdk "github.com/cosmos/cosmos-sdk/types" simtypes "github.com/cosmos/cosmos-sdk/types/simulation" "github.com/cosmos/cosmos-sdk/x/simulation" simcli "github.com/cosmos/cosmos-sdk/x/simulation/client/cli" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // SimAppChainID hardcoded chainID for simulation diff --git a/testutil/sims/simulation_helpers.go b/testutil/sims/simulation_helpers.go index e2d93e86bd47..1712d9986ac5 100644 --- a/testutil/sims/simulation_helpers.go +++ b/testutil/sims/simulation_helpers.go @@ -7,12 +7,12 @@ import ( "os" "sync" - "github.com/cosmos/cosmos-sdk/client" - dbm "github.com/cosmos/cosmos-db" "cosmossdk.io/log" storetypes "cosmossdk.io/store/types" + + "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/codec" "github.com/cosmos/cosmos-sdk/runtime" sdk "github.com/cosmos/cosmos-sdk/types" diff --git a/testutils/sims/runner.go b/testutils/sims/runner.go index cbb24002afd0..fe77f791c98b 100644 --- a/testutils/sims/runner.go +++ b/testutils/sims/runner.go @@ -6,12 +6,14 @@ import ( "path/filepath" "testing" - "github.com/cosmos/cosmos-sdk/client" + dbm "github.com/cosmos/cosmos-db" + "github.com/stretchr/testify/require" - "cosmossdk.io/log" + "cosmossdk.io/core/log" + tlog "cosmossdk.io/log" - dbm "github.com/cosmos/cosmos-db" "github.com/cosmos/cosmos-sdk/baseapp" + "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/codec" "github.com/cosmos/cosmos-sdk/runtime" @@ -21,7 +23,6 @@ import ( simtypes "github.com/cosmos/cosmos-sdk/types/simulation" "github.com/cosmos/cosmos-sdk/x/simulation" "github.com/cosmos/cosmos-sdk/x/simulation/client/cli" - "github.com/stretchr/testify/require" ) const SimAppChainID = "simulation-app" @@ -68,6 +69,7 @@ func Run[T SimulationApp]( setupStateFactory func(app T) SimStateFactory, postRunActions ...func(t *testing.T, app TestInstance[T]), ) { + t.Helper() RunWithSeeds(t, appFactory, setupStateFactory, defaultSeeds, nil, postRunActions...) } @@ -95,6 +97,7 @@ func RunWithSeeds[T SimulationApp]( fuzzSeed []byte, postRunActions ...func(t *testing.T, app TestInstance[T]), ) { + t.Helper() cfg := cli.NewConfigFromFlags() cfg.ChainID = SimAppChainID for i := range seeds { @@ -109,9 +112,9 @@ func RunWithSeeds[T SimulationApp]( testInstance := NewSimulationAppInstance(t, tCfg, appFactory) var runLogger log.Logger if cli.FlagVerboseValue { - runLogger = log.NewTestLogger(t) + runLogger = tlog.NewTestLogger(t) } else { - runLogger = log.NewTestLoggerInfo(t) + runLogger = tlog.NewTestLoggerInfo(t) } runLogger = runLogger.With("seed", tCfg.Seed) @@ -168,13 +171,14 @@ func NewSimulationAppInstance[T SimulationApp]( tCfg simtypes.Config, appFactory func(logger log.Logger, db dbm.DB, traceStore io.Writer, loadLatest bool, appOpts servertypes.AppOptions, baseAppOptions ...func(*baseapp.BaseApp)) T, ) TestInstance[T] { + t.Helper() workDir := t.TempDir() dbDir := filepath.Join(workDir, "leveldb-app-sim") var logger log.Logger if cli.FlagVerboseValue { - logger = log.NewTestLogger(t) + logger = tlog.NewTestLogger(t) } else { - logger = log.NewTestLoggerError(t) + logger = tlog.NewTestLoggerError(t) } logger = logger.With("seed", tCfg.Seed) From 01c59af8f0e652e831304fbff3c1b51dcf9e43be Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Tue, 4 Jun 2024 10:58:32 +0200 Subject: [PATCH 09/15] Update sims doc --- docs/build/building-modules/14-simulator.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/build/building-modules/14-simulator.md b/docs/build/building-modules/14-simulator.md index 78ec12c92237..fe337edb861b 100644 --- a/docs/build/building-modules/14-simulator.md +++ b/docs/build/building-modules/14-simulator.md @@ -69,10 +69,11 @@ As you can see, the weights are predefined in this case. Options exist to overri Here is how one can override the above package `simappparams`. ```go reference -https://github.com/cosmos/cosmos-sdk/blob/release/v0.50.x/Makefile#L293-L299 +https://github.com/cosmos/cosmos-sdk/blob/release/v0.51.x/Makefile#L292-L334 ``` +The SDK simulations can be executed like normal tests in Go from the shell or within an IDE. +Make sure that you pass the `-tags='sims` parameter to enable them and other params that make sense for your scenario. -For the last test a tool called [runsim](https://github.com/cosmos/tools/tree/master/cmd/runsim) is used, this is used to parallelize go test instances, provide info to Github and slack integrations to provide information to your team on how the simulations are running. ### Random proposal contents @@ -124,3 +125,12 @@ func NewCustomApp(...) { ... } ``` + +## Integration with the Go fuzzer framework + +The simulations provide deterministic behaviour already. The integration with the [Go fuzzer](https://go.dev/doc/security/fuzz/) +can be done at a high level with the deterministic pseudo random number generator where the fuzzer provides varying numbers. + +```go reference +https://github.com/cosmos/cosmos-sdk/blob/release/v0.51.x/Makefile#L352-L355 +``` \ No newline at end of file From 07f99ccf2a69877bd2068af8bdd0ad93d17423e0 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Tue, 4 Jun 2024 11:31:55 +0200 Subject: [PATCH 10/15] Address linter issues --- types/simulation/rand_util_test.go | 1 - x/simulation/simulate.go | 30 +----------------------------- 2 files changed, 1 insertion(+), 30 deletions(-) diff --git a/types/simulation/rand_util_test.go b/types/simulation/rand_util_test.go index f88fdfecfd63..d8704f69f01f 100644 --- a/types/simulation/rand_util_test.go +++ b/types/simulation/rand_util_test.go @@ -6,7 +6,6 @@ import ( "time" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "cosmossdk.io/math" diff --git a/x/simulation/simulate.go b/x/simulation/simulate.go index d5f7aff72c05..ebb0396ed83d 100644 --- a/x/simulation/simulate.go +++ b/x/simulation/simulate.go @@ -10,13 +10,12 @@ import ( "testing" "time" - "cosmossdk.io/core/log" - abci "github.com/cometbft/cometbft/abci/types" cmtproto "github.com/cometbft/cometbft/api/cometbft/types/v1" "cosmossdk.io/core/address" "cosmossdk.io/core/header" + "cosmossdk.io/core/log" "github.com/cosmos/cosmos-sdk/baseapp" "github.com/cosmos/cosmos-sdk/codec" @@ -454,30 +453,3 @@ func (s *ByteSource) Int63() int64 { return int64(s.Uint64() & rngMask) } func (s *ByteSource) Seed(seed int64) {} - -type arraySource struct { - pos int - arr []int64 - src *rand.Rand -} - -// Int63 returns a non-negative pseudo-random 63-bit integer as an int64. -func (rng *arraySource) Int63() int64 { - return int64(rng.Uint64() & rngMask) -} - -// Uint64 returns a non-negative pseudo-random 64-bit integer as an uint64. -func (rng *arraySource) Uint64() uint64 { - if rng.pos >= len(rng.arr) { - return rng.src.Uint64() - } - val := rng.arr[rng.pos] - rng.pos = rng.pos + 1 - if val < 0 { - return uint64(-val) - } - - return uint64(val) -} - -func (rng *arraySource) Seed(seed int64) {} From 5cab6001812c5306326a5e04cbad9cdc4b87d493 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Thu, 6 Jun 2024 16:10:58 +0200 Subject: [PATCH 11/15] Review feedback --- x/simulation/simulate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x/simulation/simulate.go b/x/simulation/simulate.go index ebb0396ed83d..fc2cfe8a81c2 100644 --- a/x/simulation/simulate.go +++ b/x/simulation/simulate.go @@ -306,7 +306,7 @@ func createBlockSimulator(tb testing.TB, testingMode bool, w io.Writer, params P for i := 0; i < blocksize; i++ { opAndRz = append(opAndRz, opAndR{ op: selectOp(r), - rand: simulation.DeriveRand(r), + rand: r, }) } From 647b22012d7d0c2424bc3b42dd9aabdec5f98c0b Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Fri, 7 Jun 2024 09:34:16 +0200 Subject: [PATCH 12/15] Review feedback --- x/simulation/simulate.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/x/simulation/simulate.go b/x/simulation/simulate.go index fc2cfe8a81c2..ff21f2268a9e 100644 --- a/x/simulation/simulate.go +++ b/x/simulation/simulate.go @@ -179,7 +179,7 @@ func SimulateFromSeed( res, err := app.FinalizeBlock(finalizeBlockReq) if err != nil { - return params, err + return params, fmt.Errorf("block finalization failed at height %d: %w", blockHeight, err) } ctx := app.NewContextLegacy(false, cmtproto.Header{ @@ -226,11 +226,9 @@ func SimulateFromSeed( logWriter.AddEntry(EndBlockEntry(blockHeight)) if config.Commit { - _, err := app.Commit() - if err != nil { - return params, err + if _, err := app.Commit(); err != nil { + return params, fmt.Errorf("commit failed at height %d: %w", blockHeight, err) } - } if proposerAddress == nil { From 1eccdec9524f300f9807c79d4c1a2f292888256c Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Wed, 12 Jun 2024 13:17:46 +0200 Subject: [PATCH 13/15] Stateless sims operations and better debug output --- simapp/sim_bench_test.go | 3 +- simapp/sim_test.go | 29 +++++++--- tests/sims/gov/operations_test.go | 6 +- testutils/sims/runner.go | 29 ++++++---- x/gov/simulation/operations.go | 63 +++++++++++--------- x/group/simulation/operations.go | 96 +++++++++++++++++++------------ x/simulation/log.go | 22 +++++-- x/simulation/operation.go | 21 ++++--- x/simulation/simulate.go | 43 ++++++++++---- 9 files changed, 199 insertions(+), 113 deletions(-) diff --git a/simapp/sim_bench_test.go b/simapp/sim_bench_test.go index be0286c3000f..acb7f0105418 100644 --- a/simapp/sim_bench_test.go +++ b/simapp/sim_bench_test.go @@ -59,7 +59,7 @@ func BenchmarkFullAppSimulation(b *testing.B) { app := NewSimApp(logger, db, nil, true, appOptions, interBlockCacheOpt(), baseapp.SetChainID(sims.SimAppChainID)) // run randomized simulation - simParams, simErr := simulation.SimulateFromSeed( + simParams, simErr := simulation.SimulateFromSeedX( b, log.NewNopLogger(), os.Stdout, @@ -71,6 +71,7 @@ func BenchmarkFullAppSimulation(b *testing.B) { config, app.AppCodec(), app.txConfig.SigningContext().AddressCodec(), + &simulation.DummyLogWriter{}, ) // export state and simParams before the simulation error is checked diff --git a/simapp/sim_test.go b/simapp/sim_test.go index 1e98fafd47be..2c1e1a4fb044 100644 --- a/simapp/sim_test.go +++ b/simapp/sim_test.go @@ -125,10 +125,10 @@ func TestAppSimulationAfterImport(t *testing.T) { } require.NoError(t, err) newStateFactory := setupStateFactory(newApp) - _, err = simulation.SimulateFromSeed( + _, err = simulation.SimulateFromSeedX( t, - newTestInstance.Logger, - sims.WriteToDebugLog(newTestInstance.Logger), + newTestInstance.AppLogger, + sims.WriteToDebugLog(newTestInstance.AppLogger), newApp.BaseApp, newStateFactory.AppStateFn, simtypes.RandomAccounts, @@ -137,6 +137,7 @@ func TestAppSimulationAfterImport(t *testing.T) { newTestInstance.Cfg, newStateFactory.Codec, newApp.TxConfig().SigningContext().AddressCodec(), + ti.ExecLogWriter, ) require.NoError(t, err) }) @@ -183,16 +184,30 @@ func TestAppStateDeterminism(t *testing.T) { } var mx sync.Mutex appHashResults := make(map[int64][][]byte) + appSimLogger := make(map[int64][]simulation.LogWriter) captureAndCheckHash := func(t *testing.T, ti sims.TestInstance[*SimApp]) { seed, appHash := ti.Cfg.Seed, ti.App.LastCommitID().Hash mx.Lock() - otherHashes := appHashResults[seed] - appHashResults[seed] = append(otherHashes, appHash) - t.Logf("+++ hashes: %#X\n", appHashResults) + otherHashes, execWriters := appHashResults[seed], appSimLogger[seed] + if len(otherHashes) < numTimesToRunPerSeed-1 { + appHashResults[seed], appSimLogger[seed] = append(otherHashes, appHash), append(execWriters, ti.ExecLogWriter) + } else { // cleanup + delete(appHashResults, seed) + delete(appSimLogger, seed) + } mx.Unlock() + + var failNow bool // and check that all app hashes per seed are equal for each iteration for i := 0; i < len(otherHashes); i++ { - require.Equal(t, otherHashes[i], appHash, "non-determinism in seed %d: %v\n", seed, otherHashes) + if !assert.Equal(t, otherHashes[i], appHash) { + execWriters[i].PrintLogs() + failNow = true + } + } + if failNow { + ti.ExecLogWriter.PrintLogs() + t.Fatalf("non-determinism in seed %d", seed) } } // run simulations diff --git a/tests/sims/gov/operations_test.go b/tests/sims/gov/operations_test.go index 361837d6c000..73c13f3cc8f8 100644 --- a/tests/sims/gov/operations_test.go +++ b/tests/sims/gov/operations_test.go @@ -268,7 +268,7 @@ func TestSimulateMsgDeposit(t *testing.T) { require.NoError(t, err) // execute operation - op := simulation.SimulateMsgDeposit(suite.TxConfig, suite.AccountKeeper, suite.BankKeeper, suite.GovKeeper) + op := simulation.SimulateMsgDeposit(suite.TxConfig, suite.AccountKeeper, suite.BankKeeper, suite.GovKeeper, nil) operationMsg, _, err := op(r, app.BaseApp, ctx, accounts, "") require.NoError(t, err) @@ -312,7 +312,7 @@ func TestSimulateMsgVote(t *testing.T) { require.NoError(t, err) // execute operation - op := simulation.SimulateMsgVote(suite.TxConfig, suite.AccountKeeper, suite.BankKeeper, suite.GovKeeper) + op := simulation.SimulateMsgVote(suite.TxConfig, suite.AccountKeeper, suite.BankKeeper, suite.GovKeeper, nil) operationMsg, _, err := op(r, app.BaseApp, ctx, accounts, "") require.NoError(t, err) @@ -354,7 +354,7 @@ func TestSimulateMsgVoteWeighted(t *testing.T) { require.NoError(t, err) // execute operation - op := simulation.SimulateMsgVoteWeighted(suite.TxConfig, suite.AccountKeeper, suite.BankKeeper, suite.GovKeeper) + op := simulation.SimulateMsgVoteWeighted(suite.TxConfig, suite.AccountKeeper, suite.BankKeeper, suite.GovKeeper, nil) operationMsg, _, err := op(r, app.BaseApp, ctx, accounts, "") require.NoError(t, err) diff --git a/testutils/sims/runner.go b/testutils/sims/runner.go index fe77f791c98b..888095a2d980 100644 --- a/testutils/sims/runner.go +++ b/testutils/sims/runner.go @@ -120,7 +120,7 @@ func RunWithSeeds[T SimulationApp]( app := testInstance.App stateFactory := setupStateFactory(app) - simParams, err := simulation.SimulateFromSeed( + simParams, err := simulation.SimulateFromSeedX( t, runLogger, WriteToDebugLog(runLogger), @@ -132,6 +132,7 @@ func RunWithSeeds[T SimulationApp]( tCfg, stateFactory.Codec, app.TxConfig().SigningContext().AddressCodec(), + testInstance.ExecLogWriter, ) require.NoError(t, err) err = simtestutil.CheckExportSimulation(app, tCfg, simParams) @@ -152,13 +153,15 @@ func RunWithSeeds[T SimulationApp]( // - DB: The LevelDB database for the simulation app. // - WorkDir: The temporary working directory for the simulation app. // - Cfg: The configuration flags for the simulator. -// - Logger: The logger used for logging in the app during the simulation, with seed value attached. +// - AppLogger: The logger used for logging in the app during the simulation, with seed value attached. +// - ExecLogWriter: Captures block and operation data coming from the simulation type TestInstance[T SimulationApp] struct { - App T - DB dbm.DB - WorkDir string - Cfg simtypes.Config - Logger log.Logger + App T + DB dbm.DB + WorkDir string + Cfg simtypes.Config + AppLogger log.Logger + ExecLogWriter simulation.LogWriter } // NewSimulationAppInstance initializes and returns a TestInstance of a SimulationApp. @@ -190,16 +193,18 @@ func NewSimulationAppInstance[T SimulationApp]( appOptions := make(simtestutil.AppOptionsMap) appOptions[flags.FlagHome] = workDir appOptions[server.FlagInvCheckPeriod] = cli.FlagPeriodValue + app := appFactory(logger, db, nil, true, appOptions, baseapp.SetChainID(SimAppChainID)) if !cli.FlagSigverifyTxValue { app.SetNotSigverifyTx() } return TestInstance[T]{ - App: app, - DB: db, - WorkDir: workDir, - Cfg: tCfg, - Logger: logger, + App: app, + DB: db, + WorkDir: workDir, + Cfg: tCfg, + AppLogger: logger, + ExecLogWriter: &simulation.StandardLogWriter{Seed: tCfg.Seed}, } } diff --git a/x/gov/simulation/operations.go b/x/gov/simulation/operations.go index 983a70ebc2fb..c09baf404dbd 100644 --- a/x/gov/simulation/operations.go +++ b/x/gov/simulation/operations.go @@ -3,6 +3,7 @@ package simulation import ( "math" "math/rand" + "sync/atomic" "time" sdkmath "cosmossdk.io/math" @@ -18,7 +19,7 @@ import ( "github.com/cosmos/cosmos-sdk/x/simulation" ) -var initialProposalID = uint64(100000000000000) +const unsetProposalID = 100000000000000 // Governance message types and routes var ( @@ -43,6 +44,18 @@ const ( DefaultWeightMsgCancelProposal = 5 ) +type sharedState struct { + minProposalID atomic.Uint64 +} + +func (s *sharedState) getMinProposalID() uint64 { + return s.minProposalID.Load() +} + +func (s *sharedState) setMinProposalID(id uint64) { + s.minProposalID.Store(id) +} + // WeightedOperations returns all the operations from the module with their respective weights func WeightedOperations( appParams simtypes.AppParams, @@ -119,19 +132,20 @@ func WeightedOperations( ), ) } - + state := &sharedState{} + state.setMinProposalID(unsetProposalID) wGovOps := simulation.WeightedOperations{ simulation.NewWeightedOperation( weightMsgDeposit, - SimulateMsgDeposit(txGen, ak, bk, k), + SimulateMsgDeposit(txGen, ak, bk, k, state), ), simulation.NewWeightedOperation( weightMsgVote, - SimulateMsgVote(txGen, ak, bk, k), + SimulateMsgVote(txGen, ak, bk, k, state), ), simulation.NewWeightedOperation( weightMsgVoteWeighted, - SimulateMsgVoteWeighted(txGen, ak, bk, k), + SimulateMsgVoteWeighted(txGen, ak, bk, k, state), ), simulation.NewWeightedOperation( weightMsgCancelProposal, @@ -312,7 +326,7 @@ func simulateMsgSubmitProposal( whenVote := ctx.HeaderInfo().Time.Add(time.Duration(r.Int63n(int64(votingPeriod.Seconds()))) * time.Second) fops[i] = simtypes.FutureOperation{ BlockTime: whenVote, - Op: operationSimulateMsgVote(txGen, ak, bk, k, accs[whoVotes[i]], int64(proposalID)), + Op: operationSimulateMsgVote(txGen, ak, bk, k, accs[whoVotes[i]], int64(proposalID), nil), } } @@ -326,13 +340,14 @@ func SimulateMsgDeposit( ak types.AccountKeeper, bk types.BankKeeper, k *keeper.Keeper, + s *sharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simtypes.Account, chainID string, ) (simtypes.OperationMsg, []simtypes.FutureOperation, error) { simAccount, _ := simtypes.RandomAcc(r, accs) - proposalID, ok := randomProposalID(r, k, ctx, v1.StatusDepositPeriod) + proposalID, ok := randomProposalID(r, k, ctx, v1.StatusDepositPeriod, s) if !ok { return simtypes.NoOpMsg(types.ModuleName, TypeMsgDeposit, "unable to generate proposalID"), nil, nil } @@ -392,8 +407,9 @@ func SimulateMsgVote( ak types.AccountKeeper, bk types.BankKeeper, k *keeper.Keeper, + s *sharedState, ) simtypes.Operation { - return operationSimulateMsgVote(txGen, ak, bk, k, simtypes.Account{}, -1) + return operationSimulateMsgVote(txGen, ak, bk, k, simtypes.Account{}, -1, s) } func operationSimulateMsgVote( @@ -403,6 +419,7 @@ func operationSimulateMsgVote( k *keeper.Keeper, simAccount simtypes.Account, proposalIDInt int64, + s *sharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, @@ -417,7 +434,7 @@ func operationSimulateMsgVote( switch { case proposalIDInt < 0: var ok bool - proposalID, ok = randomProposalID(r, k, ctx, v1.StatusVotingPeriod) + proposalID, ok = randomProposalID(r, k, ctx, v1.StatusVotingPeriod, s) if !ok { return simtypes.NoOpMsg(types.ModuleName, TypeMsgVote, "unable to generate proposalID"), nil, nil } @@ -459,8 +476,9 @@ func SimulateMsgVoteWeighted( ak types.AccountKeeper, bk types.BankKeeper, k *keeper.Keeper, + s *sharedState, ) simtypes.Operation { - return operationSimulateMsgVoteWeighted(txGen, ak, bk, k, simtypes.Account{}, -1) + return operationSimulateMsgVoteWeighted(txGen, ak, bk, k, simtypes.Account{}, -1, s) } func operationSimulateMsgVoteWeighted( @@ -470,6 +488,7 @@ func operationSimulateMsgVoteWeighted( k *keeper.Keeper, simAccount simtypes.Account, proposalIDInt int64, + s *sharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, @@ -484,7 +503,7 @@ func operationSimulateMsgVoteWeighted( switch { case proposalIDInt < 0: var ok bool - proposalID, ok = randomProposalID(r, k, ctx, v1.StatusVotingPeriod) + proposalID, ok = randomProposalID(r, k, ctx, v1.StatusVotingPeriod, s) if !ok { return simtypes.NoOpMsg(types.ModuleName, TypeMsgVoteWeighted, "unable to generate proposalID"), nil, nil } @@ -521,12 +540,7 @@ func operationSimulateMsgVoteWeighted( } // SimulateMsgCancelProposal generates a MsgCancelProposal. -func SimulateMsgCancelProposal( - txGen client.TxConfig, - ak types.AccountKeeper, - bk types.BankKeeper, - k *keeper.Keeper, -) simtypes.Operation { +func SimulateMsgCancelProposal(txGen client.TxConfig, ak types.AccountKeeper, bk types.BankKeeper, k *keeper.Keeper) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simtypes.Account, chainID string, @@ -663,20 +677,13 @@ func randomProposal(r *rand.Rand, k *keeper.Keeper, ctx sdk.Context) *v1.Proposa // (defined in gov GenesisState) and the latest proposal ID // that matches a given Status. // It does not provide a default ID. -func randomProposalID(r *rand.Rand, k *keeper.Keeper, ctx sdk.Context, status v1.ProposalStatus) (proposalID uint64, found bool) { +func randomProposalID(r *rand.Rand, k *keeper.Keeper, ctx sdk.Context, status v1.ProposalStatus, s *sharedState) (proposalID uint64, found bool) { proposalID, _ = k.ProposalID.Peek(ctx) - - switch { - case proposalID > initialProposalID: - // select a random ID between [initialProposalID, proposalID] + if initialProposalID := s.getMinProposalID(); initialProposalID == unsetProposalID { + s.setMinProposalID(proposalID) + } else if initialProposalID < proposalID { proposalID = uint64(simtypes.RandIntBetween(r, int(initialProposalID), int(proposalID))) - - default: - // This is called on the first call to this function - // in order to update the global variable - initialProposalID = proposalID } - proposal, err := k.Proposals.Get(ctx, proposalID) if err != nil || proposal.Status != status { return proposalID, false diff --git a/x/group/simulation/operations.go b/x/group/simulation/operations.go index df64641858cd..17dd7cf54020 100644 --- a/x/group/simulation/operations.go +++ b/x/group/simulation/operations.go @@ -5,6 +5,7 @@ import ( "fmt" "math/rand" "strings" + "sync/atomic" "time" "cosmossdk.io/core/address" @@ -21,7 +22,7 @@ import ( "github.com/cosmos/cosmos-sdk/x/simulation" ) -var initialGroupID = uint64(100000000000000) +const unsetGroupID = 100000000000000 // group message types var ( @@ -78,6 +79,18 @@ const ( WeightMsgCreateGroupWithPolicy = 50 ) +type sharedState struct { + minGroupID atomic.Uint64 +} + +func (s *sharedState) getMinGroupID() uint64 { + return s.minGroupID.Load() +} + +func (s *sharedState) setMinGroupID(id uint64) { + s.minGroupID.Store(id) +} + // WeightedOperations returns all the operations from the module with their respective weights func WeightedOperations( registry cdctypes.InterfaceRegistry, @@ -147,12 +160,15 @@ func WeightedOperations( pCdc := codec.NewProtoCodec(registry) + state := &sharedState{} + state.setMinGroupID(unsetGroupID) + // create two proposals for weightedOperations var createProposalOps simulation.WeightedOperations for i := 0; i < 2; i++ { createProposalOps = append(createProposalOps, simulation.NewWeightedOperation( weightMsgSubmitProposal, - SimulateMsgSubmitProposal(pCdc, txGen, ak, bk, k), + SimulateMsgSubmitProposal(pCdc, txGen, ak, bk, k, state), )) } @@ -163,7 +179,7 @@ func WeightedOperations( ), simulation.NewWeightedOperation( weightMsgCreateGroupPolicy, - SimulateMsgCreateGroupPolicy(pCdc, txGen, ak, bk, k), + SimulateMsgCreateGroupPolicy(pCdc, txGen, ak, bk, k, state), ), simulation.NewWeightedOperation( weightMsgCreateGroupWithPolicy, @@ -174,43 +190,43 @@ func WeightedOperations( wPostCreateProposalOps := simulation.WeightedOperations{ simulation.NewWeightedOperation( WeightMsgWithdrawProposal, - SimulateMsgWithdrawProposal(pCdc, txGen, ak, bk, k), + SimulateMsgWithdrawProposal(pCdc, txGen, ak, bk, k, state), ), simulation.NewWeightedOperation( weightMsgVote, - SimulateMsgVote(pCdc, txGen, ak, bk, k), + SimulateMsgVote(pCdc, txGen, ak, bk, k, state), ), simulation.NewWeightedOperation( weightMsgExec, - SimulateMsgExec(pCdc, txGen, ak, bk, k), + SimulateMsgExec(pCdc, txGen, ak, bk, k, state), ), simulation.NewWeightedOperation( weightMsgUpdateGroupMetadata, - SimulateMsgUpdateGroupMetadata(pCdc, txGen, ak, bk, k), + SimulateMsgUpdateGroupMetadata(pCdc, txGen, ak, bk, k, state), ), simulation.NewWeightedOperation( weightMsgUpdateGroupAdmin, - SimulateMsgUpdateGroupAdmin(pCdc, txGen, ak, bk, k), + SimulateMsgUpdateGroupAdmin(pCdc, txGen, ak, bk, k, state), ), simulation.NewWeightedOperation( weightMsgUpdateGroupMembers, - SimulateMsgUpdateGroupMembers(pCdc, txGen, ak, bk, k), + SimulateMsgUpdateGroupMembers(pCdc, txGen, ak, bk, k, state), ), simulation.NewWeightedOperation( weightMsgUpdateGroupPolicyAdmin, - SimulateMsgUpdateGroupPolicyAdmin(pCdc, txGen, ak, bk, k), + SimulateMsgUpdateGroupPolicyAdmin(pCdc, txGen, ak, bk, k, state), ), simulation.NewWeightedOperation( weightMsgUpdateGroupPolicyDecisionPolicy, - SimulateMsgUpdateGroupPolicyDecisionPolicy(pCdc, txGen, ak, bk, k), + SimulateMsgUpdateGroupPolicyDecisionPolicy(pCdc, txGen, ak, bk, k, state), ), simulation.NewWeightedOperation( weightMsgUpdateGroupPolicyMetadata, - SimulateMsgUpdateGroupPolicyMetadata(pCdc, txGen, ak, bk, k), + SimulateMsgUpdateGroupPolicyMetadata(pCdc, txGen, ak, bk, k, state), ), simulation.NewWeightedOperation( weightMsgLeaveGroup, - SimulateMsgLeaveGroup(pCdc, txGen, k, ak, bk), + SimulateMsgLeaveGroup(pCdc, txGen, k, ak, bk, state), ), } @@ -347,11 +363,12 @@ func SimulateMsgCreateGroupPolicy( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, + s *sharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, ) (simtypes.OperationMsg, []simtypes.FutureOperation, error) { - groupInfo, acc, account, err := randomGroup(r, k, ak, sdkCtx, accounts) + groupInfo, acc, account, err := randomGroup(r, k, ak, sdkCtx, accounts, s) if err != nil { return simtypes.NoOpMsg(group.ModuleName, TypeMsgCreateGroupPolicy, ""), nil, err } @@ -418,11 +435,12 @@ func SimulateMsgSubmitProposal( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, + s *sharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, ) (simtypes.OperationMsg, []simtypes.FutureOperation, error) { - g, groupPolicy, _, _, err := randomGroupPolicy(r, k, ak, sdkCtx, accounts) + g, groupPolicy, _, _, err := randomGroupPolicy(r, k, ak, sdkCtx, accounts, s) if err != nil { return simtypes.NoOpMsg(group.ModuleName, TypeMsgSubmitProposal, ""), nil, err } @@ -504,11 +522,12 @@ func SimulateMsgUpdateGroupAdmin( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, + s *sharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, ) (simtypes.OperationMsg, []simtypes.FutureOperation, error) { - groupInfo, acc, account, err := randomGroup(r, k, ak, sdkCtx, accounts) + groupInfo, acc, account, err := randomGroup(r, k, ak, sdkCtx, accounts, s) if err != nil { return simtypes.NoOpMsg(group.ModuleName, TypeMsgUpdateGroupAdmin, ""), nil, err } @@ -577,11 +596,12 @@ func SimulateMsgUpdateGroupMetadata( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, + s *sharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, ) (simtypes.OperationMsg, []simtypes.FutureOperation, error) { - groupInfo, acc, account, err := randomGroup(r, k, ak, sdkCtx, accounts) + groupInfo, acc, account, err := randomGroup(r, k, ak, sdkCtx, accounts, s) if err != nil { return simtypes.NoOpMsg(group.ModuleName, TypeMsgUpdateGroupMetadata, ""), nil, err } @@ -637,11 +657,12 @@ func SimulateMsgUpdateGroupMembers( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, + s *sharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, ) (simtypes.OperationMsg, []simtypes.FutureOperation, error) { - groupInfo, acc, account, err := randomGroup(r, k, ak, sdkCtx, accounts) + groupInfo, acc, account, err := randomGroup(r, k, ak, sdkCtx, accounts, s) if err != nil { return simtypes.NoOpMsg(group.ModuleName, TypeMsgUpdateGroupMembers, ""), nil, err } @@ -727,11 +748,12 @@ func SimulateMsgUpdateGroupPolicyAdmin( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, + s *sharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, ) (simtypes.OperationMsg, []simtypes.FutureOperation, error) { - _, groupPolicy, acc, account, err := randomGroupPolicy(r, k, ak, sdkCtx, accounts) + _, groupPolicy, acc, account, err := randomGroupPolicy(r, k, ak, sdkCtx, accounts, s) if err != nil { return simtypes.NoOpMsg(group.ModuleName, TypeMsgUpdateGroupPolicyAdmin, ""), nil, err } @@ -800,11 +822,12 @@ func SimulateMsgUpdateGroupPolicyDecisionPolicy( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, + s *sharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, ) (simtypes.OperationMsg, []simtypes.FutureOperation, error) { - _, groupPolicy, acc, account, err := randomGroupPolicy(r, k, ak, sdkCtx, accounts) + _, groupPolicy, acc, account, err := randomGroupPolicy(r, k, ak, sdkCtx, accounts, s) if err != nil { return simtypes.NoOpMsg(group.ModuleName, TypeMsgUpdateGroupPolicyDecisionPolicy, ""), nil, err } @@ -873,11 +896,12 @@ func SimulateMsgUpdateGroupPolicyMetadata( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, + s *sharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, ) (simtypes.OperationMsg, []simtypes.FutureOperation, error) { - _, groupPolicy, acc, account, err := randomGroupPolicy(r, k, ak, sdkCtx, accounts) + _, groupPolicy, acc, account, err := randomGroupPolicy(r, k, ak, sdkCtx, accounts, s) if err != nil { return simtypes.NoOpMsg(group.ModuleName, TypeMsgUpdateGroupPolicyMetadata, ""), nil, err } @@ -933,11 +957,12 @@ func SimulateMsgWithdrawProposal( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, + s *sharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, ) (simtypes.OperationMsg, []simtypes.FutureOperation, error) { - g, groupPolicy, _, _, err := randomGroupPolicy(r, k, ak, sdkCtx, accounts) + g, groupPolicy, _, _, err := randomGroupPolicy(r, k, ak, sdkCtx, accounts, s) if err != nil { return simtypes.NoOpMsg(group.ModuleName, TypeMsgWithdrawProposal, ""), nil, err } @@ -1048,11 +1073,12 @@ func SimulateMsgVote( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, + s *sharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, ) (simtypes.OperationMsg, []simtypes.FutureOperation, error) { - g, groupPolicy, _, _, err := randomGroupPolicy(r, k, ak, sdkCtx, accounts) + g, groupPolicy, _, _, err := randomGroupPolicy(r, k, ak, sdkCtx, accounts, s) if err != nil { return simtypes.NoOpMsg(group.ModuleName, TypeMsgVote, ""), nil, err } @@ -1160,11 +1186,12 @@ func SimulateMsgExec( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, + s *sharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, ) (simtypes.OperationMsg, []simtypes.FutureOperation, error) { - _, groupPolicy, acc, account, err := randomGroupPolicy(r, k, ak, sdkCtx, accounts) + _, groupPolicy, acc, account, err := randomGroupPolicy(r, k, ak, sdkCtx, accounts, s) if err != nil { return simtypes.NoOpMsg(group.ModuleName, TypeMsgExec, ""), nil, err } @@ -1244,11 +1271,12 @@ func SimulateMsgLeaveGroup( k keeper.Keeper, ak group.AccountKeeper, bk group.BankKeeper, + s *sharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, ) (simtypes.OperationMsg, []simtypes.FutureOperation, error) { - groupInfo, policyInfo, _, _, err := randomGroupPolicy(r, k, ak, sdkCtx, accounts) + groupInfo, policyInfo, _, _, err := randomGroupPolicy(r, k, ak, sdkCtx, accounts, s) if err != nil { return simtypes.NoOpMsg(group.ModuleName, TypeMsgLeaveGroup, ""), nil, err } @@ -1306,20 +1334,14 @@ func SimulateMsgLeaveGroup( } func randomGroup(r *rand.Rand, k keeper.Keeper, ak group.AccountKeeper, - ctx sdk.Context, accounts []simtypes.Account, + ctx sdk.Context, accounts []simtypes.Account, s *sharedState, ) (groupInfo *group.GroupInfo, acc simtypes.Account, account sdk.AccountI, err error) { groupID := k.GetGroupSequence(ctx) - switch { - case groupID > initialGroupID: - // select a random ID between (initialGroupID, groupID] - // if there is at least one group information, then the groupID at this time must be greater than or equal to 1 + if initialGroupID := s.getMinGroupID(); initialGroupID == unsetGroupID { + s.setMinGroupID(groupID) + } else if initialGroupID < groupID { groupID = uint64(simtypes.RandIntBetween(r, int(initialGroupID+1), int(groupID+1))) - - default: - // This is called on the first call to this function - // in order to update the global variable - initialGroupID = groupID } // when groupID is 0, it proves that SimulateMsgCreateGroup has never been called. that is, no group exists in the chain @@ -1354,9 +1376,9 @@ func randomGroup(r *rand.Rand, k keeper.Keeper, ak group.AccountKeeper, } func randomGroupPolicy(r *rand.Rand, k keeper.Keeper, ak group.AccountKeeper, - ctx sdk.Context, accounts []simtypes.Account, + ctx sdk.Context, accounts []simtypes.Account, s *sharedState, ) (groupInfo *group.GroupInfo, groupPolicyInfo *group.GroupPolicyInfo, acc simtypes.Account, account sdk.AccountI, err error) { - groupInfo, _, _, err = randomGroup(r, k, ak, ctx, accounts) + groupInfo, _, _, err = randomGroup(r, k, ak, ctx, accounts, s) if err != nil { return nil, nil, simtypes.Account{}, nil, err } diff --git a/x/simulation/log.go b/x/simulation/log.go index ff2edf1f205e..c1f9c439e10b 100644 --- a/x/simulation/log.go +++ b/x/simulation/log.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path" + "sync" "time" ) @@ -24,7 +25,11 @@ func NewLogWriter(testingmode bool) LogWriter { // log writer type StandardLogWriter struct { + Seed int64 + OpEntries []OperationEntry `json:"op_entries" yaml:"op_entries"` + wMtx sync.Mutex + written bool } // add an entry to the log writer @@ -34,7 +39,12 @@ func (lw *StandardLogWriter) AddEntry(opEntry OperationEntry) { // PrintLogs - print the logs to a simulation file func (lw *StandardLogWriter) PrintLogs() { - f := createLogFile() + lw.wMtx.Lock() + defer lw.wMtx.Unlock() + if lw.written { // print once only + return + } + f := createLogFile(lw.Seed) defer f.Close() for i := 0; i < len(lw.OpEntries); i++ { @@ -44,12 +54,16 @@ func (lw *StandardLogWriter) PrintLogs() { panic("Failed to write logs to file") } } + lw.written = true } -func createLogFile() *os.File { +func createLogFile(seed int64) *os.File { var f *os.File - - fileName := fmt.Sprintf("%d.log", time.Now().UnixMilli()) + var prefix string + if seed != 0 { + prefix = fmt.Sprintf("seed_%10d", seed) + } + fileName := fmt.Sprintf("%s--%d.log", prefix, time.Now().UnixNano()) folderPath := path.Join(os.ExpandEnv("$HOME"), ".simapp", "simulations") filePath := path.Join(folderPath, fileName) diff --git a/x/simulation/operation.go b/x/simulation/operation.go index 5594fd3f5822..8146ef15273a 100644 --- a/x/simulation/operation.go +++ b/x/simulation/operation.go @@ -4,6 +4,7 @@ import ( "encoding/json" "math/rand" "sort" + "time" "github.com/cosmos/cosmos-sdk/types/simulation" ) @@ -22,36 +23,38 @@ type OperationEntry struct { Height int64 `json:"height" yaml:"height"` Order int64 `json:"order" yaml:"order"` Operation json.RawMessage `json:"operation" yaml:"operation"` + BlockTime int64 `json:"block_time" yaml:"block_time"` } // NewOperationEntry creates a new OperationEntry instance -func NewOperationEntry(entry string, height, order int64, op json.RawMessage) OperationEntry { +func NewOperationEntry(entry string, blockTime time.Time, height, order int64, op json.RawMessage) OperationEntry { return OperationEntry{ EntryKind: entry, Height: height, Order: order, + BlockTime: blockTime.UnixNano(), Operation: op, } } // BeginBlockEntry - operation entry for begin block -func BeginBlockEntry(height int64) OperationEntry { - return NewOperationEntry(BeginBlockEntryKind, height, -1, nil) +func BeginBlockEntry(blockTime time.Time, height int64) OperationEntry { + return NewOperationEntry(BeginBlockEntryKind, blockTime, height, -1, nil) } // EndBlockEntry - operation entry for end block -func EndBlockEntry(height int64) OperationEntry { - return NewOperationEntry(EndBlockEntryKind, height, -1, nil) +func EndBlockEntry(blockTime time.Time, height int64) OperationEntry { + return NewOperationEntry(EndBlockEntryKind, blockTime, height, -1, nil) } // MsgEntry - operation entry for standard msg -func MsgEntry(height, order int64, opMsg simulation.OperationMsg) OperationEntry { - return NewOperationEntry(MsgEntryKind, height, order, opMsg.MustMarshal()) +func MsgEntry(blockTime time.Time, height, order int64, opMsg simulation.OperationMsg) OperationEntry { + return NewOperationEntry(MsgEntryKind, blockTime, height, order, opMsg.MustMarshal()) } // QueuedMsgEntry creates an operation entry for a given queued message. -func QueuedMsgEntry(height int64, opMsg simulation.OperationMsg) OperationEntry { - return NewOperationEntry(QueuedMsgEntryKind, height, -1, opMsg.MustMarshal()) +func QueuedMsgEntry(blockTime time.Time, height int64, opMsg simulation.OperationMsg) OperationEntry { + return NewOperationEntry(QueuedMsgEntryKind, blockTime, height, -1, opMsg.MustMarshal()) } // MustMarshal marshals the operation entry, panic on error. diff --git a/x/simulation/simulate.go b/x/simulation/simulate.go index ff21f2268a9e..ace9cde3a22b 100644 --- a/x/simulation/simulate.go +++ b/x/simulation/simulate.go @@ -58,7 +58,7 @@ func initChain( // SimulateFromSeed tests an application by running the provided // operations, testing the provided invariants, but using the provided config.Seed. -func SimulateFromSeed( +func SimulateFromSeed( // exists for backwards compatibility only tb testing.TB, logger log.Logger, w io.Writer, @@ -70,6 +70,26 @@ func SimulateFromSeed( config simulation.Config, cdc codec.JSONCodec, addressCodec address.Codec, +) (exportedParams Params, err error) { + mode, _, _ := getTestingMode(tb) + return SimulateFromSeedX(tb, logger, w, app, appStateFn, randAccFn, ops, blockedAddrs, config, cdc, addressCodec, NewLogWriter(mode)) +} + +// SimulateFromSeedX tests an application by running the provided +// operations, testing the provided invariants, but using the provided config.Seed. +func SimulateFromSeedX( + tb testing.TB, + logger log.Logger, + w io.Writer, + app *baseapp.BaseApp, + appStateFn simulation.AppStateFn, + randAccFn simulation.RandomAccountFn, + ops WeightedOperations, + blockedAddrs map[string]bool, + config simulation.Config, + cdc codec.JSONCodec, + addressCodec address.Codec, + logWriter LogWriter, ) (exportedParams Params, err error) { tb.Helper() // in case we have to end early, don't os.Exit so that we can run cleanup code. @@ -137,7 +157,6 @@ func SimulateFromSeed( // These are operations which have been queued by previous operations operationQueue := NewOperationQueue() - logWriter := NewLogWriter(testingMode) blockSimulator := createBlockSimulator( tb, @@ -175,7 +194,7 @@ func SimulateFromSeed( pastVoteInfos = append(pastVoteInfos, finalizeBlockReq.DecidedLastCommit.Votes) // Run the BeginBlock handler - logWriter.AddEntry(BeginBlockEntry(blockHeight)) + logWriter.AddEntry(BeginBlockEntry(blockTime, blockHeight)) res, err := app.FinalizeBlock(finalizeBlockReq) if err != nil { @@ -195,7 +214,7 @@ func SimulateFromSeed( // run queued operations; ignores block size if block size is too small numQueuedOpsRan, futureOps := runQueuedOperations( - tb, operationQueue, int(blockHeight), r, app, ctx, accs, logWriter, + tb, operationQueue, blockTime, int(blockHeight), r, app, ctx, accs, logWriter, eventStats.Tally, config.Lean, config.ChainID, ) @@ -219,12 +238,12 @@ func SimulateFromSeed( blockHeight++ + logWriter.AddEntry(EndBlockEntry(blockTime, blockHeight)) + blockTime = blockTime.Add(time.Duration(minTimePerBlock) * time.Second) blockTime = blockTime.Add(time.Duration(int64(r.Intn(int(timeDiff)))) * time.Second) proposerAddress = validators.randomProposer(r) - logWriter.AddEntry(EndBlockEntry(blockHeight)) - if config.Commit { if _, err := app.Commit(); err != nil { return params, fmt.Errorf("commit failed at height %d: %w", blockHeight, err) @@ -273,7 +292,7 @@ type blockSimFn func( // Returns a function to simulate blocks. Written like this to avoid constant // parameters being passed every time, to minimize memory overhead. -func createBlockSimulator(tb testing.TB, testingMode bool, w io.Writer, params Params, +func createBlockSimulator(tb testing.TB, printProgress bool, w io.Writer, params Params, event func(route, op, evResult string), ops WeightedOperations, operationQueue OperationQueue, timeOperationQueue []simulation.FutureOperation, logWriter LogWriter, config simulation.Config, @@ -316,7 +335,7 @@ func createBlockSimulator(tb testing.TB, testingMode bool, w io.Writer, params P opMsg.LogEvent(event) if !config.Lean || opMsg.OK { - logWriter.AddEntry(MsgEntry(header.Height, int64(i), opMsg)) + logWriter.AddEntry(MsgEntry(header.Time, header.Height, int64(i), opMsg)) } if err != nil { @@ -329,7 +348,7 @@ Comment: %s`, queueOperations(operationQueue, timeOperationQueue, futureOps) - if testingMode && opCount%50 == 0 { + if printProgress && opCount%50 == 0 { _, _ = fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ", header.Height, config.NumBlocks, opCount, blocksize) } @@ -342,7 +361,7 @@ Comment: %s`, } func runQueuedOperations(tb testing.TB, queueOps map[int][]simulation.Operation, - height int, r *rand.Rand, app *baseapp.BaseApp, + blockTime time.Time, height int, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accounts []simulation.Account, logWriter LogWriter, event func(route, op, evResult string), lean bool, chainID string, ) (numOpsRan int, allFutureOps []simulation.FutureOperation) { @@ -365,7 +384,7 @@ func runQueuedOperations(tb testing.TB, queueOps map[int][]simulation.Operation, opMsg.LogEvent(event) if !lean || opMsg.OK { - logWriter.AddEntry((QueuedMsgEntry(int64(height), opMsg))) + logWriter.AddEntry((QueuedMsgEntry(blockTime, int64(height), opMsg))) } if err != nil { @@ -395,7 +414,7 @@ func runQueuedTimeOperations(tb testing.TB, queueOps []simulation.FutureOperatio opMsg.LogEvent(event) if !lean || opMsg.OK { - logWriter.AddEntry(QueuedMsgEntry(int64(height), opMsg)) + logWriter.AddEntry(QueuedMsgEntry(currentTime, int64(height), opMsg)) } if err != nil { From 632826056250f2cbf2e5a6c4bd606a43e258252d Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Thu, 13 Jun 2024 16:25:49 +0200 Subject: [PATCH 14/15] Review feedback + linter --- simapp/sim_test.go | 9 ++++-- tests/sims/gov/operations_test.go | 6 ++-- testutils/sims/runner.go | 5 +-- types/simulation/config.go | 29 ++++++----------- x/gov/simulation/operations.go | 29 ++++++++++------- x/group/simulation/operations.go | 45 ++++++++++++++++----------- x/group/simulation/operations_test.go | 24 +++++++------- x/simulation/simulate.go | 1 + 8 files changed, 78 insertions(+), 70 deletions(-) diff --git a/simapp/sim_test.go b/simapp/sim_test.go index 2c1e1a4fb044..914b899c1af1 100644 --- a/simapp/sim_test.go +++ b/simapp/sim_test.go @@ -62,11 +62,16 @@ func setupStateFactory(app *SimApp) sims.SimStateFactory { } } +var ( + exportAllModules = []string{} + exportWithValidatorSet = []string{} +) + func TestAppImportExport(t *testing.T) { sims.Run(t, NewSimApp, setupStateFactory, func(t *testing.T, ti sims.TestInstance[*SimApp]) { app := ti.App t.Log("exporting genesis...\n") - exported, err := app.ExportAppStateAndValidators(false, []string{}, []string{}) + exported, err := app.ExportAppStateAndValidators(false, exportWithValidatorSet, exportAllModules) require.NoError(t, err) t.Log("importing genesis...\n") @@ -109,7 +114,7 @@ func TestAppSimulationAfterImport(t *testing.T) { sims.Run(t, NewSimApp, setupStateFactory, func(t *testing.T, ti sims.TestInstance[*SimApp]) { app := ti.App t.Log("exporting genesis...\n") - exported, err := app.ExportAppStateAndValidators(false, []string{}, []string{}) + exported, err := app.ExportAppStateAndValidators(false, exportWithValidatorSet, exportAllModules) require.NoError(t, err) t.Log("importing genesis...\n") diff --git a/tests/sims/gov/operations_test.go b/tests/sims/gov/operations_test.go index 73c13f3cc8f8..7e4b6a27ea69 100644 --- a/tests/sims/gov/operations_test.go +++ b/tests/sims/gov/operations_test.go @@ -268,7 +268,7 @@ func TestSimulateMsgDeposit(t *testing.T) { require.NoError(t, err) // execute operation - op := simulation.SimulateMsgDeposit(suite.TxConfig, suite.AccountKeeper, suite.BankKeeper, suite.GovKeeper, nil) + op := simulation.SimulateMsgDeposit(suite.TxConfig, suite.AccountKeeper, suite.BankKeeper, suite.GovKeeper, simulation.NewSharedState()) operationMsg, _, err := op(r, app.BaseApp, ctx, accounts, "") require.NoError(t, err) @@ -312,7 +312,7 @@ func TestSimulateMsgVote(t *testing.T) { require.NoError(t, err) // execute operation - op := simulation.SimulateMsgVote(suite.TxConfig, suite.AccountKeeper, suite.BankKeeper, suite.GovKeeper, nil) + op := simulation.SimulateMsgVote(suite.TxConfig, suite.AccountKeeper, suite.BankKeeper, suite.GovKeeper, simulation.NewSharedState()) operationMsg, _, err := op(r, app.BaseApp, ctx, accounts, "") require.NoError(t, err) @@ -354,7 +354,7 @@ func TestSimulateMsgVoteWeighted(t *testing.T) { require.NoError(t, err) // execute operation - op := simulation.SimulateMsgVoteWeighted(suite.TxConfig, suite.AccountKeeper, suite.BankKeeper, suite.GovKeeper, nil) + op := simulation.SimulateMsgVoteWeighted(suite.TxConfig, suite.AccountKeeper, suite.BankKeeper, suite.GovKeeper, simulation.NewSharedState()) operationMsg, _, err := op(r, app.BaseApp, ctx, accounts, "") require.NoError(t, err) diff --git a/testutils/sims/runner.go b/testutils/sims/runner.go index 888095a2d980..0518b1b34a8d 100644 --- a/testutils/sims/runner.go +++ b/testutils/sims/runner.go @@ -105,10 +105,7 @@ func RunWithSeeds[T SimulationApp]( t.Run(fmt.Sprintf("seed: %d", seed), func(t *testing.T) { t.Parallel() // setup environment - tCfg := cfg.Clone() - tCfg.Seed = seed - tCfg.FuzzSeed = fuzzSeed - tCfg.T = t + tCfg := cfg.With(t, seed, fuzzSeed) testInstance := NewSimulationAppInstance(t, tCfg, appFactory) var runLogger log.Logger if cli.FlagVerboseValue { diff --git a/types/simulation/config.go b/types/simulation/config.go index 59feff6dee70..6c9000d53934 100644 --- a/types/simulation/config.go +++ b/types/simulation/config.go @@ -28,23 +28,14 @@ type Config struct { T testing.TB } -func (c Config) Clone() Config { - return Config{ - GenesisFile: c.GenesisFile, - ParamsFile: c.ParamsFile, - ExportParamsPath: c.ExportParamsPath, - ExportParamsHeight: c.ExportParamsHeight, - ExportStatePath: c.ExportStatePath, - ExportStatsPath: c.ExportStatsPath, - Seed: c.Seed, - InitialBlockHeight: c.InitialBlockHeight, - GenesisTime: c.GenesisTime, - NumBlocks: c.NumBlocks, - BlockSize: c.BlockSize, - ChainID: c.ChainID, - Lean: c.Lean, - Commit: c.Commit, - DBBackend: c.DBBackend, - BlockMaxGas: c.BlockMaxGas, - } +func (c Config) shallowCopy() Config { + return c +} + +func (c Config) With(t *testing.T, seed int64, fuzzSeed []byte) Config { + r := c.shallowCopy() + r.T = t + r.Seed = seed + r.FuzzSeed = fuzzSeed + return r } diff --git a/x/gov/simulation/operations.go b/x/gov/simulation/operations.go index c09baf404dbd..a8e47b279443 100644 --- a/x/gov/simulation/operations.go +++ b/x/gov/simulation/operations.go @@ -44,15 +44,23 @@ const ( DefaultWeightMsgCancelProposal = 5 ) -type sharedState struct { +// SharedState shared state between message invocations +type SharedState struct { minProposalID atomic.Uint64 } -func (s *sharedState) getMinProposalID() uint64 { +// NewSharedState constructor +func NewSharedState() *SharedState { + r := &SharedState{} + r.setMinProposalID(unsetProposalID) + return r +} + +func (s *SharedState) getMinProposalID() uint64 { return s.minProposalID.Load() } -func (s *sharedState) setMinProposalID(id uint64) { +func (s *SharedState) setMinProposalID(id uint64) { s.minProposalID.Store(id) } @@ -132,8 +140,7 @@ func WeightedOperations( ), ) } - state := &sharedState{} - state.setMinProposalID(unsetProposalID) + state := NewSharedState() wGovOps := simulation.WeightedOperations{ simulation.NewWeightedOperation( weightMsgDeposit, @@ -340,7 +347,7 @@ func SimulateMsgDeposit( ak types.AccountKeeper, bk types.BankKeeper, k *keeper.Keeper, - s *sharedState, + s *SharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, @@ -407,7 +414,7 @@ func SimulateMsgVote( ak types.AccountKeeper, bk types.BankKeeper, k *keeper.Keeper, - s *sharedState, + s *SharedState, ) simtypes.Operation { return operationSimulateMsgVote(txGen, ak, bk, k, simtypes.Account{}, -1, s) } @@ -419,7 +426,7 @@ func operationSimulateMsgVote( k *keeper.Keeper, simAccount simtypes.Account, proposalIDInt int64, - s *sharedState, + s *SharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, @@ -476,7 +483,7 @@ func SimulateMsgVoteWeighted( ak types.AccountKeeper, bk types.BankKeeper, k *keeper.Keeper, - s *sharedState, + s *SharedState, ) simtypes.Operation { return operationSimulateMsgVoteWeighted(txGen, ak, bk, k, simtypes.Account{}, -1, s) } @@ -488,7 +495,7 @@ func operationSimulateMsgVoteWeighted( k *keeper.Keeper, simAccount simtypes.Account, proposalIDInt int64, - s *sharedState, + s *SharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, @@ -677,7 +684,7 @@ func randomProposal(r *rand.Rand, k *keeper.Keeper, ctx sdk.Context) *v1.Proposa // (defined in gov GenesisState) and the latest proposal ID // that matches a given Status. // It does not provide a default ID. -func randomProposalID(r *rand.Rand, k *keeper.Keeper, ctx sdk.Context, status v1.ProposalStatus, s *sharedState) (proposalID uint64, found bool) { +func randomProposalID(r *rand.Rand, k *keeper.Keeper, ctx sdk.Context, status v1.ProposalStatus, s *SharedState) (proposalID uint64, found bool) { proposalID, _ = k.ProposalID.Peek(ctx) if initialProposalID := s.getMinProposalID(); initialProposalID == unsetProposalID { s.setMinProposalID(proposalID) diff --git a/x/group/simulation/operations.go b/x/group/simulation/operations.go index 17dd7cf54020..2a90842681dc 100644 --- a/x/group/simulation/operations.go +++ b/x/group/simulation/operations.go @@ -79,15 +79,23 @@ const ( WeightMsgCreateGroupWithPolicy = 50 ) -type sharedState struct { +// SharedState shared state between message invocations +type SharedState struct { minGroupID atomic.Uint64 } -func (s *sharedState) getMinGroupID() uint64 { +// NewSharedState constructor +func NewSharedState() *SharedState { + r := &SharedState{} + r.setMinGroupID(unsetGroupID) + return r +} + +func (s *SharedState) getMinGroupID() uint64 { return s.minGroupID.Load() } -func (s *sharedState) setMinGroupID(id uint64) { +func (s *SharedState) setMinGroupID(id uint64) { s.minGroupID.Store(id) } @@ -160,8 +168,7 @@ func WeightedOperations( pCdc := codec.NewProtoCodec(registry) - state := &sharedState{} - state.setMinGroupID(unsetGroupID) + state := NewSharedState() // create two proposals for weightedOperations var createProposalOps simulation.WeightedOperations @@ -363,7 +370,7 @@ func SimulateMsgCreateGroupPolicy( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, - s *sharedState, + s *SharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, @@ -435,7 +442,7 @@ func SimulateMsgSubmitProposal( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, - s *sharedState, + s *SharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, @@ -522,7 +529,7 @@ func SimulateMsgUpdateGroupAdmin( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, - s *sharedState, + s *SharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, @@ -596,7 +603,7 @@ func SimulateMsgUpdateGroupMetadata( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, - s *sharedState, + s *SharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, @@ -657,7 +664,7 @@ func SimulateMsgUpdateGroupMembers( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, - s *sharedState, + s *SharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, @@ -748,7 +755,7 @@ func SimulateMsgUpdateGroupPolicyAdmin( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, - s *sharedState, + s *SharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, @@ -822,7 +829,7 @@ func SimulateMsgUpdateGroupPolicyDecisionPolicy( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, - s *sharedState, + s *SharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, @@ -896,7 +903,7 @@ func SimulateMsgUpdateGroupPolicyMetadata( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, - s *sharedState, + s *SharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, @@ -957,7 +964,7 @@ func SimulateMsgWithdrawProposal( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, - s *sharedState, + s *SharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, @@ -1073,7 +1080,7 @@ func SimulateMsgVote( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, - s *sharedState, + s *SharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, @@ -1186,7 +1193,7 @@ func SimulateMsgExec( ak group.AccountKeeper, bk group.BankKeeper, k keeper.Keeper, - s *sharedState, + s *SharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, @@ -1271,7 +1278,7 @@ func SimulateMsgLeaveGroup( k keeper.Keeper, ak group.AccountKeeper, bk group.BankKeeper, - s *sharedState, + s *SharedState, ) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, sdkCtx sdk.Context, accounts []simtypes.Account, chainID string, @@ -1334,7 +1341,7 @@ func SimulateMsgLeaveGroup( } func randomGroup(r *rand.Rand, k keeper.Keeper, ak group.AccountKeeper, - ctx sdk.Context, accounts []simtypes.Account, s *sharedState, + ctx sdk.Context, accounts []simtypes.Account, s *SharedState, ) (groupInfo *group.GroupInfo, acc simtypes.Account, account sdk.AccountI, err error) { groupID := k.GetGroupSequence(ctx) @@ -1376,7 +1383,7 @@ func randomGroup(r *rand.Rand, k keeper.Keeper, ak group.AccountKeeper, } func randomGroupPolicy(r *rand.Rand, k keeper.Keeper, ak group.AccountKeeper, - ctx sdk.Context, accounts []simtypes.Account, s *sharedState, + ctx sdk.Context, accounts []simtypes.Account, s *SharedState, ) (groupInfo *group.GroupInfo, groupPolicyInfo *group.GroupPolicyInfo, acc simtypes.Account, account sdk.AccountI, err error) { groupInfo, _, _, err = randomGroup(r, k, ak, ctx, accounts, s) if err != nil { diff --git a/x/group/simulation/operations_test.go b/x/group/simulation/operations_test.go index 19b8a1cb0f51..f67d9936fd85 100644 --- a/x/group/simulation/operations_test.go +++ b/x/group/simulation/operations_test.go @@ -193,7 +193,7 @@ func (suite *SimTestSuite) TestSimulateCreateGroupPolicy() { suite.Require().NoError(err) // execute operation - op := simulation.SimulateMsgCreateGroupPolicy(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper) + op := simulation.SimulateMsgCreateGroupPolicy(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper, simulation.NewSharedState()) operationMsg, futureOperations, err := op(r, suite.app.BaseApp, suite.ctx, accounts, "") suite.Require().NoError(err) @@ -240,7 +240,7 @@ func (suite *SimTestSuite) TestSimulateSubmitProposal() { suite.Require().NoError(err) // execute operation - op := simulation.SimulateMsgSubmitProposal(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper) + op := simulation.SimulateMsgSubmitProposal(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper, simulation.NewSharedState()) operationMsg, futureOperations, err := op(r, suite.app.BaseApp, suite.ctx, accounts, "") suite.Require().NoError(err) @@ -300,7 +300,7 @@ func (suite *SimTestSuite) TestWithdrawProposal() { suite.Require().NoError(err) // execute operation - op := simulation.SimulateMsgWithdrawProposal(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper) + op := simulation.SimulateMsgWithdrawProposal(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper, simulation.NewSharedState()) operationMsg, futureOperations, err := op(r, suite.app.BaseApp, suite.ctx, accounts, "") suite.Require().NoError(err) @@ -361,7 +361,7 @@ func (suite *SimTestSuite) TestSimulateVote() { suite.Require().NoError(err) // execute operation - op := simulation.SimulateMsgVote(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper) + op := simulation.SimulateMsgVote(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper, simulation.NewSharedState()) operationMsg, futureOperations, err := op(r, suite.app.BaseApp, suite.ctx, accounts, "") suite.Require().NoError(err) @@ -430,7 +430,7 @@ func (suite *SimTestSuite) TestSimulateExec() { suite.Require().NoError(err) // execute operation - op := simulation.SimulateMsgExec(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper) + op := simulation.SimulateMsgExec(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper, simulation.NewSharedState()) operationMsg, futureOperations, err := op(r, suite.app.BaseApp, suite.ctx, accounts, "") suite.Require().NoError(err) @@ -466,7 +466,7 @@ func (suite *SimTestSuite) TestSimulateUpdateGroupAdmin() { suite.Require().NoError(err) // execute operation - op := simulation.SimulateMsgUpdateGroupAdmin(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper) + op := simulation.SimulateMsgUpdateGroupAdmin(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper, simulation.NewSharedState()) operationMsg, futureOperations, err := op(r, suite.app.BaseApp, suite.ctx, accounts, "") suite.Require().NoError(err) @@ -502,7 +502,7 @@ func (suite *SimTestSuite) TestSimulateUpdateGroupMetadata() { suite.Require().NoError(err) // execute operation - op := simulation.SimulateMsgUpdateGroupMetadata(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper) + op := simulation.SimulateMsgUpdateGroupMetadata(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper, simulation.NewSharedState()) operationMsg, futureOperations, err := op(r, suite.app.BaseApp, suite.ctx, accounts, "") suite.Require().NoError(err) @@ -538,7 +538,7 @@ func (suite *SimTestSuite) TestSimulateUpdateGroupMembers() { suite.Require().NoError(err) // execute operation - op := simulation.SimulateMsgUpdateGroupMembers(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper) + op := simulation.SimulateMsgUpdateGroupMembers(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper, simulation.NewSharedState()) operationMsg, futureOperations, err := op(r, suite.app.BaseApp, suite.ctx, accounts, "") suite.Require().NoError(err) @@ -585,7 +585,7 @@ func (suite *SimTestSuite) TestSimulateUpdateGroupPolicyAdmin() { suite.Require().NoError(err) // execute operation - op := simulation.SimulateMsgUpdateGroupPolicyAdmin(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper) + op := simulation.SimulateMsgUpdateGroupPolicyAdmin(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper, simulation.NewSharedState()) operationMsg, futureOperations, err := op(r, suite.app.BaseApp, suite.ctx, accounts, "") suite.Require().NoError(err) @@ -632,7 +632,7 @@ func (suite *SimTestSuite) TestSimulateUpdateGroupPolicyDecisionPolicy() { suite.Require().NoError(err) // execute operation - op := simulation.SimulateMsgUpdateGroupPolicyDecisionPolicy(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper) + op := simulation.SimulateMsgUpdateGroupPolicyDecisionPolicy(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper, simulation.NewSharedState()) operationMsg, futureOperations, err := op(r, suite.app.BaseApp, suite.ctx, accounts, "") suite.Require().NoError(err) @@ -679,7 +679,7 @@ func (suite *SimTestSuite) TestSimulateUpdateGroupPolicyMetadata() { suite.Require().NoError(err) // execute operation - op := simulation.SimulateMsgUpdateGroupPolicyMetadata(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper) + op := simulation.SimulateMsgUpdateGroupPolicyMetadata(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.accountKeeper, suite.bankKeeper, suite.groupKeeper, simulation.NewSharedState()) operationMsg, futureOperations, err := op(r, suite.app.BaseApp, suite.ctx, accounts, "") suite.Require().NoError(err) @@ -742,7 +742,7 @@ func (suite *SimTestSuite) TestSimulateLeaveGroup() { require.NoError(err) // execute operation - op := simulation.SimulateMsgLeaveGroup(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.groupKeeper, suite.accountKeeper, suite.bankKeeper) + op := simulation.SimulateMsgLeaveGroup(codec.NewProtoCodec(suite.interfaceRegistry), suite.txConfig, suite.groupKeeper, suite.accountKeeper, suite.bankKeeper, simulation.NewSharedState()) operationMsg, futureOperations, err := op(r, suite.app.BaseApp, suite.ctx, accounts, "") suite.Require().NoError(err) diff --git a/x/simulation/simulate.go b/x/simulation/simulate.go index ace9cde3a22b..771382406cb5 100644 --- a/x/simulation/simulate.go +++ b/x/simulation/simulate.go @@ -71,6 +71,7 @@ func SimulateFromSeed( // exists for backwards compatibility only cdc codec.JSONCodec, addressCodec address.Codec, ) (exportedParams Params, err error) { + tb.Helper() mode, _, _ := getTestingMode(tb) return SimulateFromSeedX(tb, logger, w, app, appStateFn, randAccFn, ops, blockedAddrs, config, cdc, addressCodec, NewLogWriter(mode)) } From e79d9cf165930555d63f944857b937a58a48e4a0 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Thu, 13 Jun 2024 16:28:33 +0200 Subject: [PATCH 15/15] Code doc --- types/simulation/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/types/simulation/config.go b/types/simulation/config.go index 6c9000d53934..df0eb49cd872 100644 --- a/types/simulation/config.go +++ b/types/simulation/config.go @@ -32,6 +32,7 @@ func (c Config) shallowCopy() Config { return c } +// With sets the values of t, seed, and fuzzSeed in a copy of the Config and returns the copy. func (c Config) With(t *testing.T, seed int64, fuzzSeed []byte) Config { r := c.shallowCopy() r.T = t