diff --git a/doc.go b/doc.go index ee76bca1ece2..1ed65eb9d6f9 100644 --- a/doc.go +++ b/doc.go @@ -50,40 +50,68 @@ To execute a completely pseudo-random simulation: $ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \ -run=TestFullAppSimulation \ - -SimulationEnabled=true \ - -SimulationNumBlocks=100 \ - -SimulationBlockSize=200 \ - -SimulationCommit=true \ - -SimulationSeed=99 \ - -SimulationPeriod=5 \ + -Enabled=true \ + -NumBlocks=100 \ + -BlockSize=200 \ + -Commit=true \ + -Seed=99 \ + -Period=5 \ -v -timeout 24h To execute simulation from a genesis file: $ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \ -run=TestFullAppSimulation \ - -SimulationEnabled=true \ - -SimulationNumBlocks=100 \ - -SimulationBlockSize=200 \ - -SimulationCommit=true \ - -SimulationSeed=99 \ - -SimulationPeriod=5 \ - -SimulationGenesis=/path/to/genesis.json \ + -Enabled=true \ + -NumBlocks=100 \ + -BlockSize=200 \ + -Commit=true \ + -Seed=99 \ + -Period=5 \ + -Genesis=/path/to/genesis.json \ -v -timeout 24h -To execute simulation from a params file: +To execute simulation from a simulation params file: $ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \ -run=TestFullAppSimulation \ - -SimulationEnabled=true \ - -SimulationNumBlocks=100 \ - -SimulationBlockSize=200 \ - -SimulationCommit=true \ - -SimulationSeed=99 \ - -SimulationPeriod=5 \ - -SimulationParams=/path/to/params.json \ + -Enabled=true \ + -NumBlocks=100 \ + -BlockSize=200 \ + -Commit=true \ + -Seed=99 \ + -Period=5 \ + -Params=/path/to/params.json \ -v -timeout 24h +To export the simulation params to a file at a given block height: + + $ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \ + -run=TestFullAppSimulation \ + -Enabled=true \ + -NumBlocks=100 \ + -BlockSize=200 \ + -Commit=true \ + -Seed=99 \ + -Period=5 \ + -ExportParamsPath=/path/to/params.json \ + -ExportParamsHeight=50 \ + -v -timeout 24h + + +To export the simulation app state (i.e genesis) to a file: + + $ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \ + -run=TestFullAppSimulation \ + -Enabled=true \ + -NumBlocks=100 \ + -BlockSize=200 \ + -Commit=true \ + -Seed=99 \ + -Period=5 \ + -ExportStatePath=/path/to/genesis.json \ + v -timeout 24h + Params Params that are provided to simulation from a JSON file are used to used to set diff --git a/sim_test.go b/sim_test.go index 9a45bbd4c0d5..4427ec8583b3 100644 --- a/sim_test.go +++ b/sim_test.go @@ -31,16 +31,19 @@ import ( ) func init() { - flag.StringVar(&genesisFile, "SimulationGenesis", "", "custom simulation genesis file; cannot be used with params file") - flag.StringVar(¶msFile, "SimulationParams", "", "custom simulation params file which overrides any random params; cannot be used with genesis") - flag.Int64Var(&seed, "SimulationSeed", 42, "simulation random seed") - flag.IntVar(&numBlocks, "SimulationNumBlocks", 500, "number of blocks") - flag.IntVar(&blockSize, "SimulationBlockSize", 200, "operations per block") - flag.BoolVar(&enabled, "SimulationEnabled", false, "enable the simulation") - flag.BoolVar(&verbose, "SimulationVerbose", false, "verbose log output") - flag.BoolVar(&lean, "SimulationLean", false, "lean simulation log output") - flag.BoolVar(&commit, "SimulationCommit", false, "have the simulation commit") - flag.IntVar(&period, "SimulationPeriod", 1, "run slow invariants only once every period assertions") + flag.StringVar(&genesisFile, "Genesis", "", "custom simulation genesis file; cannot be used with params file") + flag.StringVar(¶msFile, "Params", "", "custom simulation params file which overrides any random params; cannot be used with genesis") + flag.StringVar(&exportParamsPath, "ExportParamsPath", "", "custom file path to save the exported params JSON") + flag.IntVar(&exportParamsHeight, "ExportParamsHeight", 0, "height to which export the randomly generated params") + flag.StringVar(&exportStatePath, "ExportStatePath", "", "custom file path to save the exported app state JSON") + flag.Int64Var(&seed, "Seed", 42, "simulation random seed") + flag.IntVar(&numBlocks, "NumBlocks", 500, "number of blocks") + flag.IntVar(&blockSize, "BlockSize", 200, "operations per block") + flag.BoolVar(&enabled, "Enabled", false, "enable the simulation") + flag.BoolVar(&verbose, "Verbose", false, "verbose log output") + flag.BoolVar(&lean, "Lean", false, "lean simulation log output") + flag.BoolVar(&commit, "Commit", false, "have the simulation commit") + flag.IntVar(&period, "Period", 1, "run slow invariants only once every period assertions") flag.BoolVar(&onOperation, "SimulateEveryOperation", false, "run slow invariants every operation") flag.BoolVar(&allInvariants, "PrintAllInvariants", false, "print all invariants if a broken invariant is found") } @@ -48,11 +51,15 @@ func init() { // helper function for populating input for SimulateFromSeed func getSimulateFromSeedInput(tb testing.TB, w io.Writer, app *SimApp) ( testing.TB, io.Writer, *baseapp.BaseApp, simulation.AppStateFn, int64, - simulation.WeightedOperations, sdk.Invariants, int, int, bool, bool, bool, bool, map[string]bool) { + simulation.WeightedOperations, sdk.Invariants, int, int, int, + bool, bool, bool, bool, bool, map[string]bool) { + + exportParams := exportParamsPath != "" return tb, w, app.BaseApp, appStateFn, seed, - testAndRunTxs(app), invariants(app), numBlocks, blockSize, commit, - lean, onOperation, allInvariants, app.ModuleAccountAddrs() + testAndRunTxs(app), invariants(app), + numBlocks, exportParamsHeight, blockSize, + exportParams, commit, lean, onOperation, allInvariants, app.ModuleAccountAddrs() } func appStateFn( @@ -136,6 +143,7 @@ func appStateRandomizedFn( return appState, accs, "simulation" } +// TODO: add description func testAndRunTxs(app *SimApp) []simulation.WeightedOperation { cdc := MakeCodec() ap := make(simulation.AppParams) @@ -344,7 +352,7 @@ func fauxMerkleModeOpt(bapp *baseapp.BaseApp) { } // Profile with: -// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/simapp -bench ^BenchmarkFullAppSimulation$ -SimulationCommit=true -cpuprofile cpu.out +// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/simapp -bench ^BenchmarkFullAppSimulation$ -Commit=true -cpuprofile cpu.out func BenchmarkFullAppSimulation(b *testing.B) { logger := log.NewNopLogger() @@ -358,12 +366,44 @@ func BenchmarkFullAppSimulation(b *testing.B) { app := NewSimApp(logger, db, nil, true, 0) // Run randomized simulation - // TODO parameterize numbers, save for a later PR - _, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, os.Stdout, app)) - if err != nil { - fmt.Println(err) - b.Fail() + // TODO: parameterize numbers, save for a later PR + _, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, os.Stdout, app)) + + // export state and params before the simulation error is checked + if exportStatePath != "" { + fmt.Println("Exporting app state...") + appState, _, err := app.ExportAppStateAndValidators(false, nil) + if err != nil { + fmt.Println(err) + b.Fail() + } + err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644) + if err != nil { + fmt.Println(err) + b.Fail() + } + } + + if exportParamsPath != "" { + fmt.Println("Exporting simulation params...") + paramsBz, err := json.MarshalIndent(params, "", " ") + if err != nil { + fmt.Println(err) + b.Fail() + } + + err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644) + if err != nil { + fmt.Println(err) + b.Fail() + } + } + + if simErr != nil { + fmt.Println(simErr) + b.FailNow() } + if commit { fmt.Println("GoLevelDB Stats") fmt.Println(db.Stats()["leveldb.stats"]) @@ -397,7 +437,30 @@ func TestFullAppSimulation(t *testing.T) { require.Equal(t, "SimApp", app.Name()) // Run randomized simulation - _, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app)) + _, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app)) + + // export state and params before the simulation error is checked + if exportStatePath != "" { + fmt.Println("Exporting app state...") + appState, _, err := app.ExportAppStateAndValidators(false, nil) + require.NoError(t, err) + + err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644) + require.NoError(t, err) + } + + if exportParamsPath != "" { + fmt.Println("Exporting simulation params...") + fmt.Println(params) + paramsBz, err := json.MarshalIndent(params, "", " ") + require.NoError(t, err) + + err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644) + require.NoError(t, err) + } + + require.NoError(t, simErr) + if commit { // for memdb: // fmt.Println("Database Size", db.Stats()["database.size"]) @@ -405,8 +468,6 @@ func TestFullAppSimulation(t *testing.T) { fmt.Println(db.Stats()["leveldb.stats"]) fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"]) } - - require.Nil(t, err) } func TestAppImportExport(t *testing.T) { @@ -434,7 +495,28 @@ func TestAppImportExport(t *testing.T) { require.Equal(t, "SimApp", app.Name()) // Run randomized simulation - _, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app)) + _, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app)) + + // export state and params before the simulation error is checked + if exportStatePath != "" { + fmt.Println("Exporting app state...") + appState, _, err := app.ExportAppStateAndValidators(false, nil) + require.NoError(t, err) + + err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644) + require.NoError(t, err) + } + + if exportParamsPath != "" { + fmt.Println("Exporting simulation params...") + paramsBz, err := json.MarshalIndent(params, "", " ") + require.NoError(t, err) + + err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644) + require.NoError(t, err) + } + + require.NoError(t, simErr) if commit { // for memdb: @@ -444,7 +526,6 @@ func TestAppImportExport(t *testing.T) { fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"]) } - require.Nil(t, err) fmt.Printf("Exporting genesis...\n") appState, _, err := app.ExportAppStateAndValidators(false, []string{}) @@ -530,7 +611,28 @@ func TestAppSimulationAfterImport(t *testing.T) { require.Equal(t, "SimApp", app.Name()) // Run randomized simulation - stopEarly, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app)) + stopEarly, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app)) + + // export state and params before the simulation error is checked + if exportStatePath != "" { + fmt.Println("Exporting app state...") + appState, _, err := app.ExportAppStateAndValidators(false, nil) + require.NoError(t, err) + + err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644) + require.NoError(t, err) + } + + if exportParamsPath != "" { + fmt.Println("Exporting simulation params...") + paramsBz, err := json.MarshalIndent(params, "", " ") + require.NoError(t, err) + + err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644) + require.NoError(t, err) + } + + require.NoError(t, simErr) if commit { // for memdb: @@ -540,8 +642,6 @@ func TestAppSimulationAfterImport(t *testing.T) { fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"]) } - require.Nil(t, err) - if stopEarly { // we can't export or import a zero-validator genesis fmt.Printf("We can't export or import a zero-validator genesis, exiting test...\n") @@ -572,7 +672,7 @@ func TestAppSimulationAfterImport(t *testing.T) { }) // Run randomized simulation on imported app - _, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, newApp)) + _, _, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, newApp)) require.Nil(t, err) } @@ -597,15 +697,9 @@ func TestAppStateDeterminism(t *testing.T) { // Run randomized simulation simulation.SimulateFromSeed( t, os.Stdout, app.BaseApp, appStateFn, seed, - testAndRunTxs(app), - []sdk.Invariant{}, - 50, - 100, - true, - false, - false, - false, - app.ModuleAccountAddrs(), + testAndRunTxs(app), []sdk.Invariant{}, + 50, 100, 0, + false, true, false, false, false, app.ModuleAccountAddrs(), ) appHash := app.LastCommitID().Hash appHashList[j] = appHash @@ -627,15 +721,47 @@ func BenchmarkInvariants(b *testing.B) { }() app := NewSimApp(logger, db, nil, true, 0) + exportParams := exportParamsPath != "" // 2. Run parameterized simulation (w/o invariants) - _, err := simulation.SimulateFromSeed( + _, params, simErr := simulation.SimulateFromSeed( b, ioutil.Discard, app.BaseApp, appStateFn, seed, testAndRunTxs(app), - []sdk.Invariant{}, numBlocks, blockSize, commit, lean, onOperation, false, - app.ModuleAccountAddrs(), + []sdk.Invariant{}, numBlocks, exportParamsHeight, blockSize, + exportParams, commit, lean, onOperation, false, app.ModuleAccountAddrs(), ) - if err != nil { - fmt.Println(err) + + // export state and params before the simulation error is checked + if exportStatePath != "" { + fmt.Println("Exporting app state...") + appState, _, err := app.ExportAppStateAndValidators(false, nil) + if err != nil { + fmt.Println(err) + b.Fail() + } + err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644) + if err != nil { + fmt.Println(err) + b.Fail() + } + } + + if exportParamsPath != "" { + fmt.Println("Exporting simulation params...") + paramsBz, err := json.MarshalIndent(params, "", " ") + if err != nil { + fmt.Println(err) + b.Fail() + } + + err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644) + if err != nil { + fmt.Println(err) + b.Fail() + } + } + + if simErr != nil { + fmt.Println(simErr) b.FailNow() } diff --git a/utils.go b/utils.go index a19de40f3424..f3e5d26e1047 100644 --- a/utils.go +++ b/utils.go @@ -37,18 +37,21 @@ import ( ) var ( - genesisFile string - paramsFile string - seed int64 - numBlocks int - blockSize int - enabled bool - verbose bool - lean bool - commit bool - period int - onOperation bool // TODO Remove in favor of binary search for invariant violation - allInvariants bool + genesisFile string + paramsFile string + exportParamsPath string + exportParamsHeight int + exportStatePath string + seed int64 + numBlocks int + blockSize int + enabled bool + verbose bool + lean bool + commit bool + period int + onOperation bool // TODO Remove in favor of binary search for invariant violation + allInvariants bool ) // NewSimAppUNSAFE is used for debugging purposes only.