diff --git a/.pending/improvements/sdk/4566-sim-export b/.pending/improvements/sdk/4566-sim-export new file mode 100644 index 000000000..6dfd5d421 --- /dev/null +++ b/.pending/improvements/sdk/4566-sim-export @@ -0,0 +1,2 @@ +#4566 Export simulation's parameters and app state to JSON in order to reproduce bugs +and invariants. \ No newline at end of file diff --git a/Makefile b/Makefile index 331d352e3..5afe8393b 100644 --- a/Makefile +++ b/Makefile @@ -89,17 +89,17 @@ test_race: test_sim_app_nondeterminism: @echo "Running nondeterminism test..." - @go test -mod=readonly $(SIMAPP) -run TestAppStateDeterminism -SimulationEnabled=true -v -timeout 10m + @go test -mod=readonly $(SIMAPP) -run TestAppStateDeterminism -Enabled=true -v -timeout 10m test_sim_app_custom_genesis_fast: @echo "Running custom genesis simulation..." @echo "By default, ${HOME}/.gaiad/config/genesis.json will be used." - @go test -mod=readonly $(SIMAPP) -run TestFullAppSimulation -SimulationGenesis=${HOME}/.gaiad/config/genesis.json \ - -SimulationEnabled=true -SimulationNumBlocks=100 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -SimulationPeriod=5 -v -timeout 24h + @go test -mod=readonly $(SIMAPP) -run TestFullAppSimulation -Genesis=${HOME}/.gaiad/config/genesis.json \ + -Enabled=true -NumBlocks=100 -BlockSize=200 -Commit=true -Seed=99 -Period=5 -v -timeout 24h test_sim_app_fast: @echo "Running quick application simulation. This may take several minutes..." - @go test -mod=readonly $(SIMAPP) -run TestFullAppSimulation -SimulationEnabled=true -SimulationNumBlocks=100 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -SimulationPeriod=5 -v -timeout 24h + @go test -mod=readonly $(SIMAPP) -run TestFullAppSimulation -Enabled=true -NumBlocks=100 -BlockSize=200 -Commit=true -Seed=99 -Period=5 -v -timeout 24h test_sim_app_import_export: runsim @echo "Running application import/export simulation. This may take several minutes..." @@ -121,8 +121,8 @@ test_sim_app_multi_seed: runsim test_sim_benchmark_invariants: @echo "Running simulation invariant benchmarks..." @go test -mod=readonly $(SIMAPP) -benchmem -bench=BenchmarkInvariants -run=^$ \ - -SimulationEnabled=true -SimulationNumBlocks=1000 -SimulationBlockSize=200 \ - -SimulationCommit=true -SimulationSeed=57 -v -timeout 24h + -Enabled=true -NumBlocks=1000 -BlockSize=200 \ + -Commit=true -Seed=57 -v -timeout 24h # Don't move it into tools - this will be gone once gaia has moved into the new repo runsim: $(BINDIR)/runsim @@ -137,12 +137,12 @@ SIM_COMMIT ?= true test_sim_app_benchmark: @echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!" @go test -mod=readonly -benchmem -run=^$$ $(SIMAPP) -bench ^BenchmarkFullAppSimulation$$ \ - -SimulationEnabled=true -SimulationNumBlocks=$(SIM_NUM_BLOCKS) -SimulationBlockSize=$(SIM_BLOCK_SIZE) -SimulationCommit=$(SIM_COMMIT) -timeout 24h + -Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h test_sim_app_profile: @echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!" @go test -mod=readonly -benchmem -run=^$$ $(SIMAPP) -bench ^BenchmarkFullAppSimulation$$ \ - -SimulationEnabled=true -SimulationNumBlocks=$(SIM_NUM_BLOCKS) -SimulationBlockSize=$(SIM_BLOCK_SIZE) -SimulationCommit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out + -Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out test_cover: @export VERSION=$(VERSION); bash -x tests/test_cover.sh diff --git a/simapp/doc.go b/simapp/doc.go index ee76bca1e..1ed65eb9d 100644 --- a/simapp/doc.go +++ b/simapp/doc.go @@ -50,40 +50,68 @@ To execute a completely pseudo-random simulation: $ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \ -run=TestFullAppSimulation \ - -SimulationEnabled=true \ - -SimulationNumBlocks=100 \ - -SimulationBlockSize=200 \ - -SimulationCommit=true \ - -SimulationSeed=99 \ - -SimulationPeriod=5 \ + -Enabled=true \ + -NumBlocks=100 \ + -BlockSize=200 \ + -Commit=true \ + -Seed=99 \ + -Period=5 \ -v -timeout 24h To execute simulation from a genesis file: $ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \ -run=TestFullAppSimulation \ - -SimulationEnabled=true \ - -SimulationNumBlocks=100 \ - -SimulationBlockSize=200 \ - -SimulationCommit=true \ - -SimulationSeed=99 \ - -SimulationPeriod=5 \ - -SimulationGenesis=/path/to/genesis.json \ + -Enabled=true \ + -NumBlocks=100 \ + -BlockSize=200 \ + -Commit=true \ + -Seed=99 \ + -Period=5 \ + -Genesis=/path/to/genesis.json \ -v -timeout 24h -To execute simulation from a params file: +To execute simulation from a simulation params file: $ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \ -run=TestFullAppSimulation \ - -SimulationEnabled=true \ - -SimulationNumBlocks=100 \ - -SimulationBlockSize=200 \ - -SimulationCommit=true \ - -SimulationSeed=99 \ - -SimulationPeriod=5 \ - -SimulationParams=/path/to/params.json \ + -Enabled=true \ + -NumBlocks=100 \ + -BlockSize=200 \ + -Commit=true \ + -Seed=99 \ + -Period=5 \ + -Params=/path/to/params.json \ -v -timeout 24h +To export the simulation params to a file at a given block height: + + $ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \ + -run=TestFullAppSimulation \ + -Enabled=true \ + -NumBlocks=100 \ + -BlockSize=200 \ + -Commit=true \ + -Seed=99 \ + -Period=5 \ + -ExportParamsPath=/path/to/params.json \ + -ExportParamsHeight=50 \ + -v -timeout 24h + + +To export the simulation app state (i.e genesis) to a file: + + $ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \ + -run=TestFullAppSimulation \ + -Enabled=true \ + -NumBlocks=100 \ + -BlockSize=200 \ + -Commit=true \ + -Seed=99 \ + -Period=5 \ + -ExportStatePath=/path/to/genesis.json \ + v -timeout 24h + Params Params that are provided to simulation from a JSON file are used to used to set diff --git a/simapp/sim_test.go b/simapp/sim_test.go index 9a45bbd4c..4427ec858 100644 --- a/simapp/sim_test.go +++ b/simapp/sim_test.go @@ -31,16 +31,19 @@ import ( ) func init() { - flag.StringVar(&genesisFile, "SimulationGenesis", "", "custom simulation genesis file; cannot be used with params file") - flag.StringVar(¶msFile, "SimulationParams", "", "custom simulation params file which overrides any random params; cannot be used with genesis") - flag.Int64Var(&seed, "SimulationSeed", 42, "simulation random seed") - flag.IntVar(&numBlocks, "SimulationNumBlocks", 500, "number of blocks") - flag.IntVar(&blockSize, "SimulationBlockSize", 200, "operations per block") - flag.BoolVar(&enabled, "SimulationEnabled", false, "enable the simulation") - flag.BoolVar(&verbose, "SimulationVerbose", false, "verbose log output") - flag.BoolVar(&lean, "SimulationLean", false, "lean simulation log output") - flag.BoolVar(&commit, "SimulationCommit", false, "have the simulation commit") - flag.IntVar(&period, "SimulationPeriod", 1, "run slow invariants only once every period assertions") + flag.StringVar(&genesisFile, "Genesis", "", "custom simulation genesis file; cannot be used with params file") + flag.StringVar(¶msFile, "Params", "", "custom simulation params file which overrides any random params; cannot be used with genesis") + flag.StringVar(&exportParamsPath, "ExportParamsPath", "", "custom file path to save the exported params JSON") + flag.IntVar(&exportParamsHeight, "ExportParamsHeight", 0, "height to which export the randomly generated params") + flag.StringVar(&exportStatePath, "ExportStatePath", "", "custom file path to save the exported app state JSON") + flag.Int64Var(&seed, "Seed", 42, "simulation random seed") + flag.IntVar(&numBlocks, "NumBlocks", 500, "number of blocks") + flag.IntVar(&blockSize, "BlockSize", 200, "operations per block") + flag.BoolVar(&enabled, "Enabled", false, "enable the simulation") + flag.BoolVar(&verbose, "Verbose", false, "verbose log output") + flag.BoolVar(&lean, "Lean", false, "lean simulation log output") + flag.BoolVar(&commit, "Commit", false, "have the simulation commit") + flag.IntVar(&period, "Period", 1, "run slow invariants only once every period assertions") flag.BoolVar(&onOperation, "SimulateEveryOperation", false, "run slow invariants every operation") flag.BoolVar(&allInvariants, "PrintAllInvariants", false, "print all invariants if a broken invariant is found") } @@ -48,11 +51,15 @@ func init() { // helper function for populating input for SimulateFromSeed func getSimulateFromSeedInput(tb testing.TB, w io.Writer, app *SimApp) ( testing.TB, io.Writer, *baseapp.BaseApp, simulation.AppStateFn, int64, - simulation.WeightedOperations, sdk.Invariants, int, int, bool, bool, bool, bool, map[string]bool) { + simulation.WeightedOperations, sdk.Invariants, int, int, int, + bool, bool, bool, bool, bool, map[string]bool) { + + exportParams := exportParamsPath != "" return tb, w, app.BaseApp, appStateFn, seed, - testAndRunTxs(app), invariants(app), numBlocks, blockSize, commit, - lean, onOperation, allInvariants, app.ModuleAccountAddrs() + testAndRunTxs(app), invariants(app), + numBlocks, exportParamsHeight, blockSize, + exportParams, commit, lean, onOperation, allInvariants, app.ModuleAccountAddrs() } func appStateFn( @@ -136,6 +143,7 @@ func appStateRandomizedFn( return appState, accs, "simulation" } +// TODO: add description func testAndRunTxs(app *SimApp) []simulation.WeightedOperation { cdc := MakeCodec() ap := make(simulation.AppParams) @@ -344,7 +352,7 @@ func fauxMerkleModeOpt(bapp *baseapp.BaseApp) { } // Profile with: -// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/simapp -bench ^BenchmarkFullAppSimulation$ -SimulationCommit=true -cpuprofile cpu.out +// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/simapp -bench ^BenchmarkFullAppSimulation$ -Commit=true -cpuprofile cpu.out func BenchmarkFullAppSimulation(b *testing.B) { logger := log.NewNopLogger() @@ -358,12 +366,44 @@ func BenchmarkFullAppSimulation(b *testing.B) { app := NewSimApp(logger, db, nil, true, 0) // Run randomized simulation - // TODO parameterize numbers, save for a later PR - _, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, os.Stdout, app)) - if err != nil { - fmt.Println(err) - b.Fail() + // TODO: parameterize numbers, save for a later PR + _, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, os.Stdout, app)) + + // export state and params before the simulation error is checked + if exportStatePath != "" { + fmt.Println("Exporting app state...") + appState, _, err := app.ExportAppStateAndValidators(false, nil) + if err != nil { + fmt.Println(err) + b.Fail() + } + err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644) + if err != nil { + fmt.Println(err) + b.Fail() + } } + + if exportParamsPath != "" { + fmt.Println("Exporting simulation params...") + paramsBz, err := json.MarshalIndent(params, "", " ") + if err != nil { + fmt.Println(err) + b.Fail() + } + + err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644) + if err != nil { + fmt.Println(err) + b.Fail() + } + } + + if simErr != nil { + fmt.Println(simErr) + b.FailNow() + } + if commit { fmt.Println("GoLevelDB Stats") fmt.Println(db.Stats()["leveldb.stats"]) @@ -397,7 +437,30 @@ func TestFullAppSimulation(t *testing.T) { require.Equal(t, "SimApp", app.Name()) // Run randomized simulation - _, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app)) + _, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app)) + + // export state and params before the simulation error is checked + if exportStatePath != "" { + fmt.Println("Exporting app state...") + appState, _, err := app.ExportAppStateAndValidators(false, nil) + require.NoError(t, err) + + err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644) + require.NoError(t, err) + } + + if exportParamsPath != "" { + fmt.Println("Exporting simulation params...") + fmt.Println(params) + paramsBz, err := json.MarshalIndent(params, "", " ") + require.NoError(t, err) + + err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644) + require.NoError(t, err) + } + + require.NoError(t, simErr) + if commit { // for memdb: // fmt.Println("Database Size", db.Stats()["database.size"]) @@ -405,8 +468,6 @@ func TestFullAppSimulation(t *testing.T) { fmt.Println(db.Stats()["leveldb.stats"]) fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"]) } - - require.Nil(t, err) } func TestAppImportExport(t *testing.T) { @@ -434,7 +495,28 @@ func TestAppImportExport(t *testing.T) { require.Equal(t, "SimApp", app.Name()) // Run randomized simulation - _, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app)) + _, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app)) + + // export state and params before the simulation error is checked + if exportStatePath != "" { + fmt.Println("Exporting app state...") + appState, _, err := app.ExportAppStateAndValidators(false, nil) + require.NoError(t, err) + + err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644) + require.NoError(t, err) + } + + if exportParamsPath != "" { + fmt.Println("Exporting simulation params...") + paramsBz, err := json.MarshalIndent(params, "", " ") + require.NoError(t, err) + + err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644) + require.NoError(t, err) + } + + require.NoError(t, simErr) if commit { // for memdb: @@ -444,7 +526,6 @@ func TestAppImportExport(t *testing.T) { fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"]) } - require.Nil(t, err) fmt.Printf("Exporting genesis...\n") appState, _, err := app.ExportAppStateAndValidators(false, []string{}) @@ -530,7 +611,28 @@ func TestAppSimulationAfterImport(t *testing.T) { require.Equal(t, "SimApp", app.Name()) // Run randomized simulation - stopEarly, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app)) + stopEarly, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app)) + + // export state and params before the simulation error is checked + if exportStatePath != "" { + fmt.Println("Exporting app state...") + appState, _, err := app.ExportAppStateAndValidators(false, nil) + require.NoError(t, err) + + err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644) + require.NoError(t, err) + } + + if exportParamsPath != "" { + fmt.Println("Exporting simulation params...") + paramsBz, err := json.MarshalIndent(params, "", " ") + require.NoError(t, err) + + err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644) + require.NoError(t, err) + } + + require.NoError(t, simErr) if commit { // for memdb: @@ -540,8 +642,6 @@ func TestAppSimulationAfterImport(t *testing.T) { fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"]) } - require.Nil(t, err) - if stopEarly { // we can't export or import a zero-validator genesis fmt.Printf("We can't export or import a zero-validator genesis, exiting test...\n") @@ -572,7 +672,7 @@ func TestAppSimulationAfterImport(t *testing.T) { }) // Run randomized simulation on imported app - _, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, newApp)) + _, _, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, newApp)) require.Nil(t, err) } @@ -597,15 +697,9 @@ func TestAppStateDeterminism(t *testing.T) { // Run randomized simulation simulation.SimulateFromSeed( t, os.Stdout, app.BaseApp, appStateFn, seed, - testAndRunTxs(app), - []sdk.Invariant{}, - 50, - 100, - true, - false, - false, - false, - app.ModuleAccountAddrs(), + testAndRunTxs(app), []sdk.Invariant{}, + 50, 100, 0, + false, true, false, false, false, app.ModuleAccountAddrs(), ) appHash := app.LastCommitID().Hash appHashList[j] = appHash @@ -627,15 +721,47 @@ func BenchmarkInvariants(b *testing.B) { }() app := NewSimApp(logger, db, nil, true, 0) + exportParams := exportParamsPath != "" // 2. Run parameterized simulation (w/o invariants) - _, err := simulation.SimulateFromSeed( + _, params, simErr := simulation.SimulateFromSeed( b, ioutil.Discard, app.BaseApp, appStateFn, seed, testAndRunTxs(app), - []sdk.Invariant{}, numBlocks, blockSize, commit, lean, onOperation, false, - app.ModuleAccountAddrs(), + []sdk.Invariant{}, numBlocks, exportParamsHeight, blockSize, + exportParams, commit, lean, onOperation, false, app.ModuleAccountAddrs(), ) - if err != nil { - fmt.Println(err) + + // export state and params before the simulation error is checked + if exportStatePath != "" { + fmt.Println("Exporting app state...") + appState, _, err := app.ExportAppStateAndValidators(false, nil) + if err != nil { + fmt.Println(err) + b.Fail() + } + err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644) + if err != nil { + fmt.Println(err) + b.Fail() + } + } + + if exportParamsPath != "" { + fmt.Println("Exporting simulation params...") + paramsBz, err := json.MarshalIndent(params, "", " ") + if err != nil { + fmt.Println(err) + b.Fail() + } + + err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644) + if err != nil { + fmt.Println(err) + b.Fail() + } + } + + if simErr != nil { + fmt.Println(simErr) b.FailNow() } diff --git a/simapp/utils.go b/simapp/utils.go index a19de40f3..f3e5d26e1 100644 --- a/simapp/utils.go +++ b/simapp/utils.go @@ -37,18 +37,21 @@ import ( ) var ( - genesisFile string - paramsFile string - seed int64 - numBlocks int - blockSize int - enabled bool - verbose bool - lean bool - commit bool - period int - onOperation bool // TODO Remove in favor of binary search for invariant violation - allInvariants bool + genesisFile string + paramsFile string + exportParamsPath string + exportParamsHeight int + exportStatePath string + seed int64 + numBlocks int + blockSize int + enabled bool + verbose bool + lean bool + commit bool + period int + onOperation bool // TODO Remove in favor of binary search for invariant violation + allInvariants bool ) // NewSimAppUNSAFE is used for debugging purposes only. diff --git a/x/simulation/simulate.go b/x/simulation/simulate.go index 08920e295..e2a4602d9 100644 --- a/x/simulation/simulate.go +++ b/x/simulation/simulate.go @@ -41,14 +41,15 @@ func initChain( // SimulateFromSeed tests an application by running the provided // operations, testing the provided invariants, but using the provided seed. -// TODO split this monster function up +// TODO: split this monster function up func SimulateFromSeed( tb testing.TB, w io.Writer, app *baseapp.BaseApp, appStateFn AppStateFn, seed int64, ops WeightedOperations, invariants sdk.Invariants, - numBlocks, blockSize int, commit, lean, onOperation, allInvariants bool, + numBlocks, exportParamsHeight, blockSize int, + exportParams, commit, lean, onOperation, allInvariants bool, blackListedAccs map[string]bool, -) (stopEarly bool, simError error) { +) (stopEarly bool, exportedParams Params, err error) { // in case we have to end early, don't os.Exit so that we can run cleanup code. testingMode, t, b := getTestingMode(tb) @@ -72,7 +73,7 @@ func SimulateFromSeed( // TM 0.24) Initially this is the same as the initial validator set validators, accs := initChain(r, params, accs, app, appStateFn, genesisTimestamp) if len(accs) == 0 { - return true, fmt.Errorf("must have greater than zero genesis accounts") + return true, params, fmt.Errorf("must have greater than zero genesis accounts") } // remove module account address if they exist in accs @@ -100,7 +101,7 @@ func SimulateFromSeed( go func() { receivedSignal := <-c fmt.Fprintf(w, "\nExiting early due to %s, on block %d, operation %d\n", receivedSignal, header.Height, opCount) - simError = fmt.Errorf("Exited due to %s", receivedSignal) + err = fmt.Errorf("Exited due to %s", receivedSignal) stopEarly = true }() @@ -131,12 +132,15 @@ func SimulateFromSeed( stackTrace := string(debug.Stack()) fmt.Println(stackTrace) logWriter.PrintLogs() - simError = fmt.Errorf("Simulation halted due to panic on block %d", header.Height) + err = fmt.Errorf("Simulation halted due to panic on block %d", header.Height) } }() } - // TODO split up the contents of this for loop into new functions + // set exported params to the initial state + exportedParams = params + + // TODO: split up the contents of this for loop into new functions for height := 1; height <= numBlocks && !stopEarly; height++ { // Log the header time for future lookup @@ -205,11 +209,16 @@ func SimulateFromSeed( validators = nextValidators nextValidators = updateValidators(tb, r, params, validators, res.ValidatorUpdates, eventStats.tally) + + // update the exported params + if exportParams && exportParamsHeight == height { + exportedParams = params + } } if stopEarly { eventStats.Print(w) - return true, simError + return true, exportedParams, err } fmt.Fprintf( @@ -219,7 +228,8 @@ func SimulateFromSeed( ) eventStats.Print(w) - return false, nil + + return false, exportedParams, nil } //______________________________________________________________________________