certusone
/
cosmos-sdk
mirror of https://github.com/certusone/cosmos-sdk.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge PR #4731: Save sim params and export app state to file

This commit is contained in:
Federico Kunze 2019-07-19 18:59:04 +02:00 committed by Alexander Bezobchuk
parent 43478167c6
commit 8af2230ee4
6 changed files with 261 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.pending/improvements/sdk/4566-sim-export Normal file
View File

@ -0,0 +1,2 @@
#4566 Export simulation's parameters and app state to JSON in order to reproduce bugs
and invariants.

16
Makefile
View File

@ -89,17 +89,17 @@ test_race:
test_sim_app_nondeterminism:
@echo "Running nondeterminism test..."
@go test -mod=readonly $(SIMAPP) -run TestAppStateDeterminism -SimulationEnabled=true -v -timeout 10m
@go test -mod=readonly $(SIMAPP) -run TestAppStateDeterminism -Enabled=true -v -timeout 10m
test_sim_app_custom_genesis_fast:
@echo "Running custom genesis simulation..."
@echo "By default, ${HOME}/.gaiad/config/genesis.json will be used."
@go test -mod=readonly $(SIMAPP) -run TestFullAppSimulation -SimulationGenesis=${HOME}/.gaiad/config/genesis.json \
-SimulationEnabled=true -SimulationNumBlocks=100 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -SimulationPeriod=5 -v -timeout 24h
@go test -mod=readonly $(SIMAPP) -run TestFullAppSimulation -Genesis=${HOME}/.gaiad/config/genesis.json \
-Enabled=true -NumBlocks=100 -BlockSize=200 -Commit=true -Seed=99 -Period=5 -v -timeout 24h
test_sim_app_fast:
@echo "Running quick application simulation. This may take several minutes..."
@go test -mod=readonly $(SIMAPP) -run TestFullAppSimulation -SimulationEnabled=true -SimulationNumBlocks=100 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -SimulationPeriod=5 -v -timeout 24h
@go test -mod=readonly $(SIMAPP) -run TestFullAppSimulation -Enabled=true -NumBlocks=100 -BlockSize=200 -Commit=true -Seed=99 -Period=5 -v -timeout 24h
test_sim_app_import_export: runsim
@echo "Running application import/export simulation. This may take several minutes..."
@ -121,8 +121,8 @@ test_sim_app_multi_seed: runsim
test_sim_benchmark_invariants:
@echo "Running simulation invariant benchmarks..."
@go test -mod=readonly $(SIMAPP) -benchmem -bench=BenchmarkInvariants -run=^$ \
-SimulationEnabled=true -SimulationNumBlocks=1000 -SimulationBlockSize=200 \
-SimulationCommit=true -SimulationSeed=57 -v -timeout 24h
-Enabled=true -NumBlocks=1000 -BlockSize=200 \
-Commit=true -Seed=57 -v -timeout 24h
# Don't move it into tools - this will be gone once gaia has moved into the new repo
runsim: $(BINDIR)/runsim
@ -137,12 +137,12 @@ SIM_COMMIT ?= true
test_sim_app_benchmark:
@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
@go test -mod=readonly -benchmem -run=^$$ $(SIMAPP) -bench ^BenchmarkFullAppSimulation$$ \
-SimulationEnabled=true -SimulationNumBlocks=$(SIM_NUM_BLOCKS) -SimulationBlockSize=$(SIM_BLOCK_SIZE) -SimulationCommit=$(SIM_COMMIT) -timeout 24h
-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h
test_sim_app_profile:
@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
@go test -mod=readonly -benchmem -run=^$$ $(SIMAPP) -bench ^BenchmarkFullAppSimulation$$ \
-SimulationEnabled=true -SimulationNumBlocks=$(SIM_NUM_BLOCKS) -SimulationBlockSize=$(SIM_BLOCK_SIZE) -SimulationCommit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out
-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out
test_cover:
@export VERSION=$(VERSION); bash -x tests/test_cover.sh

70
simapp/doc.go
View File

@ -50,40 +50,68 @@ To execute a completely pseudo-random simulation:
$ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \
-run=TestFullAppSimulation \
-SimulationEnabled=true \
-SimulationNumBlocks=100 \
-SimulationBlockSize=200 \
-SimulationCommit=true \
-SimulationSeed=99 \
-SimulationPeriod=5 \
-Enabled=true \
-NumBlocks=100 \
-BlockSize=200 \
-Commit=true \
-Seed=99 \
-Period=5 \
-v -timeout 24h
To execute simulation from a genesis file:
$ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \
-run=TestFullAppSimulation \
-SimulationEnabled=true \
-SimulationNumBlocks=100 \
-SimulationBlockSize=200 \
-SimulationCommit=true \
-SimulationSeed=99 \
-SimulationPeriod=5 \
-SimulationGenesis=/path/to/genesis.json \
-Enabled=true \
-NumBlocks=100 \
-BlockSize=200 \
-Commit=true \
-Seed=99 \
-Period=5 \
-Genesis=/path/to/genesis.json \
-v -timeout 24h
To execute simulation from a params file:
To execute simulation from a simulation params file:
$ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \
-run=TestFullAppSimulation \
-SimulationEnabled=true \
-SimulationNumBlocks=100 \
-SimulationBlockSize=200 \
-SimulationCommit=true \
-SimulationSeed=99 \
-SimulationPeriod=5 \
-SimulationParams=/path/to/params.json \
-Enabled=true \
-NumBlocks=100 \
-BlockSize=200 \
-Commit=true \
-Seed=99 \
-Period=5 \
-Params=/path/to/params.json \
-v -timeout 24h
To export the simulation params to a file at a given block height:
$ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \
-run=TestFullAppSimulation \
-Enabled=true \
-NumBlocks=100 \
-BlockSize=200 \
-Commit=true \
-Seed=99 \
-Period=5 \
-ExportParamsPath=/path/to/params.json \
-ExportParamsHeight=50 \
-v -timeout 24h
To export the simulation app state (i.e genesis) to a file:
$ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \
-run=TestFullAppSimulation \
-Enabled=true \
-NumBlocks=100 \
-BlockSize=200 \
-Commit=true \
-Seed=99 \
-Period=5 \
-ExportStatePath=/path/to/genesis.json \
v -timeout 24h
Params
Params that are provided to simulation from a JSON file are used to used to set

210
simapp/sim_test.go
View File

@ -31,16 +31,19 @@ import (
)
func init() {
flag.StringVar(&genesisFile, "SimulationGenesis", "", "custom simulation genesis file; cannot be used with params file")
flag.StringVar(&paramsFile, "SimulationParams", "", "custom simulation params file which overrides any random params; cannot be used with genesis")
flag.Int64Var(&seed, "SimulationSeed", 42, "simulation random seed")
flag.IntVar(&numBlocks, "SimulationNumBlocks", 500, "number of blocks")
flag.IntVar(&blockSize, "SimulationBlockSize", 200, "operations per block")
flag.BoolVar(&enabled, "SimulationEnabled", false, "enable the simulation")
flag.BoolVar(&verbose, "SimulationVerbose", false, "verbose log output")
flag.BoolVar(&lean, "SimulationLean", false, "lean simulation log output")
flag.BoolVar(&commit, "SimulationCommit", false, "have the simulation commit")
flag.IntVar(&period, "SimulationPeriod", 1, "run slow invariants only once every period assertions")
flag.StringVar(&genesisFile, "Genesis", "", "custom simulation genesis file; cannot be used with params file")
flag.StringVar(&paramsFile, "Params", "", "custom simulation params file which overrides any random params; cannot be used with genesis")
flag.StringVar(&exportParamsPath, "ExportParamsPath", "", "custom file path to save the exported params JSON")
flag.IntVar(&exportParamsHeight, "ExportParamsHeight", 0, "height to which export the randomly generated params")
flag.StringVar(&exportStatePath, "ExportStatePath", "", "custom file path to save the exported app state JSON")
flag.Int64Var(&seed, "Seed", 42, "simulation random seed")
flag.IntVar(&numBlocks, "NumBlocks", 500, "number of blocks")
flag.IntVar(&blockSize, "BlockSize", 200, "operations per block")
flag.BoolVar(&enabled, "Enabled", false, "enable the simulation")
flag.BoolVar(&verbose, "Verbose", false, "verbose log output")
flag.BoolVar(&lean, "Lean", false, "lean simulation log output")
flag.BoolVar(&commit, "Commit", false, "have the simulation commit")
flag.IntVar(&period, "Period", 1, "run slow invariants only once every period assertions")
flag.BoolVar(&onOperation, "SimulateEveryOperation", false, "run slow invariants every operation")
flag.BoolVar(&allInvariants, "PrintAllInvariants", false, "print all invariants if a broken invariant is found")
}
@ -48,11 +51,15 @@ func init() {
// helper function for populating input for SimulateFromSeed
func getSimulateFromSeedInput(tb testing.TB, w io.Writer, app *SimApp) (
testing.TB, io.Writer, *baseapp.BaseApp, simulation.AppStateFn, int64,
simulation.WeightedOperations, sdk.Invariants, int, int, bool, bool, bool, bool, map[string]bool) {
simulation.WeightedOperations, sdk.Invariants, int, int, int,
bool, bool, bool, bool, bool, map[string]bool) {
exportParams := exportParamsPath != ""
return tb, w, app.BaseApp, appStateFn, seed,
testAndRunTxs(app), invariants(app), numBlocks, blockSize, commit,
lean, onOperation, allInvariants, app.ModuleAccountAddrs()
testAndRunTxs(app), invariants(app),
numBlocks, exportParamsHeight, blockSize,
exportParams, commit, lean, onOperation, allInvariants, app.ModuleAccountAddrs()
}
func appStateFn(
@ -136,6 +143,7 @@ func appStateRandomizedFn(
return appState, accs, "simulation"
}
// TODO: add description
func testAndRunTxs(app *SimApp) []simulation.WeightedOperation {
cdc := MakeCodec()
ap := make(simulation.AppParams)
@ -344,7 +352,7 @@ func fauxMerkleModeOpt(bapp *baseapp.BaseApp) {
}
// Profile with:
// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/simapp -bench ^BenchmarkFullAppSimulation$ -SimulationCommit=true -cpuprofile cpu.out
// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/simapp -bench ^BenchmarkFullAppSimulation$ -Commit=true -cpuprofile cpu.out
func BenchmarkFullAppSimulation(b *testing.B) {
logger := log.NewNopLogger()
@ -358,12 +366,44 @@ func BenchmarkFullAppSimulation(b *testing.B) {
app := NewSimApp(logger, db, nil, true, 0)
// Run randomized simulation
// TODO parameterize numbers, save for a later PR
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, os.Stdout, app))
if err != nil {
fmt.Println(err)
b.Fail()
// TODO: parameterize numbers, save for a later PR
_, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, os.Stdout, app))
// export state and params before the simulation error is checked
if exportStatePath != "" {
fmt.Println("Exporting app state...")
appState, _, err := app.ExportAppStateAndValidators(false, nil)
if err != nil {
fmt.Println(err)
b.Fail()
}
err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644)
if err != nil {
fmt.Println(err)
b.Fail()
}
}
if exportParamsPath != "" {
fmt.Println("Exporting simulation params...")
paramsBz, err := json.MarshalIndent(params, "", " ")
if err != nil {
fmt.Println(err)
b.Fail()
}
err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644)
if err != nil {
fmt.Println(err)
b.Fail()
}
}
if simErr != nil {
fmt.Println(simErr)
b.FailNow()
}
if commit {
fmt.Println("GoLevelDB Stats")
fmt.Println(db.Stats()["leveldb.stats"])
@ -397,7 +437,30 @@ func TestFullAppSimulation(t *testing.T) {
require.Equal(t, "SimApp", app.Name())
// Run randomized simulation
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
_, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
// export state and params before the simulation error is checked
if exportStatePath != "" {
fmt.Println("Exporting app state...")
appState, _, err := app.ExportAppStateAndValidators(false, nil)
require.NoError(t, err)
err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644)
require.NoError(t, err)
}
if exportParamsPath != "" {
fmt.Println("Exporting simulation params...")
fmt.Println(params)
paramsBz, err := json.MarshalIndent(params, "", " ")
require.NoError(t, err)
err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644)
require.NoError(t, err)
}
require.NoError(t, simErr)
if commit {
// for memdb:
// fmt.Println("Database Size", db.Stats()["database.size"])
@ -405,8 +468,6 @@ func TestFullAppSimulation(t *testing.T) {
fmt.Println(db.Stats()["leveldb.stats"])
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
}
require.Nil(t, err)
}
func TestAppImportExport(t *testing.T) {
@ -434,7 +495,28 @@ func TestAppImportExport(t *testing.T) {
require.Equal(t, "SimApp", app.Name())
// Run randomized simulation
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
_, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
// export state and params before the simulation error is checked
if exportStatePath != "" {
fmt.Println("Exporting app state...")
appState, _, err := app.ExportAppStateAndValidators(false, nil)
require.NoError(t, err)
err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644)
require.NoError(t, err)
}
if exportParamsPath != "" {
fmt.Println("Exporting simulation params...")
paramsBz, err := json.MarshalIndent(params, "", " ")
require.NoError(t, err)
err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644)
require.NoError(t, err)
}
require.NoError(t, simErr)
if commit {
// for memdb:
@ -444,7 +526,6 @@ func TestAppImportExport(t *testing.T) {
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
}
require.Nil(t, err)
fmt.Printf("Exporting genesis...\n")
appState, _, err := app.ExportAppStateAndValidators(false, []string{})
@ -530,7 +611,28 @@ func TestAppSimulationAfterImport(t *testing.T) {
require.Equal(t, "SimApp", app.Name())
// Run randomized simulation
stopEarly, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
stopEarly, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
// export state and params before the simulation error is checked
if exportStatePath != "" {
fmt.Println("Exporting app state...")
appState, _, err := app.ExportAppStateAndValidators(false, nil)
require.NoError(t, err)
err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644)
require.NoError(t, err)
}
if exportParamsPath != "" {
fmt.Println("Exporting simulation params...")
paramsBz, err := json.MarshalIndent(params, "", " ")
require.NoError(t, err)
err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644)
require.NoError(t, err)
}
require.NoError(t, simErr)
if commit {
// for memdb:
@ -540,8 +642,6 @@ func TestAppSimulationAfterImport(t *testing.T) {
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
}
require.Nil(t, err)
if stopEarly {
// we can't export or import a zero-validator genesis
fmt.Printf("We can't export or import a zero-validator genesis, exiting test...\n")
@ -572,7 +672,7 @@ func TestAppSimulationAfterImport(t *testing.T) {
})
// Run randomized simulation on imported app
_, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, newApp))
_, _, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, newApp))
require.Nil(t, err)
}
@ -597,15 +697,9 @@ func TestAppStateDeterminism(t *testing.T) {
// Run randomized simulation
simulation.SimulateFromSeed(
t, os.Stdout, app.BaseApp, appStateFn, seed,
testAndRunTxs(app),
[]sdk.Invariant{},
50,
100,
true,
false,
false,
false,
app.ModuleAccountAddrs(),
testAndRunTxs(app), []sdk.Invariant{},
50, 100, 0,
false, true, false, false, false, app.ModuleAccountAddrs(),
)
appHash := app.LastCommitID().Hash
appHashList[j] = appHash
@ -627,15 +721,47 @@ func BenchmarkInvariants(b *testing.B) {
}()
app := NewSimApp(logger, db, nil, true, 0)
exportParams := exportParamsPath != ""
// 2. Run parameterized simulation (w/o invariants)
_, err := simulation.SimulateFromSeed(
_, params, simErr := simulation.SimulateFromSeed(
b, ioutil.Discard, app.BaseApp, appStateFn, seed, testAndRunTxs(app),
[]sdk.Invariant{}, numBlocks, blockSize, commit, lean, onOperation, false,
app.ModuleAccountAddrs(),
[]sdk.Invariant{}, numBlocks, exportParamsHeight, blockSize,
exportParams, commit, lean, onOperation, false, app.ModuleAccountAddrs(),
)
if err != nil {
fmt.Println(err)
// export state and params before the simulation error is checked
if exportStatePath != "" {
fmt.Println("Exporting app state...")
appState, _, err := app.ExportAppStateAndValidators(false, nil)
if err != nil {
fmt.Println(err)
b.Fail()
}
err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644)
if err != nil {
fmt.Println(err)
b.Fail()
}
}
if exportParamsPath != "" {
fmt.Println("Exporting simulation params...")
paramsBz, err := json.MarshalIndent(params, "", " ")
if err != nil {
fmt.Println(err)
b.Fail()
}
err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644)
if err != nil {
fmt.Println(err)
b.Fail()
}
}
if simErr != nil {
fmt.Println(simErr)
b.FailNow()
}

27
simapp/utils.go
View File

@ -37,18 +37,21 @@ import (
)
var (
genesisFile string
paramsFile string
seed int64
numBlocks int
blockSize int
enabled bool
verbose bool
lean bool
commit bool
period int
onOperation bool // TODO Remove in favor of binary search for invariant violation
allInvariants bool
genesisFile string
paramsFile string
exportParamsPath string
exportParamsHeight int
exportStatePath string
seed int64
numBlocks int
blockSize int
enabled bool
verbose bool
lean bool
commit bool
period int
onOperation bool // TODO Remove in favor of binary search for invariant violation
allInvariants bool
)
// NewSimAppUNSAFE is used for debugging purposes only.

28
x/simulation/simulate.go
View File

@ -41,14 +41,15 @@ func initChain(
// SimulateFromSeed tests an application by running the provided
// operations, testing the provided invariants, but using the provided seed.
// TODO split this monster function up
// TODO: split this monster function up
func SimulateFromSeed(
tb testing.TB, w io.Writer, app *baseapp.BaseApp,
appStateFn AppStateFn, seed int64, ops WeightedOperations,
invariants sdk.Invariants,
numBlocks, blockSize int, commit, lean, onOperation, allInvariants bool,
numBlocks, exportParamsHeight, blockSize int,
exportParams, commit, lean, onOperation, allInvariants bool,
blackListedAccs map[string]bool,
) (stopEarly bool, simError error) {
) (stopEarly bool, exportedParams Params, err error) {
// in case we have to end early, don't os.Exit so that we can run cleanup code.
testingMode, t, b := getTestingMode(tb)
@ -72,7 +73,7 @@ func SimulateFromSeed(
// TM 0.24) Initially this is the same as the initial validator set
validators, accs := initChain(r, params, accs, app, appStateFn, genesisTimestamp)
if len(accs) == 0 {
return true, fmt.Errorf("must have greater than zero genesis accounts")
return true, params, fmt.Errorf("must have greater than zero genesis accounts")
}
// remove module account address if they exist in accs
@ -100,7 +101,7 @@ func SimulateFromSeed(
go func() {
receivedSignal := <-c
fmt.Fprintf(w, "\nExiting early due to %s, on block %d, operation %d\n", receivedSignal, header.Height, opCount)
simError = fmt.Errorf("Exited due to %s", receivedSignal)
err = fmt.Errorf("Exited due to %s", receivedSignal)
stopEarly = true
}()
@ -131,12 +132,15 @@ func SimulateFromSeed(
stackTrace := string(debug.Stack())
fmt.Println(stackTrace)
logWriter.PrintLogs()
simError = fmt.Errorf("Simulation halted due to panic on block %d", header.Height)
err = fmt.Errorf("Simulation halted due to panic on block %d", header.Height)
}
}()
}
// TODO split up the contents of this for loop into new functions
// set exported params to the initial state
exportedParams = params
// TODO: split up the contents of this for loop into new functions
for height := 1; height <= numBlocks && !stopEarly; height++ {
// Log the header time for future lookup
@ -205,11 +209,16 @@ func SimulateFromSeed(
validators = nextValidators
nextValidators = updateValidators(tb, r, params,
validators, res.ValidatorUpdates, eventStats.tally)
// update the exported params
if exportParams && exportParamsHeight == height {
exportedParams = params
}
}
if stopEarly {
eventStats.Print(w)
return true, simError
return true, exportedParams, err
}
fmt.Fprintf(
@ -219,7 +228,8 @@ func SimulateFromSeed(
)
eventStats.Print(w)
return false, nil
return false, exportedParams, nil
}
//______________________________________________________________________________