certusone
/
cosmos-sdk
mirror of https://github.com/certusone/cosmos-sdk.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge PR #4906: Introduce Simulation Config

This commit is contained in:
Federico Kunze 2019-08-15 17:35:25 +02:00 committed by Alexander Bezobchuk
parent 7ffa95d675
commit 57cc5ae3a2
6 changed files with 362 additions and 292 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -65,6 +65,7 @@ longer panics if the store to load contains substores that we didn't explicitly
* Implement `SimulationManager` for executing modules' simulation functionalities in a modularized way
* Add `DecodeStore` to the `SimulationManager` for decoding each module's types
* (simulation) [\#4893](https://github.com/cosmos/cosmos-sdk/issues/4893) Change SimApp keepers to be public and add getter functions for keys and codec
* (simulation) [\#4906](https://github.com/cosmos/cosmos-sdk/issues/4906) Add simulation `Config` struct that wraps simulation flags
* (store) [\#4792](https://github.com/cosmos/cosmos-sdk/issues/4792) panic on non-registered store
* (types) [\#4821](https://github.com/cosmos/cosmos-sdk/issues/4821) types/errors package added with support for stacktraces. It is meant as a more feature-rich replacement for sdk.Errors in the mid-term.

278
simapp/sim_test.go
View File

@ -2,14 +2,11 @@ package simapp
import (
"encoding/json"
"flag"
"fmt"
"io"
"io/ioutil"
"math/rand"
"os"
"testing"
"time"
"github.com/stretchr/testify/require"
@ -38,135 +35,17 @@ import (
)
func init() {
flag.StringVar(&genesisFile, "Genesis", "", "custom simulation genesis file; cannot be used with params file")
flag.StringVar(&paramsFile, "Params", "", "custom simulation params file which overrides any random params; cannot be used with genesis")
flag.StringVar(&exportParamsPath, "ExportParamsPath", "", "custom file path to save the exported params JSON")
flag.IntVar(&exportParamsHeight, "ExportParamsHeight", 0, "height to which export the randomly generated params")
flag.StringVar(&exportStatePath, "ExportStatePath", "", "custom file path to save the exported app state JSON")
flag.StringVar(&exportStatsPath, "ExportStatsPath", "", "custom file path to save the exported simulation statistics JSON")
flag.Int64Var(&seed, "Seed", 42, "simulation random seed")
flag.IntVar(&initialBlockHeight, "InitialBlockHeight", 1, "initial block to start the simulation")
flag.IntVar(&numBlocks, "NumBlocks", 500, "number of new blocks to simulate from the initial block height")
flag.IntVar(&blockSize, "BlockSize", 200, "operations per block")
flag.BoolVar(&enabled, "Enabled", false, "enable the simulation")
flag.BoolVar(&verbose, "Verbose", false, "verbose log output")
flag.BoolVar(&lean, "Lean", false, "lean simulation log output")
flag.BoolVar(&commit, "Commit", false, "have the simulation commit")
flag.IntVar(&period, "Period", 1, "run slow invariants only once every period assertions")
flag.BoolVar(&onOperation, "SimulateEveryOperation", false, "run slow invariants every operation")
flag.BoolVar(&allInvariants, "PrintAllInvariants", false, "print all invariants if a broken invariant is found")
flag.Int64Var(&genesisTime, "GenesisTime", 0, "override genesis UNIX time instead of using a random UNIX time")
}
// helper function for populating input for SimulateFromSeed
// TODO: clean up this function along with the simulation refactor
func getSimulateFromSeedInput(tb testing.TB, w io.Writer, app *SimApp) (
testing.TB, io.Writer, *baseapp.BaseApp, simulation.AppStateFn, int64,
simulation.WeightedOperations, sdk.Invariants, int, int, int, int, string,
bool, bool, bool, bool, bool, map[string]bool) {
exportParams := exportParamsPath != ""
return tb, w, app.BaseApp, appStateFn, seed,
testAndRunTxs(app), invariants(app),
initialBlockHeight, numBlocks, exportParamsHeight, blockSize,
exportStatsPath, exportParams, commit, lean, onOperation, allInvariants, app.ModuleAccountAddrs()
}
func appStateFn(
r *rand.Rand, accs []simulation.Account,
) (appState json.RawMessage, simAccs []simulation.Account, chainID string, genesisTimestamp time.Time) {
cdc := MakeCodec()
if genesisTime == 0 {
genesisTimestamp = simulation.RandTimestamp(r)
} else {
genesisTimestamp = time.Unix(genesisTime, 0)
}
switch {
case paramsFile != "" && genesisFile != "":
panic("cannot provide both a genesis file and a params file")
case genesisFile != "":
appState, simAccs, chainID = AppStateFromGenesisFileFn(r, accs, genesisTimestamp)
case paramsFile != "":
appParams := make(simulation.AppParams)
bz, err := ioutil.ReadFile(paramsFile)
if err != nil {
panic(err)
}
cdc.MustUnmarshalJSON(bz, &appParams)
appState, simAccs, chainID = appStateRandomizedFn(r, accs, genesisTimestamp, appParams)
default:
appParams := make(simulation.AppParams)
appState, simAccs, chainID = appStateRandomizedFn(r, accs, genesisTimestamp, appParams)
}
return appState, simAccs, chainID, genesisTimestamp
}
// TODO refactor out random initialization code to the modules
func appStateRandomizedFn(
r *rand.Rand, accs []simulation.Account, genesisTimestamp time.Time, appParams simulation.AppParams,
) (json.RawMessage, []simulation.Account, string) {
cdc := MakeCodec()
genesisState := NewDefaultGenesisState()
var (
amount int64
numInitiallyBonded int64
)
appParams.GetOrGenerate(cdc, StakePerAccount, &amount, r,
func(r *rand.Rand) { amount = int64(r.Intn(1e12)) })
appParams.GetOrGenerate(cdc, InitiallyBondedValidators, &amount, r,
func(r *rand.Rand) { numInitiallyBonded = int64(r.Intn(250)) })
numAccs := int64(len(accs))
if numInitiallyBonded > numAccs {
numInitiallyBonded = numAccs
}
fmt.Printf(
`Selected randomly generated parameters for simulated genesis:
{
stake_per_account: "%v",
initially_bonded_validators: "%v"
}
`, amount, numInitiallyBonded,
)
GenGenesisAccounts(cdc, r, accs, genesisTimestamp, amount, numInitiallyBonded, genesisState)
GenAuthGenesisState(cdc, r, appParams, genesisState)
GenBankGenesisState(cdc, r, appParams, genesisState)
GenSupplyGenesisState(cdc, amount, numInitiallyBonded, int64(len(accs)), genesisState)
GenGovGenesisState(cdc, r, appParams, genesisState)
GenMintGenesisState(cdc, r, appParams, genesisState)
GenDistrGenesisState(cdc, r, appParams, genesisState)
stakingGen := GenStakingGenesisState(cdc, r, accs, amount, numAccs, numInitiallyBonded, appParams, genesisState)
GenSlashingGenesisState(cdc, r, stakingGen, appParams, genesisState)
appState, err := MakeCodec().MarshalJSON(genesisState)
if err != nil {
panic(err)
}
return appState, accs, "simulation"
GetSimulatorFlags()
}
// TODO: add description
func testAndRunTxs(app *SimApp) []simulation.WeightedOperation {
func testAndRunTxs(app *SimApp, config simulation.Config) []simulation.WeightedOperation {
cdc := MakeCodec()
ap := make(simulation.AppParams)
if paramsFile != "" {
bz, err := ioutil.ReadFile(paramsFile)
if config.ParamsFile != "" {
bz, err := ioutil.ReadFile(config.ParamsFile)
if err != nil {
panic(err)
}
@ -357,10 +236,10 @@ func testAndRunTxs(app *SimApp) []simulation.WeightedOperation {
func invariants(app *SimApp) []sdk.Invariant {
// TODO: fix PeriodicInvariants, it doesn't seem to call individual invariants for a period of 1
// Ref: https://github.com/cosmos/cosmos-sdk/issues/4631
if period == 1 {
if flagPeriodValue == 1 {
return app.CrisisKeeper.Invariants()
}
return simulation.PeriodicInvariants(app.CrisisKeeper.Invariants(), period, 0)
return simulation.PeriodicInvariants(app.CrisisKeeper.Invariants(), flagPeriodValue, 0)
}
// Pass this in as an option to use a dbStoreAdapter instead of an IAVLStore for simulation speed.
@ -372,6 +251,7 @@ func fauxMerkleModeOpt(bapp *baseapp.BaseApp) {
// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/simapp -bench ^BenchmarkFullAppSimulation$ -Commit=true -cpuprofile cpu.out
func BenchmarkFullAppSimulation(b *testing.B) {
logger := log.NewNopLogger()
config := NewConfigFromFlags()
var db dbm.DB
dir, _ := ioutil.TempDir("", "goleveldb-app-sim")
@ -384,24 +264,28 @@ func BenchmarkFullAppSimulation(b *testing.B) {
// Run randomized simulation
// TODO: parameterize numbers, save for a later PR
_, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, os.Stdout, app))
_, params, simErr := simulation.SimulateFromSeed(
b, os.Stdout, app.BaseApp, AppStateFn,
testAndRunTxs(app, config), invariants(app),
app.ModuleAccountAddrs(), config,
)
// export state and params before the simulation error is checked
if exportStatePath != "" {
if config.ExportStatePath != "" {
fmt.Println("Exporting app state...")
appState, _, err := app.ExportAppStateAndValidators(false, nil)
if err != nil {
fmt.Println(err)
b.Fail()
}
err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644)
err = ioutil.WriteFile(config.ExportStatePath, []byte(appState), 0644)
if err != nil {
fmt.Println(err)
b.Fail()
}
}
if exportParamsPath != "" {
if config.ExportParamsPath != "" {
fmt.Println("Exporting simulation params...")
paramsBz, err := json.MarshalIndent(params, "", " ")
if err != nil {
@ -409,7 +293,7 @@ func BenchmarkFullAppSimulation(b *testing.B) {
b.Fail()
}
err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644)
err = ioutil.WriteFile(config.ExportParamsPath, paramsBz, 0644)
if err != nil {
fmt.Println(err)
b.Fail()
@ -421,7 +305,7 @@ func BenchmarkFullAppSimulation(b *testing.B) {
b.FailNow()
}
if commit {
if config.Commit {
fmt.Println("\nGoLevelDB Stats")
fmt.Println(db.Stats()["leveldb.stats"])
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
@ -429,13 +313,14 @@ func BenchmarkFullAppSimulation(b *testing.B) {
}
func TestFullAppSimulation(t *testing.T) {
if !enabled {
if !flagEnabledValue {
t.Skip("Skipping application simulation")
}
var logger log.Logger
config := NewConfigFromFlags()
if verbose {
if flagVerboseValue {
logger = log.TestingLogger()
} else {
logger = log.NewNopLogger()
@ -454,31 +339,35 @@ func TestFullAppSimulation(t *testing.T) {
require.Equal(t, "SimApp", app.Name())
// Run randomized simulation
_, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
_, params, simErr := simulation.SimulateFromSeed(
t, os.Stdout, app.BaseApp, AppStateFn,
testAndRunTxs(app, config), invariants(app),
app.ModuleAccountAddrs(), config,
)
// export state and params before the simulation error is checked
if exportStatePath != "" {
if config.ExportStatePath != "" {
fmt.Println("Exporting app state...")
appState, _, err := app.ExportAppStateAndValidators(false, nil)
require.NoError(t, err)
err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644)
err = ioutil.WriteFile(config.ExportStatePath, []byte(appState), 0644)
require.NoError(t, err)
}
if exportParamsPath != "" {
if config.ExportParamsPath != "" {
fmt.Println("Exporting simulation params...")
fmt.Println(params)
paramsBz, err := json.MarshalIndent(params, "", " ")
require.NoError(t, err)
err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644)
err = ioutil.WriteFile(config.ExportParamsPath, paramsBz, 0644)
require.NoError(t, err)
}
require.NoError(t, simErr)
if commit {
if config.Commit {
// for memdb:
// fmt.Println("Database Size", db.Stats()["database.size"])
fmt.Println("\nGoLevelDB Stats")
@ -488,12 +377,14 @@ func TestFullAppSimulation(t *testing.T) {
}
func TestAppImportExport(t *testing.T) {
if !enabled {
if !flagEnabledValue {
t.Skip("Skipping application import/export simulation")
}
var logger log.Logger
if verbose {
config := NewConfigFromFlags()
if flagVerboseValue {
logger = log.TestingLogger()
} else {
logger = log.NewNopLogger()
@ -512,30 +403,34 @@ func TestAppImportExport(t *testing.T) {
require.Equal(t, "SimApp", app.Name())
// Run randomized simulation
_, simParams, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
_, simParams, simErr := simulation.SimulateFromSeed(
t, os.Stdout, app.BaseApp, AppStateFn,
testAndRunTxs(app, config), invariants(app),
app.ModuleAccountAddrs(), config,
)
// export state and simParams before the simulation error is checked
if exportStatePath != "" {
if config.ExportStatePath != "" {
fmt.Println("Exporting app state...")
appState, _, err := app.ExportAppStateAndValidators(false, nil)
require.NoError(t, err)
err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644)
err = ioutil.WriteFile(config.ExportStatePath, []byte(appState), 0644)
require.NoError(t, err)
}
if exportParamsPath != "" {
if config.ExportParamsPath != "" {
fmt.Println("Exporting simulation params...")
simParamsBz, err := json.MarshalIndent(simParams, "", " ")
require.NoError(t, err)
err = ioutil.WriteFile(exportParamsPath, simParamsBz, 0644)
err = ioutil.WriteFile(config.ExportParamsPath, simParamsBz, 0644)
require.NoError(t, err)
}
require.NoError(t, simErr)
if commit {
if config.Commit {
// for memdb:
// fmt.Println("Database Size", db.Stats()["database.size"])
fmt.Println("\nGoLevelDB Stats")
@ -562,9 +457,7 @@ func TestAppImportExport(t *testing.T) {
var genesisState GenesisState
err = app.cdc.UnmarshalJSON(appState, &genesisState)
if err != nil {
panic(err)
}
require.NoError(t, err)
ctxB := newApp.NewContext(true, abci.Header{Height: app.LastBlockHeight()})
newApp.mm.InitGenesis(ctxB, genesisState)
@ -611,12 +504,14 @@ func TestAppImportExport(t *testing.T) {
}
func TestAppSimulationAfterImport(t *testing.T) {
if !enabled {
if !flagEnabledValue {
t.Skip("Skipping application simulation after import")
}
var logger log.Logger
if verbose {
config := NewConfigFromFlags()
if flagVerboseValue {
logger = log.TestingLogger()
} else {
logger = log.NewNopLogger()
@ -634,30 +529,34 @@ func TestAppSimulationAfterImport(t *testing.T) {
require.Equal(t, "SimApp", app.Name())
// Run randomized simulation
stopEarly, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
stopEarly, params, simErr := simulation.SimulateFromSeed(
t, os.Stdout, app.BaseApp, AppStateFn,
testAndRunTxs(app, config), invariants(app),
app.ModuleAccountAddrs(), config,
)
// export state and params before the simulation error is checked
if exportStatePath != "" {
if config.ExportStatePath != "" {
fmt.Println("Exporting app state...")
appState, _, err := app.ExportAppStateAndValidators(false, nil)
require.NoError(t, err)
err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644)
err = ioutil.WriteFile(config.ExportStatePath, []byte(appState), 0644)
require.NoError(t, err)
}
if exportParamsPath != "" {
if config.ExportParamsPath != "" {
fmt.Println("Exporting simulation params...")
paramsBz, err := json.MarshalIndent(params, "", " ")
require.NoError(t, err)
err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644)
err = ioutil.WriteFile(config.ExportParamsPath, paramsBz, 0644)
require.NoError(t, err)
}
require.NoError(t, simErr)
if commit {
if config.Commit {
// for memdb:
// fmt.Println("Database Size", db.Stats()["database.size"])
fmt.Println("\nGoLevelDB Stats")
@ -674,9 +573,7 @@ func TestAppSimulationAfterImport(t *testing.T) {
fmt.Printf("Exporting genesis...\n")
appState, _, err := app.ExportAppStateAndValidators(true, []string{})
if err != nil {
panic(err)
}
require.NoError(t, err)
fmt.Printf("Importing genesis...\n")
@ -695,23 +592,34 @@ func TestAppSimulationAfterImport(t *testing.T) {
})
// Run randomized simulation on imported app
_, _, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, newApp))
require.Nil(t, err)
_, _, err = simulation.SimulateFromSeed(
t, os.Stdout, newApp.BaseApp, AppStateFn,
testAndRunTxs(newApp, config), invariants(newApp),
newApp.ModuleAccountAddrs(), config,
)
require.NoError(t, err)
}
// TODO: Make another test for the fuzzer itself, which just has noOp txs
// and doesn't depend on the application.
func TestAppStateDeterminism(t *testing.T) {
if !enabled {
if !flagEnabledValue {
t.Skip("Skipping application simulation")
}
config := NewConfigFromFlags()
config.InitialBlockHeight = 1
config.ExportParamsPath = ""
config.OnOperation = false
config.AllInvariants = false
numSeeds := 3
numTimesToRunPerSeed := 5
appHashList := make([]json.RawMessage, numTimesToRunPerSeed)
for i := 0; i < numSeeds; i++ {
seed := rand.Int63()
config.Seed = rand.Int63()
for j := 0; j < numTimesToRunPerSeed; j++ {
logger := log.NewNopLogger()
@ -720,29 +628,30 @@ func TestAppStateDeterminism(t *testing.T) {
fmt.Printf(
"Running non-determinism simulation; seed: %d/%d (%d), attempt: %d/%d\n",
i+1, numSeeds, seed, j+1, numTimesToRunPerSeed,
i+1, numSeeds, config.Seed, j+1, numTimesToRunPerSeed,
)
_, _, err := simulation.SimulateFromSeed(
t, os.Stdout, app.BaseApp, appStateFn, seed, testAndRunTxs(app),
[]sdk.Invariant{}, 1, numBlocks, exportParamsHeight,
blockSize, "", false, commit, lean,
false, false, app.ModuleAccountAddrs(),
t, os.Stdout, app.BaseApp, AppStateFn,
testAndRunTxs(app, config), []sdk.Invariant{},
app.ModuleAccountAddrs(), config,
)
require.NoError(t, err)
appHash := app.LastCommitID().Hash
appHashList[j] = appHash
}
for k := 1; k < numTimesToRunPerSeed; k++ {
require.Equal(t, appHashList[0], appHashList[k], "appHash list: %v", appHashList)
if j != 0 {
require.Equal(t, appHashList[0], appHashList[j], "appHash list: %v", appHashList)
}
}
}
}
func BenchmarkInvariants(b *testing.B) {
logger := log.NewNopLogger()
config := NewConfigFromFlags()
dir, _ := ioutil.TempDir("", "goleveldb-app-invariant-bench")
db, _ := sdk.NewLevelDB("simulation", dir)
@ -752,31 +661,30 @@ func BenchmarkInvariants(b *testing.B) {
}()
app := NewSimApp(logger, db, nil, true, 0)
exportParams := exportParamsPath != ""
// 2. Run parameterized simulation (w/o invariants)
_, params, simErr := simulation.SimulateFromSeed(
b, ioutil.Discard, app.BaseApp, appStateFn, seed, testAndRunTxs(app),
[]sdk.Invariant{}, initialBlockHeight, numBlocks, exportParamsHeight, blockSize,
exportStatsPath, exportParams, commit, lean, onOperation, false, app.ModuleAccountAddrs(),
b, ioutil.Discard, app.BaseApp, AppStateFn,
testAndRunTxs(app, config), []sdk.Invariant{},
app.ModuleAccountAddrs(), config,
)
// export state and params before the simulation error is checked
if exportStatePath != "" {
if config.ExportStatePath != "" {
fmt.Println("Exporting app state...")
appState, _, err := app.ExportAppStateAndValidators(false, nil)
if err != nil {
fmt.Println(err)
b.Fail()
}
err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644)
err = ioutil.WriteFile(config.ExportStatePath, []byte(appState), 0644)
if err != nil {
fmt.Println(err)
b.Fail()
}
}
if exportParamsPath != "" {
if config.ExportParamsPath != "" {
fmt.Println("Exporting simulation params...")
paramsBz, err := json.MarshalIndent(params, "", " ")
if err != nil {
@ -784,7 +692,7 @@ func BenchmarkInvariants(b *testing.B) {
b.Fail()
}
err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644)
err = ioutil.WriteFile(config.ExportParamsPath, paramsBz, 0644)
if err != nil {
fmt.Println(err)
b.Fail()
@ -805,7 +713,7 @@ func BenchmarkInvariants(b *testing.B) {
for _, cr := range app.CrisisKeeper.Routes() {
b.Run(fmt.Sprintf("%s/%s", cr.ModuleName, cr.Route), func(b *testing.B) {
if res, stop := cr.Invar(ctx); stop {
fmt.Printf("broken invariant at block %d of %d\n%s", ctx.BlockHeight()-1, numBlocks, res)
fmt.Printf("broken invariant at block %d of %d\n%s", ctx.BlockHeight()-1, config.NumBlocks, res)
b.FailNow()
}
})

142
simapp/state.go Normal file
View File

@ -0,0 +1,142 @@
package simapp
// DONTCOVER
import (
"encoding/json"
"fmt"
"io/ioutil"
"math/rand"
"time"
"github.com/tendermint/tendermint/crypto/secp256k1"
tmtypes "github.com/tendermint/tendermint/types"
"github.com/cosmos/cosmos-sdk/x/genaccounts"
"github.com/cosmos/cosmos-sdk/x/simulation"
)
// AppStateFn returns the initial application state using a genesis or the simulation parameters.
// It panics if the user provides files for both of them.
// If a file is not given for the genesis or the sim params, it creates a randomized one.
func AppStateFn(
r *rand.Rand, accs []simulation.Account, config simulation.Config,
) (appState json.RawMessage, simAccs []simulation.Account, chainID string, genesisTimestamp time.Time) {
cdc := MakeCodec()
if flagGenesisTimeValue == 0 {
genesisTimestamp = simulation.RandTimestamp(r)
} else {
genesisTimestamp = time.Unix(flagGenesisTimeValue, 0)
}
switch {
case config.ParamsFile != "" && config.GenesisFile != "":
panic("cannot provide both a genesis file and a params file")
case config.GenesisFile != "":
appState, simAccs, chainID = AppStateFromGenesisFileFn(r, config)
case config.ParamsFile != "":
appParams := make(simulation.AppParams)
bz, err := ioutil.ReadFile(config.ParamsFile)
if err != nil {
panic(err)
}
cdc.MustUnmarshalJSON(bz, &appParams)
appState, simAccs, chainID = AppStateRandomizedFn(r, accs, genesisTimestamp, appParams)
default:
appParams := make(simulation.AppParams)
appState, simAccs, chainID = AppStateRandomizedFn(r, accs, genesisTimestamp, appParams)
}
return appState, simAccs, chainID, genesisTimestamp
}
// AppStateRandomizedFn creates calls each module's GenesisState generator function
// and creates
func AppStateRandomizedFn(
r *rand.Rand, accs []simulation.Account, genesisTimestamp time.Time, appParams simulation.AppParams,
) (json.RawMessage, []simulation.Account, string) {
cdc := MakeCodec()
genesisState := NewDefaultGenesisState()
var (
amount int64
numInitiallyBonded int64
)
appParams.GetOrGenerate(cdc, StakePerAccount, &amount, r,
func(r *rand.Rand) { amount = int64(r.Intn(1e12)) })
appParams.GetOrGenerate(cdc, InitiallyBondedValidators, &amount, r,
func(r *rand.Rand) { numInitiallyBonded = int64(r.Intn(250)) })
numAccs := int64(len(accs))
if numInitiallyBonded > numAccs {
numInitiallyBonded = numAccs
}
fmt.Printf(
`Selected randomly generated parameters for simulated genesis:
{
stake_per_account: "%v",
initially_bonded_validators: "%v"
}
`, amount, numInitiallyBonded,
)
GenGenesisAccounts(cdc, r, accs, genesisTimestamp, amount, numInitiallyBonded, genesisState)
GenAuthGenesisState(cdc, r, appParams, genesisState)
GenBankGenesisState(cdc, r, appParams, genesisState)
GenSupplyGenesisState(cdc, amount, numInitiallyBonded, int64(len(accs)), genesisState)
GenGovGenesisState(cdc, r, appParams, genesisState)
GenMintGenesisState(cdc, r, appParams, genesisState)
GenDistrGenesisState(cdc, r, appParams, genesisState)
stakingGen := GenStakingGenesisState(cdc, r, accs, amount, numAccs, numInitiallyBonded, appParams, genesisState)
GenSlashingGenesisState(cdc, r, stakingGen, appParams, genesisState)
appState, err := MakeCodec().MarshalJSON(genesisState)
if err != nil {
panic(err)
}
return appState, accs, "simulation"
}
// AppStateFromGenesisFileFn util function to generate the genesis AppState
// from a genesis.json file
func AppStateFromGenesisFileFn(r *rand.Rand, config simulation.Config) (json.RawMessage, []simulation.Account, string) {
var genesis tmtypes.GenesisDoc
cdc := MakeCodec()
bytes, err := ioutil.ReadFile(config.GenesisFile)
if err != nil {
panic(err)
}
cdc.MustUnmarshalJSON(bytes, &genesis)
var appState GenesisState
cdc.MustUnmarshalJSON(genesis.AppState, &appState)
accounts := genaccounts.GetGenesisStateFromAppState(cdc, appState)
var newAccs []simulation.Account
for _, acc := range accounts {
// Pick a random private key, since we don't know the actual key
// This should be fine as it's only used for mock Tendermint validators
// and these keys are never actually used to sign by mock Tendermint.
privkeySeed := make([]byte, 15)
r.Read(privkeySeed)
privKey := secp256k1.GenPrivKeySecp256k1(privkeySeed)
newAccs = append(newAccs, simulation.Account{privKey, privKey.PubKey(), acc.Address})
}
return genesis.AppState, newAccs, genesis.ChainID
}

134
simapp/utils.go
View File

@ -3,19 +3,13 @@ package simapp
import (
"encoding/json"
"flag"
"fmt"
"io"
"io/ioutil"
"math/rand"
"time"
"github.com/tendermint/tendermint/crypto/secp256k1"
cmn "github.com/tendermint/tendermint/libs/common"
"github.com/tendermint/tendermint/libs/log"
tmtypes "github.com/tendermint/tendermint/types"
dbm "github.com/tendermint/tm-db"
"github.com/cosmos/cosmos-sdk/baseapp"
"github.com/cosmos/cosmos-sdk/codec"
sdk "github.com/cosmos/cosmos-sdk/types"
"github.com/cosmos/cosmos-sdk/x/auth"
@ -30,73 +24,76 @@ import (
"github.com/cosmos/cosmos-sdk/x/supply"
)
// List of available flags for the simulator
//---------------------------------------------------------------------
// Flags
// List of SimApp flags for the simulator
var (
genesisFile string
paramsFile string
exportParamsPath string
exportParamsHeight int
exportStatePath string
exportStatsPath string
seed int64
initialBlockHeight int
numBlocks int
blockSize int
enabled bool
verbose bool
lean bool
commit bool
period int
onOperation bool // TODO Remove in favor of binary search for invariant violation
allInvariants bool
genesisTime int64
flagGenesisFileValue string
flagParamsFileValue string
flagExportParamsPathValue string
flagExportParamsHeightValue int
flagExportStatePathValue string
flagExportStatsPathValue string
flagSeedValue int64
flagInitialBlockHeightValue int
flagNumBlocksValue int
flagBlockSizeValue int
flagLeanValue bool
flagCommitValue bool
flagOnOperationValue bool // TODO: Remove in favor of binary search for invariant violation
flagAllInvariantsValue bool
flagEnabledValue bool
flagVerboseValue bool
flagPeriodValue int
flagGenesisTimeValue int64
)
// NewSimAppUNSAFE is used for debugging purposes only.
//
// NOTE: to not use this function with non-test code
func NewSimAppUNSAFE(logger log.Logger, db dbm.DB, traceStore io.Writer, loadLatest bool,
invCheckPeriod uint, baseAppOptions ...func(*baseapp.BaseApp),
) (gapp *SimApp, keyMain, keyStaking *sdk.KVStoreKey, stakingKeeper staking.Keeper) {
// GetSimulatorFlags gets the values of all the available simulation flags
func GetSimulatorFlags() {
gapp = NewSimApp(logger, db, traceStore, loadLatest, invCheckPeriod, baseAppOptions...)
return gapp, gapp.keys[baseapp.MainStoreKey], gapp.keys[staking.StoreKey], gapp.StakingKeeper
// Config fields
flag.StringVar(&flagGenesisFileValue, "Genesis", "", "custom simulation genesis file; cannot be used with params file")
flag.StringVar(&flagParamsFileValue, "Params", "", "custom simulation params file which overrides any random params; cannot be used with genesis")
flag.StringVar(&flagExportParamsPathValue, "ExportParamsPath", "", "custom file path to save the exported params JSON")
flag.IntVar(&flagExportParamsHeightValue, "ExportParamsHeight", 0, "height to which export the randomly generated params")
flag.StringVar(&flagExportStatePathValue, "ExportStatePath", "", "custom file path to save the exported app state JSON")
flag.StringVar(&flagExportStatsPathValue, "ExportStatsPath", "", "custom file path to save the exported simulation statistics JSON")
flag.Int64Var(&flagSeedValue, "Seed", 42, "simulation random seed")
flag.IntVar(&flagInitialBlockHeightValue, "InitialBlockHeight", 1, "initial block to start the simulation")
flag.IntVar(&flagNumBlocksValue, "NumBlocks", 500, "number of new blocks to simulate from the initial block height")
flag.IntVar(&flagBlockSizeValue, "BlockSize", 200, "operations per block")
flag.BoolVar(&flagLeanValue, "Lean", false, "lean simulation log output")
flag.BoolVar(&flagCommitValue, "Commit", false, "have the simulation commit")
flag.BoolVar(&flagOnOperationValue, "SimulateEveryOperation", false, "run slow invariants every operation")
flag.BoolVar(&flagAllInvariantsValue, "PrintAllInvariants", false, "print all invariants if a broken invariant is found")
// SimApp flags
flag.BoolVar(&flagEnabledValue, "Enabled", false, "enable the simulation")
flag.BoolVar(&flagVerboseValue, "Verbose", false, "verbose log output")
flag.IntVar(&flagPeriodValue, "Period", 1, "run slow invariants only once every period assertions")
flag.Int64Var(&flagGenesisTimeValue, "GenesisTime", 0, "override genesis UNIX time instead of using a random UNIX time")
}
// AppStateFromGenesisFileFn util function to generate the genesis AppState
// from a genesis.json file
func AppStateFromGenesisFileFn(
r *rand.Rand, _ []simulation.Account, _ time.Time,
) (json.RawMessage, []simulation.Account, string) {
var genesis tmtypes.GenesisDoc
cdc := MakeCodec()
bytes, err := ioutil.ReadFile(genesisFile)
if err != nil {
panic(err)
// NewConfigFromFlags creates a simulation from the retrieved values of the flags
func NewConfigFromFlags() simulation.Config {
return simulation.Config{
GenesisFile: flagGenesisFileValue,
ParamsFile: flagParamsFileValue,
ExportParamsPath: flagExportParamsPathValue,
ExportParamsHeight: flagExportParamsHeightValue,
ExportStatePath: flagExportStatePathValue,
ExportStatsPath: flagExportStatsPathValue,
Seed: flagSeedValue,
InitialBlockHeight: flagInitialBlockHeightValue,
NumBlocks: flagNumBlocksValue,
BlockSize: flagBlockSizeValue,
Lean: flagLeanValue,
Commit: flagCommitValue,
OnOperation: flagOnOperationValue,
AllInvariants: flagAllInvariantsValue,
}
cdc.MustUnmarshalJSON(bytes, &genesis)
var appState GenesisState
cdc.MustUnmarshalJSON(genesis.AppState, &appState)
accounts := genaccounts.GetGenesisStateFromAppState(cdc, appState)
var newAccs []simulation.Account
for _, acc := range accounts {
// Pick a random private key, since we don't know the actual key
// This should be fine as it's only used for mock Tendermint validators
// and these keys are never actually used to sign by mock Tendermint.
privkeySeed := make([]byte, 15)
r.Read(privkeySeed)
privKey := secp256k1.GenPrivKeySecp256k1(privkeySeed)
newAccs = append(newAccs, simulation.Account{privKey, privKey.PubKey(), acc.Address})
}
return genesis.AppState, newAccs, genesis.ChainID
}
// GenAuthGenesisState generates a random GenesisState for auth
@ -495,6 +492,9 @@ func GenStakingGenesisState(
return stakingGenesis
}
//---------------------------------------------------------------------
// Simulation Utils
// GetSimulationLog unmarshals the KVPair's Value to the corresponding type based on the
// each's module store key and the prefix bytes of the KVPair's key.
func GetSimulationLog(storeName string, sdr sdk.StoreDecoderRegistry, cdc *codec.Codec, kvAs, kvBs []cmn.KVPair) (log string) {

23
x/simulation/config.go Normal file
View File

@ -0,0 +1,23 @@
package simulation
// Config contains the necessary configuration flags for the simulator
type Config struct {
GenesisFile string // custom simulation genesis file; cannot be used with params file
ParamsFile string // custom simulation params file which overrides any random params; cannot be used with genesis
ExportParamsPath string // custom file path to save the exported params JSON
ExportParamsHeight int //height to which export the randomly generated params
ExportStatePath string //custom file path to save the exported app state JSON
ExportStatsPath string // custom file path to save the exported simulation statistics JSON
Seed int64 // simulation random seed
InitialBlockHeight int // initial block to start the simulation
NumBlocks int // number of new blocks to simulate from the initial block height
BlockSize int // operations per block
Lean bool // lean simulation log output
Commit bool // have the simulation commit
OnOperation bool // run slow invariants every operation
AllInvariants bool // print all failed invariants if a broken invariant is found
}

76
x/simulation/simulate.go
View File

@ -18,16 +18,17 @@ import (
)
// AppStateFn returns the app state json bytes, the genesis accounts, and the chain identifier
type AppStateFn func(
r *rand.Rand, accs []Account,
) (appState json.RawMessage, accounts []Account, chainId string, genesisTimestamp time.Time)
type AppStateFn func(r *rand.Rand, accs []Account, config Config) (
appState json.RawMessage, accounts []Account, chainId string, genesisTimestamp time.Time,
)
// initialize the chain for the simulation
func initChain(
r *rand.Rand, params Params, accounts []Account, app *baseapp.BaseApp, appStateFn AppStateFn,
r *rand.Rand, params Params, accounts []Account, app *baseapp.BaseApp,
appStateFn AppStateFn, config Config,
) (mockValidators, time.Time, []Account) {
appState, accounts, chainID, genesisTimestamp := appStateFn(r, accounts)
appState, accounts, chainID, genesisTimestamp := appStateFn(r, accounts, config)
req := abci.RequestInitChain{
AppStateBytes: appState,
@ -40,23 +41,19 @@ func initChain(
}
// SimulateFromSeed tests an application by running the provided
// operations, testing the provided invariants, but using the provided seed.
// operations, testing the provided invariants, but using the provided config.Seed.
// TODO: split this monster function up
func SimulateFromSeed(
tb testing.TB, w io.Writer, app *baseapp.BaseApp,
appStateFn AppStateFn, seed int64,
ops WeightedOperations, invariants sdk.Invariants,
initialHeight, numBlocks, exportParamsHeight, blockSize int,
exportStatsPath string,
exportParams, commit, lean, onOperation, allInvariants bool,
blackListedAccs map[string]bool,
appStateFn AppStateFn, ops WeightedOperations, invariants sdk.Invariants,
blackListedAccs map[string]bool, config Config,
) (stopEarly bool, exportedParams Params, err error) {
// in case we have to end early, don't os.Exit so that we can run cleanup code.
testingMode, t, b := getTestingMode(tb)
fmt.Fprintf(w, "Starting SimulateFromSeed with randomness created with seed %d\n", int(seed))
fmt.Fprintf(w, "Starting SimulateFromSeed with randomness created with config.Seed %d\n", int(config.Seed))
r := rand.New(rand.NewSource(seed))
r := rand.New(rand.NewSource(config.Seed))
params := RandomParams(r)
fmt.Fprintf(w, "Randomized simulation params: \n%s\n", mustMarshalJSONIndent(params))
@ -66,7 +63,7 @@ func SimulateFromSeed(
// Second variable to keep pending validator set (delayed one block since
// TM 0.24) Initially this is the same as the initial validator set
validators, genesisTimestamp, accs := initChain(r, params, accs, app, appStateFn)
validators, genesisTimestamp, accs := initChain(r, params, accs, app, appStateFn, config)
if len(accs) == 0 {
return true, params, fmt.Errorf("must have greater than zero genesis accounts")
}
@ -119,8 +116,7 @@ func SimulateFromSeed(
blockSimulator := createBlockSimulator(
testingMode, tb, t, w, params, eventStats.Tally, invariants,
ops, operationQueue, timeOperationQueue,
numBlocks, blockSize, logWriter, lean, onOperation, allInvariants)
ops, operationQueue, timeOperationQueue, logWriter, config)
if !testingMode {
b.ResetTimer()
@ -136,12 +132,12 @@ func SimulateFromSeed(
}
// set exported params to the initial state
if exportParams && exportParamsHeight == 0 {
if config.ExportParamsPath != "" && config.ExportParamsHeight == 0 {
exportedParams = params
}
// TODO: split up the contents of this for loop into new functions
for height := initialHeight; height < numBlocks+initialHeight && !stopEarly; height++ {
for height := config.InitialBlockHeight; height < config.NumBlocks+config.InitialBlockHeight && !stopEarly; height++ {
// Log the header time for future lookup
pastTimes = append(pastTimes, header.Time)
@ -152,7 +148,7 @@ func SimulateFromSeed(
app.BeginBlock(request)
if testingMode {
assertAllInvariants(t, app, invariants, "BeginBlock", logWriter, allInvariants)
assertAllInvariants(t, app, invariants, "BeginBlock", logWriter, config.AllInvariants)
}
ctx := app.NewContext(false, header)
@ -160,21 +156,21 @@ func SimulateFromSeed(
// Run queued operations. Ignores blocksize if blocksize is too small
numQueuedOpsRan := runQueuedOperations(
operationQueue, int(header.Height),
tb, r, app, ctx, accs, logWriter, eventStats.Tally, lean)
tb, r, app, ctx, accs, logWriter, eventStats.Tally, config.Lean)
numQueuedTimeOpsRan := runQueuedTimeOperations(
timeOperationQueue, int(header.Height), header.Time,
tb, r, app, ctx, accs, logWriter, eventStats.Tally, lean)
tb, r, app, ctx, accs, logWriter, eventStats.Tally, config.Lean)
if testingMode && onOperation {
assertAllInvariants(t, app, invariants, "QueuedOperations", logWriter, allInvariants)
if testingMode && config.OnOperation {
assertAllInvariants(t, app, invariants, "QueuedOperations", logWriter, config.AllInvariants)
}
// run standard operations
operations := blockSimulator(r, app, ctx, accs, header)
opCount += operations + numQueuedOpsRan + numQueuedTimeOpsRan
if testingMode {
assertAllInvariants(t, app, invariants, "StandardOperations", logWriter, allInvariants)
assertAllInvariants(t, app, invariants, "StandardOperations", logWriter, config.AllInvariants)
}
res := app.EndBlock(abci.RequestEndBlock{})
@ -187,9 +183,9 @@ func SimulateFromSeed(
logWriter.AddEntry(EndBlockEntry(int64(height)))
if testingMode {
assertAllInvariants(t, app, invariants, "EndBlock", logWriter, allInvariants)
assertAllInvariants(t, app, invariants, "EndBlock", logWriter, config.AllInvariants)
}
if commit {
if config.Commit {
app.Commit()
}
@ -211,15 +207,15 @@ func SimulateFromSeed(
validators, res.ValidatorUpdates, eventStats.Tally)
// update the exported params
if exportParams && exportParamsHeight == height {
if config.ExportParamsPath != "" && config.ExportParamsHeight == height {
exportedParams = params
}
}
if stopEarly {
if exportStatsPath != "" {
if config.ExportStatsPath != "" {
fmt.Println("Exporting simulation statistics...")
eventStats.ExportJSON(exportStatsPath)
eventStats.ExportJSON(config.ExportStatsPath)
} else {
eventStats.Print(w)
}
@ -233,9 +229,9 @@ func SimulateFromSeed(
header.Height, header.Time, opCount,
)
if exportStatsPath != "" {
if config.ExportStatsPath != "" {
fmt.Println("Exporting simulation statistics...")
eventStats.ExportJSON(exportStatsPath)
eventStats.ExportJSON(config.ExportStatsPath)
} else {
eventStats.Print(w)
}
@ -253,7 +249,7 @@ type blockSimFn func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Writer, params Params,
event func(route, op, evResult string), invariants sdk.Invariants, ops WeightedOperations,
operationQueue OperationQueue, timeOperationQueue []FutureOperation,
totalNumBlocks, avgBlockSize int, logWriter LogWriter, lean, onOperation, allInvariants bool) blockSimFn {
logWriter LogWriter, config Config) blockSimFn {
lastBlockSizeState := 0 // state for [4 * uniform distribution]
blocksize := 0
@ -265,9 +261,9 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Wr
_, _ = fmt.Fprintf(
w, "\rSimulating... block %d/%d, operation %d/%d.",
header.Height, totalNumBlocks, opCount, blocksize,
header.Height, config.NumBlocks, opCount, blocksize,
)
lastBlockSizeState, blocksize = getBlockSize(r, params, lastBlockSizeState, avgBlockSize)
lastBlockSizeState, blocksize = getBlockSize(r, params, lastBlockSizeState, config.BlockSize)
type opAndR struct {
op Operation
@ -291,7 +287,7 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Wr
op, r2 := opAndR.op, opAndR.rand
opMsg, futureOps, err := op(r2, app, ctx, accounts)
opMsg.LogEvent(event)
if !lean || opMsg.OK {
if !config.Lean || opMsg.OK {
logWriter.AddEntry(MsgEntry(header.Height, int64(i), opMsg))
}
if err != nil {
@ -302,14 +298,14 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Wr
queueOperations(operationQueue, timeOperationQueue, futureOps)
if testingMode {
if onOperation {
if config.OnOperation {
fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ",
header.Height, totalNumBlocks, opCount, blocksize)
header.Height, config.NumBlocks, opCount, blocksize)
eventStr := fmt.Sprintf("operation: %v", opMsg.String())
assertAllInvariants(t, app, invariants, eventStr, logWriter, allInvariants)
assertAllInvariants(t, app, invariants, eventStr, logWriter, config.AllInvariants)
} else if opCount%50 == 0 {
fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ",
header.Height, totalNumBlocks, opCount, blocksize)
header.Height, config.NumBlocks, opCount, blocksize)
}
}
opCount++