diff --git a/Makefile b/Makefile index 983234dbc..e363891e2 100644 --- a/Makefile +++ b/Makefile @@ -163,6 +163,17 @@ test_sim_gaia_slow: @echo "Running full Gaia simulation. This may take awhile!" @go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=1000 -SimulationVerbose=true -v -timeout 24h +SIM_NUM_BLOCKS ?= 210 +SIM_BLOCK_SIZE ?= 200 +SIM_COMMIT ?= true +test_sim_gaia_benchmark: + @echo "Running Gaia benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!" + @go test -benchmem -run=^$$ github.com/cosmos/cosmos-sdk/cmd/gaia/app -bench ^BenchmarkFullGaiaSimulation$$ -SimulationEnabled=true -SimulationNumBlocks=$(SIM_NUM_BLOCKS) -SimulationBlockSize=$(SIM_BLOCK_SIZE) -SimulationCommit=$(SIM_COMMIT) -timeout 24h + +test_sim_gaia_profile: + @echo "Running Gaia benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!" + @go test -benchmem -run=^$$ github.com/cosmos/cosmos-sdk/cmd/gaia/app -bench ^BenchmarkFullGaiaSimulation$$ -SimulationEnabled=true -SimulationNumBlocks=$(SIM_NUM_BLOCKS) -SimulationBlockSize=$(SIM_BLOCK_SIZE) -SimulationCommit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out + test_cover: @bash tests/test_cover.sh diff --git a/PENDING.md b/PENDING.md index 7eb324a23..17616ced1 100644 --- a/PENDING.md +++ b/PENDING.md @@ -60,6 +60,7 @@ FEATURES * SDK * [querier] added custom querier functionality, so ABCI query requests can be handled by keepers * [simulation] \#1924 allow operations to specify future operations + * [simulation] \#1924 Add benchmarking capabilities, with makefile commands "test_sim_gaia_benchmark, test_sim_gaia_profile" * Tendermint @@ -79,7 +80,7 @@ IMPROVEMENTS * [x/stake] [#2023](https://github.com/cosmos/cosmos-sdk/pull/2023) Terminate iteration loop in `UpdateBondedValidators` and `UpdateBondedValidatorsFull` when the first revoked validator is encountered and perform a sanity check. * [x/auth] Signature verification's gas cost now accounts for pubkey type. [#2046](https://github.com/tendermint/tendermint/pull/2046) * [x/stake] [x/slashing] Ensure delegation invariants to jailed validators [#1883](https://github.com/cosmos/cosmos-sdk/issues/1883). - + * [x/stake] Improve speed of GetValidator, which was shown to be a performance bottleneck. [#2046](https://github.com/tendermint/tendermint/pull/2200) * SDK * [tools] Make get_vendor_deps deletes `.vendor-new` directories, in case scratch files are present. * [cli] \#1632 Add integration tests to ensure `basecoind init && basecoind` start sequences run successfully for both `democoin` and `basecoin` examples. diff --git a/cmd/gaia/app/sim_test.go b/cmd/gaia/app/sim_test.go index 559b8b4b9..3878166b3 100644 --- a/cmd/gaia/app/sim_test.go +++ b/cmd/gaia/app/sim_test.go @@ -3,7 +3,9 @@ package app import ( "encoding/json" "flag" + "fmt" "math/rand" + "os" "testing" "github.com/stretchr/testify/require" @@ -15,6 +17,7 @@ import ( "github.com/cosmos/cosmos-sdk/baseapp" sdk "github.com/cosmos/cosmos-sdk/types" banksim "github.com/cosmos/cosmos-sdk/x/bank/simulation" + "github.com/cosmos/cosmos-sdk/x/gov" govsim "github.com/cosmos/cosmos-sdk/x/gov/simulation" "github.com/cosmos/cosmos-sdk/x/mock/simulation" slashingsim "github.com/cosmos/cosmos-sdk/x/slashing/simulation" @@ -28,6 +31,7 @@ var ( blockSize int enabled bool verbose bool + commit bool ) func init() { @@ -36,6 +40,7 @@ func init() { flag.IntVar(&blockSize, "SimulationBlockSize", 200, "Operations per block") flag.BoolVar(&enabled, "SimulationEnabled", false, "Enable the simulation") flag.BoolVar(&verbose, "SimulationVerbose", false, "Verbose log output") + flag.BoolVar(&commit, "SimulationCommit", false, "Have the simulation commit") } func appStateFn(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage { @@ -49,7 +54,7 @@ func appStateFn(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json Coins: coins, }) } - + govGenesis := gov.DefaultGenesisState() // Default genesis state stakeGenesis := stake.DefaultGenesisState() var validators []stake.Validator @@ -73,6 +78,7 @@ func appStateFn(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json genesis := GenesisState{ Accounts: genesisAccounts, StakeData: stakeGenesis, + GovData: govGenesis, } // Marshal genesis @@ -112,6 +118,39 @@ func invariants(app *GaiaApp) []simulation.Invariant { } } +// Profile with: +// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/cmd/gaia/app -bench ^BenchmarkFullGaiaSimulation$ -SimulationCommit=true -cpuprofile cpu.out +func BenchmarkFullGaiaSimulation(b *testing.B) { + // Setup Gaia application + var logger log.Logger + logger = log.NewNopLogger() + var db dbm.DB + dir := os.TempDir() + db, _ = dbm.NewGoLevelDB("Simulation", dir) + defer func() { + db.Close() + os.RemoveAll(dir) + }() + app := NewGaiaApp(logger, db, nil) + + // Run randomized simulation + // TODO parameterize numbers, save for a later PR + simulation.SimulateFromSeed( + b, app.BaseApp, appStateFn, seed, + testAndRunTxs(app), + []simulation.RandSetup{}, + invariants(app), // these shouldn't get ran + numBlocks, + blockSize, + commit, + ) + if commit { + fmt.Println("GoLevelDB Stats") + fmt.Println(db.Stats()["leveldb.stats"]) + fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"]) + } +} + func TestFullGaiaSimulation(t *testing.T) { if !enabled { t.Skip("Skipping Gaia simulation") @@ -136,9 +175,11 @@ func TestFullGaiaSimulation(t *testing.T) { invariants(app), numBlocks, blockSize, - false, + commit, ) - + if commit { + fmt.Println("Database Size", db.Stats()["database.size"]) + } } // TODO: Make another test for the fuzzer itself, which just has noOp txs diff --git a/x/bank/simulation/msgs.go b/x/bank/simulation/msgs.go index e3f3ab77b..c53916a61 100644 --- a/x/bank/simulation/msgs.go +++ b/x/bank/simulation/msgs.go @@ -21,7 +21,7 @@ import ( // SimulateSingleInputMsgSend tests and runs a single msg send, with one input and one output, where both // accounts already exist. func SimulateSingleInputMsgSend(mapper auth.AccountMapper) simulation.Operation { - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOps []simulation.FutureOperation, err sdk.Error) { + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOps []simulation.FutureOperation, err sdk.Error) { fromKey := simulation.RandomKey(r, keys) fromAddr := sdk.AccAddress(fromKey.PubKey().Address()) toKey := simulation.RandomKey(r, keys) @@ -58,7 +58,7 @@ func SimulateSingleInputMsgSend(mapper auth.AccountMapper) simulation.Operation Inputs: []bank.Input{bank.NewInput(fromAddr, coins)}, Outputs: []bank.Output{bank.NewOutput(toAddr, coins)}, } - sendAndVerifyMsgSend(t, app, mapper, msg, ctx, log, []crypto.PrivKey{fromKey}) + sendAndVerifyMsgSend(tb, app, mapper, msg, ctx, log, []crypto.PrivKey{fromKey}) event("bank/sendAndVerifyMsgSend/ok") return action, nil, nil @@ -66,7 +66,7 @@ func SimulateSingleInputMsgSend(mapper auth.AccountMapper) simulation.Operation } // Sends and verifies the transition of a msg send. This fails if there are repeated inputs or outputs -func sendAndVerifyMsgSend(t *testing.T, app *baseapp.BaseApp, mapper auth.AccountMapper, msg bank.MsgSend, ctx sdk.Context, log string, privkeys []crypto.PrivKey) { +func sendAndVerifyMsgSend(tb testing.TB, app *baseapp.BaseApp, mapper auth.AccountMapper, msg bank.MsgSend, ctx sdk.Context, log string, privkeys []crypto.PrivKey) { initialInputAddrCoins := make([]sdk.Coins, len(msg.Inputs)) initialOutputAddrCoins := make([]sdk.Coins, len(msg.Outputs)) AccountNumbers := make([]int64, len(msg.Inputs)) @@ -91,12 +91,12 @@ func sendAndVerifyMsgSend(t *testing.T, app *baseapp.BaseApp, mapper auth.Accoun // TODO: Do this in a more 'canonical' way fmt.Println(res) fmt.Println(log) - t.FailNow() + tb.FailNow() } for i := 0; i < len(msg.Inputs); i++ { terminalInputCoins := mapper.GetAccount(ctx, msg.Inputs[i].Address).GetCoins() - require.Equal(t, + require.Equal(tb, initialInputAddrCoins[i].Minus(msg.Inputs[i].Coins), terminalInputCoins, fmt.Sprintf("Input #%d had an incorrect amount of coins\n%s", i, log), @@ -104,11 +104,9 @@ func sendAndVerifyMsgSend(t *testing.T, app *baseapp.BaseApp, mapper auth.Accoun } for i := 0; i < len(msg.Outputs); i++ { terminalOutputCoins := mapper.GetAccount(ctx, msg.Outputs[i].Address).GetCoins() - require.Equal(t, - initialOutputAddrCoins[i].Plus(msg.Outputs[i].Coins), - terminalOutputCoins, - fmt.Sprintf("Output #%d had an incorrect amount of coins\n%s", i, log), - ) + if !terminalOutputCoins.IsEqual(initialOutputAddrCoins[i].Plus(msg.Outputs[i].Coins)) { + tb.Fatalf("Output #%d had an incorrect amount of coins\n%s", i, log) + } } } diff --git a/x/gov/simulation/msgs.go b/x/gov/simulation/msgs.go index 0b1530138..eca8accae 100644 --- a/x/gov/simulation/msgs.go +++ b/x/gov/simulation/msgs.go @@ -5,8 +5,6 @@ import ( "math/rand" "testing" - "github.com/stretchr/testify/require" - "github.com/tendermint/tendermint/crypto" "github.com/cosmos/cosmos-sdk/baseapp" @@ -23,24 +21,15 @@ const ( // SimulateMsgSubmitProposal simulates a msg Submit Proposal // Note: Currently doesn't ensure that the proposal txt is in JSON form func SimulateMsgSubmitProposal(k gov.Keeper, sk stake.Keeper) simulation.Operation { - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOps []simulation.FutureOperation, err sdk.Error) { - key := simulation.RandomKey(r, keys) - addr := sdk.AccAddress(key.PubKey().Address()) - deposit := randomDeposit(r) - msg := gov.NewMsgSubmitProposal( - simulation.RandStringOfLength(r, 5), - simulation.RandStringOfLength(r, 5), - gov.ProposalTypeText, - addr, - deposit, - ) - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + handler := gov.NewHandler(k) + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOps []simulation.FutureOperation, err sdk.Error) { + msg := simulationCreateMsgSubmitProposal(tb, r, keys, log) ctx, write := ctx.CacheContext() - result := gov.NewHandler(k)(ctx, msg) + result := handler(ctx, msg) if result.IsOK() { // Update pool to keep invariants pool := sk.GetPool(ctx) - pool.LooseTokens = pool.LooseTokens.Sub(sdk.NewDecFromInt(deposit.AmountOf(denom))) + pool.LooseTokens = pool.LooseTokens.Sub(sdk.NewDecFromInt(msg.InitialDeposit.AmountOf(denom))) sk.SetPool(ctx, pool) write() } @@ -50,9 +39,26 @@ func SimulateMsgSubmitProposal(k gov.Keeper, sk stake.Keeper) simulation.Operati } } +func simulationCreateMsgSubmitProposal(tb testing.TB, r *rand.Rand, keys []crypto.PrivKey, log string) gov.MsgSubmitProposal { + key := simulation.RandomKey(r, keys) + addr := sdk.AccAddress(key.PubKey().Address()) + deposit := randomDeposit(r) + msg := gov.NewMsgSubmitProposal( + simulation.RandStringOfLength(r, 5), + simulation.RandStringOfLength(r, 5), + gov.ProposalTypeText, + addr, + deposit, + ) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } + return msg +} + // SimulateMsgDeposit func SimulateMsgDeposit(k gov.Keeper, sk stake.Keeper) simulation.Operation { - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { key := simulation.RandomKey(r, keys) addr := sdk.AccAddress(key.PubKey().Address()) proposalID, ok := randomProposalID(r, k, ctx) @@ -61,7 +67,9 @@ func SimulateMsgDeposit(k gov.Keeper, sk stake.Keeper) simulation.Operation { } deposit := randomDeposit(r) msg := gov.NewMsgDeposit(addr, proposalID, deposit) - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() result := gov.NewHandler(k)(ctx, msg) if result.IsOK() { @@ -79,7 +87,7 @@ func SimulateMsgDeposit(k gov.Keeper, sk stake.Keeper) simulation.Operation { // SimulateMsgVote func SimulateMsgVote(k gov.Keeper, sk stake.Keeper) simulation.Operation { - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { key := simulation.RandomKey(r, keys) addr := sdk.AccAddress(key.PubKey().Address()) proposalID, ok := randomProposalID(r, k, ctx) @@ -88,7 +96,9 @@ func SimulateMsgVote(k gov.Keeper, sk stake.Keeper) simulation.Operation { } option := randomVotingOption(r) msg := gov.NewMsgVote(addr, proposalID, option) - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() result := gov.NewHandler(k)(ctx, msg) if result.IsOK() { diff --git a/x/mock/simulation/random_simulate_blocks.go b/x/mock/simulation/random_simulate_blocks.go index a99b3a4c9..fc7543280 100644 --- a/x/mock/simulation/random_simulate_blocks.go +++ b/x/mock/simulation/random_simulate_blocks.go @@ -5,6 +5,7 @@ import ( "fmt" "math" "math/rand" + "os" "sort" "testing" "time" @@ -16,7 +17,6 @@ import ( "github.com/cosmos/cosmos-sdk/baseapp" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/mock" - "github.com/stretchr/testify/require" ) // Simulate tests application by sending random messages. @@ -28,34 +28,10 @@ func Simulate( SimulateFromSeed(t, app, appStateFn, time, ops, setups, invariants, numBlocks, blockSize, commit) } -// SimulateFromSeed tests an application by running the provided -// operations, testing the provided invariants, but using the provided seed. -func SimulateFromSeed( - t *testing.T, app *baseapp.BaseApp, appStateFn func(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage, seed int64, ops []Operation, setups []RandSetup, - invariants []Invariant, numBlocks int, blockSize int, commit bool, -) { - log := fmt.Sprintf("Starting SimulateFromSeed with randomness created with seed %d", int(seed)) - r := rand.New(rand.NewSource(seed)) - - unixTime := r.Int63n(int64(math.Pow(2, 40))) - - // Set the timestamp for simulation - timestamp := time.Unix(unixTime, 0) - log = fmt.Sprintf("%s\nStarting the simulation from time %v, unixtime %v", log, timestamp.UTC().Format(time.UnixDate), timestamp.Unix()) - fmt.Printf("%s\n", log) - timeDiff := maxTimePerBlock - minTimePerBlock - - keys, accs := mock.GeneratePrivKeyAddressPairsFromRand(r, numKeys) - - // Setup event stats - events := make(map[string]uint) - event := func(what string) { - log += "\nevent - " + what - events[what]++ - } - +func initChain(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress, setups []RandSetup, app *baseapp.BaseApp, + appStateFn func(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage) (validators map[string]mockValidator) { res := app.InitChain(abci.RequestInitChain{AppStateBytes: appStateFn(r, keys, accs)}) - validators := make(map[string]mockValidator) + validators = make(map[string]mockValidator) for _, validator := range res.Validators { validators[string(validator.Address)] = mockValidator{validator, GetMemberOfInitialState(r, initialLivenessWeightings)} } @@ -64,85 +40,161 @@ func SimulateFromSeed( setups[i](r, keys) } + return +} + +func randTimestamp(r *rand.Rand) time.Time { + unixTime := r.Int63n(int64(math.Pow(2, 40))) + return time.Unix(unixTime, 0) +} + +// SimulateFromSeed tests an application by running the provided +// operations, testing the provided invariants, but using the provided seed. +func SimulateFromSeed( + tb testing.TB, app *baseapp.BaseApp, appStateFn func(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage, seed int64, ops []Operation, setups []RandSetup, + invariants []Invariant, numBlocks int, blockSize int, commit bool, +) { + testingMode, t, b := getTestingMode(tb) + log := fmt.Sprintf("Starting SimulateFromSeed with randomness created with seed %d", int(seed)) + r := rand.New(rand.NewSource(seed)) + timestamp := randTimestamp(r) + log = updateLog(testingMode, log, "Starting the simulation from time %v, unixtime %v", timestamp.UTC().Format(time.UnixDate), timestamp.Unix()) + fmt.Printf("%s\n", log) + timeDiff := maxTimePerBlock - minTimePerBlock + + keys, accs := mock.GeneratePrivKeyAddressPairsFromRand(r, numKeys) + + // Setup event stats + events := make(map[string]uint) + event := func(what string) { + log = updateLog(testingMode, log, "event - %s", what) + events[what]++ + } + + validators := initChain(r, keys, accs, setups, app, appStateFn) + header := abci.Header{Height: 0, Time: timestamp} opCount := 0 var pastTimes []time.Time var pastSigningValidators [][]abci.SigningValidator - request := RandomRequestBeginBlock(t, r, validators, livenessTransitionMatrix, evidenceFraction, pastTimes, pastSigningValidators, event, header, log) + request := RandomRequestBeginBlock(r, validators, livenessTransitionMatrix, evidenceFraction, pastTimes, pastSigningValidators, event, header, log) // These are operations which have been queued by previous operations operationQueue := make(map[int][]Operation) - for i := 0; i < numBlocks; i++ { + if !testingMode { + b.ResetTimer() + } + blockSimulator := createBlockSimulator(testingMode, tb, t, event, invariants, ops, operationQueue, numBlocks) + for i := 0; i < numBlocks; i++ { // Log the header time for future lookup pastTimes = append(pastTimes, header.Time) pastSigningValidators = append(pastSigningValidators, request.LastCommitInfo.Validators) // Run the BeginBlock handler app.BeginBlock(request) + log = updateLog(testingMode, log, "BeginBlock") - log += "\nBeginBlock" - - // Make sure invariants hold at beginning of block - AssertAllInvariants(t, app, invariants, log) + if testingMode { + // Make sure invariants hold at beginning of block + AssertAllInvariants(t, app, invariants, log) + } ctx := app.NewContext(false, header) + thisBlockSize := getBlockSize(r, blockSize) - var thisBlockSize int - load := r.Float64() - switch { - case load < 0.33: - thisBlockSize = 0 - case load < 0.66: - thisBlockSize = r.Intn(blockSize * 2) - default: - thisBlockSize = r.Intn(blockSize * 4) - } // Run queued operations. Ignores blocksize if blocksize is too small - log, numQueuedOpsRan := runQueuedOperations(operationQueue, int(header.Height), t, r, app, ctx, keys, log, event) + log, numQueuedOpsRan := runQueuedOperations(operationQueue, int(header.Height), tb, r, app, ctx, keys, log, event) opCount += numQueuedOpsRan thisBlockSize -= numQueuedOpsRan - for j := 0; j < thisBlockSize; j++ { - logUpdate, futureOps, err := ops[r.Intn(len(ops))](t, r, app, ctx, keys, log, event) - log += "\n" + logUpdate - queueOperations(operationQueue, futureOps) - - require.Nil(t, err, log) - if onOperation { - AssertAllInvariants(t, app, invariants, log) - } - if opCount%200 == 0 { - fmt.Printf("\rSimulating... block %d/%d, operation %d.", header.Height, numBlocks, opCount) - } - opCount++ - } + log, operations := blockSimulator(thisBlockSize, r, app, ctx, keys, log, header) + opCount += operations res := app.EndBlock(abci.RequestEndBlock{}) header.Height++ header.Time = header.Time.Add(time.Duration(minTimePerBlock) * time.Second).Add(time.Duration(int64(r.Intn(int(timeDiff)))) * time.Second) + log = updateLog(testingMode, log, "EndBlock") - log += "\nEndBlock" - - // Make sure invariants hold at end of block - AssertAllInvariants(t, app, invariants, log) - + if testingMode { + // Make sure invariants hold at end of block + AssertAllInvariants(t, app, invariants, log) + } if commit { app.Commit() } // Generate a random RequestBeginBlock with the current validator set for the next block - request = RandomRequestBeginBlock(t, r, validators, livenessTransitionMatrix, evidenceFraction, pastTimes, pastSigningValidators, event, header, log) + request = RandomRequestBeginBlock(r, validators, livenessTransitionMatrix, evidenceFraction, pastTimes, pastSigningValidators, event, header, log) // Update the validator set - validators = updateValidators(t, r, validators, res.ValidatorUpdates, event) + validators = updateValidators(tb, r, validators, res.ValidatorUpdates, event) } - fmt.Printf("\nSimulation complete. Final height (blocks): %d, final time (seconds): %v\n", header.Height, header.Time) + fmt.Printf("\nSimulation complete. Final height (blocks): %d, final time (seconds), : %v, operations ran %d\n", header.Height, header.Time, opCount) DisplayEvents(events) } +// Returns a function to simulate blocks. Written like this to avoid constant parameters being passed everytime, to minimize +// memory overhead +func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, event func(string), invariants []Invariant, ops []Operation, operationQueue map[int][]Operation, totalNumBlocks int) func( + blocksize int, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, privKeys []crypto.PrivKey, log string, header abci.Header) (updatedLog string, opCount int) { + return func(blocksize int, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, + keys []crypto.PrivKey, log string, header abci.Header) (updatedLog string, opCount int) { + for j := 0; j < blocksize; j++ { + logUpdate, futureOps, err := ops[r.Intn(len(ops))](tb, r, app, ctx, keys, log, event) + log = updateLog(testingMode, log, logUpdate) + if err != nil { + tb.Fatalf("error on operation %d within block %d, %v, log %s", header.Height, opCount, err, log) + } + + queueOperations(operationQueue, futureOps) + if testingMode { + if onOperation { + AssertAllInvariants(t, app, invariants, log) + } + if opCount%50 == 0 { + fmt.Printf("\rSimulating... block %d/%d, operation %d/%d.", header.Height, totalNumBlocks, opCount, blocksize) + } + } + opCount++ + } + return log, opCount + } +} + +func getTestingMode(tb testing.TB) (testingMode bool, t *testing.T, b *testing.B) { + testingMode = false + if _t, ok := tb.(*testing.T); ok { + t = _t + testingMode = true + } else { + b = tb.(*testing.B) + } + return +} + +func updateLog(testingMode bool, log string, update string, args ...interface{}) (updatedLog string) { + if testingMode { + update = fmt.Sprintf(update, args...) + return fmt.Sprintf("%s\n%s", log, update) + } + return "" +} + +func getBlockSize(r *rand.Rand, blockSize int) int { + load := r.Float64() + switch { + case load < 0.33: + return 0 + case load < 0.66: + return r.Intn(blockSize * 2) + default: + return r.Intn(blockSize * 4) + } +} + // adds all future operations into the operation queue. func queueOperations(queuedOperations map[int][]Operation, futureOperations []FutureOperation) { if futureOperations == nil { @@ -157,7 +209,7 @@ func queueOperations(queuedOperations map[int][]Operation, futureOperations []Fu } } -func runQueuedOperations(queueOperations map[int][]Operation, height int, t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, +func runQueuedOperations(queueOperations map[int][]Operation, height int, tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, privKeys []crypto.PrivKey, log string, event func(string)) (updatedLog string, numOpsRan int) { updatedLog = log if queuedOps, ok := queueOperations[height]; ok { @@ -166,9 +218,12 @@ func runQueuedOperations(queueOperations map[int][]Operation, height int, t *tes // For now, queued operations cannot queue more operations. // If a need arises for us to support queued messages to queue more messages, this can // be changed. - logUpdate, _, err := queuedOps[i](t, r, app, ctx, privKeys, updatedLog, event) - updatedLog += "\n" + logUpdate - require.Nil(t, err, updatedLog) + logUpdate, _, err := queuedOps[i](tb, r, app, ctx, privKeys, updatedLog, event) + updatedLog = fmt.Sprintf("%s\n%s", updatedLog, logUpdate) + if err != nil { + fmt.Fprint(os.Stderr, updatedLog) + tb.FailNow() + } } delete(queueOperations, height) return updatedLog, numOps @@ -188,7 +243,7 @@ func getKeys(validators map[string]mockValidator) []string { } // RandomRequestBeginBlock generates a list of signing validators according to the provided list of validators, signing fraction, and evidence fraction -func RandomRequestBeginBlock(t *testing.T, r *rand.Rand, validators map[string]mockValidator, livenessTransitions TransitionMatrix, evidenceFraction float64, +func RandomRequestBeginBlock(r *rand.Rand, validators map[string]mockValidator, livenessTransitions TransitionMatrix, evidenceFraction float64, pastTimes []time.Time, pastSigningValidators [][]abci.SigningValidator, event func(string), header abci.Header, log string) abci.RequestBeginBlock { if len(validators) == 0 { return abci.RequestBeginBlock{Header: header} @@ -220,6 +275,7 @@ func RandomRequestBeginBlock(t *testing.T, r *rand.Rand, validators map[string]m } i++ } + // TODO: Determine capacity before allocation evidence := make([]abci.Evidence, 0) // Anything but the first block if len(pastTimes) > 0 { @@ -264,11 +320,19 @@ func AssertAllInvariants(t *testing.T, app *baseapp.BaseApp, tests []Invariant, } // updateValidators mimicks Tendermint's update logic -func updateValidators(t *testing.T, r *rand.Rand, current map[string]mockValidator, updates []abci.Validator, event func(string)) map[string]mockValidator { +func updateValidators(tb testing.TB, r *rand.Rand, current map[string]mockValidator, updates []abci.Validator, event func(string)) map[string]mockValidator { for _, update := range updates { switch { case update.Power == 0: - require.NotNil(t, current[string(update.PubKey.Data)], "tried to delete a nonexistent validator") + // // TEMPORARY DEBUG CODE TO PROVE THAT THE OLD METHOD WAS BROKEN + // // (i.e. didn't catch in the event of problem) + // if val, ok := tb.(*testing.T); ok { + // require.NotNil(val, current[string(update.PubKey.Data)]) + // } + // // CORRECT CHECK + // if _, ok := current[string(update.PubKey.Data)]; !ok { + // tb.Fatalf("tried to delete a nonexistent validator") + // } event("endblock/validatorupdates/kicked") delete(current, string(update.PubKey.Data)) default: diff --git a/x/mock/simulation/types.go b/x/mock/simulation/types.go index 2516e07ae..2f91a4f26 100644 --- a/x/mock/simulation/types.go +++ b/x/mock/simulation/types.go @@ -23,7 +23,7 @@ type ( // Operations can optionally provide a list of "FutureOperations" to run later // These will be ran at the beginning of the corresponding block. Operation func( - t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, + tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, privKeys []crypto.PrivKey, log string, event func(string), ) (action string, futureOperations []FutureOperation, err sdk.Error) diff --git a/x/slashing/simulation/msgs.go b/x/slashing/simulation/msgs.go index dc7f63ba6..9cbb2c48a 100644 --- a/x/slashing/simulation/msgs.go +++ b/x/slashing/simulation/msgs.go @@ -5,8 +5,6 @@ import ( "math/rand" "testing" - "github.com/stretchr/testify/require" - "github.com/tendermint/tendermint/crypto" "github.com/cosmos/cosmos-sdk/baseapp" @@ -17,11 +15,13 @@ import ( // SimulateMsgUnjail func SimulateMsgUnjail(k slashing.Keeper) simulation.Operation { - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { key := simulation.RandomKey(r, keys) address := sdk.ValAddress(key.PubKey().Address()) msg := slashing.NewMsgUnjail(address) - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() result := slashing.NewHandler(k)(ctx, msg) if result.IsOK() { diff --git a/x/stake/keeper/validator.go b/x/stake/keeper/validator.go index b42418686..47a5d5f66 100644 --- a/x/stake/keeper/validator.go +++ b/x/stake/keeper/validator.go @@ -2,6 +2,7 @@ package keeper import ( "bytes" + "container/list" "fmt" abci "github.com/tendermint/tendermint/abci/types" @@ -11,6 +12,19 @@ import ( "github.com/cosmos/cosmos-sdk/x/stake/types" ) +// Cache the amino decoding of validators, as it can be the case that repeated slashing calls +// cause many calls to GetValidator, which were shown to throttle the state machine in our +// simulation. Note this is quite biased though, as the simulator does more slashes than a +// live chain should, however we require the slashing to be fast as noone pays gas for it. +type cachedValidator struct { + val types.Validator + marshalled string // marshalled amino bytes for the validator object (not operator address) +} + +// validatorCache-key: validator amino bytes +var validatorCache = make(map[string]cachedValidator, 500) +var validatorCacheList = list.New() + // get a single validator func (k Keeper) GetValidator(ctx sdk.Context, addr sdk.ValAddress) (validator types.Validator, found bool) { store := ctx.KVStore(k.storeKey) @@ -18,6 +32,28 @@ func (k Keeper) GetValidator(ctx sdk.Context, addr sdk.ValAddress) (validator ty if value == nil { return validator, false } + + // If these amino encoded bytes are in the cache, return the cached validator + strValue := string(value) + if val, ok := validatorCache[strValue]; ok { + valToReturn := val.val + // Doesn't mutate the cache's value + valToReturn.Operator = addr + return valToReturn, true + } + + // amino bytes weren't found in cache, so amino unmarshal and add it to the cache + validator = types.MustUnmarshalValidator(k.cdc, addr, value) + cachedVal := cachedValidator{validator, strValue} + validatorCache[strValue] = cachedValidator{validator, strValue} + validatorCacheList.PushBack(cachedVal) + + // if the cache is too big, pop off the last element from it + if validatorCacheList.Len() > 500 { + valToRemove := validatorCacheList.Remove(validatorCacheList.Front()).(cachedValidator) + delete(validatorCache, valToRemove.marshalled) + } + validator = types.MustUnmarshalValidator(k.cdc, addr, value) return validator, true } diff --git a/x/stake/simulation/msgs.go b/x/stake/simulation/msgs.go index cd95a2c27..af42dba49 100644 --- a/x/stake/simulation/msgs.go +++ b/x/stake/simulation/msgs.go @@ -5,8 +5,6 @@ import ( "math/rand" "testing" - "github.com/stretchr/testify/require" - "github.com/cosmos/cosmos-sdk/baseapp" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/auth" @@ -19,9 +17,8 @@ import ( // SimulateMsgCreateValidator func SimulateMsgCreateValidator(m auth.AccountMapper, k stake.Keeper) simulation.Operation { - - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, - keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + handler := stake.NewHandler(k) + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { denom := k.GetParams(ctx).BondDenom description := stake.Description{ @@ -44,9 +41,11 @@ func SimulateMsgCreateValidator(m auth.AccountMapper, k stake.Keeper) simulation PubKey: pubkey, Delegation: sdk.NewCoin(denom, amount), } - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() - result := stake.NewHandler(k)(ctx, msg) + result := handler(ctx, msg) if result.IsOK() { write() } @@ -59,9 +58,8 @@ func SimulateMsgCreateValidator(m auth.AccountMapper, k stake.Keeper) simulation // SimulateMsgEditValidator func SimulateMsgEditValidator(k stake.Keeper) simulation.Operation { - - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, - keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + handler := stake.NewHandler(k) + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { description := stake.Description{ Moniker: simulation.RandStringOfLength(r, 10), @@ -76,9 +74,11 @@ func SimulateMsgEditValidator(k stake.Keeper) simulation.Operation { Description: description, ValidatorAddr: address, } - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() - result := stake.NewHandler(k)(ctx, msg) + result := handler(ctx, msg) if result.IsOK() { write() } @@ -90,9 +90,8 @@ func SimulateMsgEditValidator(k stake.Keeper) simulation.Operation { // SimulateMsgDelegate func SimulateMsgDelegate(m auth.AccountMapper, k stake.Keeper) simulation.Operation { - - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, - keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + handler := stake.NewHandler(k) + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { denom := k.GetParams(ctx).BondDenom validatorKey := simulation.RandomKey(r, keys) @@ -111,9 +110,11 @@ func SimulateMsgDelegate(m auth.AccountMapper, k stake.Keeper) simulation.Operat ValidatorAddr: validatorAddress, Delegation: sdk.NewCoin(denom, amount), } - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() - result := stake.NewHandler(k)(ctx, msg) + result := handler(ctx, msg) if result.IsOK() { write() } @@ -125,9 +126,8 @@ func SimulateMsgDelegate(m auth.AccountMapper, k stake.Keeper) simulation.Operat // SimulateMsgBeginUnbonding func SimulateMsgBeginUnbonding(m auth.AccountMapper, k stake.Keeper) simulation.Operation { - - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, - keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + handler := stake.NewHandler(k) + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { denom := k.GetParams(ctx).BondDenom validatorKey := simulation.RandomKey(r, keys) @@ -146,9 +146,11 @@ func SimulateMsgBeginUnbonding(m auth.AccountMapper, k stake.Keeper) simulation. ValidatorAddr: validatorAddress, SharesAmount: sdk.NewDecFromInt(amount), } - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() - result := stake.NewHandler(k)(ctx, msg) + result := handler(ctx, msg) if result.IsOK() { write() } @@ -160,9 +162,8 @@ func SimulateMsgBeginUnbonding(m auth.AccountMapper, k stake.Keeper) simulation. // SimulateMsgCompleteUnbonding func SimulateMsgCompleteUnbonding(k stake.Keeper) simulation.Operation { - - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, - keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + handler := stake.NewHandler(k) + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { validatorKey := simulation.RandomKey(r, keys) validatorAddress := sdk.ValAddress(validatorKey.PubKey().Address()) @@ -172,9 +173,11 @@ func SimulateMsgCompleteUnbonding(k stake.Keeper) simulation.Operation { DelegatorAddr: delegatorAddress, ValidatorAddr: validatorAddress, } - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() - result := stake.NewHandler(k)(ctx, msg) + result := handler(ctx, msg) if result.IsOK() { write() } @@ -186,9 +189,8 @@ func SimulateMsgCompleteUnbonding(k stake.Keeper) simulation.Operation { // SimulateMsgBeginRedelegate func SimulateMsgBeginRedelegate(m auth.AccountMapper, k stake.Keeper) simulation.Operation { - - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, - keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + handler := stake.NewHandler(k) + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { denom := k.GetParams(ctx).BondDenom sourceValidatorKey := simulation.RandomKey(r, keys) @@ -211,9 +213,11 @@ func SimulateMsgBeginRedelegate(m auth.AccountMapper, k stake.Keeper) simulation ValidatorDstAddr: destValidatorAddress, SharesAmount: sdk.NewDecFromInt(amount), } - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() - result := stake.NewHandler(k)(ctx, msg) + result := handler(ctx, msg) if result.IsOK() { write() } @@ -225,9 +229,8 @@ func SimulateMsgBeginRedelegate(m auth.AccountMapper, k stake.Keeper) simulation // SimulateMsgCompleteRedelegate func SimulateMsgCompleteRedelegate(k stake.Keeper) simulation.Operation { - - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, - keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + handler := stake.NewHandler(k) + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { validatorSrcKey := simulation.RandomKey(r, keys) validatorSrcAddress := sdk.ValAddress(validatorSrcKey.PubKey().Address()) @@ -240,9 +243,11 @@ func SimulateMsgCompleteRedelegate(k stake.Keeper) simulation.Operation { ValidatorSrcAddr: validatorSrcAddress, ValidatorDstAddr: validatorDstAddress, } - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() - result := stake.NewHandler(k)(ctx, msg) + result := handler(ctx, msg) if result.IsOK() { write() }