Merge PR #2187: simulation: Add benchmarking

Makefile

@@ -163,6 +163,17 @@ test_sim_gaia_slow:
 	@echo "Running full Gaia simulation. This may take awhile!"
 	@go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=1000 -SimulationVerbose=true -v -timeout 24h
 
+SIM_NUM_BLOCKS ?= 210
+SIM_BLOCK_SIZE ?= 200
+SIM_COMMIT ?= true
+test_sim_gaia_benchmark:
+	@echo "Running Gaia benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
+	@go test -benchmem -run=^$$ github.com/cosmos/cosmos-sdk/cmd/gaia/app -bench ^BenchmarkFullGaiaSimulation$$  -SimulationEnabled=true -SimulationNumBlocks=$(SIM_NUM_BLOCKS) -SimulationBlockSize=$(SIM_BLOCK_SIZE) -SimulationCommit=$(SIM_COMMIT) -timeout 24h
+
+test_sim_gaia_profile:
+	@echo "Running Gaia benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
+	@go test -benchmem -run=^$$ github.com/cosmos/cosmos-sdk/cmd/gaia/app -bench ^BenchmarkFullGaiaSimulation$$ -SimulationEnabled=true -SimulationNumBlocks=$(SIM_NUM_BLOCKS) -SimulationBlockSize=$(SIM_BLOCK_SIZE) -SimulationCommit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out
+
 test_cover:
 	@bash tests/test_cover.sh
 

PENDING.md

@@ -60,6 +60,7 @@ FEATURES
 * SDK
   * [querier] added custom querier functionality, so ABCI query requests can be handled by keepers
   * [simulation] \#1924 allow operations to specify future operations
+  * [simulation] \#1924 Add benchmarking capabilities, with makefile commands "test_sim_gaia_benchmark, test_sim_gaia_profile"
 
 * Tendermint
 
@@ -79,7 +80,7 @@ IMPROVEMENTS
     * [x/stake] [#2023](https://github.com/cosmos/cosmos-sdk/pull/2023) Terminate iteration loop in `UpdateBondedValidators` and `UpdateBondedValidatorsFull` when the first revoked validator is encountered and perform a sanity check.
     * [x/auth] Signature verification's gas cost now accounts for pubkey type. [#2046](https://github.com/tendermint/tendermint/pull/2046)
     * [x/stake] [x/slashing] Ensure delegation invariants to jailed validators [#1883](https://github.com/cosmos/cosmos-sdk/issues/1883).
-
+    * [x/stake] Improve speed of GetValidator, which was shown to be a performance bottleneck. [#2046](https://github.com/tendermint/tendermint/pull/2200)
 * SDK
     * [tools] Make get_vendor_deps deletes `.vendor-new` directories, in case scratch files are present.
     * [cli] \#1632 Add integration tests to ensure `basecoind init && basecoind` start sequences run successfully for both `democoin` and `basecoin` examples.

cmd/gaia/app/sim_test.go

@@ -3,7 +3,9 @@ package app
 import (
 	"encoding/json"
 	"flag"
+	"fmt"
 	"math/rand"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -15,6 +17,7 @@ import (
 	"github.com/cosmos/cosmos-sdk/baseapp"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	banksim "github.com/cosmos/cosmos-sdk/x/bank/simulation"
+	"github.com/cosmos/cosmos-sdk/x/gov"
 	govsim "github.com/cosmos/cosmos-sdk/x/gov/simulation"
 	"github.com/cosmos/cosmos-sdk/x/mock/simulation"
 	slashingsim "github.com/cosmos/cosmos-sdk/x/slashing/simulation"
@@ -28,6 +31,7 @@ var (
 	blockSize int
 	enabled   bool
 	verbose   bool
+	commit    bool
 )
 
 func init() {
@@ -36,6 +40,7 @@ func init() {
 	flag.IntVar(&blockSize, "SimulationBlockSize", 200, "Operations per block")
 	flag.BoolVar(&enabled, "SimulationEnabled", false, "Enable the simulation")
 	flag.BoolVar(&verbose, "SimulationVerbose", false, "Verbose log output")
+	flag.BoolVar(&commit, "SimulationCommit", false, "Have the simulation commit")
 }
 
 func appStateFn(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage {
@@ -49,7 +54,7 @@ func appStateFn(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json
 			Coins:   coins,
 		})
 	}
-
+	govGenesis := gov.DefaultGenesisState()
 	// Default genesis state
 	stakeGenesis := stake.DefaultGenesisState()
 	var validators []stake.Validator
@@ -73,6 +78,7 @@ func appStateFn(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json
 	genesis := GenesisState{
 		Accounts:  genesisAccounts,
 		StakeData: stakeGenesis,
+		GovData:   govGenesis,
 	}
 
 	// Marshal genesis
@@ -112,6 +118,39 @@ func invariants(app *GaiaApp) []simulation.Invariant {
 	}
 }
 
+// Profile with:
+// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/cmd/gaia/app -bench ^BenchmarkFullGaiaSimulation$ -SimulationCommit=true -cpuprofile cpu.out
+func BenchmarkFullGaiaSimulation(b *testing.B) {
+	// Setup Gaia application
+	var logger log.Logger
+	logger = log.NewNopLogger()
+	var db dbm.DB
+	dir := os.TempDir()
+	db, _ = dbm.NewGoLevelDB("Simulation", dir)
+	defer func() {
+		db.Close()
+		os.RemoveAll(dir)
+	}()
+	app := NewGaiaApp(logger, db, nil)
+
+	// Run randomized simulation
+	// TODO parameterize numbers, save for a later PR
+	simulation.SimulateFromSeed(
+		b, app.BaseApp, appStateFn, seed,
+		testAndRunTxs(app),
+		[]simulation.RandSetup{},
+		invariants(app), // these shouldn't get ran
+		numBlocks,
+		blockSize,
+		commit,
+	)
+	if commit {
+		fmt.Println("GoLevelDB Stats")
+		fmt.Println(db.Stats()["leveldb.stats"])
+		fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
+	}
+}
+
 func TestFullGaiaSimulation(t *testing.T) {
 	if !enabled {
 		t.Skip("Skipping Gaia simulation")
@@ -136,9 +175,11 @@ func TestFullGaiaSimulation(t *testing.T) {
 		invariants(app),
 		numBlocks,
 		blockSize,
-		false,
+		commit,
 	)
-
+	if commit {
+		fmt.Println("Database Size", db.Stats()["database.size"])
+	}
 }
 
 // TODO: Make another test for the fuzzer itself, which just has noOp txs

x/bank/simulation/msgs.go

@@ -21,7 +21,7 @@ import (
 // SimulateSingleInputMsgSend tests and runs a single msg send, with one input and one output, where both
 // accounts already exist.
 func SimulateSingleInputMsgSend(mapper auth.AccountMapper) simulation.Operation {
-	return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOps []simulation.FutureOperation, err sdk.Error) {
+	return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOps []simulation.FutureOperation, err sdk.Error) {
 		fromKey := simulation.RandomKey(r, keys)
 		fromAddr := sdk.AccAddress(fromKey.PubKey().Address())
 		toKey := simulation.RandomKey(r, keys)
@@ -58,7 +58,7 @@ func SimulateSingleInputMsgSend(mapper auth.AccountMapper) simulation.Operation
 			Inputs:  []bank.Input{bank.NewInput(fromAddr, coins)},
 			Outputs: []bank.Output{bank.NewOutput(toAddr, coins)},
 		}
-		sendAndVerifyMsgSend(t, app, mapper, msg, ctx, log, []crypto.PrivKey{fromKey})
+		sendAndVerifyMsgSend(tb, app, mapper, msg, ctx, log, []crypto.PrivKey{fromKey})
 		event("bank/sendAndVerifyMsgSend/ok")
 
 		return action, nil, nil
@@ -66,7 +66,7 @@ func SimulateSingleInputMsgSend(mapper auth.AccountMapper) simulation.Operation
 }
 
 // Sends and verifies the transition of a msg send. This fails if there are repeated inputs or outputs
-func sendAndVerifyMsgSend(t *testing.T, app *baseapp.BaseApp, mapper auth.AccountMapper, msg bank.MsgSend, ctx sdk.Context, log string, privkeys []crypto.PrivKey) {
+func sendAndVerifyMsgSend(tb testing.TB, app *baseapp.BaseApp, mapper auth.AccountMapper, msg bank.MsgSend, ctx sdk.Context, log string, privkeys []crypto.PrivKey) {
 	initialInputAddrCoins := make([]sdk.Coins, len(msg.Inputs))
 	initialOutputAddrCoins := make([]sdk.Coins, len(msg.Outputs))
 	AccountNumbers := make([]int64, len(msg.Inputs))
@@ -91,12 +91,12 @@ func sendAndVerifyMsgSend(t *testing.T, app *baseapp.BaseApp, mapper auth.Accoun
 		// TODO: Do this in a more 'canonical' way
 		fmt.Println(res)
 		fmt.Println(log)
-		t.FailNow()
+		tb.FailNow()
 	}
 
 	for i := 0; i < len(msg.Inputs); i++ {
 		terminalInputCoins := mapper.GetAccount(ctx, msg.Inputs[i].Address).GetCoins()
-		require.Equal(t,
+		require.Equal(tb,
 			initialInputAddrCoins[i].Minus(msg.Inputs[i].Coins),
 			terminalInputCoins,
 			fmt.Sprintf("Input #%d had an incorrect amount of coins\n%s", i, log),
@@ -104,11 +104,9 @@ func sendAndVerifyMsgSend(t *testing.T, app *baseapp.BaseApp, mapper auth.Accoun
 	}
 	for i := 0; i < len(msg.Outputs); i++ {
 		terminalOutputCoins := mapper.GetAccount(ctx, msg.Outputs[i].Address).GetCoins()
-		require.Equal(t,
+		if !terminalOutputCoins.IsEqual(initialOutputAddrCoins[i].Plus(msg.Outputs[i].Coins)) {
-			initialOutputAddrCoins[i].Plus(msg.Outputs[i].Coins),
+			tb.Fatalf("Output #%d had an incorrect amount of coins\n%s", i, log)
-			terminalOutputCoins,
+		}
-			fmt.Sprintf("Output #%d had an incorrect amount of coins\n%s", i, log),
-		)
 	}
 }
 

x/gov/simulation/msgs.go

@@ -5,8 +5,6 @@ import (
 	"math/rand"
 	"testing"
 
-	"github.com/stretchr/testify/require"
-
 	"github.com/tendermint/tendermint/crypto"
 
 	"github.com/cosmos/cosmos-sdk/baseapp"
@@ -23,24 +21,15 @@ const (
 // SimulateMsgSubmitProposal simulates a msg Submit Proposal
 // Note: Currently doesn't ensure that the proposal txt is in JSON form
 func SimulateMsgSubmitProposal(k gov.Keeper, sk stake.Keeper) simulation.Operation {
-	return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOps []simulation.FutureOperation, err sdk.Error) {
+	handler := gov.NewHandler(k)
-		key := simulation.RandomKey(r, keys)
+	return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOps []simulation.FutureOperation, err sdk.Error) {
-		addr := sdk.AccAddress(key.PubKey().Address())
+		msg := simulationCreateMsgSubmitProposal(tb, r, keys, log)
-		deposit := randomDeposit(r)
-		msg := gov.NewMsgSubmitProposal(
-			simulation.RandStringOfLength(r, 5),
-			simulation.RandStringOfLength(r, 5),
-			gov.ProposalTypeText,
-			addr,
-			deposit,
-		)
-		require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
 		ctx, write := ctx.CacheContext()
-		result := gov.NewHandler(k)(ctx, msg)
+		result := handler(ctx, msg)
 		if result.IsOK() {
 			// Update pool to keep invariants
 			pool := sk.GetPool(ctx)
-			pool.LooseTokens = pool.LooseTokens.Sub(sdk.NewDecFromInt(deposit.AmountOf(denom)))
+			pool.LooseTokens = pool.LooseTokens.Sub(sdk.NewDecFromInt(msg.InitialDeposit.AmountOf(denom)))
 			sk.SetPool(ctx, pool)
 			write()
 		}
@@ -50,9 +39,26 @@ func SimulateMsgSubmitProposal(k gov.Keeper, sk stake.Keeper) simulation.Operati
 	}
 }
 
+func simulationCreateMsgSubmitProposal(tb testing.TB, r *rand.Rand, keys []crypto.PrivKey, log string) gov.MsgSubmitProposal {
+	key := simulation.RandomKey(r, keys)
+	addr := sdk.AccAddress(key.PubKey().Address())
+	deposit := randomDeposit(r)
+	msg := gov.NewMsgSubmitProposal(
+		simulation.RandStringOfLength(r, 5),
+		simulation.RandStringOfLength(r, 5),
+		gov.ProposalTypeText,
+		addr,
+		deposit,
+	)
+	if msg.ValidateBasic() != nil {
+		tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log)
+	}
+	return msg
+}
+
 // SimulateMsgDeposit
 func SimulateMsgDeposit(k gov.Keeper, sk stake.Keeper) simulation.Operation {
-	return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
+	return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
 		key := simulation.RandomKey(r, keys)
 		addr := sdk.AccAddress(key.PubKey().Address())
 		proposalID, ok := randomProposalID(r, k, ctx)
@@ -61,7 +67,9 @@ func SimulateMsgDeposit(k gov.Keeper, sk stake.Keeper) simulation.Operation {
 		}
 		deposit := randomDeposit(r)
 		msg := gov.NewMsgDeposit(addr, proposalID, deposit)
-		require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
+		if msg.ValidateBasic() != nil {
+			tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log)
+		}
 		ctx, write := ctx.CacheContext()
 		result := gov.NewHandler(k)(ctx, msg)
 		if result.IsOK() {
@@ -79,7 +87,7 @@ func SimulateMsgDeposit(k gov.Keeper, sk stake.Keeper) simulation.Operation {
 
 // SimulateMsgVote
 func SimulateMsgVote(k gov.Keeper, sk stake.Keeper) simulation.Operation {
-	return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
+	return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
 		key := simulation.RandomKey(r, keys)
 		addr := sdk.AccAddress(key.PubKey().Address())
 		proposalID, ok := randomProposalID(r, k, ctx)
@@ -88,7 +96,9 @@ func SimulateMsgVote(k gov.Keeper, sk stake.Keeper) simulation.Operation {
 		}
 		option := randomVotingOption(r)
 		msg := gov.NewMsgVote(addr, proposalID, option)
-		require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
+		if msg.ValidateBasic() != nil {
+			tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log)
+		}
 		ctx, write := ctx.CacheContext()
 		result := gov.NewHandler(k)(ctx, msg)
 		if result.IsOK() {

x/mock/simulation/random_simulate_blocks.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"math"
 	"math/rand"
+	"os"
 	"sort"
 	"testing"
 	"time"
@@ -16,7 +17,6 @@ import (
 	"github.com/cosmos/cosmos-sdk/baseapp"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/cosmos/cosmos-sdk/x/mock"
-	"github.com/stretchr/testify/require"
 )
 
 // Simulate tests application by sending random messages.
@@ -28,34 +28,10 @@ func Simulate(
 	SimulateFromSeed(t, app, appStateFn, time, ops, setups, invariants, numBlocks, blockSize, commit)
 }
 
-// SimulateFromSeed tests an application by running the provided
+func initChain(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress, setups []RandSetup, app *baseapp.BaseApp,
-// operations, testing the provided invariants, but using the provided seed.
+	appStateFn func(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage) (validators map[string]mockValidator) {
-func SimulateFromSeed(
-	t *testing.T, app *baseapp.BaseApp, appStateFn func(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage, seed int64, ops []Operation, setups []RandSetup,
-	invariants []Invariant, numBlocks int, blockSize int, commit bool,
-) {
-	log := fmt.Sprintf("Starting SimulateFromSeed with randomness created with seed %d", int(seed))
-	r := rand.New(rand.NewSource(seed))
-
-	unixTime := r.Int63n(int64(math.Pow(2, 40)))
-
-	// Set the timestamp for simulation
-	timestamp := time.Unix(unixTime, 0)
-	log = fmt.Sprintf("%s\nStarting the simulation from time %v, unixtime %v", log, timestamp.UTC().Format(time.UnixDate), timestamp.Unix())
-	fmt.Printf("%s\n", log)
-	timeDiff := maxTimePerBlock - minTimePerBlock
-
-	keys, accs := mock.GeneratePrivKeyAddressPairsFromRand(r, numKeys)
-
-	// Setup event stats
-	events := make(map[string]uint)
-	event := func(what string) {
-		log += "\nevent - " + what
-		events[what]++
-	}
-
 	res := app.InitChain(abci.RequestInitChain{AppStateBytes: appStateFn(r, keys, accs)})
-	validators := make(map[string]mockValidator)
+	validators = make(map[string]mockValidator)
 	for _, validator := range res.Validators {
 		validators[string(validator.Address)] = mockValidator{validator, GetMemberOfInitialState(r, initialLivenessWeightings)}
 	}
@@ -64,85 +40,161 @@ func SimulateFromSeed(
 		setups[i](r, keys)
 	}
 
+	return
+}
+
+func randTimestamp(r *rand.Rand) time.Time {
+	unixTime := r.Int63n(int64(math.Pow(2, 40)))
+	return time.Unix(unixTime, 0)
+}
+
+// SimulateFromSeed tests an application by running the provided
+// operations, testing the provided invariants, but using the provided seed.
+func SimulateFromSeed(
+	tb testing.TB, app *baseapp.BaseApp, appStateFn func(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage, seed int64, ops []Operation, setups []RandSetup,
+	invariants []Invariant, numBlocks int, blockSize int, commit bool,
+) {
+	testingMode, t, b := getTestingMode(tb)
+	log := fmt.Sprintf("Starting SimulateFromSeed with randomness created with seed %d", int(seed))
+	r := rand.New(rand.NewSource(seed))
+	timestamp := randTimestamp(r)
+	log = updateLog(testingMode, log, "Starting the simulation from time %v, unixtime %v", timestamp.UTC().Format(time.UnixDate), timestamp.Unix())
+	fmt.Printf("%s\n", log)
+	timeDiff := maxTimePerBlock - minTimePerBlock
+
+	keys, accs := mock.GeneratePrivKeyAddressPairsFromRand(r, numKeys)
+
+	// Setup event stats
+	events := make(map[string]uint)
+	event := func(what string) {
+		log = updateLog(testingMode, log, "event - %s", what)
+		events[what]++
+	}
+
+	validators := initChain(r, keys, accs, setups, app, appStateFn)
+
 	header := abci.Header{Height: 0, Time: timestamp}
 	opCount := 0
 
 	var pastTimes []time.Time
 	var pastSigningValidators [][]abci.SigningValidator
 
-	request := RandomRequestBeginBlock(t, r, validators, livenessTransitionMatrix, evidenceFraction, pastTimes, pastSigningValidators, event, header, log)
+	request := RandomRequestBeginBlock(r, validators, livenessTransitionMatrix, evidenceFraction, pastTimes, pastSigningValidators, event, header, log)
 	// These are operations which have been queued by previous operations
 	operationQueue := make(map[int][]Operation)
 
-	for i := 0; i < numBlocks; i++ {
+	if !testingMode {
+		b.ResetTimer()
+	}
+	blockSimulator := createBlockSimulator(testingMode, tb, t, event, invariants, ops, operationQueue, numBlocks)
 
+	for i := 0; i < numBlocks; i++ {
 		// Log the header time for future lookup
 		pastTimes = append(pastTimes, header.Time)
 		pastSigningValidators = append(pastSigningValidators, request.LastCommitInfo.Validators)
 
 		// Run the BeginBlock handler
 		app.BeginBlock(request)
+		log = updateLog(testingMode, log, "BeginBlock")
 
-		log += "\nBeginBlock"
+		if testingMode {
-
+			// Make sure invariants hold at beginning of block
-		// Make sure invariants hold at beginning of block
+			AssertAllInvariants(t, app, invariants, log)
-		AssertAllInvariants(t, app, invariants, log)
+		}
 
 		ctx := app.NewContext(false, header)
+		thisBlockSize := getBlockSize(r, blockSize)
 
-		var thisBlockSize int
-		load := r.Float64()
-		switch {
-		case load < 0.33:
-			thisBlockSize = 0
-		case load < 0.66:
-			thisBlockSize = r.Intn(blockSize * 2)
-		default:
-			thisBlockSize = r.Intn(blockSize * 4)
-		}
 		// Run queued operations. Ignores blocksize if blocksize is too small
-		log, numQueuedOpsRan := runQueuedOperations(operationQueue, int(header.Height), t, r, app, ctx, keys, log, event)
+		log, numQueuedOpsRan := runQueuedOperations(operationQueue, int(header.Height), tb, r, app, ctx, keys, log, event)
 		opCount += numQueuedOpsRan
 		thisBlockSize -= numQueuedOpsRan
-		for j := 0; j < thisBlockSize; j++ {
+		log, operations := blockSimulator(thisBlockSize, r, app, ctx, keys, log, header)
-			logUpdate, futureOps, err := ops[r.Intn(len(ops))](t, r, app, ctx, keys, log, event)
+		opCount += operations
-			log += "\n" + logUpdate
-			queueOperations(operationQueue, futureOps)
-
-			require.Nil(t, err, log)
-			if onOperation {
-				AssertAllInvariants(t, app, invariants, log)
-			}
-			if opCount%200 == 0 {
-				fmt.Printf("\rSimulating... block %d/%d, operation %d.", header.Height, numBlocks, opCount)
-			}
-			opCount++
-		}
 
 		res := app.EndBlock(abci.RequestEndBlock{})
 		header.Height++
 		header.Time = header.Time.Add(time.Duration(minTimePerBlock) * time.Second).Add(time.Duration(int64(r.Intn(int(timeDiff)))) * time.Second)
+		log = updateLog(testingMode, log, "EndBlock")
 
-		log += "\nEndBlock"
+		if testingMode {
-
+			// Make sure invariants hold at end of block
-		// Make sure invariants hold at end of block
+			AssertAllInvariants(t, app, invariants, log)
-		AssertAllInvariants(t, app, invariants, log)
+		}
-
 		if commit {
 			app.Commit()
 		}
 
 		// Generate a random RequestBeginBlock with the current validator set for the next block
-		request = RandomRequestBeginBlock(t, r, validators, livenessTransitionMatrix, evidenceFraction, pastTimes, pastSigningValidators, event, header, log)
+		request = RandomRequestBeginBlock(r, validators, livenessTransitionMatrix, evidenceFraction, pastTimes, pastSigningValidators, event, header, log)
 
 		// Update the validator set
-		validators = updateValidators(t, r, validators, res.ValidatorUpdates, event)
+		validators = updateValidators(tb, r, validators, res.ValidatorUpdates, event)
 	}
 
-	fmt.Printf("\nSimulation complete. Final height (blocks): %d, final time (seconds): %v\n", header.Height, header.Time)
+	fmt.Printf("\nSimulation complete. Final height (blocks): %d, final time (seconds), : %v, operations ran %d\n", header.Height, header.Time, opCount)
 	DisplayEvents(events)
 }
 
+// Returns a function to simulate blocks. Written like this to avoid constant parameters being passed everytime, to minimize
+// memory overhead
+func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, event func(string), invariants []Invariant, ops []Operation, operationQueue map[int][]Operation, totalNumBlocks int) func(
+	blocksize int, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, privKeys []crypto.PrivKey, log string, header abci.Header) (updatedLog string, opCount int) {
+	return func(blocksize int, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
+		keys []crypto.PrivKey, log string, header abci.Header) (updatedLog string, opCount int) {
+		for j := 0; j < blocksize; j++ {
+			logUpdate, futureOps, err := ops[r.Intn(len(ops))](tb, r, app, ctx, keys, log, event)
+			log = updateLog(testingMode, log, logUpdate)
+			if err != nil {
+				tb.Fatalf("error on operation %d within block %d, %v, log %s", header.Height, opCount, err, log)
+			}
+
+			queueOperations(operationQueue, futureOps)
+			if testingMode {
+				if onOperation {
+					AssertAllInvariants(t, app, invariants, log)
+				}
+				if opCount%50 == 0 {
+					fmt.Printf("\rSimulating... block %d/%d, operation %d/%d.", header.Height, totalNumBlocks, opCount, blocksize)
+				}
+			}
+			opCount++
+		}
+		return log, opCount
+	}
+}
+
+func getTestingMode(tb testing.TB) (testingMode bool, t *testing.T, b *testing.B) {
+	testingMode = false
+	if _t, ok := tb.(*testing.T); ok {
+		t = _t
+		testingMode = true
+	} else {
+		b = tb.(*testing.B)
+	}
+	return
+}
+
+func updateLog(testingMode bool, log string, update string, args ...interface{}) (updatedLog string) {
+	if testingMode {
+		update = fmt.Sprintf(update, args...)
+		return fmt.Sprintf("%s\n%s", log, update)
+	}
+	return ""
+}
+
+func getBlockSize(r *rand.Rand, blockSize int) int {
+	load := r.Float64()
+	switch {
+	case load < 0.33:
+		return 0
+	case load < 0.66:
+		return r.Intn(blockSize * 2)
+	default:
+		return r.Intn(blockSize * 4)
+	}
+}
+
 // adds all future operations into the operation queue.
 func queueOperations(queuedOperations map[int][]Operation, futureOperations []FutureOperation) {
 	if futureOperations == nil {
@@ -157,7 +209,7 @@ func queueOperations(queuedOperations map[int][]Operation, futureOperations []Fu
 	}
 }
 
-func runQueuedOperations(queueOperations map[int][]Operation, height int, t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
+func runQueuedOperations(queueOperations map[int][]Operation, height int, tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
 	privKeys []crypto.PrivKey, log string, event func(string)) (updatedLog string, numOpsRan int) {
 	updatedLog = log
 	if queuedOps, ok := queueOperations[height]; ok {
@@ -166,9 +218,12 @@ func runQueuedOperations(queueOperations map[int][]Operation, height int, t *tes
 			// For now, queued operations cannot queue more operations.
 			// If a need arises for us to support queued messages to queue more messages, this can
 			// be changed.
-			logUpdate, _, err := queuedOps[i](t, r, app, ctx, privKeys, updatedLog, event)
+			logUpdate, _, err := queuedOps[i](tb, r, app, ctx, privKeys, updatedLog, event)
-			updatedLog += "\n" + logUpdate
+			updatedLog = fmt.Sprintf("%s\n%s", updatedLog, logUpdate)
-			require.Nil(t, err, updatedLog)
+			if err != nil {
+				fmt.Fprint(os.Stderr, updatedLog)
+				tb.FailNow()
+			}
 		}
 		delete(queueOperations, height)
 		return updatedLog, numOps
@@ -188,7 +243,7 @@ func getKeys(validators map[string]mockValidator) []string {
 }
 
 // RandomRequestBeginBlock generates a list of signing validators according to the provided list of validators, signing fraction, and evidence fraction
-func RandomRequestBeginBlock(t *testing.T, r *rand.Rand, validators map[string]mockValidator, livenessTransitions TransitionMatrix, evidenceFraction float64,
+func RandomRequestBeginBlock(r *rand.Rand, validators map[string]mockValidator, livenessTransitions TransitionMatrix, evidenceFraction float64,
 	pastTimes []time.Time, pastSigningValidators [][]abci.SigningValidator, event func(string), header abci.Header, log string) abci.RequestBeginBlock {
 	if len(validators) == 0 {
 		return abci.RequestBeginBlock{Header: header}
@@ -220,6 +275,7 @@ func RandomRequestBeginBlock(t *testing.T, r *rand.Rand, validators map[string]m
 		}
 		i++
 	}
+	// TODO: Determine capacity before allocation
 	evidence := make([]abci.Evidence, 0)
 	// Anything but the first block
 	if len(pastTimes) > 0 {
@@ -264,11 +320,19 @@ func AssertAllInvariants(t *testing.T, app *baseapp.BaseApp, tests []Invariant,
 }
 
 // updateValidators mimicks Tendermint's update logic
-func updateValidators(t *testing.T, r *rand.Rand, current map[string]mockValidator, updates []abci.Validator, event func(string)) map[string]mockValidator {
+func updateValidators(tb testing.TB, r *rand.Rand, current map[string]mockValidator, updates []abci.Validator, event func(string)) map[string]mockValidator {
 	for _, update := range updates {
 		switch {
 		case update.Power == 0:
-			require.NotNil(t, current[string(update.PubKey.Data)], "tried to delete a nonexistent validator")
+			// // TEMPORARY DEBUG CODE TO PROVE THAT THE OLD METHOD WAS BROKEN
+			// // (i.e. didn't catch in the event of problem)
+			// if val, ok := tb.(*testing.T); ok {
+			// 	require.NotNil(val, current[string(update.PubKey.Data)])
+			// }
+			// // CORRECT CHECK
+			// if _, ok := current[string(update.PubKey.Data)]; !ok {
+			// 	tb.Fatalf("tried to delete a nonexistent validator")
+			// }
 			event("endblock/validatorupdates/kicked")
 			delete(current, string(update.PubKey.Data))
 		default:

x/mock/simulation/types.go

@@ -23,7 +23,7 @@ type (
 	// Operations can optionally provide a list of "FutureOperations" to run later
 	// These will be ran at the beginning of the corresponding block.
 	Operation func(
-		t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
+		tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
 		privKeys []crypto.PrivKey, log string, event func(string),
 	) (action string, futureOperations []FutureOperation, err sdk.Error)
 

x/slashing/simulation/msgs.go

@@ -5,8 +5,6 @@ import (
 	"math/rand"
 	"testing"
 
-	"github.com/stretchr/testify/require"
-
 	"github.com/tendermint/tendermint/crypto"
 
 	"github.com/cosmos/cosmos-sdk/baseapp"
@@ -17,11 +15,13 @@ import (
 
 // SimulateMsgUnjail
 func SimulateMsgUnjail(k slashing.Keeper) simulation.Operation {
-	return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
+	return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
 		key := simulation.RandomKey(r, keys)
 		address := sdk.ValAddress(key.PubKey().Address())
 		msg := slashing.NewMsgUnjail(address)
-		require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
+		if msg.ValidateBasic() != nil {
+			tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log)
+		}
 		ctx, write := ctx.CacheContext()
 		result := slashing.NewHandler(k)(ctx, msg)
 		if result.IsOK() {

x/stake/keeper/validator.go

@@ -2,6 +2,7 @@ package keeper
 
 import (
 	"bytes"
+	"container/list"
 	"fmt"
 
 	abci "github.com/tendermint/tendermint/abci/types"
@@ -11,6 +12,19 @@ import (
 	"github.com/cosmos/cosmos-sdk/x/stake/types"
 )
 
+// Cache the amino decoding of validators, as it can be the case that repeated slashing calls
+// cause many calls to GetValidator, which were shown to throttle the state machine in our
+// simulation. Note this is quite biased though, as the simulator does more slashes than a
+// live chain should, however we require the slashing to be fast as noone pays gas for it.
+type cachedValidator struct {
+	val        types.Validator
+	marshalled string // marshalled amino bytes for the validator object (not operator address)
+}
+
+// validatorCache-key: validator amino bytes
+var validatorCache = make(map[string]cachedValidator, 500)
+var validatorCacheList = list.New()
+
 // get a single validator
 func (k Keeper) GetValidator(ctx sdk.Context, addr sdk.ValAddress) (validator types.Validator, found bool) {
 	store := ctx.KVStore(k.storeKey)
@@ -18,6 +32,28 @@ func (k Keeper) GetValidator(ctx sdk.Context, addr sdk.ValAddress) (validator ty
 	if value == nil {
 		return validator, false
 	}
+
+	// If these amino encoded bytes are in the cache, return the cached validator
+	strValue := string(value)
+	if val, ok := validatorCache[strValue]; ok {
+		valToReturn := val.val
+		// Doesn't mutate the cache's value
+		valToReturn.Operator = addr
+		return valToReturn, true
+	}
+
+	// amino bytes weren't found in cache, so amino unmarshal and add it to the cache
+	validator = types.MustUnmarshalValidator(k.cdc, addr, value)
+	cachedVal := cachedValidator{validator, strValue}
+	validatorCache[strValue] = cachedValidator{validator, strValue}
+	validatorCacheList.PushBack(cachedVal)
+
+	// if the cache is too big, pop off the last element from it
+	if validatorCacheList.Len() > 500 {
+		valToRemove := validatorCacheList.Remove(validatorCacheList.Front()).(cachedValidator)
+		delete(validatorCache, valToRemove.marshalled)
+	}
+
 	validator = types.MustUnmarshalValidator(k.cdc, addr, value)
 	return validator, true
 }

x/stake/simulation/msgs.go

@@ -5,8 +5,6 @@ import (
 	"math/rand"
 	"testing"
 
-	"github.com/stretchr/testify/require"
-
 	"github.com/cosmos/cosmos-sdk/baseapp"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/cosmos/cosmos-sdk/x/auth"
@@ -19,9 +17,8 @@ import (
 
 // SimulateMsgCreateValidator
 func SimulateMsgCreateValidator(m auth.AccountMapper, k stake.Keeper) simulation.Operation {
-
+	handler := stake.NewHandler(k)
-	return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
+	return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
-		keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
 
 		denom := k.GetParams(ctx).BondDenom
 		description := stake.Description{
@@ -44,9 +41,11 @@ func SimulateMsgCreateValidator(m auth.AccountMapper, k stake.Keeper) simulation
 			PubKey:        pubkey,
 			Delegation:    sdk.NewCoin(denom, amount),
 		}
-		require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
+		if msg.ValidateBasic() != nil {
+			tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log)
+		}
 		ctx, write := ctx.CacheContext()
-		result := stake.NewHandler(k)(ctx, msg)
+		result := handler(ctx, msg)
 		if result.IsOK() {
 			write()
 		}
@@ -59,9 +58,8 @@ func SimulateMsgCreateValidator(m auth.AccountMapper, k stake.Keeper) simulation
 
 // SimulateMsgEditValidator
 func SimulateMsgEditValidator(k stake.Keeper) simulation.Operation {
-
+	handler := stake.NewHandler(k)
-	return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
+	return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
-		keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
 
 		description := stake.Description{
 			Moniker:  simulation.RandStringOfLength(r, 10),
@@ -76,9 +74,11 @@ func SimulateMsgEditValidator(k stake.Keeper) simulation.Operation {
 			Description:   description,
 			ValidatorAddr: address,
 		}
-		require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
+		if msg.ValidateBasic() != nil {
+			tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log)
+		}
 		ctx, write := ctx.CacheContext()
-		result := stake.NewHandler(k)(ctx, msg)
+		result := handler(ctx, msg)
 		if result.IsOK() {
 			write()
 		}
@@ -90,9 +90,8 @@ func SimulateMsgEditValidator(k stake.Keeper) simulation.Operation {
 
 // SimulateMsgDelegate
 func SimulateMsgDelegate(m auth.AccountMapper, k stake.Keeper) simulation.Operation {
-
+	handler := stake.NewHandler(k)
-	return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
+	return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
-		keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
 
 		denom := k.GetParams(ctx).BondDenom
 		validatorKey := simulation.RandomKey(r, keys)
@@ -111,9 +110,11 @@ func SimulateMsgDelegate(m auth.AccountMapper, k stake.Keeper) simulation.Operat
 			ValidatorAddr: validatorAddress,
 			Delegation:    sdk.NewCoin(denom, amount),
 		}
-		require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
+		if msg.ValidateBasic() != nil {
+			tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log)
+		}
 		ctx, write := ctx.CacheContext()
-		result := stake.NewHandler(k)(ctx, msg)
+		result := handler(ctx, msg)
 		if result.IsOK() {
 			write()
 		}
@@ -125,9 +126,8 @@ func SimulateMsgDelegate(m auth.AccountMapper, k stake.Keeper) simulation.Operat
 
 // SimulateMsgBeginUnbonding
 func SimulateMsgBeginUnbonding(m auth.AccountMapper, k stake.Keeper) simulation.Operation {
-
+	handler := stake.NewHandler(k)
-	return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
+	return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
-		keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
 
 		denom := k.GetParams(ctx).BondDenom
 		validatorKey := simulation.RandomKey(r, keys)
@@ -146,9 +146,11 @@ func SimulateMsgBeginUnbonding(m auth.AccountMapper, k stake.Keeper) simulation.
 			ValidatorAddr: validatorAddress,
 			SharesAmount:  sdk.NewDecFromInt(amount),
 		}
-		require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
+		if msg.ValidateBasic() != nil {
+			tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log)
+		}
 		ctx, write := ctx.CacheContext()
-		result := stake.NewHandler(k)(ctx, msg)
+		result := handler(ctx, msg)
 		if result.IsOK() {
 			write()
 		}
@@ -160,9 +162,8 @@ func SimulateMsgBeginUnbonding(m auth.AccountMapper, k stake.Keeper) simulation.
 
 // SimulateMsgCompleteUnbonding
 func SimulateMsgCompleteUnbonding(k stake.Keeper) simulation.Operation {
-
+	handler := stake.NewHandler(k)
-	return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
+	return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
-		keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
 
 		validatorKey := simulation.RandomKey(r, keys)
 		validatorAddress := sdk.ValAddress(validatorKey.PubKey().Address())
@@ -172,9 +173,11 @@ func SimulateMsgCompleteUnbonding(k stake.Keeper) simulation.Operation {
 			DelegatorAddr: delegatorAddress,
 			ValidatorAddr: validatorAddress,
 		}
-		require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
+		if msg.ValidateBasic() != nil {
+			tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log)
+		}
 		ctx, write := ctx.CacheContext()
-		result := stake.NewHandler(k)(ctx, msg)
+		result := handler(ctx, msg)
 		if result.IsOK() {
 			write()
 		}
@@ -186,9 +189,8 @@ func SimulateMsgCompleteUnbonding(k stake.Keeper) simulation.Operation {
 
 // SimulateMsgBeginRedelegate
 func SimulateMsgBeginRedelegate(m auth.AccountMapper, k stake.Keeper) simulation.Operation {
-
+	handler := stake.NewHandler(k)
-	return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
+	return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
-		keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
 
 		denom := k.GetParams(ctx).BondDenom
 		sourceValidatorKey := simulation.RandomKey(r, keys)
@@ -211,9 +213,11 @@ func SimulateMsgBeginRedelegate(m auth.AccountMapper, k stake.Keeper) simulation
 			ValidatorDstAddr: destValidatorAddress,
 			SharesAmount:     sdk.NewDecFromInt(amount),
 		}
-		require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
+		if msg.ValidateBasic() != nil {
+			tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log)
+		}
 		ctx, write := ctx.CacheContext()
-		result := stake.NewHandler(k)(ctx, msg)
+		result := handler(ctx, msg)
 		if result.IsOK() {
 			write()
 		}
@@ -225,9 +229,8 @@ func SimulateMsgBeginRedelegate(m auth.AccountMapper, k stake.Keeper) simulation
 
 // SimulateMsgCompleteRedelegate
 func SimulateMsgCompleteRedelegate(k stake.Keeper) simulation.Operation {
-
+	handler := stake.NewHandler(k)
-	return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
+	return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
-		keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
 
 		validatorSrcKey := simulation.RandomKey(r, keys)
 		validatorSrcAddress := sdk.ValAddress(validatorSrcKey.PubKey().Address())
@@ -240,9 +243,11 @@ func SimulateMsgCompleteRedelegate(k stake.Keeper) simulation.Operation {
 			ValidatorSrcAddr: validatorSrcAddress,
 			ValidatorDstAddr: validatorDstAddress,
 		}
-		require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
+		if msg.ValidateBasic() != nil {
+			tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log)
+		}
 		ctx, write := ctx.CacheContext()
-		result := stake.NewHandler(k)(ctx, msg)
+		result := handler(ctx, msg)
 		if result.IsOK() {
 			write()
 		}