Merge PR #4079: Implement Invariant Benchmarks

.pending/improvements/sdk/3914-Implement-invariant-benchmarks-and-add-target-to-makefile

@@ -0,0 +1 @@
+#3914 Implement invariant benchmarks and add target to makefile.

Makefile

@@ -181,6 +181,12 @@ test_sim_gaia_multi_seed:
 	@echo "Running multi-seed Gaia simulation. This may take awhile!"
 	@bash scripts/multisim.sh 400 5 TestFullGaiaSimulation
 
+test_sim_benchmark_invariants:
+	@echo "Running simulation invariant benchmarks..."
+	@go test -mod=readonly ./cmd/gaia/app -benchmem -bench=BenchmarkInvariants -run=^$ \
+	-SimulationEnabled=true -SimulationNumBlocks=1000 -SimulationBlockSize=200 \
+	-SimulationCommit=true -SimulationSeed=57 -v -timeout 24h
+
 SIM_NUM_BLOCKS ?= 500
 SIM_BLOCK_SIZE ?= 200
 SIM_COMMIT ?= true
@@ -268,5 +274,5 @@ test_cover lint benchmark devdoc_init devdoc devdoc_save devdoc_update \
 build-linux build-docker-gaiadnode localnet-start localnet-stop \
 format check-ledger test_sim_gaia_nondeterminism test_sim_modules test_sim_gaia_fast \
 test_sim_gaia_custom_genesis_fast test_sim_gaia_custom_genesis_multi_seed \
-test_sim_gaia_multi_seed test_sim_gaia_import_export \
+test_sim_gaia_multi_seed test_sim_gaia_import_export test_sim_benchmark_invariants \
 go-mod-cache

cmd/gaia/app/sim_test.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"math/rand"
 	"os"
@@ -61,11 +62,11 @@ func init() {
 }
 
 // helper function for populating input for SimulateFromSeed
-func getSimulateFromSeedInput(tb testing.TB, app *GaiaApp) (
+func getSimulateFromSeedInput(tb testing.TB, w io.Writer, app *GaiaApp) (
-	testing.TB, *baseapp.BaseApp, simulation.AppStateFn, int64,
+	testing.TB, io.Writer, *baseapp.BaseApp, simulation.AppStateFn, int64,
 	simulation.WeightedOperations, sdk.Invariants, int, int, bool, bool) {
 
-	return tb, app.BaseApp, appStateFn, seed,
+	return tb, w, app.BaseApp, appStateFn, seed,
 		testAndRunTxs(app), invariants(app), numBlocks, blockSize, commit, lean
 }
 
@@ -318,7 +319,7 @@ func BenchmarkFullGaiaSimulation(b *testing.B) {
 
 	// Run randomized simulation
 	// TODO parameterize numbers, save for a later PR
-	_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, app))
+	_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, os.Stdout, app))
 	if err != nil {
 		fmt.Println(err)
 		b.Fail()
@@ -353,7 +354,7 @@ func TestFullGaiaSimulation(t *testing.T) {
 	require.Equal(t, "GaiaApp", app.Name())
 
 	// Run randomized simulation
-	_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, app))
+	_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
 	if commit {
 		// for memdb:
 		// fmt.Println("Database Size", db.Stats()["database.size"])
@@ -387,7 +388,7 @@ func TestGaiaImportExport(t *testing.T) {
 	require.Equal(t, "GaiaApp", app.Name())
 
 	// Run randomized simulation
-	_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, app))
+	_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
 
 	if commit {
 		// for memdb:
@@ -477,7 +478,7 @@ func TestGaiaSimulationAfterImport(t *testing.T) {
 	require.Equal(t, "GaiaApp", app.Name())
 
 	// Run randomized simulation
-	stopEarly, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, app))
+	stopEarly, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
 
 	if commit {
 		// for memdb:
@@ -516,7 +517,7 @@ func TestGaiaSimulationAfterImport(t *testing.T) {
 	})
 
 	// Run randomized simulation on imported app
-	_, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, newApp))
+	_, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, newApp))
 	require.Nil(t, err)
 
 }
@@ -541,7 +542,7 @@ func TestAppStateDeterminism(t *testing.T) {
 
 			// Run randomized simulation
 			simulation.SimulateFromSeed(
-				t, app.BaseApp, appStateFn, seed,
+				t, os.Stdout, app.BaseApp, appStateFn, seed,
 				testAndRunTxs(app),
 				[]sdk.Invariant{},
 				50,
@@ -557,3 +558,42 @@ func TestAppStateDeterminism(t *testing.T) {
 		}
 	}
 }
+
+func BenchmarkInvariants(b *testing.B) {
+	// 1. Setup a simulated Gaia application
+	logger := log.NewNopLogger()
+	dir, _ := ioutil.TempDir("", "goleveldb-gaia-invariant-bench")
+	db, _ := sdk.NewLevelDB("simulation", dir)
+
+	defer func() {
+		db.Close()
+		os.RemoveAll(dir)
+	}()
+
+	app := NewGaiaApp(logger, db, nil, true, false)
+
+	// 2. Run parameterized simulation (w/o invariants)
+	_, err := simulation.SimulateFromSeed(
+		b, ioutil.Discard, app.BaseApp, appStateFn, seed, testAndRunTxs(app),
+		[]sdk.Invariant{}, numBlocks, blockSize, commit, lean,
+	)
+	if err != nil {
+		fmt.Println(err)
+		b.FailNow()
+	}
+
+	ctx := app.NewContext(true, abci.Header{Height: app.LastBlockHeight() + 1})
+
+	// 3. Benchmark each invariant separately
+	//
+	// NOTE: We use the crisis keeper as it has all the invariants registered with
+	// their respective metadata which makes it useful for testing/benchmarking.
+	for _, cr := range app.crisisKeeper.Routes() {
+		b.Run(fmt.Sprintf("%s/%s", cr.ModuleName, cr.Route), func(b *testing.B) {
+			if err := cr.Invar(ctx); err != nil {
+				fmt.Println(err)
+				b.FailNow()
+			}
+		})
+	}
+}

types/invariant.go

@@ -6,5 +6,5 @@ package types
 // The simulator will then halt and print the logs.
 type Invariant func(ctx Context) error
 
-// group of Invarient
+// Invariants defines a group of invariants
 type Invariants []Invariant

x/simulation/event_stats.go

@@ -2,6 +2,7 @@ package simulation
 
 import (
 	"fmt"
+	"io"
 	"sort"
 )
 
@@ -17,14 +18,16 @@ func (es eventStats) tally(eventDesc string) {
 }
 
 // Pretty-print events as a table
-func (es eventStats) Print() {
+func (es eventStats) Print(w io.Writer) {
 	var keys []string
 	for key := range es {
 		keys = append(keys, key)
 	}
+
 	sort.Strings(keys)
-	fmt.Printf("Event statistics: \n")
+	fmt.Fprintf(w, "Event statistics: \n")
+
 	for _, key := range keys {
-		fmt.Printf("  % 60s => %d\n", key, es[key])
+		fmt.Fprintf(w, "  % 60s => %d\n", key, es[key])
 	}
 }

x/simulation/simulate.go

@@ -3,6 +3,7 @@ package simulation
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"math/rand"
 	"os"
 	"os/signal"
@@ -26,7 +27,7 @@ func Simulate(t *testing.T, app *baseapp.BaseApp,
 	invariants sdk.Invariants, numBlocks, blockSize int, commit, lean bool) (bool, error) {
 
 	time := time.Now().UnixNano()
-	return SimulateFromSeed(t, app, appStateFn, time, ops,
+	return SimulateFromSeed(t, os.Stdout, app, appStateFn, time, ops,
 		invariants, numBlocks, blockSize, commit, lean)
 }
 
@@ -51,19 +52,20 @@ func initChain(
 // SimulateFromSeed tests an application by running the provided
 // operations, testing the provided invariants, but using the provided seed.
 // TODO split this monster function up
-func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
+func SimulateFromSeed(
+	tb testing.TB, w io.Writer, app *baseapp.BaseApp,
 	appStateFn AppStateFn, seed int64, ops WeightedOperations,
 	invariants sdk.Invariants,
-	numBlocks, blockSize int, commit, lean bool) (stopEarly bool, simError error) {
+	numBlocks, blockSize int, commit, lean bool,
+) (stopEarly bool, simError error) {
 
 	// in case we have to end early, don't os.Exit so that we can run cleanup code.
 	testingMode, t, b := getTestingMode(tb)
-	fmt.Printf("Starting SimulateFromSeed with randomness "+
+	fmt.Fprintf(w, "Starting SimulateFromSeed with randomness created with seed %d\n", int(seed))
-		"created with seed %d\n", int(seed))
 
 	r := rand.New(rand.NewSource(seed))
 	params := RandomParams(r) // := DefaultParams()
-	fmt.Printf("Randomized simulation params: %+v\n", params)
+	fmt.Fprintf(w, "Randomized simulation params: %+v\n", params)
 
 	genesisTimestamp := RandTimestamp(r)
 	fmt.Printf("Starting the simulation from time %v, unixtime %v\n",
@@ -94,8 +96,7 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
 	signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGINT)
 	go func() {
 		receivedSignal := <-c
-		fmt.Printf("\nExiting early due to %s, on block %d, operation %d\n",
+		fmt.Fprintf(w, "\nExiting early due to %s, on block %d, operation %d\n", receivedSignal, header.Height, opCount)
-			receivedSignal, header.Height, opCount)
 		simError = fmt.Errorf("Exited due to %s", receivedSignal)
 		stopEarly = true
 	}()
@@ -113,7 +114,7 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
 	logWriter := NewLogWriter(testingMode)
 
 	blockSimulator := createBlockSimulator(
-		testingMode, tb, t, params, eventStats.tally, invariants,
+		testingMode, tb, t, w, params, eventStats.tally, invariants,
 		ops, operationQueue, timeOperationQueue,
 		numBlocks, blockSize, logWriter, lean)
 
@@ -123,13 +124,11 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
 		// Recover logs in case of panic
 		defer func() {
 			if r := recover(); r != nil {
-				fmt.Println("Panic with err\n", r)
+				fmt.Fprintf(w, "panic with err: %v\n", r)
 				stackTrace := string(debug.Stack())
 				fmt.Println(stackTrace)
 				logWriter.PrintLogs()
-				simError = fmt.Errorf(
+				simError = fmt.Errorf("Simulation halted due to panic on block %d", header.Height)
-					"Simulation halted due to panic on block %d",
-					header.Height)
 			}
 		}()
 	}
@@ -188,8 +187,7 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
 		}
 
 		if header.ProposerAddress == nil {
-			fmt.Printf("\nSimulation stopped early as all validators " +
+			fmt.Fprintf(w, "\nSimulation stopped early as all validators have been unbonded; nobody left to propose a block!\n")
-				"have been unbonded, there is nobody left propose a block!\n")
 			stopEarly = true
 			break
 		}
@@ -207,14 +205,17 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
 	}
 
 	if stopEarly {
-		eventStats.Print()
+		eventStats.Print(w)
 		return true, simError
 	}
-	fmt.Printf("\nSimulation complete. Final height (blocks): %d, "+
-		"final time (seconds), : %v, operations ran %d\n",
-		header.Height, header.Time, opCount)
 
-	eventStats.Print()
+	fmt.Fprintf(
+		w,
+		"\nSimulation complete; Final height (blocks): %d, final time (seconds): %v, operations ran: %d\n",
+		header.Height, header.Time, opCount,
+	)
+
+	eventStats.Print(w)
 	return false, nil
 }
 
@@ -225,7 +226,7 @@ type blockSimFn func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
 
 // Returns a function to simulate blocks. Written like this to avoid constant
 // parameters being passed everytime, to minimize memory overhead.
-func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, params Params,
+func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Writer, params Params,
 	event func(string), invariants sdk.Invariants, ops WeightedOperations,
 	operationQueue OperationQueue, timeOperationQueue []FutureOperation,
 	totalNumBlocks, avgBlockSize int, logWriter LogWriter, lean bool) blockSimFn {
@@ -237,7 +238,7 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, params
 	return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
 		accounts []Account, header abci.Header) (opCount int) {
 
-		fmt.Printf("\rSimulating... block %d/%d, operation %d/%d. ",
+		fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ",
 			header.Height, totalNumBlocks, opCount, blocksize)
 		lastBlocksizeState, blocksize = getBlockSize(r, params, lastBlocksizeState, avgBlockSize)
 
@@ -277,7 +278,7 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, params
 					assertAllInvariants(t, app, invariants, eventStr, logWriter)
 				}
 				if opCount%50 == 0 {
-					fmt.Printf("\rSimulating... block %d/%d, operation %d/%d. ",
+					fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ",
 						header.Height, totalNumBlocks, opCount, blocksize)
 				}
 			}