From 722d122da7cfa77e00afacabe2624fc65c50cb35 Mon Sep 17 00:00:00 2001 From: Alexander Bezobchuk Date: Fri, 12 Apr 2019 18:52:16 -0400 Subject: [PATCH] Merge PR #4079: Implement Invariant Benchmarks --- ...iant-benchmarks-and-add-target-to-makefile | 1 + Makefile | 8 ++- cmd/gaia/app/sim_test.go | 58 ++++++++++++++++--- types/invariant.go | 2 +- x/simulation/event_stats.go | 9 ++- x/simulation/simulate.go | 47 +++++++-------- 6 files changed, 88 insertions(+), 37 deletions(-) create mode 100644 .pending/improvements/sdk/3914-Implement-invariant-benchmarks-and-add-target-to-makefile diff --git a/.pending/improvements/sdk/3914-Implement-invariant-benchmarks-and-add-target-to-makefile b/.pending/improvements/sdk/3914-Implement-invariant-benchmarks-and-add-target-to-makefile new file mode 100644 index 000000000..46b548d30 --- /dev/null +++ b/.pending/improvements/sdk/3914-Implement-invariant-benchmarks-and-add-target-to-makefile @@ -0,0 +1 @@ +#3914 Implement invariant benchmarks and add target to makefile. diff --git a/Makefile b/Makefile index c0fab2f6b..f59244501 100644 --- a/Makefile +++ b/Makefile @@ -181,6 +181,12 @@ test_sim_gaia_multi_seed: @echo "Running multi-seed Gaia simulation. This may take awhile!" @bash scripts/multisim.sh 400 5 TestFullGaiaSimulation +test_sim_benchmark_invariants: + @echo "Running simulation invariant benchmarks..." + @go test -mod=readonly ./cmd/gaia/app -benchmem -bench=BenchmarkInvariants -run=^$ \ + -SimulationEnabled=true -SimulationNumBlocks=1000 -SimulationBlockSize=200 \ + -SimulationCommit=true -SimulationSeed=57 -v -timeout 24h + SIM_NUM_BLOCKS ?= 500 SIM_BLOCK_SIZE ?= 200 SIM_COMMIT ?= true @@ -268,5 +274,5 @@ test_cover lint benchmark devdoc_init devdoc devdoc_save devdoc_update \ build-linux build-docker-gaiadnode localnet-start localnet-stop \ format check-ledger test_sim_gaia_nondeterminism test_sim_modules test_sim_gaia_fast \ test_sim_gaia_custom_genesis_fast test_sim_gaia_custom_genesis_multi_seed \ -test_sim_gaia_multi_seed test_sim_gaia_import_export \ +test_sim_gaia_multi_seed test_sim_gaia_import_export test_sim_benchmark_invariants \ go-mod-cache diff --git a/cmd/gaia/app/sim_test.go b/cmd/gaia/app/sim_test.go index da4131fcb..dbeacc11a 100644 --- a/cmd/gaia/app/sim_test.go +++ b/cmd/gaia/app/sim_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "flag" "fmt" + "io" "io/ioutil" "math/rand" "os" @@ -61,11 +62,11 @@ func init() { } // helper function for populating input for SimulateFromSeed -func getSimulateFromSeedInput(tb testing.TB, app *GaiaApp) ( - testing.TB, *baseapp.BaseApp, simulation.AppStateFn, int64, +func getSimulateFromSeedInput(tb testing.TB, w io.Writer, app *GaiaApp) ( + testing.TB, io.Writer, *baseapp.BaseApp, simulation.AppStateFn, int64, simulation.WeightedOperations, sdk.Invariants, int, int, bool, bool) { - return tb, app.BaseApp, appStateFn, seed, + return tb, w, app.BaseApp, appStateFn, seed, testAndRunTxs(app), invariants(app), numBlocks, blockSize, commit, lean } @@ -318,7 +319,7 @@ func BenchmarkFullGaiaSimulation(b *testing.B) { // Run randomized simulation // TODO parameterize numbers, save for a later PR - _, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, app)) + _, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, os.Stdout, app)) if err != nil { fmt.Println(err) b.Fail() @@ -353,7 +354,7 @@ func TestFullGaiaSimulation(t *testing.T) { require.Equal(t, "GaiaApp", app.Name()) // Run randomized simulation - _, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, app)) + _, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app)) if commit { // for memdb: // fmt.Println("Database Size", db.Stats()["database.size"]) @@ -387,7 +388,7 @@ func TestGaiaImportExport(t *testing.T) { require.Equal(t, "GaiaApp", app.Name()) // Run randomized simulation - _, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, app)) + _, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app)) if commit { // for memdb: @@ -477,7 +478,7 @@ func TestGaiaSimulationAfterImport(t *testing.T) { require.Equal(t, "GaiaApp", app.Name()) // Run randomized simulation - stopEarly, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, app)) + stopEarly, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app)) if commit { // for memdb: @@ -516,7 +517,7 @@ func TestGaiaSimulationAfterImport(t *testing.T) { }) // Run randomized simulation on imported app - _, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, newApp)) + _, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, newApp)) require.Nil(t, err) } @@ -541,7 +542,7 @@ func TestAppStateDeterminism(t *testing.T) { // Run randomized simulation simulation.SimulateFromSeed( - t, app.BaseApp, appStateFn, seed, + t, os.Stdout, app.BaseApp, appStateFn, seed, testAndRunTxs(app), []sdk.Invariant{}, 50, @@ -557,3 +558,42 @@ func TestAppStateDeterminism(t *testing.T) { } } } + +func BenchmarkInvariants(b *testing.B) { + // 1. Setup a simulated Gaia application + logger := log.NewNopLogger() + dir, _ := ioutil.TempDir("", "goleveldb-gaia-invariant-bench") + db, _ := sdk.NewLevelDB("simulation", dir) + + defer func() { + db.Close() + os.RemoveAll(dir) + }() + + app := NewGaiaApp(logger, db, nil, true, false) + + // 2. Run parameterized simulation (w/o invariants) + _, err := simulation.SimulateFromSeed( + b, ioutil.Discard, app.BaseApp, appStateFn, seed, testAndRunTxs(app), + []sdk.Invariant{}, numBlocks, blockSize, commit, lean, + ) + if err != nil { + fmt.Println(err) + b.FailNow() + } + + ctx := app.NewContext(true, abci.Header{Height: app.LastBlockHeight() + 1}) + + // 3. Benchmark each invariant separately + // + // NOTE: We use the crisis keeper as it has all the invariants registered with + // their respective metadata which makes it useful for testing/benchmarking. + for _, cr := range app.crisisKeeper.Routes() { + b.Run(fmt.Sprintf("%s/%s", cr.ModuleName, cr.Route), func(b *testing.B) { + if err := cr.Invar(ctx); err != nil { + fmt.Println(err) + b.FailNow() + } + }) + } +} diff --git a/types/invariant.go b/types/invariant.go index 86c9024fa..aa8aca566 100644 --- a/types/invariant.go +++ b/types/invariant.go @@ -6,5 +6,5 @@ package types // The simulator will then halt and print the logs. type Invariant func(ctx Context) error -// group of Invarient +// Invariants defines a group of invariants type Invariants []Invariant diff --git a/x/simulation/event_stats.go b/x/simulation/event_stats.go index f54eef4cc..ad8acddb2 100644 --- a/x/simulation/event_stats.go +++ b/x/simulation/event_stats.go @@ -2,6 +2,7 @@ package simulation import ( "fmt" + "io" "sort" ) @@ -17,14 +18,16 @@ func (es eventStats) tally(eventDesc string) { } // Pretty-print events as a table -func (es eventStats) Print() { +func (es eventStats) Print(w io.Writer) { var keys []string for key := range es { keys = append(keys, key) } + sort.Strings(keys) - fmt.Printf("Event statistics: \n") + fmt.Fprintf(w, "Event statistics: \n") + for _, key := range keys { - fmt.Printf(" % 60s => %d\n", key, es[key]) + fmt.Fprintf(w, " % 60s => %d\n", key, es[key]) } } diff --git a/x/simulation/simulate.go b/x/simulation/simulate.go index 3eb25ccbc..86e238a12 100644 --- a/x/simulation/simulate.go +++ b/x/simulation/simulate.go @@ -3,6 +3,7 @@ package simulation import ( "encoding/json" "fmt" + "io" "math/rand" "os" "os/signal" @@ -26,7 +27,7 @@ func Simulate(t *testing.T, app *baseapp.BaseApp, invariants sdk.Invariants, numBlocks, blockSize int, commit, lean bool) (bool, error) { time := time.Now().UnixNano() - return SimulateFromSeed(t, app, appStateFn, time, ops, + return SimulateFromSeed(t, os.Stdout, app, appStateFn, time, ops, invariants, numBlocks, blockSize, commit, lean) } @@ -51,19 +52,20 @@ func initChain( // SimulateFromSeed tests an application by running the provided // operations, testing the provided invariants, but using the provided seed. // TODO split this monster function up -func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp, +func SimulateFromSeed( + tb testing.TB, w io.Writer, app *baseapp.BaseApp, appStateFn AppStateFn, seed int64, ops WeightedOperations, invariants sdk.Invariants, - numBlocks, blockSize int, commit, lean bool) (stopEarly bool, simError error) { + numBlocks, blockSize int, commit, lean bool, +) (stopEarly bool, simError error) { // in case we have to end early, don't os.Exit so that we can run cleanup code. testingMode, t, b := getTestingMode(tb) - fmt.Printf("Starting SimulateFromSeed with randomness "+ - "created with seed %d\n", int(seed)) + fmt.Fprintf(w, "Starting SimulateFromSeed with randomness created with seed %d\n", int(seed)) r := rand.New(rand.NewSource(seed)) params := RandomParams(r) // := DefaultParams() - fmt.Printf("Randomized simulation params: %+v\n", params) + fmt.Fprintf(w, "Randomized simulation params: %+v\n", params) genesisTimestamp := RandTimestamp(r) fmt.Printf("Starting the simulation from time %v, unixtime %v\n", @@ -94,8 +96,7 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp, signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGINT) go func() { receivedSignal := <-c - fmt.Printf("\nExiting early due to %s, on block %d, operation %d\n", - receivedSignal, header.Height, opCount) + fmt.Fprintf(w, "\nExiting early due to %s, on block %d, operation %d\n", receivedSignal, header.Height, opCount) simError = fmt.Errorf("Exited due to %s", receivedSignal) stopEarly = true }() @@ -113,7 +114,7 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp, logWriter := NewLogWriter(testingMode) blockSimulator := createBlockSimulator( - testingMode, tb, t, params, eventStats.tally, invariants, + testingMode, tb, t, w, params, eventStats.tally, invariants, ops, operationQueue, timeOperationQueue, numBlocks, blockSize, logWriter, lean) @@ -123,13 +124,11 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp, // Recover logs in case of panic defer func() { if r := recover(); r != nil { - fmt.Println("Panic with err\n", r) + fmt.Fprintf(w, "panic with err: %v\n", r) stackTrace := string(debug.Stack()) fmt.Println(stackTrace) logWriter.PrintLogs() - simError = fmt.Errorf( - "Simulation halted due to panic on block %d", - header.Height) + simError = fmt.Errorf("Simulation halted due to panic on block %d", header.Height) } }() } @@ -188,8 +187,7 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp, } if header.ProposerAddress == nil { - fmt.Printf("\nSimulation stopped early as all validators " + - "have been unbonded, there is nobody left propose a block!\n") + fmt.Fprintf(w, "\nSimulation stopped early as all validators have been unbonded; nobody left to propose a block!\n") stopEarly = true break } @@ -207,14 +205,17 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp, } if stopEarly { - eventStats.Print() + eventStats.Print(w) return true, simError } - fmt.Printf("\nSimulation complete. Final height (blocks): %d, "+ - "final time (seconds), : %v, operations ran %d\n", - header.Height, header.Time, opCount) - eventStats.Print() + fmt.Fprintf( + w, + "\nSimulation complete; Final height (blocks): %d, final time (seconds): %v, operations ran: %d\n", + header.Height, header.Time, opCount, + ) + + eventStats.Print(w) return false, nil } @@ -225,7 +226,7 @@ type blockSimFn func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, // Returns a function to simulate blocks. Written like this to avoid constant // parameters being passed everytime, to minimize memory overhead. -func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, params Params, +func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Writer, params Params, event func(string), invariants sdk.Invariants, ops WeightedOperations, operationQueue OperationQueue, timeOperationQueue []FutureOperation, totalNumBlocks, avgBlockSize int, logWriter LogWriter, lean bool) blockSimFn { @@ -237,7 +238,7 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, params return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accounts []Account, header abci.Header) (opCount int) { - fmt.Printf("\rSimulating... block %d/%d, operation %d/%d. ", + fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ", header.Height, totalNumBlocks, opCount, blocksize) lastBlocksizeState, blocksize = getBlockSize(r, params, lastBlocksizeState, avgBlockSize) @@ -277,7 +278,7 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, params assertAllInvariants(t, app, invariants, eventStr, logWriter) } if opCount%50 == 0 { - fmt.Printf("\rSimulating... block %d/%d, operation %d/%d. ", + fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ", header.Height, totalNumBlocks, opCount, blocksize) } }