Merge PR #4079: Implement Invariant Benchmarks
This commit is contained in:
parent
3c88ddc2f5
commit
722d122da7
|
@ -0,0 +1 @@
|
|||
#3914 Implement invariant benchmarks and add target to makefile.
|
8
Makefile
8
Makefile
|
@ -181,6 +181,12 @@ test_sim_gaia_multi_seed:
|
|||
@echo "Running multi-seed Gaia simulation. This may take awhile!"
|
||||
@bash scripts/multisim.sh 400 5 TestFullGaiaSimulation
|
||||
|
||||
test_sim_benchmark_invariants:
|
||||
@echo "Running simulation invariant benchmarks..."
|
||||
@go test -mod=readonly ./cmd/gaia/app -benchmem -bench=BenchmarkInvariants -run=^$ \
|
||||
-SimulationEnabled=true -SimulationNumBlocks=1000 -SimulationBlockSize=200 \
|
||||
-SimulationCommit=true -SimulationSeed=57 -v -timeout 24h
|
||||
|
||||
SIM_NUM_BLOCKS ?= 500
|
||||
SIM_BLOCK_SIZE ?= 200
|
||||
SIM_COMMIT ?= true
|
||||
|
@ -268,5 +274,5 @@ test_cover lint benchmark devdoc_init devdoc devdoc_save devdoc_update \
|
|||
build-linux build-docker-gaiadnode localnet-start localnet-stop \
|
||||
format check-ledger test_sim_gaia_nondeterminism test_sim_modules test_sim_gaia_fast \
|
||||
test_sim_gaia_custom_genesis_fast test_sim_gaia_custom_genesis_multi_seed \
|
||||
test_sim_gaia_multi_seed test_sim_gaia_import_export \
|
||||
test_sim_gaia_multi_seed test_sim_gaia_import_export test_sim_benchmark_invariants \
|
||||
go-mod-cache
|
||||
|
|
|
@ -4,6 +4,7 @@ import (
|
|||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
|
@ -61,11 +62,11 @@ func init() {
|
|||
}
|
||||
|
||||
// helper function for populating input for SimulateFromSeed
|
||||
func getSimulateFromSeedInput(tb testing.TB, app *GaiaApp) (
|
||||
testing.TB, *baseapp.BaseApp, simulation.AppStateFn, int64,
|
||||
func getSimulateFromSeedInput(tb testing.TB, w io.Writer, app *GaiaApp) (
|
||||
testing.TB, io.Writer, *baseapp.BaseApp, simulation.AppStateFn, int64,
|
||||
simulation.WeightedOperations, sdk.Invariants, int, int, bool, bool) {
|
||||
|
||||
return tb, app.BaseApp, appStateFn, seed,
|
||||
return tb, w, app.BaseApp, appStateFn, seed,
|
||||
testAndRunTxs(app), invariants(app), numBlocks, blockSize, commit, lean
|
||||
}
|
||||
|
||||
|
@ -318,7 +319,7 @@ func BenchmarkFullGaiaSimulation(b *testing.B) {
|
|||
|
||||
// Run randomized simulation
|
||||
// TODO parameterize numbers, save for a later PR
|
||||
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, app))
|
||||
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, os.Stdout, app))
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
b.Fail()
|
||||
|
@ -353,7 +354,7 @@ func TestFullGaiaSimulation(t *testing.T) {
|
|||
require.Equal(t, "GaiaApp", app.Name())
|
||||
|
||||
// Run randomized simulation
|
||||
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, app))
|
||||
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
|
||||
if commit {
|
||||
// for memdb:
|
||||
// fmt.Println("Database Size", db.Stats()["database.size"])
|
||||
|
@ -387,7 +388,7 @@ func TestGaiaImportExport(t *testing.T) {
|
|||
require.Equal(t, "GaiaApp", app.Name())
|
||||
|
||||
// Run randomized simulation
|
||||
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, app))
|
||||
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
|
||||
|
||||
if commit {
|
||||
// for memdb:
|
||||
|
@ -477,7 +478,7 @@ func TestGaiaSimulationAfterImport(t *testing.T) {
|
|||
require.Equal(t, "GaiaApp", app.Name())
|
||||
|
||||
// Run randomized simulation
|
||||
stopEarly, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, app))
|
||||
stopEarly, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
|
||||
|
||||
if commit {
|
||||
// for memdb:
|
||||
|
@ -516,7 +517,7 @@ func TestGaiaSimulationAfterImport(t *testing.T) {
|
|||
})
|
||||
|
||||
// Run randomized simulation on imported app
|
||||
_, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, newApp))
|
||||
_, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, newApp))
|
||||
require.Nil(t, err)
|
||||
|
||||
}
|
||||
|
@ -541,7 +542,7 @@ func TestAppStateDeterminism(t *testing.T) {
|
|||
|
||||
// Run randomized simulation
|
||||
simulation.SimulateFromSeed(
|
||||
t, app.BaseApp, appStateFn, seed,
|
||||
t, os.Stdout, app.BaseApp, appStateFn, seed,
|
||||
testAndRunTxs(app),
|
||||
[]sdk.Invariant{},
|
||||
50,
|
||||
|
@ -557,3 +558,42 @@ func TestAppStateDeterminism(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkInvariants(b *testing.B) {
|
||||
// 1. Setup a simulated Gaia application
|
||||
logger := log.NewNopLogger()
|
||||
dir, _ := ioutil.TempDir("", "goleveldb-gaia-invariant-bench")
|
||||
db, _ := sdk.NewLevelDB("simulation", dir)
|
||||
|
||||
defer func() {
|
||||
db.Close()
|
||||
os.RemoveAll(dir)
|
||||
}()
|
||||
|
||||
app := NewGaiaApp(logger, db, nil, true, false)
|
||||
|
||||
// 2. Run parameterized simulation (w/o invariants)
|
||||
_, err := simulation.SimulateFromSeed(
|
||||
b, ioutil.Discard, app.BaseApp, appStateFn, seed, testAndRunTxs(app),
|
||||
[]sdk.Invariant{}, numBlocks, blockSize, commit, lean,
|
||||
)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
b.FailNow()
|
||||
}
|
||||
|
||||
ctx := app.NewContext(true, abci.Header{Height: app.LastBlockHeight() + 1})
|
||||
|
||||
// 3. Benchmark each invariant separately
|
||||
//
|
||||
// NOTE: We use the crisis keeper as it has all the invariants registered with
|
||||
// their respective metadata which makes it useful for testing/benchmarking.
|
||||
for _, cr := range app.crisisKeeper.Routes() {
|
||||
b.Run(fmt.Sprintf("%s/%s", cr.ModuleName, cr.Route), func(b *testing.B) {
|
||||
if err := cr.Invar(ctx); err != nil {
|
||||
fmt.Println(err)
|
||||
b.FailNow()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
@ -6,5 +6,5 @@ package types
|
|||
// The simulator will then halt and print the logs.
|
||||
type Invariant func(ctx Context) error
|
||||
|
||||
// group of Invarient
|
||||
// Invariants defines a group of invariants
|
||||
type Invariants []Invariant
|
||||
|
|
|
@ -2,6 +2,7 @@ package simulation
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
)
|
||||
|
||||
|
@ -17,14 +18,16 @@ func (es eventStats) tally(eventDesc string) {
|
|||
}
|
||||
|
||||
// Pretty-print events as a table
|
||||
func (es eventStats) Print() {
|
||||
func (es eventStats) Print(w io.Writer) {
|
||||
var keys []string
|
||||
for key := range es {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
|
||||
sort.Strings(keys)
|
||||
fmt.Printf("Event statistics: \n")
|
||||
fmt.Fprintf(w, "Event statistics: \n")
|
||||
|
||||
for _, key := range keys {
|
||||
fmt.Printf(" % 60s => %d\n", key, es[key])
|
||||
fmt.Fprintf(w, " % 60s => %d\n", key, es[key])
|
||||
}
|
||||
}
|
||||
|
|
|
@ -3,6 +3,7 @@ package simulation
|
|||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"os"
|
||||
"os/signal"
|
||||
|
@ -26,7 +27,7 @@ func Simulate(t *testing.T, app *baseapp.BaseApp,
|
|||
invariants sdk.Invariants, numBlocks, blockSize int, commit, lean bool) (bool, error) {
|
||||
|
||||
time := time.Now().UnixNano()
|
||||
return SimulateFromSeed(t, app, appStateFn, time, ops,
|
||||
return SimulateFromSeed(t, os.Stdout, app, appStateFn, time, ops,
|
||||
invariants, numBlocks, blockSize, commit, lean)
|
||||
}
|
||||
|
||||
|
@ -51,19 +52,20 @@ func initChain(
|
|||
// SimulateFromSeed tests an application by running the provided
|
||||
// operations, testing the provided invariants, but using the provided seed.
|
||||
// TODO split this monster function up
|
||||
func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
|
||||
func SimulateFromSeed(
|
||||
tb testing.TB, w io.Writer, app *baseapp.BaseApp,
|
||||
appStateFn AppStateFn, seed int64, ops WeightedOperations,
|
||||
invariants sdk.Invariants,
|
||||
numBlocks, blockSize int, commit, lean bool) (stopEarly bool, simError error) {
|
||||
numBlocks, blockSize int, commit, lean bool,
|
||||
) (stopEarly bool, simError error) {
|
||||
|
||||
// in case we have to end early, don't os.Exit so that we can run cleanup code.
|
||||
testingMode, t, b := getTestingMode(tb)
|
||||
fmt.Printf("Starting SimulateFromSeed with randomness "+
|
||||
"created with seed %d\n", int(seed))
|
||||
fmt.Fprintf(w, "Starting SimulateFromSeed with randomness created with seed %d\n", int(seed))
|
||||
|
||||
r := rand.New(rand.NewSource(seed))
|
||||
params := RandomParams(r) // := DefaultParams()
|
||||
fmt.Printf("Randomized simulation params: %+v\n", params)
|
||||
fmt.Fprintf(w, "Randomized simulation params: %+v\n", params)
|
||||
|
||||
genesisTimestamp := RandTimestamp(r)
|
||||
fmt.Printf("Starting the simulation from time %v, unixtime %v\n",
|
||||
|
@ -94,8 +96,7 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
|
|||
signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGINT)
|
||||
go func() {
|
||||
receivedSignal := <-c
|
||||
fmt.Printf("\nExiting early due to %s, on block %d, operation %d\n",
|
||||
receivedSignal, header.Height, opCount)
|
||||
fmt.Fprintf(w, "\nExiting early due to %s, on block %d, operation %d\n", receivedSignal, header.Height, opCount)
|
||||
simError = fmt.Errorf("Exited due to %s", receivedSignal)
|
||||
stopEarly = true
|
||||
}()
|
||||
|
@ -113,7 +114,7 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
|
|||
logWriter := NewLogWriter(testingMode)
|
||||
|
||||
blockSimulator := createBlockSimulator(
|
||||
testingMode, tb, t, params, eventStats.tally, invariants,
|
||||
testingMode, tb, t, w, params, eventStats.tally, invariants,
|
||||
ops, operationQueue, timeOperationQueue,
|
||||
numBlocks, blockSize, logWriter, lean)
|
||||
|
||||
|
@ -123,13 +124,11 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
|
|||
// Recover logs in case of panic
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
fmt.Println("Panic with err\n", r)
|
||||
fmt.Fprintf(w, "panic with err: %v\n", r)
|
||||
stackTrace := string(debug.Stack())
|
||||
fmt.Println(stackTrace)
|
||||
logWriter.PrintLogs()
|
||||
simError = fmt.Errorf(
|
||||
"Simulation halted due to panic on block %d",
|
||||
header.Height)
|
||||
simError = fmt.Errorf("Simulation halted due to panic on block %d", header.Height)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
@ -188,8 +187,7 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
|
|||
}
|
||||
|
||||
if header.ProposerAddress == nil {
|
||||
fmt.Printf("\nSimulation stopped early as all validators " +
|
||||
"have been unbonded, there is nobody left propose a block!\n")
|
||||
fmt.Fprintf(w, "\nSimulation stopped early as all validators have been unbonded; nobody left to propose a block!\n")
|
||||
stopEarly = true
|
||||
break
|
||||
}
|
||||
|
@ -207,14 +205,17 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
|
|||
}
|
||||
|
||||
if stopEarly {
|
||||
eventStats.Print()
|
||||
eventStats.Print(w)
|
||||
return true, simError
|
||||
}
|
||||
fmt.Printf("\nSimulation complete. Final height (blocks): %d, "+
|
||||
"final time (seconds), : %v, operations ran %d\n",
|
||||
header.Height, header.Time, opCount)
|
||||
|
||||
eventStats.Print()
|
||||
fmt.Fprintf(
|
||||
w,
|
||||
"\nSimulation complete; Final height (blocks): %d, final time (seconds): %v, operations ran: %d\n",
|
||||
header.Height, header.Time, opCount,
|
||||
)
|
||||
|
||||
eventStats.Print(w)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
|
@ -225,7 +226,7 @@ type blockSimFn func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
|
|||
|
||||
// Returns a function to simulate blocks. Written like this to avoid constant
|
||||
// parameters being passed everytime, to minimize memory overhead.
|
||||
func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, params Params,
|
||||
func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Writer, params Params,
|
||||
event func(string), invariants sdk.Invariants, ops WeightedOperations,
|
||||
operationQueue OperationQueue, timeOperationQueue []FutureOperation,
|
||||
totalNumBlocks, avgBlockSize int, logWriter LogWriter, lean bool) blockSimFn {
|
||||
|
@ -237,7 +238,7 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, params
|
|||
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
|
||||
accounts []Account, header abci.Header) (opCount int) {
|
||||
|
||||
fmt.Printf("\rSimulating... block %d/%d, operation %d/%d. ",
|
||||
fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ",
|
||||
header.Height, totalNumBlocks, opCount, blocksize)
|
||||
lastBlocksizeState, blocksize = getBlockSize(r, params, lastBlocksizeState, avgBlockSize)
|
||||
|
||||
|
@ -277,7 +278,7 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, params
|
|||
assertAllInvariants(t, app, invariants, eventStr, logWriter)
|
||||
}
|
||||
if opCount%50 == 0 {
|
||||
fmt.Printf("\rSimulating... block %d/%d, operation %d/%d. ",
|
||||
fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ",
|
||||
header.Height, totalNumBlocks, opCount, blocksize)
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue