certusone
/
cosmos-sdk
mirror of https://github.com/certusone/cosmos-sdk.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge PR #4079: Implement Invariant Benchmarks

This commit is contained in:
Alexander Bezobchuk 2019-04-12 18:52:16 -04:00 committed by GitHub
parent 3c88ddc2f5
commit 722d122da7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 88 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.pending/improvements/sdk/3914-Implement-invariant-benchmarks-and-add-target-to-makefile Normal file
View File

@ -0,0 +1 @@
#3914 Implement invariant benchmarks and add target to makefile.

8
Makefile
View File

@ -181,6 +181,12 @@ test_sim_gaia_multi_seed:
@echo "Running multi-seed Gaia simulation. This may take awhile!"
@bash scripts/multisim.sh 400 5 TestFullGaiaSimulation
test_sim_benchmark_invariants:
@echo "Running simulation invariant benchmarks..."
@go test -mod=readonly ./cmd/gaia/app -benchmem -bench=BenchmarkInvariants -run=^$ \
-SimulationEnabled=true -SimulationNumBlocks=1000 -SimulationBlockSize=200 \
-SimulationCommit=true -SimulationSeed=57 -v -timeout 24h
SIM_NUM_BLOCKS ?= 500
SIM_BLOCK_SIZE ?= 200
SIM_COMMIT ?= true
@ -268,5 +274,5 @@ test_cover lint benchmark devdoc_init devdoc devdoc_save devdoc_update \
build-linux build-docker-gaiadnode localnet-start localnet-stop \
format check-ledger test_sim_gaia_nondeterminism test_sim_modules test_sim_gaia_fast \
test_sim_gaia_custom_genesis_fast test_sim_gaia_custom_genesis_multi_seed \
test_sim_gaia_multi_seed test_sim_gaia_import_export \
test_sim_gaia_multi_seed test_sim_gaia_import_export test_sim_benchmark_invariants \
go-mod-cache

58
cmd/gaia/app/sim_test.go
View File

@ -4,6 +4,7 @@ import (
"encoding/json"
"flag"
"fmt"
"io"
"io/ioutil"
"math/rand"
"os"
@ -61,11 +62,11 @@ func init() {
}
// helper function for populating input for SimulateFromSeed
func getSimulateFromSeedInput(tb testing.TB, app *GaiaApp) (
testing.TB, *baseapp.BaseApp, simulation.AppStateFn, int64,
func getSimulateFromSeedInput(tb testing.TB, w io.Writer, app *GaiaApp) (
testing.TB, io.Writer, *baseapp.BaseApp, simulation.AppStateFn, int64,
simulation.WeightedOperations, sdk.Invariants, int, int, bool, bool) {
return tb, app.BaseApp, appStateFn, seed,
return tb, w, app.BaseApp, appStateFn, seed,
testAndRunTxs(app), invariants(app), numBlocks, blockSize, commit, lean
}
@ -318,7 +319,7 @@ func BenchmarkFullGaiaSimulation(b *testing.B) {
// Run randomized simulation
// TODO parameterize numbers, save for a later PR
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, app))
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, os.Stdout, app))
if err != nil {
fmt.Println(err)
b.Fail()
@ -353,7 +354,7 @@ func TestFullGaiaSimulation(t *testing.T) {
require.Equal(t, "GaiaApp", app.Name())
// Run randomized simulation
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, app))
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
if commit {
// for memdb:
// fmt.Println("Database Size", db.Stats()["database.size"])
@ -387,7 +388,7 @@ func TestGaiaImportExport(t *testing.T) {
require.Equal(t, "GaiaApp", app.Name())
// Run randomized simulation
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, app))
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
if commit {
// for memdb:
@ -477,7 +478,7 @@ func TestGaiaSimulationAfterImport(t *testing.T) {
require.Equal(t, "GaiaApp", app.Name())
// Run randomized simulation
stopEarly, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, app))
stopEarly, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
if commit {
// for memdb:
@ -516,7 +517,7 @@ func TestGaiaSimulationAfterImport(t *testing.T) {
})
// Run randomized simulation on imported app
_, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, newApp))
_, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, newApp))
require.Nil(t, err)
}
@ -541,7 +542,7 @@ func TestAppStateDeterminism(t *testing.T) {
// Run randomized simulation
simulation.SimulateFromSeed(
t, app.BaseApp, appStateFn, seed,
t, os.Stdout, app.BaseApp, appStateFn, seed,
testAndRunTxs(app),
[]sdk.Invariant{},
50,
@ -557,3 +558,42 @@ func TestAppStateDeterminism(t *testing.T) {
}
}
}
func BenchmarkInvariants(b *testing.B) {
// 1. Setup a simulated Gaia application
logger := log.NewNopLogger()
dir, _ := ioutil.TempDir("", "goleveldb-gaia-invariant-bench")
db, _ := sdk.NewLevelDB("simulation", dir)
defer func() {
db.Close()
os.RemoveAll(dir)
}()
app := NewGaiaApp(logger, db, nil, true, false)
// 2. Run parameterized simulation (w/o invariants)
_, err := simulation.SimulateFromSeed(
b, ioutil.Discard, app.BaseApp, appStateFn, seed, testAndRunTxs(app),
[]sdk.Invariant{}, numBlocks, blockSize, commit, lean,
)
if err != nil {
fmt.Println(err)
b.FailNow()
}
ctx := app.NewContext(true, abci.Header{Height: app.LastBlockHeight() + 1})
// 3. Benchmark each invariant separately
//
// NOTE: We use the crisis keeper as it has all the invariants registered with
// their respective metadata which makes it useful for testing/benchmarking.
for _, cr := range app.crisisKeeper.Routes() {
b.Run(fmt.Sprintf("%s/%s", cr.ModuleName, cr.Route), func(b *testing.B) {
if err := cr.Invar(ctx); err != nil {
fmt.Println(err)
b.FailNow()
}
})
}
}

2
types/invariant.go
View File

@ -6,5 +6,5 @@ package types
// The simulator will then halt and print the logs.
type Invariant func(ctx Context) error
// group of Invarient
// Invariants defines a group of invariants
type Invariants []Invariant

9
x/simulation/event_stats.go
View File

@ -2,6 +2,7 @@ package simulation
import (
"fmt"
"io"
"sort"
)
@ -17,14 +18,16 @@ func (es eventStats) tally(eventDesc string) {
}
// Pretty-print events as a table
func (es eventStats) Print() {
func (es eventStats) Print(w io.Writer) {
var keys []string
for key := range es {
keys = append(keys, key)
}
sort.Strings(keys)
fmt.Printf("Event statistics: \n")
fmt.Fprintf(w, "Event statistics: \n")
for _, key := range keys {
fmt.Printf(" % 60s => %d\n", key, es[key])
fmt.Fprintf(w, " % 60s => %d\n", key, es[key])
}
}

47
x/simulation/simulate.go
View File

@ -3,6 +3,7 @@ package simulation
import (
"encoding/json"
"fmt"
"io"
"math/rand"
"os"
"os/signal"
@ -26,7 +27,7 @@ func Simulate(t *testing.T, app *baseapp.BaseApp,
invariants sdk.Invariants, numBlocks, blockSize int, commit, lean bool) (bool, error) {
time := time.Now().UnixNano()
return SimulateFromSeed(t, app, appStateFn, time, ops,
return SimulateFromSeed(t, os.Stdout, app, appStateFn, time, ops,
invariants, numBlocks, blockSize, commit, lean)
}
@ -51,19 +52,20 @@ func initChain(
// SimulateFromSeed tests an application by running the provided
// operations, testing the provided invariants, but using the provided seed.
// TODO split this monster function up
func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
func SimulateFromSeed(
tb testing.TB, w io.Writer, app *baseapp.BaseApp,
appStateFn AppStateFn, seed int64, ops WeightedOperations,
invariants sdk.Invariants,
numBlocks, blockSize int, commit, lean bool) (stopEarly bool, simError error) {
numBlocks, blockSize int, commit, lean bool,
) (stopEarly bool, simError error) {
// in case we have to end early, don't os.Exit so that we can run cleanup code.
testingMode, t, b := getTestingMode(tb)
fmt.Printf("Starting SimulateFromSeed with randomness "+
"created with seed %d\n", int(seed))
fmt.Fprintf(w, "Starting SimulateFromSeed with randomness created with seed %d\n", int(seed))
r := rand.New(rand.NewSource(seed))
params := RandomParams(r) // := DefaultParams()
fmt.Printf("Randomized simulation params: %+v\n", params)
fmt.Fprintf(w, "Randomized simulation params: %+v\n", params)
genesisTimestamp := RandTimestamp(r)
fmt.Printf("Starting the simulation from time %v, unixtime %v\n",
@ -94,8 +96,7 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGINT)
go func() {
receivedSignal := <-c
fmt.Printf("\nExiting early due to %s, on block %d, operation %d\n",
receivedSignal, header.Height, opCount)
fmt.Fprintf(w, "\nExiting early due to %s, on block %d, operation %d\n", receivedSignal, header.Height, opCount)
simError = fmt.Errorf("Exited due to %s", receivedSignal)
stopEarly = true
}()
@ -113,7 +114,7 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
logWriter := NewLogWriter(testingMode)
blockSimulator := createBlockSimulator(
testingMode, tb, t, params, eventStats.tally, invariants,
testingMode, tb, t, w, params, eventStats.tally, invariants,
ops, operationQueue, timeOperationQueue,
numBlocks, blockSize, logWriter, lean)
@ -123,13 +124,11 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
// Recover logs in case of panic
defer func() {
if r := recover(); r != nil {
fmt.Println("Panic with err\n", r)
fmt.Fprintf(w, "panic with err: %v\n", r)
stackTrace := string(debug.Stack())
fmt.Println(stackTrace)
logWriter.PrintLogs()
simError = fmt.Errorf(
"Simulation halted due to panic on block %d",
header.Height)
simError = fmt.Errorf("Simulation halted due to panic on block %d", header.Height)
}
}()
}
@ -188,8 +187,7 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
}
if header.ProposerAddress == nil {
fmt.Printf("\nSimulation stopped early as all validators " +
"have been unbonded, there is nobody left propose a block!\n")
fmt.Fprintf(w, "\nSimulation stopped early as all validators have been unbonded; nobody left to propose a block!\n")
stopEarly = true
break
}
@ -207,14 +205,17 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
}
if stopEarly {
eventStats.Print()
eventStats.Print(w)
return true, simError
}
fmt.Printf("\nSimulation complete. Final height (blocks): %d, "+
"final time (seconds), : %v, operations ran %d\n",
header.Height, header.Time, opCount)
eventStats.Print()
fmt.Fprintf(
w,
"\nSimulation complete; Final height (blocks): %d, final time (seconds): %v, operations ran: %d\n",
header.Height, header.Time, opCount,
)
eventStats.Print(w)
return false, nil
}
@ -225,7 +226,7 @@ type blockSimFn func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
// Returns a function to simulate blocks. Written like this to avoid constant
// parameters being passed everytime, to minimize memory overhead.
func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, params Params,
func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Writer, params Params,
event func(string), invariants sdk.Invariants, ops WeightedOperations,
operationQueue OperationQueue, timeOperationQueue []FutureOperation,
totalNumBlocks, avgBlockSize int, logWriter LogWriter, lean bool) blockSimFn {
@ -237,7 +238,7 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, params
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
accounts []Account, header abci.Header) (opCount int) {
fmt.Printf("\rSimulating... block %d/%d, operation %d/%d. ",
fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ",
header.Height, totalNumBlocks, opCount, blocksize)
lastBlocksizeState, blocksize = getBlockSize(r, params, lastBlocksizeState, avgBlockSize)
@ -277,7 +278,7 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, params
assertAllInvariants(t, app, invariants, eventStr, logWriter)
}
if opCount%50 == 0 {
fmt.Printf("\rSimulating... block %d/%d, operation %d/%d. ",
fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ",
header.Height, totalNumBlocks, opCount, blocksize)
}
}