Merge PR #2285: simulation: Write logs on panic
This commit is contained in:
parent
2a49406e9b
commit
962b04d985
|
@ -107,6 +107,7 @@ IMPROVEMENTS
|
||||||
* [store] \#1952, \#2281 Update IAVL dependency to v0.11.0
|
* [store] \#1952, \#2281 Update IAVL dependency to v0.11.0
|
||||||
* [simulation] Make timestamps randomized [#2153](https://github.com/cosmos/cosmos-sdk/pull/2153)
|
* [simulation] Make timestamps randomized [#2153](https://github.com/cosmos/cosmos-sdk/pull/2153)
|
||||||
* [simulation] Make logs not just pure strings, speeding it up by a large factor at greater block heights \#2282
|
* [simulation] Make logs not just pure strings, speeding it up by a large factor at greater block heights \#2282
|
||||||
|
* [simulation] Logs get written to file if large, and also get printed on panics \#2285
|
||||||
|
|
||||||
* Tendermint
|
* Tendermint
|
||||||
|
|
||||||
|
|
|
@ -131,7 +131,7 @@ func BenchmarkFullGaiaSimulation(b *testing.B) {
|
||||||
|
|
||||||
// Run randomized simulation
|
// Run randomized simulation
|
||||||
// TODO parameterize numbers, save for a later PR
|
// TODO parameterize numbers, save for a later PR
|
||||||
simulation.SimulateFromSeed(
|
err := simulation.SimulateFromSeed(
|
||||||
b, app.BaseApp, appStateFn, seed,
|
b, app.BaseApp, appStateFn, seed,
|
||||||
testAndRunTxs(app),
|
testAndRunTxs(app),
|
||||||
[]simulation.RandSetup{},
|
[]simulation.RandSetup{},
|
||||||
|
@ -140,6 +140,10 @@ func BenchmarkFullGaiaSimulation(b *testing.B) {
|
||||||
blockSize,
|
blockSize,
|
||||||
commit,
|
commit,
|
||||||
)
|
)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println(err)
|
||||||
|
b.Fail()
|
||||||
|
}
|
||||||
if commit {
|
if commit {
|
||||||
fmt.Println("GoLevelDB Stats")
|
fmt.Println("GoLevelDB Stats")
|
||||||
fmt.Println(db.Stats()["leveldb.stats"])
|
fmt.Println(db.Stats()["leveldb.stats"])
|
||||||
|
@ -164,7 +168,7 @@ func TestFullGaiaSimulation(t *testing.T) {
|
||||||
require.Equal(t, "GaiaApp", app.Name())
|
require.Equal(t, "GaiaApp", app.Name())
|
||||||
|
|
||||||
// Run randomized simulation
|
// Run randomized simulation
|
||||||
simulation.SimulateFromSeed(
|
err := simulation.SimulateFromSeed(
|
||||||
t, app.BaseApp, appStateFn, seed,
|
t, app.BaseApp, appStateFn, seed,
|
||||||
testAndRunTxs(app),
|
testAndRunTxs(app),
|
||||||
[]simulation.RandSetup{},
|
[]simulation.RandSetup{},
|
||||||
|
@ -176,6 +180,7 @@ func TestFullGaiaSimulation(t *testing.T) {
|
||||||
if commit {
|
if commit {
|
||||||
fmt.Println("Database Size", db.Stats()["database.size"])
|
fmt.Println("Database Size", db.Stats()["database.size"])
|
||||||
}
|
}
|
||||||
|
require.Nil(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: Make another test for the fuzzer itself, which just has noOp txs
|
// TODO: Make another test for the fuzzer itself, which just has noOp txs
|
||||||
|
|
|
@ -7,6 +7,7 @@ import (
|
||||||
"math/rand"
|
"math/rand"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
|
"runtime/debug"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
|
@ -26,9 +27,9 @@ import (
|
||||||
func Simulate(
|
func Simulate(
|
||||||
t *testing.T, app *baseapp.BaseApp, appStateFn func(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage, ops []Operation, setups []RandSetup,
|
t *testing.T, app *baseapp.BaseApp, appStateFn func(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage, ops []Operation, setups []RandSetup,
|
||||||
invariants []Invariant, numBlocks int, blockSize int, commit bool,
|
invariants []Invariant, numBlocks int, blockSize int, commit bool,
|
||||||
) {
|
) error {
|
||||||
time := time.Now().UnixNano()
|
time := time.Now().UnixNano()
|
||||||
SimulateFromSeed(t, app, appStateFn, time, ops, setups, invariants, numBlocks, blockSize, commit)
|
return SimulateFromSeed(t, app, appStateFn, time, ops, setups, invariants, numBlocks, blockSize, commit)
|
||||||
}
|
}
|
||||||
|
|
||||||
func initChain(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress, setups []RandSetup, app *baseapp.BaseApp,
|
func initChain(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress, setups []RandSetup, app *baseapp.BaseApp,
|
||||||
|
@ -56,7 +57,9 @@ func randTimestamp(r *rand.Rand) time.Time {
|
||||||
func SimulateFromSeed(
|
func SimulateFromSeed(
|
||||||
tb testing.TB, app *baseapp.BaseApp, appStateFn func(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage, seed int64, ops []Operation, setups []RandSetup,
|
tb testing.TB, app *baseapp.BaseApp, appStateFn func(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage, seed int64, ops []Operation, setups []RandSetup,
|
||||||
invariants []Invariant, numBlocks int, blockSize int, commit bool,
|
invariants []Invariant, numBlocks int, blockSize int, commit bool,
|
||||||
) {
|
) (simError error) {
|
||||||
|
// in case we have to end early, don't os.Exit so that we can run cleanup code.
|
||||||
|
stopEarly := false
|
||||||
testingMode, t, b := getTestingMode(tb)
|
testingMode, t, b := getTestingMode(tb)
|
||||||
fmt.Printf("Starting SimulateFromSeed with randomness created with seed %d\n", int(seed))
|
fmt.Printf("Starting SimulateFromSeed with randomness created with seed %d\n", int(seed))
|
||||||
r := rand.New(rand.NewSource(seed))
|
r := rand.New(rand.NewSource(seed))
|
||||||
|
@ -79,12 +82,12 @@ func SimulateFromSeed(
|
||||||
|
|
||||||
// Setup code to catch SIGTERM's
|
// Setup code to catch SIGTERM's
|
||||||
c := make(chan os.Signal)
|
c := make(chan os.Signal)
|
||||||
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
|
signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGINT)
|
||||||
go func() {
|
go func() {
|
||||||
<-c
|
receivedSignal := <-c
|
||||||
fmt.Printf("Exiting early due to SIGTERM, on block %d, operation %d\n", header.Height, opCount)
|
fmt.Printf("Exiting early due to %s, on block %d, operation %d\n", receivedSignal, header.Height, opCount)
|
||||||
DisplayEvents(events)
|
simError = fmt.Errorf("Exited due to %s", receivedSignal)
|
||||||
os.Exit(128 + int(syscall.SIGTERM))
|
stopEarly = true
|
||||||
}()
|
}()
|
||||||
|
|
||||||
var pastTimes []time.Time
|
var pastTimes []time.Time
|
||||||
|
@ -95,15 +98,27 @@ func SimulateFromSeed(
|
||||||
operationQueue := make(map[int][]Operation)
|
operationQueue := make(map[int][]Operation)
|
||||||
var blockLogBuilders []*strings.Builder
|
var blockLogBuilders []*strings.Builder
|
||||||
|
|
||||||
if !testingMode {
|
if testingMode {
|
||||||
b.ResetTimer()
|
|
||||||
} else {
|
|
||||||
blockLogBuilders = make([]*strings.Builder, numBlocks)
|
blockLogBuilders = make([]*strings.Builder, numBlocks)
|
||||||
}
|
}
|
||||||
displayLogs := logPrinter(testingMode, blockLogBuilders)
|
displayLogs := logPrinter(testingMode, blockLogBuilders)
|
||||||
blockSimulator := createBlockSimulator(testingMode, tb, t, event, invariants, ops, operationQueue, numBlocks, displayLogs)
|
blockSimulator := createBlockSimulator(testingMode, tb, t, event, invariants, ops, operationQueue, numBlocks, displayLogs)
|
||||||
|
if !testingMode {
|
||||||
|
b.ResetTimer()
|
||||||
|
} else {
|
||||||
|
// Recover logs in case of panic
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
fmt.Println("Panic with err\n", r)
|
||||||
|
stackTrace := string(debug.Stack())
|
||||||
|
fmt.Println(stackTrace)
|
||||||
|
displayLogs()
|
||||||
|
simError = fmt.Errorf("Simulation halted due to panic on block %d", header.Height)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
for i := 0; i < numBlocks; i++ {
|
for i := 0; i < numBlocks && !stopEarly; i++ {
|
||||||
// Log the header time for future lookup
|
// Log the header time for future lookup
|
||||||
pastTimes = append(pastTimes, header.Time)
|
pastTimes = append(pastTimes, header.Time)
|
||||||
pastSigningValidators = append(pastSigningValidators, request.LastCommitInfo.Validators)
|
pastSigningValidators = append(pastSigningValidators, request.LastCommitInfo.Validators)
|
||||||
|
@ -122,10 +137,9 @@ func SimulateFromSeed(
|
||||||
|
|
||||||
// Run queued operations. Ignores blocksize if blocksize is too small
|
// Run queued operations. Ignores blocksize if blocksize is too small
|
||||||
numQueuedOpsRan := runQueuedOperations(operationQueue, int(header.Height), tb, r, app, ctx, keys, logWriter, displayLogs, event)
|
numQueuedOpsRan := runQueuedOperations(operationQueue, int(header.Height), tb, r, app, ctx, keys, logWriter, displayLogs, event)
|
||||||
opCount += numQueuedOpsRan
|
|
||||||
thisBlockSize -= numQueuedOpsRan
|
thisBlockSize -= numQueuedOpsRan
|
||||||
operations := blockSimulator(thisBlockSize, r, app, ctx, keys, header, logWriter)
|
operations := blockSimulator(thisBlockSize, r, app, ctx, keys, header, logWriter)
|
||||||
opCount += operations
|
opCount += operations + numQueuedOpsRan
|
||||||
|
|
||||||
res := app.EndBlock(abci.RequestEndBlock{})
|
res := app.EndBlock(abci.RequestEndBlock{})
|
||||||
header.Height++
|
header.Height++
|
||||||
|
@ -146,9 +160,13 @@ func SimulateFromSeed(
|
||||||
// Update the validator set
|
// Update the validator set
|
||||||
validators = updateValidators(tb, r, validators, res.ValidatorUpdates, event)
|
validators = updateValidators(tb, r, validators, res.ValidatorUpdates, event)
|
||||||
}
|
}
|
||||||
|
if stopEarly {
|
||||||
|
DisplayEvents(events)
|
||||||
|
return
|
||||||
|
}
|
||||||
fmt.Printf("\nSimulation complete. Final height (blocks): %d, final time (seconds), : %v, operations ran %d\n", header.Height, header.Time, opCount)
|
fmt.Printf("\nSimulation complete. Final height (blocks): %d, final time (seconds), : %v, operations ran %d\n", header.Height, header.Time, opCount)
|
||||||
DisplayEvents(events)
|
DisplayEvents(events)
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns a function to simulate blocks. Written like this to avoid constant parameters being passed everytime, to minimize
|
// Returns a function to simulate blocks. Written like this to avoid constant parameters being passed everytime, to minimize
|
||||||
|
|
|
@ -3,9 +3,11 @@ package simulation
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
|
"os"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/tendermint/tendermint/crypto"
|
"github.com/tendermint/tendermint/crypto"
|
||||||
|
|
||||||
|
@ -94,15 +96,36 @@ func assertAllInvariants(t *testing.T, app *baseapp.BaseApp, invariants []Invari
|
||||||
func logPrinter(testingmode bool, logs []*strings.Builder) func() {
|
func logPrinter(testingmode bool, logs []*strings.Builder) func() {
|
||||||
if testingmode {
|
if testingmode {
|
||||||
return func() {
|
return func() {
|
||||||
|
numLoggers := 0
|
||||||
for i := 0; i < len(logs); i++ {
|
for i := 0; i < len(logs); i++ {
|
||||||
// We're passed the last created block
|
// We're passed the last created block
|
||||||
if logs[i] == nil {
|
if logs[i] == nil {
|
||||||
return
|
numLoggers = i - 1
|
||||||
|
break
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
var f *os.File
|
||||||
|
if numLoggers > 10 {
|
||||||
|
fileName := fmt.Sprintf("simulation_log_%s.txt", time.Now().Format("2006-01-02 15:04:05"))
|
||||||
|
fmt.Printf("Too many logs to display, instead writing to %s\n", fileName)
|
||||||
|
f, _ = os.Create(fileName)
|
||||||
|
}
|
||||||
|
for i := 0; i < numLoggers; i++ {
|
||||||
|
if f != nil {
|
||||||
|
_, err := f.WriteString(fmt.Sprintf("Begin block %d\n", i))
|
||||||
|
if err != nil {
|
||||||
|
panic("Failed to write logs to file")
|
||||||
|
}
|
||||||
|
_, err = f.WriteString((*logs[i]).String())
|
||||||
|
if err != nil {
|
||||||
|
panic("Failed to write logs to file")
|
||||||
|
}
|
||||||
|
} else {
|
||||||
fmt.Printf("Begin block %d\n", i)
|
fmt.Printf("Begin block %d\n", i)
|
||||||
fmt.Println((*logs[i]).String())
|
fmt.Println((*logs[i]).String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
return func() {}
|
return func() {}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue