Merge PR #4161: New experimental simulation runner
This commit is contained in:
Alessio Treglia
Alexander Bezobchuk
parent
df6f2d6e25
commit
4bf9d2b3ec
|
|
@ -196,7 +196,8 @@ jobs:
|
|||
command: |
|
||||
export PATH="$GOBIN:$PATH"
|
||||
export GO111MODULE=on
|
||||
cmd/gaia/contrib/sim/multisim.sh 500 50 TestFullGaiaSimulation
|
||||
make runsim
|
||||
runsim 500 50 TestFullGaiaSimulation
|
||||
|
||||
test_sim_gaia_multi_seed:
|
||||
<<: *linux_defaults
|
||||
|
|
@ -214,7 +215,8 @@ jobs:
|
|||
command: |
|
||||
export PATH="$GOBIN:$PATH"
|
||||
export GO111MODULE=on
|
||||
cmd/gaia/contrib/sim/multisim.sh 50 10 TestFullGaiaSimulation
|
||||
make runsim
|
||||
runsim 50 10 TestFullGaiaSimulation
|
||||
|
||||
test_cover:
|
||||
<<: *linux_defaults
|
||||
|
|
|
|||
21
Makefile
21
Makefile
|
|
@ -165,22 +165,22 @@ test_sim_gaia_fast:
|
|||
@echo "Running quick Gaia simulation. This may take several minutes..."
|
||||
@go test -mod=readonly ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=100 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -SimulationPeriod=5 -v -timeout 24h
|
||||
|
||||
test_sim_gaia_import_export:
|
||||
test_sim_gaia_import_export: runsim
|
||||
@echo "Running Gaia import/export simulation. This may take several minutes..."
|
||||
@bash cmd/gaia/contrib/sim/multisim.sh 50 5 TestGaiaImportExport
|
||||
$(GOBIN)/runsim 50 5 TestGaiaImportExport
|
||||
|
||||
test_sim_gaia_simulation_after_import:
|
||||
test_sim_gaia_simulation_after_import: runsim
|
||||
@echo "Running Gaia simulation-after-import. This may take several minutes..."
|
||||
@bash cmd/gaia/contrib/sim/multisim.sh 50 5 TestGaiaSimulationAfterImport
|
||||
$(GOBIN)/runsim 50 5 TestGaiaSimulationAfterImport
|
||||
|
||||
test_sim_gaia_custom_genesis_multi_seed:
|
||||
test_sim_gaia_custom_genesis_multi_seed: runsim
|
||||
@echo "Running multi-seed custom genesis simulation..."
|
||||
@echo "By default, ${HOME}/.gaiad/config/genesis.json will be used."
|
||||
@bash cmd/gaia/contrib/sim/multisim.sh 400 5 TestFullGaiaSimulation ${HOME}/.gaiad/config/genesis.json
|
||||
$(GOBIN)/runsim -g ${HOME}/.gaiad/config/genesis.json 400 5 TestFullGaiaSimulation
|
||||
|
||||
test_sim_gaia_multi_seed:
|
||||
test_sim_gaia_multi_seed: runsim
|
||||
@echo "Running multi-seed Gaia simulation. This may take awhile!"
|
||||
@bash cmd/gaia/contrib/sim/multisim.sh 400 5 TestFullGaiaSimulation
|
||||
$(GOBIN)/runsim 400 5 TestFullGaiaSimulation
|
||||
|
||||
test_sim_benchmark_invariants:
|
||||
@echo "Running simulation invariant benchmarks..."
|
||||
|
|
@ -188,6 +188,11 @@ test_sim_benchmark_invariants:
|
|||
-SimulationEnabled=true -SimulationNumBlocks=1000 -SimulationBlockSize=200 \
|
||||
-SimulationCommit=true -SimulationSeed=57 -v -timeout 24h
|
||||
|
||||
# Don't move it into tools - this will be gone once gaia has moved into the new repo
|
||||
runsim: $(GOBIN)/runsim
|
||||
$(GOBIN)/runsim: cmd/gaia/contrib/runsim/main.go
|
||||
go install github.com/cosmos/cosmos-sdk/cmd/gaia/contrib/runsim
|
||||
|
||||
SIM_NUM_BLOCKS ?= 500
|
||||
SIM_BLOCK_SIZE ?= 200
|
||||
SIM_COMMIT ?= true
|
||||
|
|
|
|||
|
|
@ -0,0 +1,228 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
// default seeds
|
||||
seeds = []int{
|
||||
1, 2, 4, 7, 32, 123, 124, 582, 1893, 2989,
|
||||
3012, 4728, 37827, 981928, 87821, 891823782,
|
||||
989182, 89182391, 11, 22, 44, 77, 99, 2020,
|
||||
3232, 123123, 124124, 582582, 18931893,
|
||||
29892989, 30123012, 47284728, 37827,
|
||||
}
|
||||
|
||||
// goroutine-safe process map
|
||||
procs map[int]*os.Process
|
||||
mutex *sync.Mutex
|
||||
|
||||
// results channel
|
||||
results chan bool
|
||||
|
||||
// command line arguments and options
|
||||
jobs int
|
||||
blocks string
|
||||
period string
|
||||
testname string
|
||||
genesis string
|
||||
|
||||
// logs temporary directory
|
||||
tempdir string
|
||||
)
|
||||
|
||||
func init() {
|
||||
log.SetPrefix("")
|
||||
log.SetFlags(0)
|
||||
|
||||
procs = map[int]*os.Process{}
|
||||
mutex = &sync.Mutex{}
|
||||
flag.IntVar(&jobs, "j", 10, "Number of parallel processes")
|
||||
flag.StringVar(&genesis, "g", "", "Genesis file")
|
||||
|
||||
flag.Usage = func() {
|
||||
fmt.Fprintf(flag.CommandLine.Output(),
|
||||
`Usage: %s [-j maxprocs] [-g genesis.json] [blocks] [period] [testname]
|
||||
Run simulations in parallel
|
||||
|
||||
`, filepath.Base(os.Args[0]))
|
||||
flag.PrintDefaults()
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
var err error
|
||||
|
||||
flag.Parse()
|
||||
if flag.NArg() != 3 {
|
||||
log.Fatal("wrong number of arguments")
|
||||
}
|
||||
|
||||
// prepare input channel
|
||||
queue := make(chan int, len(seeds))
|
||||
for _, seed := range seeds {
|
||||
queue <- seed
|
||||
}
|
||||
close(queue)
|
||||
|
||||
// jobs cannot be > len(seeds)
|
||||
if jobs > len(seeds) {
|
||||
jobs = len(seeds)
|
||||
}
|
||||
results = make(chan bool, len(seeds))
|
||||
|
||||
// setup signal handling
|
||||
sigs := make(chan os.Signal, 1)
|
||||
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
go func() {
|
||||
_ = <-sigs
|
||||
fmt.Println()
|
||||
|
||||
// drain the queue
|
||||
log.Printf("Draining seeds queue...")
|
||||
for seed := range queue {
|
||||
log.Printf("%d", seed)
|
||||
}
|
||||
log.Printf("Kill all remaining processes...")
|
||||
killAllProcs()
|
||||
os.Exit(1)
|
||||
}()
|
||||
|
||||
// initialise common test parameters
|
||||
blocks = flag.Arg(0)
|
||||
period = flag.Arg(1)
|
||||
testname = flag.Arg(2)
|
||||
tempdir, err = ioutil.TempDir("", "")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// set up worker pool
|
||||
wg := sync.WaitGroup{}
|
||||
for workerId := 0; workerId < jobs; workerId++ {
|
||||
wg.Add(1)
|
||||
|
||||
go func(workerId int) {
|
||||
defer wg.Done()
|
||||
worker(workerId, queue)
|
||||
}(workerId)
|
||||
}
|
||||
|
||||
// idiomatic hack required to use wg.Wait() with select
|
||||
waitCh := make(chan struct{})
|
||||
go func() {
|
||||
defer close(waitCh)
|
||||
wg.Wait()
|
||||
}()
|
||||
|
||||
wait:
|
||||
for {
|
||||
select {
|
||||
case <-waitCh:
|
||||
break wait
|
||||
case <-time.After(1 * time.Minute):
|
||||
fmt.Println(".")
|
||||
}
|
||||
}
|
||||
|
||||
// analyse results and exit with 1 on first error
|
||||
close(results)
|
||||
for rc := range results {
|
||||
if !rc {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
func buildCommand(testname, blocks, period, genesis string, seed int) string {
|
||||
return fmt.Sprintf("go test github.com/cosmos/cosmos-sdk/cmd/gaia/app -run %s -SimulationEnabled=true "+
|
||||
"-SimulationNumBlocks=%s -SimulationGenesis=%s "+
|
||||
"-SimulationVerbose=true -SimulationCommit=true -SimulationSeed=%d -SimulationPeriod=%s -v -timeout 24h",
|
||||
testname, blocks, genesis, seed, period)
|
||||
}
|
||||
|
||||
func makeCmd(cmdStr string) *exec.Cmd {
|
||||
cmdSlice := strings.Split(cmdStr, " ")
|
||||
return exec.Command(cmdSlice[0], cmdSlice[1:]...)
|
||||
}
|
||||
|
||||
func makeFilename(seed int) string {
|
||||
return fmt.Sprintf("gaia-simulation-seed-%d-date-%s", seed, time.Now().Format("01-02-2006_15:04:05.000000000"))
|
||||
}
|
||||
|
||||
func worker(id int, seeds <-chan int) {
|
||||
log.Printf("[W%d] Worker is up and running", id)
|
||||
for seed := range seeds {
|
||||
if err := spawnProc(id, seed); err != nil {
|
||||
results <- false
|
||||
log.Printf("[W%d] Seed %d: FAILED", id, seed)
|
||||
log.Printf("To reproduce run: %s", buildCommand(testname, blocks, period, genesis, seed))
|
||||
} else {
|
||||
log.Printf("[W%d] Seed %d: OK", id, seed)
|
||||
}
|
||||
}
|
||||
log.Printf("[W%d] no seeds left, shutting down", id)
|
||||
}
|
||||
|
||||
func spawnProc(workerId int, seed int) error {
|
||||
stderrFile, _ := os.Create(filepath.Join(tempdir, makeFilename(seed)+".stderr"))
|
||||
stdoutFile, _ := os.Create(filepath.Join(tempdir, makeFilename(seed)+".stdout"))
|
||||
s := buildCommand(testname, blocks, period, genesis, seed)
|
||||
cmd := makeCmd(s)
|
||||
cmd.Stdout = stdoutFile
|
||||
cmd.Stderr = stderrFile
|
||||
err := cmd.Start()
|
||||
if err != nil {
|
||||
log.Printf("couldn't start %q", s)
|
||||
return err
|
||||
}
|
||||
log.Printf("[W%d] Spawned simulation with pid %d [seed=%d stdout=%s stderr=%s]",
|
||||
workerId, cmd.Process.Pid, seed, stdoutFile.Name(), stderrFile.Name())
|
||||
pushProcess(cmd.Process)
|
||||
defer popProcess(cmd.Process)
|
||||
return cmd.Wait()
|
||||
}
|
||||
|
||||
func pushProcess(proc *os.Process) {
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
procs[proc.Pid] = proc
|
||||
}
|
||||
|
||||
func popProcess(proc *os.Process) {
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
if _, ok := procs[proc.Pid]; ok {
|
||||
delete(procs, proc.Pid)
|
||||
}
|
||||
}
|
||||
|
||||
func killAllProcs() {
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
for _, proc := range procs {
|
||||
checkSignal(proc, syscall.SIGTERM)
|
||||
checkSignal(proc, syscall.SIGKILL)
|
||||
}
|
||||
}
|
||||
|
||||
func checkSignal(proc *os.Process, signal syscall.Signal) {
|
||||
if err := proc.Signal(signal); err != nil {
|
||||
log.Printf("Failed to send %s to PID %d", signal, proc.Pid)
|
||||
}
|
||||
}
|
||||
|
|
@ -3,6 +3,10 @@
|
|||
########################################
|
||||
### Simulations
|
||||
|
||||
runsim: $(GOBIN)/runsim
|
||||
$(GOBIN)/runsim: contrib/runsim/main.go
|
||||
go install github.com/cosmos/cosmos-sdk/cmd/gaia/contrib/runsim
|
||||
|
||||
sim-gaia-nondeterminism:
|
||||
@echo "Running nondeterminism test..."
|
||||
@go test -mod=readonly ./cmd/gaia/app -run TestAppStateDeterminism -SimulationEnabled=true -v -timeout 10m
|
||||
|
|
@ -17,22 +21,22 @@ sim-gaia-fast:
|
|||
@echo "Running quick Gaia simulation. This may take several minutes..."
|
||||
@go test -mod=readonly github.com/cosmos/cosmos-sdk/cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=100 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -SimulationPeriod=5 -v -timeout 24h
|
||||
|
||||
sim-gaia-import-export:
|
||||
sim-gaia-import-export: runsim
|
||||
@echo "Running Gaia import/export simulation. This may take several minutes..."
|
||||
@bash contrib/sim/multisim.sh 50 5 TestGaiaImportExport
|
||||
$(GOBIN)/runsim 50 5 TestGaiaImportExport
|
||||
|
||||
sim-gaia-simulation-after-import:
|
||||
sim-gaia-simulation-after-import: runsim
|
||||
@echo "Running Gaia simulation-after-import. This may take several minutes..."
|
||||
@bash contrib/sim/multisim.sh 50 5 TestGaiaSimulationAfterImport
|
||||
$(GOBIN)/runsim 50 5 TestGaiaSimulationAfterImport
|
||||
|
||||
sim-gaia-custom-genesis-multi-seed:
|
||||
sim-gaia-custom-genesis-multi-seed: runsim
|
||||
@echo "Running multi-seed custom genesis simulation..."
|
||||
@echo "By default, ${HOME}/.gaiad/config/genesis.json will be used."
|
||||
@bash contrib/sim/multisim.sh 400 5 TestFullGaiaSimulation ${HOME}/.gaiad/config/genesis.json
|
||||
$(GOBIN)/runsim -g ${HOME}/.gaiad/config/genesis.json 400 5 TestFullGaiaSimulation
|
||||
|
||||
sim-gaia-multi-seed:
|
||||
sim-gaia-multi-seed: runsim
|
||||
@echo "Running multi-seed Gaia simulation. This may take awhile!"
|
||||
@bash contrib/sim/multisim.sh 400 5 TestFullGaiaSimulation
|
||||
$(GOBIN)/runsim 400 5 TestFullGaiaSimulation
|
||||
|
||||
sim-benchmark-invariants:
|
||||
@echo "Running simulation invariant benchmarks..."
|
||||
|
|
|
|||
Loading…
Reference in New Issue