certusone
/
cosmos-sdk
mirror of https://github.com/certusone/cosmos-sdk.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

[runsim] push logs to S3 (#4666)

This commit is contained in:
mircea-c 2019-07-02 12:57:19 -04:00 committed by Alessio Treglia
parent 179476d6b0
commit 0d3b4ba0d6
4 changed files with 105 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
contrib/runsim/main.go
View File

@ -52,6 +52,7 @@ var (
genesis string
exitOnFail bool
githubConfig string
gitRevision string
slackConfig string
// integration with Slack and Github
@ -81,12 +82,13 @@ func init() {
flag.StringVar(&genesis, "g", "", "Genesis file")
flag.StringVar(&seedOverrideList, "seeds", "", "run the supplied comma-separated list of seeds instead of defaults")
flag.BoolVar(&exitOnFail, "e", false, "Exit on fail during multi-sim, print error")
flag.StringVar(&gitRevision, "rev", "", "git revision")
flag.StringVar(&githubConfig, "github", "", "Report results to Github's PR")
flag.StringVar(&slackConfig, "slack", "", "Report results to slack channel")
flag.Usage = func() {
_, _ = fmt.Fprintf(flag.CommandLine.Output(),
`Usage: %s [-j maxprocs] [-seeds comma-separated-seed-list] [-g genesis.json] [-e] [-github token,pr-url] [-slack token,channel,thread] [package] [blocks] [period] [testname]
`Usage: %s [-j maxprocs] [-seeds comma-separated-seed-list] [-rev git-commmit-hash] [-g genesis.json] [-e] [-github token,pr-url] [-slack token,channel,thread] [package] [blocks] [period] [testname]
Run simulations in parallel`, filepath.Base(os.Args[0]))
flag.PrintDefaults()
}
@ -209,11 +211,11 @@ wait:
os.Exit(0)
}
func buildCommand(testname, blocks, period, genesis string, seed int) string {
func buildCommand(testName, blocks, period, genesis string, seed int) string {
return fmt.Sprintf("go test %s -run %s -SimulationEnabled=true "+
"-SimulationNumBlocks=%s -SimulationGenesis=%s "+
"-SimulationVerbose=true -SimulationCommit=true -SimulationSeed=%d -SimulationPeriod=%s -v -timeout 24h",
pkgName, testname, blocks, genesis, seed, period)
pkgName, testName, blocks, genesis, seed, period)
}
func makeCmd(cmdStr string) *exec.Cmd {
@ -228,7 +230,8 @@ func makeFilename(seed int) string {
func worker(id int, seeds <-chan int) {
log.Printf("[W%d] Worker is up and running", id)
for seed := range seeds {
if err := spawnProc(id, seed); err != nil {
stdOut, stdErr, err := spawnProc(id, seed)
if err != nil {
results <- false
log.Printf("[W%d] Seed %d: FAILED", id, seed)
log.Printf("To reproduce run: %s", buildCommand(testname, blocks, period, genesis, seed))
@ -242,12 +245,13 @@ func worker(id int, seeds <-chan int) {
} else {
log.Printf("[W%d] Seed %d: OK", id, seed)
}
pushLogs(stdOut, stdErr, gitRevision)
}
log.Printf("[W%d] no seeds left, shutting down", id)
}
func spawnProc(workerID int, seed int) error {
func spawnProc(workerID int, seed int) (*os.File, *os.File, error) {
stderrFile, _ := os.Create(filepath.Join(tempdir, makeFilename(seed)+".stderr"))
stdoutFile, _ := os.Create(filepath.Join(tempdir, makeFilename(seed)+".stdout"))
s := buildCommand(testname, blocks, period, genesis, seed)
@ -261,7 +265,7 @@ func spawnProc(workerID int, seed int) error {
} else {
stderr, err = cmd.StderrPipe()
if err != nil {
return err
return nil, nil, err
}
}
sc := bufio.NewScanner(stderr)
@ -269,7 +273,7 @@ func spawnProc(workerID int, seed int) error {
err = cmd.Start()
if err != nil {
log.Printf("couldn't start %q", s)
return err
return nil, nil, err
}
log.Printf("[W%d] Spawned simulation with pid %d [seed=%d stdout=%s stderr=%s]",
workerID, cmd.Process.Pid, seed, stdoutFile.Name(), stderrFile.Name())
@ -285,7 +289,7 @@ func spawnProc(workerID int, seed int) error {
fmt.Printf("stderr: %s\n", sc.Text())
}
}
return err
return stdoutFile, stderrFile, err
}
func pushProcess(proc *os.Process) {

105
contrib/runsim/notification.go
View File

@ -1,10 +1,97 @@
package main
import (
"github.com/nlopes/slack"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/nlopes/slack"
)
const (
logBucketPrefix = "sim-logs-"
awsRegion = "us-east-1"
)
var (
simTimeStamp = time.Now().Format("01-02-2006_15:05:05")
)
func awsErrHandler(err error) {
if awsErr, ok := err.(awserr.Error); ok {
switch awsErr.Code() {
default:
log.Println(awsErr.Error())
}
} else {
log.Println(err.Error())
}
}
func makeObjKey(folderName string, fileName string) string {
return fmt.Sprintf("%s/%s/%s", folderName, simTimeStamp, fileName)
}
func putObj(fileHandle *os.File, svc *s3.S3, folderName string, bucketName string) {
_, _ = fileHandle.Seek(0, 0)
stdOutObjInput := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(fileHandle),
Bucket: aws.String(bucketName),
Key: aws.String(makeObjKey(folderName, filepath.Base(fileHandle.Name()))),
}
if output, err := svc.PutObject(stdOutObjInput); err != nil {
awsErrHandler(err)
} else {
log.Printf("Log file pushed: %s", output.String())
}
}
func pushLogs(stdOut *os.File, stdErr *os.File, folderName string) {
var logBucket *string
sessionS3 := s3.New(session.Must(session.NewSession(&aws.Config{
Region: aws.String(awsRegion),
})))
if listBucketsOutput, err := sessionS3.ListBuckets(&s3.ListBucketsInput{}); err != nil {
awsErrHandler(err)
} else {
for _, bucket := range listBucketsOutput.Buckets {
if strings.Contains(*bucket.Name, logBucketPrefix) {
logBucket = bucket.Name
putObj(stdOut, sessionS3, folderName, *logBucket)
putObj(stdErr, sessionS3, folderName, *logBucket)
break
}
}
}
if logBucket == nil {
log.Println("Log bucket not found")
}
}
func slackMessage(token string, channel string, threadTS *string, message string) {
client := slack.New(token)
if threadTS != nil {
_, _, err := client.PostMessage(channel, slack.MsgOptionText(message, false), slack.MsgOptionTS(*threadTS))
if err != nil {
log.Printf("ERROR: %v", err)
}
} else {
_, _, err := client.PostMessage(channel, slack.MsgOptionText(message, false))
if err != nil {
log.Printf("ERROR: %v", err)
}
}
}
//type GithubPayload struct {
// Issue struct {
// Number int `json:"number"`
@ -32,22 +119,6 @@ import (
// } `json:"head"`
//}
func slackMessage(token string, channel string, threadTS *string, message string) {
client := slack.New(token)
if threadTS != nil {
_, _, err := client.PostMessage(channel, slack.MsgOptionText(message, false), slack.MsgOptionTS(*threadTS))
if err != nil {
log.Printf("ERROR: %v", err)
}
} else {
_, _, err := client.PostMessage(channel, slack.MsgOptionText(message, false))
if err != nil {
log.Printf("ERROR: %v", err)
}
}
}
//func createCheckRun(client *github.Client, payload GithubPayload, pr PullRequestDetails) error {
// var opt github.CreateCheckRunOptions
// opt.Name = "Test Check"

1
go.mod
View File

@ -1,6 +1,7 @@
module github.com/cosmos/cosmos-sdk
require (
github.com/aws/aws-sdk-go v1.20.11
github.com/bartekn/go-bip39 v0.0.0-20171116152956-a05967ea095d
github.com/bgentry/speakeasy v0.1.0
github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d

4
go.sum
View File

@ -7,6 +7,8 @@ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBA
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/aws/aws-sdk-go v1.20.11 h1:xDc2f/8KmwPW7WkuB0kDUCEP4jpx1PIMMMZkav6cbU4=
github.com/aws/aws-sdk-go v1.20.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/bartekn/go-bip39 v0.0.0-20171116152956-a05967ea095d h1:1aAija9gr0Hyv4KfQcRcwlmFIrhkDmIj2dz5bkg/s/8=
github.com/bartekn/go-bip39 v0.0.0-20171116152956-a05967ea095d/go.mod h1:icNx/6QdFblhsEjZehARqbNumymUT/ydwlLojFdv7Sk=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
@ -82,6 +84,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U=
github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ=
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=