[runsim] push logs to S3 (#4666)
This commit is contained in:
parent
179476d6b0
commit
0d3b4ba0d6
|
@ -52,6 +52,7 @@ var (
|
|||
genesis string
|
||||
exitOnFail bool
|
||||
githubConfig string
|
||||
gitRevision string
|
||||
slackConfig string
|
||||
|
||||
// integration with Slack and Github
|
||||
|
@ -81,12 +82,13 @@ func init() {
|
|||
flag.StringVar(&genesis, "g", "", "Genesis file")
|
||||
flag.StringVar(&seedOverrideList, "seeds", "", "run the supplied comma-separated list of seeds instead of defaults")
|
||||
flag.BoolVar(&exitOnFail, "e", false, "Exit on fail during multi-sim, print error")
|
||||
flag.StringVar(&gitRevision, "rev", "", "git revision")
|
||||
flag.StringVar(&githubConfig, "github", "", "Report results to Github's PR")
|
||||
flag.StringVar(&slackConfig, "slack", "", "Report results to slack channel")
|
||||
|
||||
flag.Usage = func() {
|
||||
_, _ = fmt.Fprintf(flag.CommandLine.Output(),
|
||||
`Usage: %s [-j maxprocs] [-seeds comma-separated-seed-list] [-g genesis.json] [-e] [-github token,pr-url] [-slack token,channel,thread] [package] [blocks] [period] [testname]
|
||||
`Usage: %s [-j maxprocs] [-seeds comma-separated-seed-list] [-rev git-commmit-hash] [-g genesis.json] [-e] [-github token,pr-url] [-slack token,channel,thread] [package] [blocks] [period] [testname]
|
||||
Run simulations in parallel`, filepath.Base(os.Args[0]))
|
||||
flag.PrintDefaults()
|
||||
}
|
||||
|
@ -209,11 +211,11 @@ wait:
|
|||
os.Exit(0)
|
||||
}
|
||||
|
||||
func buildCommand(testname, blocks, period, genesis string, seed int) string {
|
||||
func buildCommand(testName, blocks, period, genesis string, seed int) string {
|
||||
return fmt.Sprintf("go test %s -run %s -SimulationEnabled=true "+
|
||||
"-SimulationNumBlocks=%s -SimulationGenesis=%s "+
|
||||
"-SimulationVerbose=true -SimulationCommit=true -SimulationSeed=%d -SimulationPeriod=%s -v -timeout 24h",
|
||||
pkgName, testname, blocks, genesis, seed, period)
|
||||
pkgName, testName, blocks, genesis, seed, period)
|
||||
}
|
||||
|
||||
func makeCmd(cmdStr string) *exec.Cmd {
|
||||
|
@ -228,7 +230,8 @@ func makeFilename(seed int) string {
|
|||
func worker(id int, seeds <-chan int) {
|
||||
log.Printf("[W%d] Worker is up and running", id)
|
||||
for seed := range seeds {
|
||||
if err := spawnProc(id, seed); err != nil {
|
||||
stdOut, stdErr, err := spawnProc(id, seed)
|
||||
if err != nil {
|
||||
results <- false
|
||||
log.Printf("[W%d] Seed %d: FAILED", id, seed)
|
||||
log.Printf("To reproduce run: %s", buildCommand(testname, blocks, period, genesis, seed))
|
||||
|
@ -242,12 +245,13 @@ func worker(id int, seeds <-chan int) {
|
|||
} else {
|
||||
log.Printf("[W%d] Seed %d: OK", id, seed)
|
||||
}
|
||||
pushLogs(stdOut, stdErr, gitRevision)
|
||||
}
|
||||
|
||||
log.Printf("[W%d] no seeds left, shutting down", id)
|
||||
}
|
||||
|
||||
func spawnProc(workerID int, seed int) error {
|
||||
func spawnProc(workerID int, seed int) (*os.File, *os.File, error) {
|
||||
stderrFile, _ := os.Create(filepath.Join(tempdir, makeFilename(seed)+".stderr"))
|
||||
stdoutFile, _ := os.Create(filepath.Join(tempdir, makeFilename(seed)+".stdout"))
|
||||
s := buildCommand(testname, blocks, period, genesis, seed)
|
||||
|
@ -261,7 +265,7 @@ func spawnProc(workerID int, seed int) error {
|
|||
} else {
|
||||
stderr, err = cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
sc := bufio.NewScanner(stderr)
|
||||
|
@ -269,7 +273,7 @@ func spawnProc(workerID int, seed int) error {
|
|||
err = cmd.Start()
|
||||
if err != nil {
|
||||
log.Printf("couldn't start %q", s)
|
||||
return err
|
||||
return nil, nil, err
|
||||
}
|
||||
log.Printf("[W%d] Spawned simulation with pid %d [seed=%d stdout=%s stderr=%s]",
|
||||
workerID, cmd.Process.Pid, seed, stdoutFile.Name(), stderrFile.Name())
|
||||
|
@ -285,7 +289,7 @@ func spawnProc(workerID int, seed int) error {
|
|||
fmt.Printf("stderr: %s\n", sc.Text())
|
||||
}
|
||||
}
|
||||
return err
|
||||
return stdoutFile, stderrFile, err
|
||||
}
|
||||
|
||||
func pushProcess(proc *os.Process) {
|
||||
|
|
|
@ -1,10 +1,97 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"github.com/nlopes/slack"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/nlopes/slack"
|
||||
)
|
||||
|
||||
const (
|
||||
logBucketPrefix = "sim-logs-"
|
||||
awsRegion = "us-east-1"
|
||||
)
|
||||
|
||||
var (
|
||||
simTimeStamp = time.Now().Format("01-02-2006_15:05:05")
|
||||
)
|
||||
|
||||
func awsErrHandler(err error) {
|
||||
if awsErr, ok := err.(awserr.Error); ok {
|
||||
switch awsErr.Code() {
|
||||
default:
|
||||
log.Println(awsErr.Error())
|
||||
}
|
||||
} else {
|
||||
log.Println(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func makeObjKey(folderName string, fileName string) string {
|
||||
return fmt.Sprintf("%s/%s/%s", folderName, simTimeStamp, fileName)
|
||||
}
|
||||
|
||||
func putObj(fileHandle *os.File, svc *s3.S3, folderName string, bucketName string) {
|
||||
_, _ = fileHandle.Seek(0, 0)
|
||||
|
||||
stdOutObjInput := &s3.PutObjectInput{
|
||||
Body: aws.ReadSeekCloser(fileHandle),
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(makeObjKey(folderName, filepath.Base(fileHandle.Name()))),
|
||||
}
|
||||
if output, err := svc.PutObject(stdOutObjInput); err != nil {
|
||||
awsErrHandler(err)
|
||||
} else {
|
||||
log.Printf("Log file pushed: %s", output.String())
|
||||
}
|
||||
}
|
||||
|
||||
func pushLogs(stdOut *os.File, stdErr *os.File, folderName string) {
|
||||
var logBucket *string
|
||||
|
||||
sessionS3 := s3.New(session.Must(session.NewSession(&aws.Config{
|
||||
Region: aws.String(awsRegion),
|
||||
})))
|
||||
if listBucketsOutput, err := sessionS3.ListBuckets(&s3.ListBucketsInput{}); err != nil {
|
||||
awsErrHandler(err)
|
||||
} else {
|
||||
for _, bucket := range listBucketsOutput.Buckets {
|
||||
if strings.Contains(*bucket.Name, logBucketPrefix) {
|
||||
logBucket = bucket.Name
|
||||
putObj(stdOut, sessionS3, folderName, *logBucket)
|
||||
putObj(stdErr, sessionS3, folderName, *logBucket)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if logBucket == nil {
|
||||
log.Println("Log bucket not found")
|
||||
}
|
||||
}
|
||||
|
||||
func slackMessage(token string, channel string, threadTS *string, message string) {
|
||||
client := slack.New(token)
|
||||
if threadTS != nil {
|
||||
_, _, err := client.PostMessage(channel, slack.MsgOptionText(message, false), slack.MsgOptionTS(*threadTS))
|
||||
if err != nil {
|
||||
log.Printf("ERROR: %v", err)
|
||||
}
|
||||
} else {
|
||||
_, _, err := client.PostMessage(channel, slack.MsgOptionText(message, false))
|
||||
if err != nil {
|
||||
log.Printf("ERROR: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//type GithubPayload struct {
|
||||
// Issue struct {
|
||||
// Number int `json:"number"`
|
||||
|
@ -32,22 +119,6 @@ import (
|
|||
// } `json:"head"`
|
||||
//}
|
||||
|
||||
func slackMessage(token string, channel string, threadTS *string, message string) {
|
||||
client := slack.New(token)
|
||||
if threadTS != nil {
|
||||
_, _, err := client.PostMessage(channel, slack.MsgOptionText(message, false), slack.MsgOptionTS(*threadTS))
|
||||
if err != nil {
|
||||
log.Printf("ERROR: %v", err)
|
||||
}
|
||||
} else {
|
||||
_, _, err := client.PostMessage(channel, slack.MsgOptionText(message, false))
|
||||
if err != nil {
|
||||
log.Printf("ERROR: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//func createCheckRun(client *github.Client, payload GithubPayload, pr PullRequestDetails) error {
|
||||
// var opt github.CreateCheckRunOptions
|
||||
// opt.Name = "Test Check"
|
||||
|
|
1
go.mod
1
go.mod
|
@ -1,6 +1,7 @@
|
|||
module github.com/cosmos/cosmos-sdk
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go v1.20.11
|
||||
github.com/bartekn/go-bip39 v0.0.0-20171116152956-a05967ea095d
|
||||
github.com/bgentry/speakeasy v0.1.0
|
||||
github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d
|
||||
|
|
4
go.sum
4
go.sum
|
@ -7,6 +7,8 @@ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBA
|
|||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/aws/aws-sdk-go v1.20.11 h1:xDc2f/8KmwPW7WkuB0kDUCEP4jpx1PIMMMZkav6cbU4=
|
||||
github.com/aws/aws-sdk-go v1.20.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/bartekn/go-bip39 v0.0.0-20171116152956-a05967ea095d h1:1aAija9gr0Hyv4KfQcRcwlmFIrhkDmIj2dz5bkg/s/8=
|
||||
github.com/bartekn/go-bip39 v0.0.0-20171116152956-a05967ea095d/go.mod h1:icNx/6QdFblhsEjZehARqbNumymUT/ydwlLojFdv7Sk=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
|
||||
|
@ -82,6 +84,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
|
|||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U=
|
||||
github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ=
|
||||
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
|
||||
|
|
Loading…
Reference in New Issue