From 0d3b4ba0d68407f252e34d591376367492d96998 Mon Sep 17 00:00:00 2001 From: mircea-c Date: Tue, 2 Jul 2019 12:57:19 -0400 Subject: [PATCH] [runsim] push logs to S3 (#4666) --- contrib/runsim/main.go | 20 ++++--- contrib/runsim/notification.go | 105 +++++++++++++++++++++++++++------ go.mod | 1 + go.sum | 4 ++ 4 files changed, 105 insertions(+), 25 deletions(-) diff --git a/contrib/runsim/main.go b/contrib/runsim/main.go index c10eefdbe..fbd1eb0bc 100644 --- a/contrib/runsim/main.go +++ b/contrib/runsim/main.go @@ -52,6 +52,7 @@ var ( genesis string exitOnFail bool githubConfig string + gitRevision string slackConfig string // integration with Slack and Github @@ -81,12 +82,13 @@ func init() { flag.StringVar(&genesis, "g", "", "Genesis file") flag.StringVar(&seedOverrideList, "seeds", "", "run the supplied comma-separated list of seeds instead of defaults") flag.BoolVar(&exitOnFail, "e", false, "Exit on fail during multi-sim, print error") + flag.StringVar(&gitRevision, "rev", "", "git revision") flag.StringVar(&githubConfig, "github", "", "Report results to Github's PR") flag.StringVar(&slackConfig, "slack", "", "Report results to slack channel") flag.Usage = func() { _, _ = fmt.Fprintf(flag.CommandLine.Output(), - `Usage: %s [-j maxprocs] [-seeds comma-separated-seed-list] [-g genesis.json] [-e] [-github token,pr-url] [-slack token,channel,thread] [package] [blocks] [period] [testname] + `Usage: %s [-j maxprocs] [-seeds comma-separated-seed-list] [-rev git-commmit-hash] [-g genesis.json] [-e] [-github token,pr-url] [-slack token,channel,thread] [package] [blocks] [period] [testname] Run simulations in parallel`, filepath.Base(os.Args[0])) flag.PrintDefaults() } @@ -209,11 +211,11 @@ wait: os.Exit(0) } -func buildCommand(testname, blocks, period, genesis string, seed int) string { +func buildCommand(testName, blocks, period, genesis string, seed int) string { return fmt.Sprintf("go test %s -run %s -SimulationEnabled=true "+ "-SimulationNumBlocks=%s -SimulationGenesis=%s "+ "-SimulationVerbose=true -SimulationCommit=true -SimulationSeed=%d -SimulationPeriod=%s -v -timeout 24h", - pkgName, testname, blocks, genesis, seed, period) + pkgName, testName, blocks, genesis, seed, period) } func makeCmd(cmdStr string) *exec.Cmd { @@ -228,7 +230,8 @@ func makeFilename(seed int) string { func worker(id int, seeds <-chan int) { log.Printf("[W%d] Worker is up and running", id) for seed := range seeds { - if err := spawnProc(id, seed); err != nil { + stdOut, stdErr, err := spawnProc(id, seed) + if err != nil { results <- false log.Printf("[W%d] Seed %d: FAILED", id, seed) log.Printf("To reproduce run: %s", buildCommand(testname, blocks, period, genesis, seed)) @@ -242,12 +245,13 @@ func worker(id int, seeds <-chan int) { } else { log.Printf("[W%d] Seed %d: OK", id, seed) } + pushLogs(stdOut, stdErr, gitRevision) } log.Printf("[W%d] no seeds left, shutting down", id) } -func spawnProc(workerID int, seed int) error { +func spawnProc(workerID int, seed int) (*os.File, *os.File, error) { stderrFile, _ := os.Create(filepath.Join(tempdir, makeFilename(seed)+".stderr")) stdoutFile, _ := os.Create(filepath.Join(tempdir, makeFilename(seed)+".stdout")) s := buildCommand(testname, blocks, period, genesis, seed) @@ -261,7 +265,7 @@ func spawnProc(workerID int, seed int) error { } else { stderr, err = cmd.StderrPipe() if err != nil { - return err + return nil, nil, err } } sc := bufio.NewScanner(stderr) @@ -269,7 +273,7 @@ func spawnProc(workerID int, seed int) error { err = cmd.Start() if err != nil { log.Printf("couldn't start %q", s) - return err + return nil, nil, err } log.Printf("[W%d] Spawned simulation with pid %d [seed=%d stdout=%s stderr=%s]", workerID, cmd.Process.Pid, seed, stdoutFile.Name(), stderrFile.Name()) @@ -285,7 +289,7 @@ func spawnProc(workerID int, seed int) error { fmt.Printf("stderr: %s\n", sc.Text()) } } - return err + return stdoutFile, stderrFile, err } func pushProcess(proc *os.Process) { diff --git a/contrib/runsim/notification.go b/contrib/runsim/notification.go index 4d792cff9..5668cfd8d 100644 --- a/contrib/runsim/notification.go +++ b/contrib/runsim/notification.go @@ -1,10 +1,97 @@ package main import ( - "github.com/nlopes/slack" + "fmt" "log" + "os" + "path/filepath" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/nlopes/slack" ) +const ( + logBucketPrefix = "sim-logs-" + awsRegion = "us-east-1" +) + +var ( + simTimeStamp = time.Now().Format("01-02-2006_15:05:05") +) + +func awsErrHandler(err error) { + if awsErr, ok := err.(awserr.Error); ok { + switch awsErr.Code() { + default: + log.Println(awsErr.Error()) + } + } else { + log.Println(err.Error()) + } +} + +func makeObjKey(folderName string, fileName string) string { + return fmt.Sprintf("%s/%s/%s", folderName, simTimeStamp, fileName) +} + +func putObj(fileHandle *os.File, svc *s3.S3, folderName string, bucketName string) { + _, _ = fileHandle.Seek(0, 0) + + stdOutObjInput := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(fileHandle), + Bucket: aws.String(bucketName), + Key: aws.String(makeObjKey(folderName, filepath.Base(fileHandle.Name()))), + } + if output, err := svc.PutObject(stdOutObjInput); err != nil { + awsErrHandler(err) + } else { + log.Printf("Log file pushed: %s", output.String()) + } +} + +func pushLogs(stdOut *os.File, stdErr *os.File, folderName string) { + var logBucket *string + + sessionS3 := s3.New(session.Must(session.NewSession(&aws.Config{ + Region: aws.String(awsRegion), + }))) + if listBucketsOutput, err := sessionS3.ListBuckets(&s3.ListBucketsInput{}); err != nil { + awsErrHandler(err) + } else { + for _, bucket := range listBucketsOutput.Buckets { + if strings.Contains(*bucket.Name, logBucketPrefix) { + logBucket = bucket.Name + putObj(stdOut, sessionS3, folderName, *logBucket) + putObj(stdErr, sessionS3, folderName, *logBucket) + break + } + } + } + if logBucket == nil { + log.Println("Log bucket not found") + } +} + +func slackMessage(token string, channel string, threadTS *string, message string) { + client := slack.New(token) + if threadTS != nil { + _, _, err := client.PostMessage(channel, slack.MsgOptionText(message, false), slack.MsgOptionTS(*threadTS)) + if err != nil { + log.Printf("ERROR: %v", err) + } + } else { + _, _, err := client.PostMessage(channel, slack.MsgOptionText(message, false)) + if err != nil { + log.Printf("ERROR: %v", err) + } + } +} + //type GithubPayload struct { // Issue struct { // Number int `json:"number"` @@ -32,22 +119,6 @@ import ( // } `json:"head"` //} -func slackMessage(token string, channel string, threadTS *string, message string) { - client := slack.New(token) - if threadTS != nil { - _, _, err := client.PostMessage(channel, slack.MsgOptionText(message, false), slack.MsgOptionTS(*threadTS)) - if err != nil { - log.Printf("ERROR: %v", err) - } - } else { - _, _, err := client.PostMessage(channel, slack.MsgOptionText(message, false)) - if err != nil { - log.Printf("ERROR: %v", err) - } - } - -} - //func createCheckRun(client *github.Client, payload GithubPayload, pr PullRequestDetails) error { // var opt github.CreateCheckRunOptions // opt.Name = "Test Check" diff --git a/go.mod b/go.mod index 6c9013094..54d5f43d6 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,7 @@ module github.com/cosmos/cosmos-sdk require ( + github.com/aws/aws-sdk-go v1.20.11 github.com/bartekn/go-bip39 v0.0.0-20171116152956-a05967ea095d github.com/bgentry/speakeasy v0.1.0 github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d diff --git a/go.sum b/go.sum index f9ed61b07..fe4a3153d 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBA github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/aws/aws-sdk-go v1.20.11 h1:xDc2f/8KmwPW7WkuB0kDUCEP4jpx1PIMMMZkav6cbU4= +github.com/aws/aws-sdk-go v1.20.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/bartekn/go-bip39 v0.0.0-20171116152956-a05967ea095d h1:1aAija9gr0Hyv4KfQcRcwlmFIrhkDmIj2dz5bkg/s/8= github.com/bartekn/go-bip39 v0.0.0-20171116152956-a05967ea095d/go.mod h1:icNx/6QdFblhsEjZehARqbNumymUT/ydwlLojFdv7Sk= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= @@ -82,6 +84,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=