runsim pushes logs to S3 bucket (#4677)
This commit is contained in:
parent
1bfac99c35
commit
74915f1e2a
|
@ -52,7 +52,7 @@ var (
|
||||||
genesis string
|
genesis string
|
||||||
exitOnFail bool
|
exitOnFail bool
|
||||||
githubConfig string
|
githubConfig string
|
||||||
gitRevision string
|
logObjKey string
|
||||||
slackConfig string
|
slackConfig string
|
||||||
|
|
||||||
// integration with Slack and Github
|
// integration with Slack and Github
|
||||||
|
@ -82,7 +82,7 @@ func init() {
|
||||||
flag.StringVar(&genesis, "g", "", "Genesis file")
|
flag.StringVar(&genesis, "g", "", "Genesis file")
|
||||||
flag.StringVar(&seedOverrideList, "seeds", "", "run the supplied comma-separated list of seeds instead of defaults")
|
flag.StringVar(&seedOverrideList, "seeds", "", "run the supplied comma-separated list of seeds instead of defaults")
|
||||||
flag.BoolVar(&exitOnFail, "e", false, "Exit on fail during multi-sim, print error")
|
flag.BoolVar(&exitOnFail, "e", false, "Exit on fail during multi-sim, print error")
|
||||||
flag.StringVar(&gitRevision, "rev", "", "git revision")
|
flag.StringVar(&logObjKey, "log", "", "S3 object key for log files")
|
||||||
flag.StringVar(&githubConfig, "github", "", "Report results to Github's PR")
|
flag.StringVar(&githubConfig, "github", "", "Report results to Github's PR")
|
||||||
flag.StringVar(&slackConfig, "slack", "", "Report results to slack channel")
|
flag.StringVar(&slackConfig, "slack", "", "Report results to slack channel")
|
||||||
|
|
||||||
|
@ -224,7 +224,16 @@ func makeCmd(cmdStr string) *exec.Cmd {
|
||||||
}
|
}
|
||||||
|
|
||||||
func makeFilename(seed int) string {
|
func makeFilename(seed int) string {
|
||||||
return fmt.Sprintf("app-simulation-seed-%d-date-%s", seed, time.Now().Format("01-02-2006_15:04:05.000000000"))
|
return fmt.Sprintf("app-simulation-seed-%d-date-%s", seed, time.Now().Format("01-02-2006_150405"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeFailSlackMsg(seed int, stdoutKey, stderrKey, bucket string, logsPushed bool) string {
|
||||||
|
if logsPushed {
|
||||||
|
return fmt.Sprintf("*Seed %s: FAILED*. *<https://%s.s3.amazonaws.com/%s|stdout>* *<https://%s.s3.amazonaws.com/%s|stderr>*\nTo reproduce run: ```\n%s\n```",
|
||||||
|
strconv.Itoa(seed), bucket, stdoutKey, bucket, stderrKey, buildCommand(testname, blocks, period, genesis, seed))
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("*Seed %s: FAILED*. \nTo reproduce run: ```\n%s\n```\n*Could not upload logs:* ```\n%s\n```",
|
||||||
|
strconv.Itoa(seed), buildCommand(testname, blocks, period, genesis, seed), bucket)
|
||||||
}
|
}
|
||||||
|
|
||||||
func worker(id int, seeds <-chan int) {
|
func worker(id int, seeds <-chan int) {
|
||||||
|
@ -236,18 +245,25 @@ func worker(id int, seeds <-chan int) {
|
||||||
log.Printf("[W%d] Seed %d: FAILED", id, seed)
|
log.Printf("[W%d] Seed %d: FAILED", id, seed)
|
||||||
log.Printf("To reproduce run: %s", buildCommand(testname, blocks, period, genesis, seed))
|
log.Printf("To reproduce run: %s", buildCommand(testname, blocks, period, genesis, seed))
|
||||||
if slackConfigSupplied() {
|
if slackConfigSupplied() {
|
||||||
slackMessage(slackToken, slackChannel, nil, "Seed "+strconv.Itoa(seed)+" failed. To reproduce, run: "+buildCommand(testname, blocks, period, genesis, seed))
|
objKeys, bucket, err := pushLogs(stdOut, stdErr, logObjKey)
|
||||||
|
if err != nil {
|
||||||
|
slackMessage(slackToken, slackChannel, nil, makeFailSlackMsg(seed, "", "", err.Error(), false))
|
||||||
|
}
|
||||||
|
slackMessage(slackToken, slackChannel, nil, makeFailSlackMsg(seed, objKeys[0], objKeys[1], *bucket, true))
|
||||||
}
|
}
|
||||||
if exitOnFail {
|
if exitOnFail {
|
||||||
log.Printf("\bERROR OUTPUT \n\n%s", err)
|
log.Printf("\bERROR OUTPUT \n\n%s", err)
|
||||||
panic("halting simulations")
|
panic("halting simulations")
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
log.Printf("[W%d] Seed %d: OK", id, seed)
|
|
||||||
}
|
}
|
||||||
pushLogs(stdOut, stdErr, gitRevision)
|
log.Printf("[W%d] Seed %d: OK", id, seed)
|
||||||
|
if slackConfigSupplied() {
|
||||||
|
_, _, err = pushLogs(stdOut, stdErr, logObjKey)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Printf("[W%d] no seeds left, shutting down", id)
|
log.Printf("[W%d] no seeds left, shutting down", id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -6,7 +6,6 @@ import (
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/aws/aws-sdk-go/aws"
|
"github.com/aws/aws-sdk-go/aws"
|
||||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||||
|
@ -20,61 +19,65 @@ const (
|
||||||
awsRegion = "us-east-1"
|
awsRegion = "us-east-1"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
func awsErrHandler(err error) error {
|
||||||
simTimeStamp = time.Now().Format("01-02-2006_15:05:05")
|
|
||||||
)
|
|
||||||
|
|
||||||
func awsErrHandler(err error) {
|
|
||||||
if awsErr, ok := err.(awserr.Error); ok {
|
if awsErr, ok := err.(awserr.Error); ok {
|
||||||
switch awsErr.Code() {
|
switch awsErr.Code() {
|
||||||
default:
|
default:
|
||||||
log.Println(awsErr.Error())
|
return awsErr
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
log.Println(err.Error())
|
|
||||||
}
|
}
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func makeObjKey(folderName string, fileName string) string {
|
func makeObjKey(objKeyPrefix string, fileName string) string {
|
||||||
return fmt.Sprintf("%s/%s/%s", folderName, simTimeStamp, fileName)
|
return fmt.Sprintf("%s/%s", objKeyPrefix, fileName)
|
||||||
}
|
}
|
||||||
|
|
||||||
func putObj(fileHandle *os.File, svc *s3.S3, folderName string, bucketName string) {
|
// putObjects attempts to upload to an S3 bucket the content of each file from fileHandles.
|
||||||
_, _ = fileHandle.Seek(0, 0)
|
// File descriptors have their read offset set to 0 to ensure all the content is uploaded.
|
||||||
|
// Each file will become an S3 bucket object that can be accessed via its object key.
|
||||||
stdOutObjInput := &s3.PutObjectInput{
|
//
|
||||||
Body: aws.ReadSeekCloser(fileHandle),
|
// Function returns the list of object keys and an error, if any.
|
||||||
Bucket: aws.String(bucketName),
|
func putObjects(svc *s3.S3, objKeyPrefix string, bucketName string, fileHandles ...*os.File) ([]string, error) {
|
||||||
Key: aws.String(makeObjKey(folderName, filepath.Base(fileHandle.Name()))),
|
objKeys := make([]string, len(fileHandles))
|
||||||
}
|
for index, fileHandle := range fileHandles {
|
||||||
if output, err := svc.PutObject(stdOutObjInput); err != nil {
|
_, _ = fileHandle.Seek(0, 0)
|
||||||
awsErrHandler(err)
|
objKey := makeObjKey(objKeyPrefix, filepath.Base(fileHandle.Name()))
|
||||||
} else {
|
stdOutObjInput := &s3.PutObjectInput{
|
||||||
log.Printf("Log file pushed: %s", output.String())
|
Body: aws.ReadSeekCloser(fileHandle),
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objKey),
|
||||||
|
}
|
||||||
|
_, err := svc.PutObject(stdOutObjInput)
|
||||||
|
if err != nil {
|
||||||
|
return nil, awsErrHandler(err)
|
||||||
|
}
|
||||||
|
objKeys[index] = objKey
|
||||||
}
|
}
|
||||||
|
return objKeys, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func pushLogs(stdOut *os.File, stdErr *os.File, folderName string) {
|
func pushLogs(stdOut *os.File, stdErr *os.File, folderName string) ([]string, *string, error) {
|
||||||
var logBucket *string
|
var logBucket *string
|
||||||
|
|
||||||
sessionS3 := s3.New(session.Must(session.NewSession(&aws.Config{
|
sessionS3 := s3.New(session.Must(session.NewSession(&aws.Config{
|
||||||
Region: aws.String(awsRegion),
|
Region: aws.String(awsRegion),
|
||||||
})))
|
})))
|
||||||
if listBucketsOutput, err := sessionS3.ListBuckets(&s3.ListBucketsInput{}); err != nil {
|
listBucketsOutput, err := sessionS3.ListBuckets(&s3.ListBucketsInput{})
|
||||||
awsErrHandler(err)
|
if err != nil {
|
||||||
} else {
|
return nil, nil, awsErrHandler(err)
|
||||||
for _, bucket := range listBucketsOutput.Buckets {
|
}
|
||||||
if strings.Contains(*bucket.Name, logBucketPrefix) {
|
for _, bucket := range listBucketsOutput.Buckets {
|
||||||
logBucket = bucket.Name
|
if strings.Contains(*bucket.Name, logBucketPrefix) {
|
||||||
putObj(stdOut, sessionS3, folderName, *logBucket)
|
logBucket = bucket.Name
|
||||||
putObj(stdErr, sessionS3, folderName, *logBucket)
|
objKeys, err := putObjects(sessionS3, folderName, *logBucket, stdOut, stdErr)
|
||||||
break
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
return objKeys, bucket.Name, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if logBucket == nil {
|
return nil, nil, nil
|
||||||
log.Println("Log bucket not found")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func slackMessage(token string, channel string, threadTS *string, message string) {
|
func slackMessage(token string, channel string, threadTS *string, message string) {
|
||||||
|
|
Loading…
Reference in New Issue