R4R: added slack notification to runsim (#4547)

This commit is contained in:
Alessio Treglia 2019-06-26 08:11:42 +01:00 committed by GitHub
parent 891eb8eec5
commit 908c5cf4be
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 276 additions and 15 deletions

View File

@ -125,7 +125,7 @@ test_sim_benchmark_invariants:
# Don't move it into tools - this will be gone once gaia has moved into the new repo
runsim: $(BINDIR)/runsim
$(BINDIR)/runsim: contrib/runsim/main.go
$(BINDIR)/runsim: contrib/runsim/main.go contrib/runsim/notification.go
go install github.com/cosmos/cosmos-sdk/contrib/runsim
SIM_NUM_BLOCKS ?= 500

View File

@ -12,12 +12,18 @@ import (
"os/signal"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
"syscall"
"time"
)
const (
GithubConfigSep = ","
SlackConfigSep = ","
)
var (
// default seeds
seeds = []int{
@ -28,6 +34,7 @@ var (
29892989, 30123012, 47284728, 7601778, 8090485,
977367484, 491163361, 424254581, 673398983,
}
seedOverrideList = ""
// goroutine-safe process map
procs map[int]*os.Process
@ -37,13 +44,20 @@ var (
results chan bool
// command line arguments and options
jobs = runtime.GOMAXPROCS(0)
pkgName string
blocks string
period string
testname string
genesis string
exitOnFail bool
jobs = runtime.GOMAXPROCS(0)
pkgName string
blocks string
period string
testname string
genesis string
exitOnFail bool
githubConfig string
slackConfig string
// integration with Slack and Github
slackToken string
slackChannel string
slackThread string
// logs temporary directory
tempdir string
@ -53,19 +67,27 @@ func init() {
log.SetPrefix("")
log.SetFlags(0)
runsimLogfile, err := os.OpenFile("sim_log_file", os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
if err != nil {
log.Printf("ERROR: opening log file: %v", err.Error())
} else {
log.SetOutput(io.MultiWriter(os.Stdout, runsimLogfile))
}
procs = map[int]*os.Process{}
mutex = &sync.Mutex{}
flag.IntVar(&jobs, "j", jobs, "Number of parallel processes")
flag.StringVar(&genesis, "g", "", "Genesis file")
flag.StringVar(&seedOverrideList, "seeds", "", "run the supplied comma-separated list of seeds instead of defaults")
flag.BoolVar(&exitOnFail, "e", false, "Exit on fail during multi-sim, print error")
flag.StringVar(&githubConfig, "github", "", "Report results to Github's PR")
flag.StringVar(&slackConfig, "slack", "", "Report results to slack channel")
flag.Usage = func() {
fmt.Fprintf(flag.CommandLine.Output(),
`Usage: %s [-j maxprocs] [-g genesis.json] [-e] [package] [blocks] [period] [testname]
Run simulations in parallel
`, filepath.Base(os.Args[0]))
_, _ = fmt.Fprintf(flag.CommandLine.Output(),
`Usage: %s [-j maxprocs] [-seeds comma-separated-seed-list] [-g genesis.json] [-e] [-github token,pr-url] [-slack token,channel,thread] [package] [blocks] [period] [testname]
Run simulations in parallel`, filepath.Base(os.Args[0]))
flag.PrintDefaults()
}
}
@ -78,7 +100,29 @@ func main() {
log.Fatal("wrong number of arguments")
}
// prepare input channel
if githubConfig != "" {
opts := strings.Split(githubConfig, GithubConfigSep)
if len(opts) != 2 {
log.Fatal("incorrect github config string format")
}
}
if slackConfig != "" {
opts := strings.Split(slackConfig, SlackConfigSep)
if len(opts) != 3 {
log.Fatal("incorrect slack config string format")
}
slackToken, slackChannel, slackThread = opts[0], opts[1], opts[2]
}
seedOverrideList = strings.TrimSpace(seedOverrideList)
if seedOverrideList != "" {
seeds, err = makeSeedList(seedOverrideList)
if err != nil {
log.Fatal(err)
}
}
queue := make(chan int, len(seeds))
for _, seed := range seeds {
queue <- seed
@ -155,7 +199,13 @@ wait:
os.Exit(1)
}
}
if slackConfigSupplied() {
seedStrings := make([]string, len(seeds))
for i, seed := range seeds {
seedStrings[i] = fmt.Sprintf("%d", seed)
}
slackMessage(slackToken, slackChannel, &slackThread, fmt.Sprintf("Finished running simulation for seeds: %s", strings.Join(seedStrings, " ")))
}
os.Exit(0)
}
@ -182,6 +232,9 @@ func worker(id int, seeds <-chan int) {
results <- false
log.Printf("[W%d] Seed %d: FAILED", id, seed)
log.Printf("To reproduce run: %s", buildCommand(testname, blocks, period, genesis, seed))
if slackConfigSupplied() {
slackMessage(slackToken, slackChannel, nil, "Seed "+strconv.Itoa(seed)+" failed. To reproduce, run: "+buildCommand(testname, blocks, period, genesis, seed))
}
if exitOnFail {
log.Printf("\bERROR OUTPUT \n\n%s", err)
panic("halting simulations")
@ -190,6 +243,7 @@ func worker(id int, seeds <-chan int) {
log.Printf("[W%d] Seed %d: OK", id, seed)
}
}
log.Printf("[W%d] no seeds left, shutting down", id)
}
@ -262,3 +316,21 @@ func checkSignal(proc *os.Process, signal syscall.Signal) {
log.Printf("Failed to send %s to PID %d", signal, proc.Pid)
}
}
func makeSeedList(seeds string) ([]int, error) {
strSeedsLst := strings.Split(seeds, ",")
if len(strSeedsLst) == 0 {
return nil, fmt.Errorf("seeds was empty")
}
intSeeds := make([]int, len(strSeedsLst))
for i, seedstr := range strSeedsLst {
intSeed, err := strconv.Atoi(strings.TrimSpace(seedstr))
if err != nil {
return nil, fmt.Errorf("cannot convert seed to integer: %v", err)
}
intSeeds[i] = intSeed
}
return intSeeds, nil
}
func slackConfigSupplied() bool { return slackConfig != "" }

View File

@ -0,0 +1,170 @@
package main
import (
"github.com/nlopes/slack"
"log"
)
//type GithubPayload struct {
// Issue struct {
// Number int `json:"number"`
// Pull struct {
// Url string `json:"url,omitempty"`
// } `json:"pull_request,omitempty"`
// } `json:"issue"`
//
// Comment struct {
// Body string `json:"body"`
// } `json:"comment"`
//
// Repository struct {
// Name string `json:"name"`
// Owner struct {
// Login string `json:"login"`
// } `json:"owner"`
// } `json:"repository"`
//}
//
//type PullRequestDetails struct {
// Head struct {
// Ref string `json:"ref"`
// Sha string `json:"sha"`
// } `json:"head"`
//}
func slackMessage(token string, channel string, threadTS *string, message string) {
client := slack.New(token)
if threadTS != nil {
_, _, err := client.PostMessage(channel, slack.MsgOptionText(message, false), slack.MsgOptionTS(*threadTS))
if err != nil {
log.Printf("ERROR: %v", err)
}
} else {
_, _, err := client.PostMessage(channel, slack.MsgOptionText(message, false))
if err != nil {
log.Printf("ERROR: %v", err)
}
}
}
//func createCheckRun(client *github.Client, payload GithubPayload, pr PullRequestDetails) error {
// var opt github.CreateCheckRunOptions
// opt.Name = "Test Check"
// opt.HeadBranch = pr.Head.Ref
// opt.HeadSHA = pr.Head.Sha
//
// checkRUn, resp, err := client.Checks.CreateCheckRun(context.Background(), payload.Repository.Owner.Login, payload.Repository.Name, opt)
// log.Printf("%v", resp)
// log.Printf("%v", checkRUn)
// if err != nil {
// log.Printf("ERROR: CreateCheckRun: %v", err.Error())
// return err
// }
// return err
//}
//
//func getPrDetails(prUrl string) (*PullRequestDetails, error) {
// request, err := http.Get(prUrl)
// if err != nil {
// return nil, err
// }
//
// var details PullRequestDetails
// if err := json.NewDecoder(request.Body).Decode(&details); err != nil {
// return nil, err
// }
//
// return &details, nil
//}
//
//func updateCheckRun(client *github.Client, payload GithubPayload, pr PullRequestDetails) error {
// status := "completed"
// conclusion := "success"
// var opt github.UpdateCheckRunOptions
// opt.Name = "Test Check"
// opt.Status = &status
// opt.Conclusion = &conclusion
// ts := github.Timestamp{Time: time.Now()}
// opt.CompletedAt = &ts
//
// updatedCheck, resp, err := client.Checks.UpdateCheckRun(context.Background(), payload.Repository.Owner.Login, payload.Repository.Name, 136693316, opt)
// log.Printf("%v", updatedCheck)
// log.Printf("%v", resp)
// if err != nil {
// log.Printf("ERROR: UpdateCheckRun: %v", err.Error())
// return err
// }
// return nil
//}
//func githubCheckHandler(request events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) {
// response := events.APIGatewayProxyResponse{StatusCode: 200}
// var comment GithubPayload
// if err := json.NewDecoder(bytes.NewBufferString(request.Body)).Decode(&comment); err != nil {
// response.StatusCode = 500
// response.Body = err.Error()
// return response, err
// }
//
// itr, err := ghinstallation.NewKeyFromFile(http.DefaultTransport, 30867, 997580, "github-integration/gaia-sim.2019-05-16.private-key.pem")
// if err != nil {
// response.StatusCode = 500
// response.Body = err.Error()
// log.Printf("AuthError: %v", err)
// return response, err
// }
// client := github.NewClient(&http.Client{Transport: itr})
// message := "App comment"
// issue := new(github.IssueComment)
// issue.Body = &message
//
// if comment.Comment.Body == "Start sim" && comment.Issue.Pull.Url != "" {
// prDetails, err := getPrDetails(comment.Issue.Pull.Url)
// if err != nil {
// response.StatusCode = 500
// response.Body = err.Error()
// log.Printf("ERROR: getPrDetails: %v", err.Error())
// return response, err
// }
// log.Printf("%v", prDetails)
//
// if err := createCheckRun(client, comment, *prDetails); err != nil {
// response.StatusCode = 500
// response.Body = err.Error()
// return response, err
// }
//
// comments, resp, err := client.Issues.CreateComment(context.Background(),
// comment.Repository.Owner.Login, comment.Repository.Name, comment.Issue.Number, issue)
//
// log.Printf("%v", resp)
// log.Printf("%v", comments)
// if err != nil {
// log.Printf("ERROR: CreateComment: %v", err.Error())
// response.StatusCode = 500
// response.Body = err.Error()
// return response, err
// }
// }
//
// if comment.Comment.Body == "Update check" && comment.Issue.Pull.Url != "" {
// prDetails, err := getPrDetails(comment.Issue.Pull.Url)
// if err != nil {
// response.StatusCode = 500
// response.Body = err.Error()
// log.Printf("ERROR: getPrDetails: %v", err.Error())
// return response, err
// }
// log.Printf("%v", prDetails)
//
// if err := updateCheckRun(client, comment, *prDetails); err != nil {
// response.StatusCode = 500
// response.Body = err.Error()
// log.Printf("ERROR: getPrDetails: %v", err.Error())
// return response, err
// }
// }
//
// return response, nil
//}

5
go.mod
View File

@ -2,21 +2,26 @@ module github.com/cosmos/cosmos-sdk
require (
github.com/VividCortex/gohistogram v1.0.0 // indirect
github.com/aws/aws-lambda-go v1.11.1
github.com/bartekn/go-bip39 v0.0.0-20171116152956-a05967ea095d
github.com/bgentry/speakeasy v0.1.0
github.com/bradleyfalzon/ghinstallation v0.1.2
github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d
github.com/cosmos/go-bip39 v0.0.0-20180618194314-52158e4697b8
github.com/cosmos/ledger-cosmos-go v0.10.3
github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
github.com/fortytw2/leaktest v1.3.0 // indirect
github.com/go-logfmt/logfmt v0.4.0 // indirect
github.com/gogo/protobuf v1.1.1
github.com/golang/mock v1.3.1-0.20190508161146-9fa652df1129
github.com/golang/protobuf v1.3.0
github.com/golang/snappy v0.0.1 // indirect
github.com/google/go-github/v26 v26.0.2
github.com/gorilla/mux v1.7.0
github.com/gorilla/websocket v1.4.0 // indirect
github.com/jmhodges/levigo v1.0.0 // indirect
github.com/mattn/go-isatty v0.0.6
github.com/nlopes/slack v0.5.0
github.com/pelletier/go-toml v1.2.0
github.com/pkg/errors v0.8.1
github.com/prometheus/client_golang v0.9.2 // indirect

14
go.sum
View File

@ -6,6 +6,8 @@ github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/aws/aws-lambda-go v1.11.1 h1:wuOnhS5aqzPOWns71FO35PtbtBKHr4MYsPVt5qXLSfI=
github.com/aws/aws-lambda-go v1.11.1/go.mod h1:Rr2SMTLeSMKgD45uep9V/NP8tnbCcySgu04cx0k/6cw=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/bartekn/go-bip39 v0.0.0-20171116152956-a05967ea095d h1:1aAija9gr0Hyv4KfQcRcwlmFIrhkDmIj2dz5bkg/s/8=
github.com/bartekn/go-bip39 v0.0.0-20171116152956-a05967ea095d/go.mod h1:icNx/6QdFblhsEjZehARqbNumymUT/ydwlLojFdv7Sk=
@ -13,6 +15,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLM
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bradleyfalzon/ghinstallation v0.1.2 h1:9fdqVadlvEX/EUts5/aIGvx2ujKnGNIMcuCuUrM6s6Q=
github.com/bradleyfalzon/ghinstallation v0.1.2/go.mod h1:VQsLlCoNa54/CNXcc2DuCfNZrZxqQcyPeqKUugF/2h8=
github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d h1:xG8Pj6Y6J760xwETNmMzmlt38QSwz0BLp1cZ09g27uw=
github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d/go.mod h1:d3C0AkH6BRcvO8T0UEPu53cnw4IbV63x1bEjildYhO0=
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
@ -39,6 +43,8 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
@ -64,6 +70,10 @@ github.com/golang/protobuf v1.3.0 h1:kbxbvI4Un1LUWKxufD+BiE6AEExYYgkQLQmLFqA1LFk
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-github/v26 v26.0.2 h1:tiPWJNapF2n6Mxbmue/g0uDkSuWf34nzlsNp4Ym7qb8=
github.com/google/go-github/v26 v26.0.2/go.mod h1:v6/FmX9au22j4CtYxnMhJJkP+JfOQDXALk7hI+MPDNM=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck=
github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U=
@ -96,6 +106,8 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nlopes/slack v0.5.0 h1:NbIae8Kd0NpqaEI3iUrsuS0KbcEDhzhc939jLW5fNm0=
github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@ -146,6 +158,7 @@ github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
@ -209,6 +222,7 @@ gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=