commit 5e864389db356ec8c0d54f91390ad0b05ab7fc2b Author: Hendrik Hofstadt Date: Mon Sep 24 23:37:57 2018 +0200 initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f9be139 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +vendor/ +.idea/ +*.iml \ No newline at end of file diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 0000000..13c7202 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,396 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + branch = "master" + name = "github.com/beorn7/perks" + packages = ["quantile"] + revision = "3a771d992973f24aa725d07868b467d1ddfceafb" + +[[projects]] + branch = "master" + name = "github.com/btcsuite/btcd" + packages = ["btcec"] + revision = "2a560b2036bee5e3679ec2133eb6520b2f195213" + +[[projects]] + name = "github.com/certifi/gocertifi" + packages = ["."] + revision = "deb3ae2ef2610fde3330947281941c562861188b" + version = "2018.01.18" + +[[projects]] + name = "github.com/davecgh/go-spew" + packages = ["spew"] + revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" + version = "v1.1.1" + +[[projects]] + name = "github.com/ebuchman/fail-test" + packages = ["."] + revision = "95f809107225be108efcf10a3509e4ea6ceef3c4" + +[[projects]] + branch = "master" + name = "github.com/getsentry/raven-go" + packages = ["."] + revision = "084a9de9eb0361fbd5ded14b55c84e5493a5d7f6" + +[[projects]] + name = "github.com/go-kit/kit" + packages = [ + "log", + "log/level", + "log/term", + "metrics", + "metrics/discard", + "metrics/internal/lv", + "metrics/prometheus" + ] + revision = "4dc7be5d2d12881735283bcab7352178e190fc71" + version = "v0.6.0" + +[[projects]] + name = "github.com/go-logfmt/logfmt" + packages = ["."] + revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5" + version = "v0.3.0" + +[[projects]] + name = "github.com/go-pg/pg" + packages = [ + ".", + "internal", + "internal/parser", + "internal/pool", + "orm", + "types" + ] + revision = "514bed76d8f579d6ff8d40294fa77e476a5c1b3f" + version = "v6.15.0" + +[[projects]] + name = "github.com/go-stack/stack" + packages = ["."] + revision = "2fee6af1a9795aafbe0253a0cfbdf668e1fb8a9a" + version = "v1.8.0" + +[[projects]] + name = "github.com/gogo/protobuf" + packages = [ + "gogoproto", + "jsonpb", + "proto", + "protoc-gen-gogo/descriptor", + "sortkeys", + "types" + ] + revision = "636bf0302bc95575d69441b25a2603156ffdddf1" + version = "v1.1.1" + +[[projects]] + name = "github.com/golang/protobuf" + packages = [ + "proto", + "ptypes", + "ptypes/any", + "ptypes/duration", + "ptypes/timestamp" + ] + revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" + version = "v1.1.0" + +[[projects]] + branch = "master" + name = "github.com/golang/snappy" + packages = ["."] + revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a" + +[[projects]] + name = "github.com/gorilla/websocket" + packages = ["."] + revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b" + version = "v1.2.0" + +[[projects]] + branch = "master" + name = "github.com/jinzhu/inflection" + packages = ["."] + revision = "04140366298a54a039076d798123ffa108fff46c" + +[[projects]] + branch = "master" + name = "github.com/jmhodges/levigo" + packages = ["."] + revision = "c42d9e0ca023e2198120196f842701bb4c55d7b9" + +[[projects]] + branch = "master" + name = "github.com/kr/logfmt" + packages = ["."] + revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0" + +[[projects]] + name = "github.com/matttproud/golang_protobuf_extensions" + packages = ["pbutil"] + revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" + version = "v1.0.1" + +[[projects]] + name = "github.com/pkg/errors" + packages = ["."] + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + +[[projects]] + name = "github.com/prometheus/client_golang" + packages = [ + "prometheus", + "prometheus/promhttp" + ] + revision = "ae27198cdd90bf12cd134ad79d1366a6cf49f632" + +[[projects]] + branch = "master" + name = "github.com/prometheus/client_model" + packages = ["go"] + revision = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f" + +[[projects]] + branch = "master" + name = "github.com/prometheus/common" + packages = [ + "expfmt", + "internal/bitbucket.org/ww/goautoneg", + "model" + ] + revision = "c7de2306084e37d54b8be01f3541a8464345e9a5" + +[[projects]] + branch = "master" + name = "github.com/prometheus/procfs" + packages = [ + ".", + "internal/util", + "nfs", + "xfs" + ] + revision = "418d78d0b9a7b7de3a6bbc8a23def624cc977bb2" + +[[projects]] + name = "github.com/rcrowley/go-metrics" + packages = ["."] + revision = "e2704e165165ec55d062f5919b4b29494e9fa790" + +[[projects]] + branch = "master" + name = "github.com/syndtr/goleveldb" + packages = [ + "leveldb", + "leveldb/cache", + "leveldb/comparer", + "leveldb/errors", + "leveldb/filter", + "leveldb/iterator", + "leveldb/journal", + "leveldb/memdb", + "leveldb/opt", + "leveldb/storage", + "leveldb/table", + "leveldb/util" + ] + revision = "ae2bd5eed72d46b28834ec3f60db3a3ebedd8dbd" + +[[projects]] + name = "github.com/tendermint/btcd" + packages = ["btcec"] + revision = "e5840949ff4fff0c56f9b6a541e22b63581ea9df" + +[[projects]] + branch = "master" + name = "github.com/tendermint/ed25519" + packages = [ + ".", + "edwards25519", + "extra25519" + ] + revision = "d8387025d2b9d158cf4efb07e7ebf814bcce2057" + +[[projects]] + name = "github.com/tendermint/go-amino" + packages = ["."] + revision = "faa6e731944e2b7b6a46ad202902851e8ce85bee" + version = "v0.12.0" + +[[projects]] + name = "github.com/tendermint/tendermint" + packages = [ + "abci/client", + "abci/example/code", + "abci/example/kvstore", + "abci/types", + "blockchain", + "config", + "consensus", + "consensus/types", + "crypto", + "crypto/ed25519", + "crypto/encoding/amino", + "crypto/merkle", + "crypto/multisig", + "crypto/multisig/bitarray", + "crypto/secp256k1", + "crypto/tmhash", + "evidence", + "libs/autofile", + "libs/clist", + "libs/common", + "libs/db", + "libs/events", + "libs/flowrate", + "libs/log", + "libs/pubsub", + "libs/pubsub/query", + "mempool", + "node", + "p2p", + "p2p/conn", + "p2p/pex", + "p2p/upnp", + "privval", + "proxy", + "rpc/client", + "rpc/core", + "rpc/core/types", + "rpc/grpc", + "rpc/lib", + "rpc/lib/client", + "rpc/lib/server", + "rpc/lib/types", + "state", + "state/txindex", + "state/txindex/kv", + "state/txindex/null", + "types", + "types/time", + "version" + ] + revision = "d419fffe18531317c28c29a292ad7d253f6cafdf" + version = "v0.24.0" + +[[projects]] + branch = "master" + name = "github.com/vmihailenco/sasl" + packages = ["."] + revision = "58bfd21040084b3e377ef748e559e2bd3e7c826e" + +[[projects]] + branch = "master" + name = "golang.org/x/crypto" + packages = [ + "chacha20poly1305", + "curve25519", + "hkdf", + "internal/chacha20", + "internal/subtle", + "nacl/box", + "nacl/secretbox", + "pbkdf2", + "poly1305", + "ripemd160", + "salsa20/salsa" + ] + revision = "0e37d006457bf46f9e6692014ba72ef82c33022c" + +[[projects]] + name = "golang.org/x/net" + packages = [ + "context", + "http/httpguts", + "http2", + "http2/hpack", + "idna", + "internal/timeseries", + "netutil", + "publicsuffix", + "trace" + ] + revision = "292b43bbf7cb8d35ddf40f8d5100ef3837cced3f" + +[[projects]] + branch = "master" + name = "golang.org/x/sys" + packages = ["cpu"] + revision = "d47a0f3392421c5624713c9a19fe781f651f8a50" + +[[projects]] + name = "golang.org/x/text" + packages = [ + "collate", + "collate/build", + "internal/colltab", + "internal/gen", + "internal/tag", + "internal/triegen", + "internal/ucd", + "language", + "secure/bidirule", + "transform", + "unicode/bidi", + "unicode/cldr", + "unicode/norm", + "unicode/rangetable" + ] + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" + +[[projects]] + branch = "master" + name = "google.golang.org/genproto" + packages = ["googleapis/rpc/status"] + revision = "c3f76f3b92d1ffa4c58a9ff842a58b8877655e0f" + +[[projects]] + name = "google.golang.org/grpc" + packages = [ + ".", + "balancer", + "balancer/base", + "balancer/roundrobin", + "codes", + "connectivity", + "credentials", + "encoding", + "encoding/proto", + "grpclog", + "internal", + "internal/backoff", + "internal/channelz", + "internal/grpcrand", + "keepalive", + "metadata", + "naming", + "peer", + "resolver", + "resolver/dns", + "resolver/passthrough", + "stats", + "status", + "tap", + "transport" + ] + revision = "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + version = "v1.13.0" + +[[projects]] + name = "gopkg.in/resty.v1" + packages = ["."] + revision = "d4920dcf5b7689548a6db640278a9b35a5b48ec6" + version = "v1.9.1" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "a5e5308283adc534de6be188f80748bee9e9db9f0835fa8ce51f7ed184d016de" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..c1e99f1 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,46 @@ +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true + + +[[constraint]] + name = "github.com/tendermint/tendermint" + version = "0.24.0" + +[prune] + go-tests = true + unused-packages = true + +[[constraint]] + name = "github.com/go-pg/pg" + version = "6.15.0" + +[[constraint]] + name = "gopkg.in/resty.v1" + version = "1.9.1" + +[[constraint]] + branch = "master" + name = "github.com/getsentry/raven-go" diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..cb86a6f --- /dev/null +++ b/Makefile @@ -0,0 +1,6 @@ +get_vendor_deps: + go get -u -v github.com/golang/dep/cmd/dep + dep ensure -v + +install: + go install ./ \ No newline at end of file diff --git a/alerter/main.go b/alerter/main.go new file mode 100644 index 0000000..1908cf5 --- /dev/null +++ b/alerter/main.go @@ -0,0 +1,96 @@ +package main + +import ( + "github.com/certusone/chain_exporter/types" + "github.com/getsentry/raven-go" + "github.com/go-pg/pg" + "github.com/pkg/errors" + "os" + "strconv" + "time" +) + +type ( + Monitor struct { + db *pg.DB + } +) + +func main() { + + if os.Getenv("DB_HOST") == "" { + panic(errors.New("DB_HOST needs to be set")) + } + if os.Getenv("DB_USER") == "" { + panic(errors.New("DB_USER needs to be set")) + } + if os.Getenv("DB_PW") == "" { + panic(errors.New("DB_PW needs to be set")) + } + if os.Getenv("RAVEN_DSN") == "" { + panic(errors.New("RAVEN_DSN needs to be set")) + } + + raven.SetDSN(os.Getenv("RAVEN_DSN")) + + db := pg.Connect(&pg.Options{ + Addr: os.Getenv("DB_HOST"), + User: os.Getenv("DB_USER"), + Password: os.Getenv("DB_PW"), + }) + defer db.Close() + + monitor := &Monitor{db} + for { + select { + case <-time.Tick(time.Second): + err := monitor.sync() + if err != nil { + panic(err) + } + } + } +} + +func (m *Monitor) sync() error { + println("syncing") + var misses []*types.MissInfo + err := m.db.Model(&types.MissInfo{}).Where("alerted = FALSE").Select(&misses) + if err != nil { + panic(err) + } + for _, miss := range misses { + raven.CaptureError(errors.New("Missed block"), map[string]string{"height": strconv.FormatInt(miss.Height, 10), "time": miss.Time.String(), "address": miss.Address}) + miss.Alerted = true + _, err = m.db.Model(miss).Where("id = ?", miss.ID).Update() + if err != nil { + return err + } + } + + var proposals []*types.Proposal + err = m.db.Model(&types.Proposal{}).Where("alerted = FALSE").Select(&proposals) + if err != nil { + panic(err) + } + for _, proposal := range proposals { + if proposal.ProposalStatus == "Passed" || proposal.ProposalStatus == "Rejected" { + proposal.Alerted = true + _, err = m.db.Model(proposal).Where("id = ?", proposal.ID).Update() + if err != nil { + return err + } + continue + } + + raven.CaptureMessage("New governance proposal: "+proposal.Title+"\nDescription: "+proposal.Description+"\nStartHeight: "+proposal.VotingStartBlock, map[string]string{"height": strconv.FormatInt(proposal.Height, 10), "type": proposal.Type}) + + proposal.Alerted = true + _, err = m.db.Model(proposal).Where("id = ?", proposal.ID).Update() + if err != nil { + return err + } + } + + return nil +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..c8a77fb --- /dev/null +++ b/main.go @@ -0,0 +1,216 @@ +package main + +import ( + "encoding/json" + "fmt" + ctypes "github.com/certusone/chain_exporter/types" + "github.com/go-pg/pg" + "github.com/go-pg/pg/orm" + "github.com/pkg/errors" + "github.com/tendermint/tendermint/rpc/client" + "github.com/tendermint/tendermint/types" + "gopkg.in/resty.v1" + "os" + "time" +) + +type ( + Monitor struct { + client *client.HTTP + db *pg.DB + } +) + +func main() { + if os.Getenv("GAIA_URL") == "" { + panic(errors.New("GAIA_URL needs to be set")) + } + if os.Getenv("DB_HOST") == "" { + panic(errors.New("DB_HOST needs to be set")) + } + if os.Getenv("DB_USER") == "" { + panic(errors.New("DB_USER needs to be set")) + } + if os.Getenv("DB_PW") == "" { + panic(errors.New("DB_PW needs to be set")) + } + if os.Getenv("LCD_URL") == "" { + panic(errors.New("LCD_URL needs to be set")) + } + + tClient := client.NewHTTP(os.Getenv("GAIA_URL"), "/websocket") + + db := pg.Connect(&pg.Options{ + Addr: os.Getenv("DB_HOST"), + User: os.Getenv("DB_USER"), + Password: os.Getenv("DB_PW"), + }) + defer db.Close() + + err := createSchema(db) + if err != nil { + //panic(err) + } + monitor := &Monitor{tClient, db} + go func() { + for { + err = monitor.sync() + if err != nil { + panic(err) + } + time.Sleep(time.Second) + } + }() + + for { + select { + case <-time.Tick(time.Second): + err := monitor.getGovernance() + if err != nil { + panic(err) + } + } + } +} + +func createSchema(db *pg.DB) error { + for _, model := range []interface{}{(*ctypes.BlockInfo)(nil), (*ctypes.EvidenceInfo)(nil), (*ctypes.MissInfo)(nil), (*ctypes.Proposal)(nil)} { + err := db.CreateTable(model, &orm.CreateTableOptions{}) + if err != nil { + return err + } + } + return nil +} + +func (m *Monitor) sync() error { + var blocks []ctypes.BlockInfo + err := m.db.Model(&blocks).Order("height DESC").Limit(1).Select() + if err != nil { + return err + } + bestHeight := int64(1) + if len(blocks) > 0 { + bestHeight = blocks[0].Height + } + + status, err := m.client.Status() + if err != nil { + return err + } + maxHeight := status.SyncInfo.LatestBlockHeight + + for i := bestHeight + 1; i <= maxHeight; i++ { + err = m.ingestBlock(i) + if err != nil { + return err + } + fmt.Printf("synced block %d/%d \n", i, maxHeight) + } + return nil +} + +func (m *Monitor) ingestBlock(height int64) error { + prevHeight := height - 1 + + // Get Data + validators, err := m.client.Validators(&prevHeight) + if err != nil { + return err + } + + block, err := m.client.Block(&prevHeight) + if err != nil { + return err + } + + nextBlock, err := m.client.Block(&height) + if err != nil { + return err + } + + blockInfo := new(ctypes.BlockInfo) + blockInfo.ID = nextBlock.BlockMeta.Header.LastBlockID.String() + blockInfo.Height = height + blockInfo.Time = nextBlock.BlockMeta.Header.Time + blockInfo.Proposer = block.Block.ProposerAddress.String() + + // Identify missed validators + missedValidators := make([]*ctypes.MissInfo, 0) + + // Parse + for i, validator := range validators.Validators { + if nextBlock.Block.LastCommit.Precommits[i] == nil { + missed := &ctypes.MissInfo{ + Height: block.BlockMeta.Header.Height, + Address: validator.Address.String(), + Alerted: false, + Time: block.BlockMeta.Header.Time, + } + missedValidators = append(missedValidators, missed) + continue + } + } + + // Collect evidence + evidenceInfo := make([]*ctypes.EvidenceInfo, 0) + for _, evidence := range nextBlock.Block.Evidence.Evidence { + evInfo := &ctypes.EvidenceInfo{} + evInfo.Address = types.Address(evidence.Address()).String() + evInfo.Height = evidence.Height() + evidenceInfo = append(evidenceInfo, evInfo) + } + + // Insert in DB + err = m.db.RunInTransaction(func(tx *pg.Tx) error { + err = tx.Insert(blockInfo) + if err != nil { + return err + } + if len(evidenceInfo) > 0 { + err = tx.Insert(&evidenceInfo) + if err != nil { + return err + } + } + if len(missedValidators) > 0 { + err = tx.Insert(&missedValidators) + if err != nil { + return err + } + } + return nil + }) + if err != nil { + return err + } + + return nil +} + +func (m *Monitor) getGovernance() error { + resp, err := resty.R().Get(os.Getenv("LCD_URL") + "/gov/proposals") + if err != nil { + return err + } + + var proposals []*ctypes.Proposal + err = json.Unmarshal(resp.Body(), &proposals) + if err != nil { + return err + } + + for _, proposal := range proposals { + proposal.ID = proposal.Details.ProposalID + proposal.Height = proposal.Details.SubmitBlock + proposal.Alerted = false + proposal.Description = proposal.Details.Description + proposal.ProposalStatus = proposal.Details.ProposalStatus + proposal.ProposalType = proposal.Details.ProposalType + proposal.Title = proposal.Details.Title + proposal.VotingStartBlock = proposal.Details.VotingStartBlock + } + + _, err = m.db.Model(&proposals).OnConflict("DO NOTHING").Insert() + return err +} diff --git a/types/types.go b/types/types.go new file mode 100644 index 0000000..2556dd8 --- /dev/null +++ b/types/types.go @@ -0,0 +1,48 @@ +package types + +import ( + "time" +) + +type ( + BlockInfo struct { + ID string + Height int64 + Proposer string + Time time.Time + } + + EvidenceInfo struct { + Address string + Height int64 + } + + MissInfo struct { + ID int64 + Address string + Height int64 + Alerted bool `sql:",default:false,notnull"` + Time time.Time + } + + Proposal struct { + ID string + Type string `json:"type"` + Height int64 + Alerted bool `sql:",default:false,notnull"` + Title string + Description string + ProposalType string + ProposalStatus string + VotingStartBlock string + Details struct { + ProposalID string `json:"proposal_id"` + Title string `json:"title"` + Description string `json:"description"` + ProposalType string `json:"proposal_type"` + ProposalStatus string `json:"proposal_status"` + SubmitBlock int64 `json:"submit_block,string"` + VotingStartBlock string `json:"voting_start_block"` + } `json:"value"` + } +)