diff --git a/tm-monitor/Makefile b/tm-monitor/Makefile index d94379ca..b5f46012 100644 --- a/tm-monitor/Makefile +++ b/tm-monitor/Makefile @@ -3,6 +3,7 @@ VERSION := $(shell perl -ne '/^var version.*"([^"]+)".*$$/ && print "v$$1\n"' ma GOTOOLS = \ github.com/Masterminds/glide \ github.com/mitchellh/gox +PACKAGES=$(shell go list ./... | grep -v '/vendor/') tools: go get -v $(GOTOOLS) @@ -17,7 +18,7 @@ install: go install -ldflags "-X main.version=${VERSION}" test: - go test + @go test $(PACKAGES) build-all: tools gox -verbose \ @@ -41,7 +42,6 @@ build-docker: docker build -t "tendermint/monitor" . clean: - rm -f ./tm-monitor.log rm -f ./tm-monitor rm -rf ./dist diff --git a/tm-monitor/glide.lock b/tm-monitor/glide.lock index 040bc126..7be1bdf4 100644 --- a/tm-monitor/glide.lock +++ b/tm-monitor/glide.lock @@ -1,5 +1,5 @@ -hash: 3315dcf12e2554e2927f2a0907f2547cd86d5c8926461d055662a7bee88caa4a -updated: 2017-03-07T08:38:45.613512657Z +hash: d21d1f12681cd4ab5b7f0efd7bf00c1d5f7021b1ae6e8700c11bca6822337079 +updated: 2017-03-16T10:01:58.079646405Z imports: - name: github.com/btcsuite/btcd version: 583684b21bfbde9b5fc4403916fd7c807feb0289 @@ -7,6 +7,12 @@ imports: - btcec - name: github.com/BurntSushi/toml version: 99064174e013895bbd9b025c31100bd1d9b590ca +- name: github.com/go-kit/kit + version: b6f30a2e0632f5722fb26d8765d726335b79d3e6 + subpackages: + - log +- name: github.com/go-logfmt/logfmt + version: 390ab7935ee28ec6b286364bba9b4dd6410cb3d5 - name: github.com/go-stack/stack version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82 - name: github.com/golang/protobuf @@ -19,10 +25,14 @@ imports: version: 3ab3a8b8831546bd18fd182c20687ca853b2bb13 - name: github.com/jmhodges/levigo version: c42d9e0ca023e2198120196f842701bb4c55d7b9 +- name: github.com/kr/logfmt + version: b84e30acd515aadc4b783ad4ff83aff3299bdfe0 - name: github.com/mattn/go-colorable version: d898aa9fb31c91f35dd28ca75db377eff023c076 - name: github.com/mattn/go-isatty version: dda3de49cbfcec471bd7a70e6cc01fcc3ff90109 +- name: github.com/pkg/errors + version: bfd5150e4e41705ded2129ec33379de1cb90b513 - name: github.com/rcrowley/go-metrics version: 1f30fe9094a513ce4c700b9a54458bbb0c96996c - name: github.com/stretchr/testify diff --git a/tm-monitor/glide.yaml b/tm-monitor/glide.yaml index e881af99..14bc82fb 100644 --- a/tm-monitor/glide.yaml +++ b/tm-monitor/glide.yaml @@ -16,3 +16,8 @@ import: subpackages: - client - package: github.com/tendermint/log15 +- package: github.com/go-kit/kit + subpackages: + - log + - term +- package: github.com/pkg/errors diff --git a/tm-monitor/main.go b/tm-monitor/main.go index f2640b63..720e4d30 100644 --- a/tm-monitor/main.go +++ b/tm-monitor/main.go @@ -6,22 +6,21 @@ import ( "os" "strings" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/term" cmn "github.com/tendermint/go-common" - logger "github.com/tendermint/go-logger" - log15 "github.com/tendermint/log15" - em "github.com/tendermint/tools/tm-monitor/eventmeter" + monitor "github.com/tendermint/tools/tm-monitor/monitor" ) var version = "0.3.0.pre" -var log = logger.New() +var logger = log.NewNopLogger() func main() { var listenAddr string - var verbose, noton bool + var noton bool flag.StringVar(&listenAddr, "listen-addr", "tcp://0.0.0.0:46670", "HTTP and Websocket server listen address") - flag.BoolVar(&verbose, "v", false, "verbose logging") flag.BoolVar(¬on, "no-ton", false, "Do not show ton (table of nodes)") flag.Usage = func() { @@ -29,7 +28,7 @@ func main() { applications, collecting and providing various statistics to the user. Usage: - tm-monitor [-v] [-no-ton] [-listen-addr="tcp://0.0.0.0:46670"] [endpoints] + tm-monitor [-no-ton] [-listen-addr="tcp://0.0.0.0:46670"] [endpoints] Examples: # monitor single instance @@ -48,17 +47,28 @@ Examples: os.Exit(1) } + if noton { + // Color errors red + colorFn := func(keyvals ...interface{}) term.FgBgColor { + for i := 1; i < len(keyvals); i += 2 { + if _, ok := keyvals[i].(error); ok { + return term.FgBgColor{Fg: term.White, Bg: term.Red} + } + } + return term.FgBgColor{} + } + + logger = term.NewLogger(os.Stdout, log.NewLogfmtLogger, colorFn) + } + m := startMonitor(flag.Arg(0)) startRPC(listenAddr, m) var ton *Ton if !noton { - logToFile("tm-monitor.log", verbose) ton = NewTon(m) ton.Start() - } else { - logToStdout(verbose) } cmn.TrapSignal(func() { @@ -69,50 +79,21 @@ Examples: }) } -func startMonitor(endpoints string) *Monitor { - m := NewMonitor() +func startMonitor(endpoints string) *monitor.Monitor { + m := monitor.NewMonitor() + m.SetLogger(log.With(logger, "component", "monitor")) for _, e := range strings.Split(endpoints, ",") { - if err := m.Monitor(NewNode(e)); err != nil { - log.Crit(err.Error()) - os.Exit(1) + n := monitor.NewNode(e) + n.SetLogger(log.With(logger, "node", e)) + if err := m.Monitor(n); err != nil { + panic(err) } } if err := m.Start(); err != nil { - log.Crit(err.Error()) - os.Exit(1) + panic(err) } return m } - -func logToStdout(verbose bool) { - if verbose { - log.SetHandler(logger.LvlFilterHandler( - logger.LvlDebug, - logger.BypassHandler(), - )) - } else { - log.SetHandler(logger.LvlFilterHandler( - logger.LvlInfo, - logger.BypassHandler(), - )) - } - em.Log = log -} - -func logToFile(filename string, verbose bool) { - if verbose { - log.SetHandler(logger.LvlFilterHandler( - logger.LvlDebug, - log15.Must.FileHandler(filename, log15.LogfmtFormat()), - )) - } else { - log.SetHandler(logger.LvlFilterHandler( - logger.LvlInfo, - log15.Must.FileHandler(filename, log15.LogfmtFormat()), - )) - } - em.Log = log -} diff --git a/tm-monitor/monitor.go b/tm-monitor/monitor/monitor.go similarity index 91% rename from tm-monitor/monitor.go rename to tm-monitor/monitor/monitor.go index e21c6713..e70c001f 100644 --- a/tm-monitor/monitor.go +++ b/tm-monitor/monitor/monitor.go @@ -1,9 +1,12 @@ -package main +package monitor import ( + "fmt" "math/rand" "time" + "github.com/go-kit/kit/log" + "github.com/pkg/errors" tmtypes "github.com/tendermint/tendermint/types" ) @@ -23,6 +26,8 @@ type Monitor struct { recalculateNetworkUptimeEvery time.Duration numValidatorsUpdateInterval time.Duration + + logger log.Logger } // NewMonitor creates new instance of a Monitor. You can provide options to @@ -38,6 +43,7 @@ func NewMonitor(options ...func(*Monitor)) *Monitor { nodeQuit: make(map[string]chan struct{}), recalculateNetworkUptimeEvery: 10 * time.Second, numValidatorsUpdateInterval: 5 * time.Second, + logger: log.NewNopLogger(), } for _, option := range options { @@ -61,6 +67,11 @@ func SetNumValidatorsUpdateInterval(d time.Duration) func(m *Monitor) { } } +// SetLogger lets you set your own logger +func (m *Monitor) SetLogger(l log.Logger) { + m.logger = l +} + // Monitor begins to monitor the node `n`. The node will be started and added // to the monitor. func (m *Monitor) Monitor(n *Node) error { @@ -116,6 +127,8 @@ func (m *Monitor) Stop() { // main loop where we listen for events from the node func (m *Monitor) listen(nodeName string, blockCh <-chan tmtypes.Header, blockLatencyCh <-chan float64, disconnectCh <-chan bool, quit <-chan struct{}) { + logger := log.With(m.logger, "node", nodeName) + for { select { case <-quit: @@ -133,6 +146,7 @@ func (m *Monitor) listen(nodeName string, blockCh <-chan tmtypes.Header, blockLa m.Network.NodeIsOnline(nodeName) } case <-time.After(nodeLivenessTimeout): + logger.Log("event", fmt.Sprintf("node was not responding for %v", nodeLivenessTimeout)) m.Network.NodeIsDown(nodeName) } } @@ -176,7 +190,7 @@ func (m *Monitor) updateNumValidatorLoop() { if i == randomNodeIndex { height, num, err = n.NumValidators() if err != nil { - log.Debug(err.Error()) + m.logger.Log("err", errors.Wrap(err, "update num validators failed")) } break } diff --git a/tm-monitor/monitor_test.go b/tm-monitor/monitor/monitor_test.go similarity index 96% rename from tm-monitor/monitor_test.go rename to tm-monitor/monitor/monitor_test.go index 17ec0a04..1a4dd711 100644 --- a/tm-monitor/monitor_test.go +++ b/tm-monitor/monitor/monitor_test.go @@ -1,4 +1,4 @@ -package main_test +package monitor_test import ( "testing" @@ -10,8 +10,8 @@ import ( crypto "github.com/tendermint/go-crypto" ctypes "github.com/tendermint/tendermint/rpc/core/types" tmtypes "github.com/tendermint/tendermint/types" - monitor "github.com/tendermint/tools/tm-monitor" mock "github.com/tendermint/tools/tm-monitor/mock" + monitor "github.com/tendermint/tools/tm-monitor/monitor" ) func TestMonitorUpdatesNumberOfValidators(t *testing.T) { diff --git a/tm-monitor/network.go b/tm-monitor/monitor/network.go similarity index 96% rename from tm-monitor/network.go rename to tm-monitor/monitor/network.go index 03750f64..bb776973 100644 --- a/tm-monitor/network.go +++ b/tm-monitor/monitor/network.go @@ -1,4 +1,4 @@ -package main +package monitor import ( "sync" @@ -74,11 +74,9 @@ func (n *Network) NewBlock(b tmtypes.Header) { defer n.mu.Unlock() if n.Height >= uint64(b.Height) { - log.Debug("Received new block with height <= current", "received", b.Height, "current", n.Height) return } - log.Debug("Received new block", "height", b.Height, "ntxs", b.NumTxs) n.Height = uint64(b.Height) n.blockTimeMeter.Mark(1) diff --git a/tm-monitor/network_test.go b/tm-monitor/monitor/network_test.go similarity index 96% rename from tm-monitor/network_test.go rename to tm-monitor/monitor/network_test.go index d2ca1741..d0d2c93a 100644 --- a/tm-monitor/network_test.go +++ b/tm-monitor/monitor/network_test.go @@ -1,4 +1,4 @@ -package main_test +package monitor_test import ( "testing" @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" tmtypes "github.com/tendermint/tendermint/types" - monitor "github.com/tendermint/tools/tm-monitor" + monitor "github.com/tendermint/tools/tm-monitor/monitor" ) func TestNetworkNewBlock(t *testing.T) { diff --git a/tm-monitor/node.go b/tm-monitor/monitor/node.go similarity index 90% rename from tm-monitor/node.go rename to tm-monitor/monitor/node.go index 15a119de..8f3071d1 100644 --- a/tm-monitor/node.go +++ b/tm-monitor/monitor/node.go @@ -1,11 +1,12 @@ -package main +package monitor import ( "encoding/json" - "fmt" "math" "time" + "github.com/go-kit/kit/log" + "github.com/pkg/errors" crypto "github.com/tendermint/go-crypto" events "github.com/tendermint/go-events" rpc_client "github.com/tendermint/go-rpc/client" @@ -46,6 +47,8 @@ type Node struct { checkIsValidatorInterval time.Duration quit chan struct{} + + logger log.Logger } func NewNode(rpcAddr string, options ...func(*Node)) *Node { @@ -62,6 +65,7 @@ func NewNodeWithEventMeterAndRpcClient(rpcAddr string, em eventMeter, rpcClient Name: rpcAddr, quit: make(chan struct{}), checkIsValidatorInterval: 5 * time.Second, + logger: log.NewNopLogger(), } for _, option := range options { @@ -91,6 +95,11 @@ func (n *Node) NotifyAboutDisconnects(ch chan<- bool) { n.disconnectCh = ch } +// SetLogger lets you set your own logger +func (n *Node) SetLogger(l log.Logger) { + n.logger = l +} + func (n *Node) Start() error { if err := n.em.Start(); err != nil { return err @@ -127,6 +136,7 @@ func newBlockCallback(n *Node) em.EventCallbackFunc { block := data.(tmtypes.EventDataNewBlockHeader).Header n.Height = uint64(block.Height) + n.logger.Log("event", "new block", "height", block.Height, "numTxs", block.NumTxs) if n.blockCh != nil { n.blockCh <- *block @@ -138,6 +148,8 @@ func newBlockCallback(n *Node) em.EventCallbackFunc { func latencyCallback(n *Node) em.LatencyCallbackFunc { return func(latency float64) { n.BlockLatency = latency / 1000000.0 // ns to ms + n.logger.Log("event", "new block latency", "latency", n.BlockLatency) + if n.blockLatencyCh != nil { n.blockLatencyCh <- latency } @@ -148,14 +160,18 @@ func latencyCallback(n *Node) em.LatencyCallbackFunc { func disconnectCallback(n *Node) em.DisconnectCallbackFunc { return func() { n.Online = false + n.logger.Log("status", "down") + if n.disconnectCh != nil { n.disconnectCh <- true } if err := n.RestartBackOff(); err != nil { - log.Error(err.Error()) + n.logger.Log("err", errors.Wrap(err, "restart failed")) } else { n.Online = true + n.logger.Log("status", "online") + if n.disconnectCh != nil { n.disconnectCh <- false } @@ -171,7 +187,7 @@ func (n *Node) RestartBackOff() error { time.Sleep(d * time.Second) if err := n.Start(); err != nil { - log.Debug("Can't connect to node %v due to %v", n, err) + n.logger.Log("err", errors.Wrap(err, "restart failed")) } else { // TODO: authenticate pubkey return nil @@ -180,7 +196,7 @@ func (n *Node) RestartBackOff() error { attempt++ if attempt > maxRestarts { - return fmt.Errorf("Reached max restarts for node %v", n) + return errors.New("Reached max restarts") } } } @@ -223,7 +239,7 @@ func (n *Node) checkIsValidator() { } } } else { - log.Debug(err.Error()) + n.logger.Log("err", errors.Wrap(err, "check is validator failed")) } } diff --git a/tm-monitor/node_test.go b/tm-monitor/monitor/node_test.go similarity index 96% rename from tm-monitor/node_test.go rename to tm-monitor/monitor/node_test.go index 15afe271..a379701c 100644 --- a/tm-monitor/node_test.go +++ b/tm-monitor/monitor/node_test.go @@ -1,4 +1,4 @@ -package main_test +package monitor_test import ( "testing" @@ -9,9 +9,9 @@ import ( crypto "github.com/tendermint/go-crypto" ctypes "github.com/tendermint/tendermint/rpc/core/types" tmtypes "github.com/tendermint/tendermint/types" - monitor "github.com/tendermint/tools/tm-monitor" em "github.com/tendermint/tools/tm-monitor/eventmeter" mock "github.com/tendermint/tools/tm-monitor/mock" + monitor "github.com/tendermint/tools/tm-monitor/monitor" ) const ( diff --git a/tm-monitor/rpc.go b/tm-monitor/rpc.go index 67169f6b..127179e2 100644 --- a/tm-monitor/rpc.go +++ b/tm-monitor/rpc.go @@ -5,9 +5,10 @@ import ( "net/http" rpc "github.com/tendermint/go-rpc/server" + monitor "github.com/tendermint/tools/tm-monitor/monitor" ) -func startRPC(listenAddr string, m *Monitor) { +func startRPC(listenAddr string, m *monitor.Monitor) { routes := routes(m) // serve http and ws @@ -20,7 +21,7 @@ func startRPC(listenAddr string, m *Monitor) { } } -func routes(m *Monitor) map[string]*rpc.RPCFunc { +func routes(m *monitor.Monitor) map[string]*rpc.RPCFunc { return map[string]*rpc.RPCFunc{ "status": rpc.NewRPCFunc(RPCStatus(m), ""), "status/network": rpc.NewRPCFunc(RPCNetworkStatus(m), ""), @@ -35,9 +36,9 @@ func routes(m *Monitor) map[string]*rpc.RPCFunc { } // RPCStatus returns common statistics for the network and statistics per node. -func RPCStatus(m *Monitor) interface{} { +func RPCStatus(m *monitor.Monitor) interface{} { return func() (networkAndNodes, error) { - values := make([]*Node, len(m.Nodes)) + values := make([]*monitor.Node, len(m.Nodes)) i := 0 for _, v := range m.Nodes { values[i] = v @@ -49,15 +50,15 @@ func RPCStatus(m *Monitor) interface{} { } // RPCNetworkStatus returns common statistics for the network. -func RPCNetworkStatus(m *Monitor) interface{} { - return func() (*Network, error) { +func RPCNetworkStatus(m *monitor.Monitor) interface{} { + return func() (*monitor.Network, error) { return m.Network, nil } } // RPCNodeStatus returns statistics for the given node. -func RPCNodeStatus(m *Monitor) interface{} { - return func(name string) (*Node, error) { +func RPCNodeStatus(m *monitor.Monitor) interface{} { + return func(name string) (*monitor.Node, error) { if n, ok := m.Nodes[name]; ok { return n, nil } @@ -66,9 +67,9 @@ func RPCNodeStatus(m *Monitor) interface{} { } // RPCMonitor allows to dynamically add a endpoint to under the monitor. -func RPCMonitor(m *Monitor) interface{} { - return func(endpoint string) (*Node, error) { - n := NewNode(endpoint) +func RPCMonitor(m *monitor.Monitor) interface{} { + return func(endpoint string) (*monitor.Node, error) { + n := monitor.NewNode(endpoint) if err := m.Monitor(n); err != nil { return nil, err } @@ -77,7 +78,7 @@ func RPCMonitor(m *Monitor) interface{} { } // RPCUnmonitor removes the given endpoint from under the monitor. -func RPCUnmonitor(m *Monitor) interface{} { +func RPCUnmonitor(m *monitor.Monitor) interface{} { return func(endpoint string) (bool, error) { if n, ok := m.Nodes[endpoint]; ok { m.Unmonitor(n) @@ -121,6 +122,6 @@ func RPCUnmonitor(m *Monitor) interface{} { //--> types type networkAndNodes struct { - Network *Network `json:"network"` - Nodes []*Node `json:"nodes"` + Network *monitor.Network `json:"network"` + Nodes []*monitor.Node `json:"nodes"` } diff --git a/tm-monitor/ton.go b/tm-monitor/ton.go index 5ffaf6e9..8052db2e 100644 --- a/tm-monitor/ton.go +++ b/tm-monitor/ton.go @@ -6,6 +6,8 @@ import ( "os" "text/tabwriter" "time" + + monitor "github.com/tendermint/tools/tm-monitor/monitor" ) const ( @@ -24,14 +26,14 @@ const ( // Ton was inspired by [Linux top // program](https://en.wikipedia.org/wiki/Top_(software)) as the name suggests. type Ton struct { - monitor *Monitor + monitor *monitor.Monitor RefreshRate time.Duration Output io.Writer quit chan struct{} } -func NewTon(m *Monitor) *Ton { +func NewTon(m *monitor.Monitor) *Ton { return &Ton{ RefreshRate: defaultRefreshRate, Output: os.Stdout,