separate http server for metrics

config/config.go

@@ -50,6 +50,7 @@ type Config struct {
 	Mempool         *MempoolConfig         `mapstructure:"mempool"`
 	Consensus       *ConsensusConfig       `mapstructure:"consensus"`
 	TxIndex         *TxIndexConfig         `mapstructure:"tx_index"`
+	Instrumentation *InstrumentationConfig `mapstructure:"instrumentation"`
 }
 
 // DefaultConfig returns a default configuration for a Tendermint node
@@ -61,6 +62,7 @@ func DefaultConfig() *Config {
 		Mempool:         DefaultMempoolConfig(),
 		Consensus:       DefaultConsensusConfig(),
 		TxIndex:         DefaultTxIndexConfig(),
+		Instrumentation: DefaultInstrumentationConfig(),
 	}
 }
 
@@ -73,6 +75,7 @@ func TestConfig() *Config {
 		Mempool:         TestMempoolConfig(),
 		Consensus:       TestConsensusConfig(),
 		TxIndex:         TestTxIndexConfig(),
+		Instrumentation: TestInstrumentationConfig(),
 	}
 }
 
@@ -142,10 +145,6 @@ type BaseConfig struct {
 
 	// Database directory
 	DBPath string `mapstructure:"db_dir"`
-
-	// When true, metrics are served under `/metrics` using a Prometheus client
-	// Check out the documentation for the list of available metrics.
-	Monitoring bool `mapstructure:"monitoring"`
 }
 
 // DefaultBaseConfig returns a default base configuration for a Tendermint node
@@ -163,7 +162,6 @@ func DefaultBaseConfig() BaseConfig {
 		FilterPeers:       false,
 		DBBackend:         "leveldb",
 		DBPath:            "data",
-		Monitoring:        false,
 	}
 }
 
@@ -578,6 +576,35 @@ func TestTxIndexConfig() *TxIndexConfig {
 	return DefaultTxIndexConfig()
 }
 
+//-----------------------------------------------------------------------------
+// InstrumentationConfig
+
+// InstrumentationConfig defines the configuration for metrics reporting.
+type InstrumentationConfig struct {
+	// When true, Prometheus metrics are served under /metrics on
+	// PrometheusListenAddr.
+	// Check out the documentation for the list of available metrics.
+	Prometheus bool `mapstructure:"prometheus"`
+
+	// Address to listen for Prometheus collector(s) connections.
+	PrometheusListenAddr string `mapstructure:"prometheus_listen_addr"`
+}
+
+// DefaultInstrumentationConfig returns a default configuration for metrics
+// reporting.
+func DefaultInstrumentationConfig() *InstrumentationConfig {
+	return &InstrumentationConfig{
+		Prometheus:           false,
+		PrometheusListenAddr: ":26660",
+	}
+}
+
+// TestInstrumentationConfig returns a default configuration for metrics
+// reporting.
+func TestInstrumentationConfig() *InstrumentationConfig {
+	return DefaultInstrumentationConfig()
+}
+
 //-----------------------------------------------------------------------------
 // Utils
 

config/toml.go

@@ -107,10 +107,6 @@ prof_laddr = "{{ .BaseConfig.ProfListenAddress }}"
 # so the app can decide if we should keep the connection or not
 filter_peers = {{ .BaseConfig.FilterPeers }}
 
-# When true, metrics are served under /metrics using a Prometheus client
-# Check out the documentation for the list of available metrics.
-monitoring = {{ .BaseConfig.Monitoring }}
-
 ##### advanced configuration options #####
 
 ##### rpc server configuration options #####
@@ -236,6 +232,17 @@ index_tags = "{{ .TxIndex.IndexTags }}"
 # desirable (see the comment above). IndexTags has a precedence over
 # IndexAllTags (i.e. when given both, IndexTags will be indexed).
 index_all_tags = {{ .TxIndex.IndexAllTags }}
+
+##### instrumentation configuration options #####
+[instrumentation]
+
+# When true, Prometheus metrics are served under /metrics on
+# PrometheusListenAddr.
+# Check out the documentation for the list of available metrics.
+prometheus = {{ .Instrumentation.Prometheus }}
+
+# Address to listen for Prometheus collector(s) connections
+prometheus_listen_addr = "{{ .Instrumentation.PrometheusListenAddr }}"
 `
 
 /****** these are for test settings ***********/

node/node.go

@@ -2,6 +2,7 @@ package node
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"net"
@@ -210,6 +211,7 @@ type Node struct {
 	rpcListeners     []net.Listener         // rpc servers
 	txIndexer        txindex.TxIndexer
 	indexerService   *txindex.IndexerService
+	prometheusSrv    *http.Server
 }
 
 // NewNode returns a new, ready to go, Tendermint Node.
@@ -311,7 +313,7 @@ func NewNode(config *cfg.Config,
 		p2pMetrics   *p2p.Metrics
 		memplMetrics *mempl.Metrics
 	)
-	if config.BaseConfig.Monitoring {
+	if config.Instrumentation.Prometheus {
 		csMetrics, p2pMetrics, memplMetrics = metricsProvider()
 	} else {
 		csMetrics, p2pMetrics, memplMetrics = NopMetricsProvider()
@@ -520,6 +522,10 @@ func (n *Node) OnStart() error {
 		n.rpcListeners = listeners
 	}
 
+	if n.config.Instrumentation.Prometheus {
+		n.prometheusSrv = n.StartPrometheusServer(n.config.Instrumentation.PrometheusListenAddr)
+	}
+
 	// Start the switch (the P2P server).
 	err = n.sw.Start()
 	if err != nil {
@@ -561,6 +567,13 @@ func (n *Node) OnStop() {
 			n.Logger.Error("Error stopping priv validator socket client", "err", err)
 		}
 	}
+
+	if n.prometheusSrv != nil {
+		if err := n.prometheusSrv.Shutdown(context.Background()); err != nil {
+			// Error from closing listeners, or context timeout:
+			n.Logger.Error("Prometheus HTTP server Shutdown", "err", err)
+		}
+	}
 }
 
 // RunForever waits for an interrupt signal and stops the node.
@@ -615,9 +628,6 @@ func (n *Node) startRPC() ([]net.Listener, error) {
 		wm := rpcserver.NewWebsocketManager(rpccore.Routes, coreCodec, rpcserver.EventSubscriber(n.eventBus))
 		wm.SetLogger(rpcLogger.With("protocol", "websocket"))
 		mux.HandleFunc("/websocket", wm.WebsocketHandler)
-		if n.config.BaseConfig.Monitoring {
-			mux.Handle("/metrics", promhttp.Handler())
-		}
 		rpcserver.RegisterRPCFuncs(mux, rpccore.Routes, coreCodec, rpcLogger)
 		listener, err := rpcserver.StartHTTPServer(listenAddr, mux, rpcLogger)
 		if err != nil {
@@ -639,6 +649,22 @@ func (n *Node) startRPC() ([]net.Listener, error) {
 	return listeners, nil
 }
 
+// StartPrometheusServer starts a Prometheus HTTP server, listening for metrics
+// collectors on addr.
+func (n *Node) StartPrometheusServer(addr string) *http.Server {
+	srv := &http.Server{
+		Addr:    addr,
+		Handler: promhttp.Handler(),
+	}
+	go func(s *http.Server, logger log.Logger) {
+		if err := s.ListenAndServe(); err != http.ErrServerClosed {
+			// Error starting or closing listener:
+			logger.Error("Prometheus HTTP server ListenAndServe", "err", err)
+		}
+	}(srv, n.Logger)
+	return srv
+}
+
 // Switch returns the Node's Switch.
 func (n *Node) Switch() *p2p.Switch {
 	return n.sw