limit the number of requests in flights for Prometheus server (#1927)

* limit the number of requests in flights for Prometheus server

Closes #1804

Default to 1 because usually there's just one collector.

* config: Up default for prom connections

CHANGELOG.md

@@ -5,6 +5,10 @@
 IMPROVEMENT
 - [docs] Include `ecosystem.json` and `tendermint-bft.md` from deprecated `aib-data` repository.
 
+IMPROVEMENTS:
+- [config] Add `instrumentation.max_open_connections`, which limits the number
+  of requests in flight to Prometheus server (if enabled). Default: 3.
+
 ## 0.22.0
 
 *July 2nd, 2018*

Gopkg.lock

@@ -180,13 +180,13 @@
   version = "v1.0.0"
 
 [[projects]]
+  branch = "master"
   name = "github.com/prometheus/client_golang"
   packages = [
     "prometheus",
     "prometheus/promhttp"
   ]
-  revision = "c5b7fccd204277076155f10851dad72b76a49317"
-  version = "v0.8.0"
+  revision = "d6a9817c4afc94d51115e4a30d449056a3fbf547"
 
 [[projects]]
   branch = "master"
@@ -414,6 +414,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "71753a9d4ece4252d23941f116f5ff66c0d5da730a099e5a9867491d223ed93b"
+  inputs-digest = "6e854634d6c203278ce83bef7725cecbcf90023b0d0e440fb3374acedacbd5ad"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -88,7 +88,7 @@
 
 [[constraint]]
   name = "github.com/prometheus/client_golang"
-  version = "0.8.0"
+  branch = "master"
 
 [[constraint]]
   branch = "master"

config/config.go

@@ -606,6 +606,12 @@ type InstrumentationConfig struct {
 
 	// Address to listen for Prometheus collector(s) connections.
 	PrometheusListenAddr string `mapstructure:"prometheus_listen_addr"`
+
+	// Maximum number of simultaneous connections.
+	// If you want to accept more significant number than the default, make sure
+	// you increase your OS limits.
+	// 0 - unlimited.
+	MaxOpenConnections int `mapstructure:"max_open_connections"`
 }
 
 // DefaultInstrumentationConfig returns a default configuration for metrics
@@ -614,6 +620,7 @@ func DefaultInstrumentationConfig() *InstrumentationConfig {
 	return &InstrumentationConfig{
 		Prometheus:           false,
 		PrometheusListenAddr: ":26660",
+		MaxOpenConnections:   3,
 	}
 }
 

config/toml.go

@@ -262,6 +262,12 @@ prometheus = {{ .Instrumentation.Prometheus }}
 
 # Address to listen for Prometheus collector(s) connections
 prometheus_listen_addr = "{{ .Instrumentation.PrometheusListenAddr }}"
+
+# Maximum number of simultaneous connections.
+# If you want to accept more significant number than the default, make sure
+# you increase your OS limits.
+# 0 - unlimited.
+max_open_connections = {{ .Instrumentation.MaxOpenConnections }}
 `
 
 /****** these are for test settings ***********/

docs/tendermint-core/configuration.md

@@ -209,4 +209,10 @@ prometheus = false
 
 # Address to listen for Prometheus collector(s) connections
 prometheus_listen_addr = ":26660"
+
+# Maximum number of simultaneous connections.
+# If you want to accept a more significant number than the default, make sure
+# you increase your OS limits.
+# 0 - unlimited.
+max_open_connections = 3
 ```

node/node.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"net/http"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 
 	amino "github.com/tendermint/go-amino"
@@ -599,8 +600,13 @@ func (n *Node) startRPC() ([]net.Listener, error) {
 // collectors on addr.
 func (n *Node) startPrometheusServer(addr string) *http.Server {
 	srv := &http.Server{
-		Addr:    addr,
-		Handler: promhttp.Handler(),
+		Addr: addr,
+		Handler: promhttp.InstrumentMetricHandler(
+			prometheus.DefaultRegisterer, promhttp.HandlerFor(
+				prometheus.DefaultGatherer,
+				promhttp.HandlerOpts{MaxRequestsInFlight: n.config.Instrumentation.MaxOpenConnections},
+			),
+		),
 	}
 	go func() {
 		if err := srv.ListenAndServe(); err != http.ErrServerClosed {