From 4de9d42e4cb07b825fadb69abc983c1a5846d5e0 Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Tue, 10 Jul 2018 15:49:48 +0400
Subject: [PATCH] limit the number of requests in flights for Prometheus server
 (#1927)

* limit the number of requests in flights for Prometheus server

Closes #1804

Default to 1 because usually there's just one collector.

* config: Up default for prom connections
---
 CHANGELOG.md                          |  4 ++++
 Gopkg.lock                            |  6 +++---
 Gopkg.toml                            |  2 +-
 config/config.go                      |  7 +++++++
 config/toml.go                        |  6 ++++++
 docs/tendermint-core/configuration.md |  6 ++++++
 node/node.go                          | 10 ++++++++--
 7 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b6df7698..76b91f96 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,10 @@
 IMPROVEMENT
 - [docs] Include `ecosystem.json` and `tendermint-bft.md` from deprecated `aib-data` repository.
 
+IMPROVEMENTS:
+- [config] Add `instrumentation.max_open_connections`, which limits the number
+  of requests in flight to Prometheus server (if enabled). Default: 3.
+
 ## 0.22.0
 
 *July 2nd, 2018*
diff --git a/Gopkg.lock b/Gopkg.lock
index 17b74d74..b1beaa20 100644
--- a/Gopkg.lock
+++ b/Gopkg.lock
@@ -180,13 +180,13 @@
   version = "v1.0.0"
 
 [[projects]]
+  branch = "master"
   name = "github.com/prometheus/client_golang"
   packages = [
     "prometheus",
     "prometheus/promhttp"
   ]
-  revision = "c5b7fccd204277076155f10851dad72b76a49317"
-  version = "v0.8.0"
+  revision = "d6a9817c4afc94d51115e4a30d449056a3fbf547"
 
 [[projects]]
   branch = "master"
@@ -414,6 +414,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "71753a9d4ece4252d23941f116f5ff66c0d5da730a099e5a9867491d223ed93b"
+  inputs-digest = "6e854634d6c203278ce83bef7725cecbcf90023b0d0e440fb3374acedacbd5ad"
   solver-name = "gps-cdcl"
   solver-version = 1
diff --git a/Gopkg.toml b/Gopkg.toml
index 28394b8b..ecce0e41 100644
--- a/Gopkg.toml
+++ b/Gopkg.toml
@@ -88,7 +88,7 @@
 
 [[constraint]]
   name = "github.com/prometheus/client_golang"
-  version = "0.8.0"
+  branch = "master"
 
 [[constraint]]
   branch = "master"
diff --git a/config/config.go b/config/config.go
index 22cecf98..2df8eb8e 100644
--- a/config/config.go
+++ b/config/config.go
@@ -606,6 +606,12 @@ type InstrumentationConfig struct {
 
 	// Address to listen for Prometheus collector(s) connections.
 	PrometheusListenAddr string `mapstructure:"prometheus_listen_addr"`
+
+	// Maximum number of simultaneous connections.
+	// If you want to accept more significant number than the default, make sure
+	// you increase your OS limits.
+	// 0 - unlimited.
+	MaxOpenConnections int `mapstructure:"max_open_connections"`
 }
 
 // DefaultInstrumentationConfig returns a default configuration for metrics
@@ -614,6 +620,7 @@ func DefaultInstrumentationConfig() *InstrumentationConfig {
 	return &InstrumentationConfig{
 		Prometheus:           false,
 		PrometheusListenAddr: ":26660",
+		MaxOpenConnections:   3,
 	}
 }
 
diff --git a/config/toml.go b/config/toml.go
index 084325ba..858d9b31 100644
--- a/config/toml.go
+++ b/config/toml.go
@@ -262,6 +262,12 @@ prometheus = {{ .Instrumentation.Prometheus }}
 
 # Address to listen for Prometheus collector(s) connections
 prometheus_listen_addr = "{{ .Instrumentation.PrometheusListenAddr }}"
+
+# Maximum number of simultaneous connections.
+# If you want to accept more significant number than the default, make sure
+# you increase your OS limits.
+# 0 - unlimited.
+max_open_connections = {{ .Instrumentation.MaxOpenConnections }}
 `
 
 /****** these are for test settings ***********/
diff --git a/docs/tendermint-core/configuration.md b/docs/tendermint-core/configuration.md
index e298dfd8..0453bdad 100644
--- a/docs/tendermint-core/configuration.md
+++ b/docs/tendermint-core/configuration.md
@@ -209,4 +209,10 @@ prometheus = false
 
 # Address to listen for Prometheus collector(s) connections
 prometheus_listen_addr = ":26660"
+
+# Maximum number of simultaneous connections.
+# If you want to accept a more significant number than the default, make sure
+# you increase your OS limits.
+# 0 - unlimited.
+max_open_connections = 3
 ```
diff --git a/node/node.go b/node/node.go
index 0780891e..9f6428ec 100644
--- a/node/node.go
+++ b/node/node.go
@@ -8,6 +8,7 @@ import (
 	"net"
 	"net/http"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 
 	amino "github.com/tendermint/go-amino"
@@ -599,8 +600,13 @@ func (n *Node) startRPC() ([]net.Listener, error) {
 // collectors on addr.
 func (n *Node) startPrometheusServer(addr string) *http.Server {
 	srv := &http.Server{
-		Addr:    addr,
-		Handler: promhttp.Handler(),
+		Addr: addr,
+		Handler: promhttp.InstrumentMetricHandler(
+			prometheus.DefaultRegisterer, promhttp.HandlerFor(
+				prometheus.DefaultGatherer,
+				promhttp.HandlerOpts{MaxRequestsInFlight: n.config.Instrumentation.MaxOpenConnections},
+			),
+		),
 	}
 	go func() {
 		if err := srv.ListenAndServe(); err != http.ErrServerClosed {