limit the number of requests in flights for Prometheus server (#1927)
* limit the number of requests in flights for Prometheus server Closes #1804 Default to 1 because usually there's just one collector. * config: Up default for prom connections
This commit is contained in:
parent
ce33914f70
commit
4de9d42e4c
|
@ -5,6 +5,10 @@
|
||||||
IMPROVEMENT
|
IMPROVEMENT
|
||||||
- [docs] Include `ecosystem.json` and `tendermint-bft.md` from deprecated `aib-data` repository.
|
- [docs] Include `ecosystem.json` and `tendermint-bft.md` from deprecated `aib-data` repository.
|
||||||
|
|
||||||
|
IMPROVEMENTS:
|
||||||
|
- [config] Add `instrumentation.max_open_connections`, which limits the number
|
||||||
|
of requests in flight to Prometheus server (if enabled). Default: 3.
|
||||||
|
|
||||||
## 0.22.0
|
## 0.22.0
|
||||||
|
|
||||||
*July 2nd, 2018*
|
*July 2nd, 2018*
|
||||||
|
|
|
@ -180,13 +180,13 @@
|
||||||
version = "v1.0.0"
|
version = "v1.0.0"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
|
branch = "master"
|
||||||
name = "github.com/prometheus/client_golang"
|
name = "github.com/prometheus/client_golang"
|
||||||
packages = [
|
packages = [
|
||||||
"prometheus",
|
"prometheus",
|
||||||
"prometheus/promhttp"
|
"prometheus/promhttp"
|
||||||
]
|
]
|
||||||
revision = "c5b7fccd204277076155f10851dad72b76a49317"
|
revision = "d6a9817c4afc94d51115e4a30d449056a3fbf547"
|
||||||
version = "v0.8.0"
|
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
|
@ -414,6 +414,6 @@
|
||||||
[solve-meta]
|
[solve-meta]
|
||||||
analyzer-name = "dep"
|
analyzer-name = "dep"
|
||||||
analyzer-version = 1
|
analyzer-version = 1
|
||||||
inputs-digest = "71753a9d4ece4252d23941f116f5ff66c0d5da730a099e5a9867491d223ed93b"
|
inputs-digest = "6e854634d6c203278ce83bef7725cecbcf90023b0d0e440fb3374acedacbd5ad"
|
||||||
solver-name = "gps-cdcl"
|
solver-name = "gps-cdcl"
|
||||||
solver-version = 1
|
solver-version = 1
|
||||||
|
|
|
@ -88,7 +88,7 @@
|
||||||
|
|
||||||
[[constraint]]
|
[[constraint]]
|
||||||
name = "github.com/prometheus/client_golang"
|
name = "github.com/prometheus/client_golang"
|
||||||
version = "0.8.0"
|
branch = "master"
|
||||||
|
|
||||||
[[constraint]]
|
[[constraint]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
|
|
|
@ -606,6 +606,12 @@ type InstrumentationConfig struct {
|
||||||
|
|
||||||
// Address to listen for Prometheus collector(s) connections.
|
// Address to listen for Prometheus collector(s) connections.
|
||||||
PrometheusListenAddr string `mapstructure:"prometheus_listen_addr"`
|
PrometheusListenAddr string `mapstructure:"prometheus_listen_addr"`
|
||||||
|
|
||||||
|
// Maximum number of simultaneous connections.
|
||||||
|
// If you want to accept more significant number than the default, make sure
|
||||||
|
// you increase your OS limits.
|
||||||
|
// 0 - unlimited.
|
||||||
|
MaxOpenConnections int `mapstructure:"max_open_connections"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// DefaultInstrumentationConfig returns a default configuration for metrics
|
// DefaultInstrumentationConfig returns a default configuration for metrics
|
||||||
|
@ -614,6 +620,7 @@ func DefaultInstrumentationConfig() *InstrumentationConfig {
|
||||||
return &InstrumentationConfig{
|
return &InstrumentationConfig{
|
||||||
Prometheus: false,
|
Prometheus: false,
|
||||||
PrometheusListenAddr: ":26660",
|
PrometheusListenAddr: ":26660",
|
||||||
|
MaxOpenConnections: 3,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -262,6 +262,12 @@ prometheus = {{ .Instrumentation.Prometheus }}
|
||||||
|
|
||||||
# Address to listen for Prometheus collector(s) connections
|
# Address to listen for Prometheus collector(s) connections
|
||||||
prometheus_listen_addr = "{{ .Instrumentation.PrometheusListenAddr }}"
|
prometheus_listen_addr = "{{ .Instrumentation.PrometheusListenAddr }}"
|
||||||
|
|
||||||
|
# Maximum number of simultaneous connections.
|
||||||
|
# If you want to accept more significant number than the default, make sure
|
||||||
|
# you increase your OS limits.
|
||||||
|
# 0 - unlimited.
|
||||||
|
max_open_connections = {{ .Instrumentation.MaxOpenConnections }}
|
||||||
`
|
`
|
||||||
|
|
||||||
/****** these are for test settings ***********/
|
/****** these are for test settings ***********/
|
||||||
|
|
|
@ -209,4 +209,10 @@ prometheus = false
|
||||||
|
|
||||||
# Address to listen for Prometheus collector(s) connections
|
# Address to listen for Prometheus collector(s) connections
|
||||||
prometheus_listen_addr = ":26660"
|
prometheus_listen_addr = ":26660"
|
||||||
|
|
||||||
|
# Maximum number of simultaneous connections.
|
||||||
|
# If you want to accept a more significant number than the default, make sure
|
||||||
|
# you increase your OS limits.
|
||||||
|
# 0 - unlimited.
|
||||||
|
max_open_connections = 3
|
||||||
```
|
```
|
||||||
|
|
10
node/node.go
10
node/node.go
|
@ -8,6 +8,7 @@ import (
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||||
|
|
||||||
amino "github.com/tendermint/go-amino"
|
amino "github.com/tendermint/go-amino"
|
||||||
|
@ -599,8 +600,13 @@ func (n *Node) startRPC() ([]net.Listener, error) {
|
||||||
// collectors on addr.
|
// collectors on addr.
|
||||||
func (n *Node) startPrometheusServer(addr string) *http.Server {
|
func (n *Node) startPrometheusServer(addr string) *http.Server {
|
||||||
srv := &http.Server{
|
srv := &http.Server{
|
||||||
Addr: addr,
|
Addr: addr,
|
||||||
Handler: promhttp.Handler(),
|
Handler: promhttp.InstrumentMetricHandler(
|
||||||
|
prometheus.DefaultRegisterer, promhttp.HandlerFor(
|
||||||
|
prometheus.DefaultGatherer,
|
||||||
|
promhttp.HandlerOpts{MaxRequestsInFlight: n.config.Instrumentation.MaxOpenConnections},
|
||||||
|
),
|
||||||
|
),
|
||||||
}
|
}
|
||||||
go func() {
|
go func() {
|
||||||
if err := srv.ListenAndServe(); err != http.ErrServerClosed {
|
if err := srv.ListenAndServe(); err != http.ErrServerClosed {
|
||||||
|
|
Loading…
Reference in New Issue