limit the number of requests in flights for Prometheus server (#1927)
* limit the number of requests in flights for Prometheus server Closes #1804 Default to 1 because usually there's just one collector. * config: Up default for prom connections
This commit is contained in:
parent
ce33914f70
commit
4de9d42e4c
|
@ -5,6 +5,10 @@
|
|||
IMPROVEMENT
|
||||
- [docs] Include `ecosystem.json` and `tendermint-bft.md` from deprecated `aib-data` repository.
|
||||
|
||||
IMPROVEMENTS:
|
||||
- [config] Add `instrumentation.max_open_connections`, which limits the number
|
||||
of requests in flight to Prometheus server (if enabled). Default: 3.
|
||||
|
||||
## 0.22.0
|
||||
|
||||
*July 2nd, 2018*
|
||||
|
|
|
@ -180,13 +180,13 @@
|
|||
version = "v1.0.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/prometheus/client_golang"
|
||||
packages = [
|
||||
"prometheus",
|
||||
"prometheus/promhttp"
|
||||
]
|
||||
revision = "c5b7fccd204277076155f10851dad72b76a49317"
|
||||
version = "v0.8.0"
|
||||
revision = "d6a9817c4afc94d51115e4a30d449056a3fbf547"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
|
@ -414,6 +414,6 @@
|
|||
[solve-meta]
|
||||
analyzer-name = "dep"
|
||||
analyzer-version = 1
|
||||
inputs-digest = "71753a9d4ece4252d23941f116f5ff66c0d5da730a099e5a9867491d223ed93b"
|
||||
inputs-digest = "6e854634d6c203278ce83bef7725cecbcf90023b0d0e440fb3374acedacbd5ad"
|
||||
solver-name = "gps-cdcl"
|
||||
solver-version = 1
|
||||
|
|
|
@ -88,7 +88,7 @@
|
|||
|
||||
[[constraint]]
|
||||
name = "github.com/prometheus/client_golang"
|
||||
version = "0.8.0"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
|
|
|
@ -606,6 +606,12 @@ type InstrumentationConfig struct {
|
|||
|
||||
// Address to listen for Prometheus collector(s) connections.
|
||||
PrometheusListenAddr string `mapstructure:"prometheus_listen_addr"`
|
||||
|
||||
// Maximum number of simultaneous connections.
|
||||
// If you want to accept more significant number than the default, make sure
|
||||
// you increase your OS limits.
|
||||
// 0 - unlimited.
|
||||
MaxOpenConnections int `mapstructure:"max_open_connections"`
|
||||
}
|
||||
|
||||
// DefaultInstrumentationConfig returns a default configuration for metrics
|
||||
|
@ -614,6 +620,7 @@ func DefaultInstrumentationConfig() *InstrumentationConfig {
|
|||
return &InstrumentationConfig{
|
||||
Prometheus: false,
|
||||
PrometheusListenAddr: ":26660",
|
||||
MaxOpenConnections: 3,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -262,6 +262,12 @@ prometheus = {{ .Instrumentation.Prometheus }}
|
|||
|
||||
# Address to listen for Prometheus collector(s) connections
|
||||
prometheus_listen_addr = "{{ .Instrumentation.PrometheusListenAddr }}"
|
||||
|
||||
# Maximum number of simultaneous connections.
|
||||
# If you want to accept more significant number than the default, make sure
|
||||
# you increase your OS limits.
|
||||
# 0 - unlimited.
|
||||
max_open_connections = {{ .Instrumentation.MaxOpenConnections }}
|
||||
`
|
||||
|
||||
/****** these are for test settings ***********/
|
||||
|
|
|
@ -209,4 +209,10 @@ prometheus = false
|
|||
|
||||
# Address to listen for Prometheus collector(s) connections
|
||||
prometheus_listen_addr = ":26660"
|
||||
|
||||
# Maximum number of simultaneous connections.
|
||||
# If you want to accept a more significant number than the default, make sure
|
||||
# you increase your OS limits.
|
||||
# 0 - unlimited.
|
||||
max_open_connections = 3
|
||||
```
|
||||
|
|
10
node/node.go
10
node/node.go
|
@ -8,6 +8,7 @@ import (
|
|||
"net"
|
||||
"net/http"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
|
||||
amino "github.com/tendermint/go-amino"
|
||||
|
@ -599,8 +600,13 @@ func (n *Node) startRPC() ([]net.Listener, error) {
|
|||
// collectors on addr.
|
||||
func (n *Node) startPrometheusServer(addr string) *http.Server {
|
||||
srv := &http.Server{
|
||||
Addr: addr,
|
||||
Handler: promhttp.Handler(),
|
||||
Addr: addr,
|
||||
Handler: promhttp.InstrumentMetricHandler(
|
||||
prometheus.DefaultRegisterer, promhttp.HandlerFor(
|
||||
prometheus.DefaultGatherer,
|
||||
promhttp.HandlerOpts{MaxRequestsInFlight: n.config.Instrumentation.MaxOpenConnections},
|
||||
),
|
||||
),
|
||||
}
|
||||
go func() {
|
||||
if err := srv.ListenAndServe(); err != http.ErrServerClosed {
|
||||
|
|
Loading…
Reference in New Issue