metrics: Implement IP access control on Prometheus scrape endpoint
This commit is contained in:
parent
59da774f22
commit
d08cdbe5f7
|
@ -676,6 +676,12 @@ dependencies = [
|
||||||
"cfg-if 1.0.0",
|
"cfg-if 1.0.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ipnet"
|
||||||
|
version = "2.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "itoa"
|
name = "itoa"
|
||||||
version = "0.4.7"
|
version = "0.4.7"
|
||||||
|
@ -719,12 +725,16 @@ dependencies = [
|
||||||
"ed25519-zebra",
|
"ed25519-zebra",
|
||||||
"funty",
|
"funty",
|
||||||
"group",
|
"group",
|
||||||
|
"hyper",
|
||||||
|
"ipnet",
|
||||||
"jubjub",
|
"jubjub",
|
||||||
"libc",
|
"libc",
|
||||||
"metrics",
|
"metrics",
|
||||||
"metrics-exporter-prometheus",
|
"metrics-exporter-prometheus",
|
||||||
"rand_core",
|
"rand_core",
|
||||||
"subtle",
|
"subtle",
|
||||||
|
"thiserror",
|
||||||
|
"tokio",
|
||||||
"tracing",
|
"tracing",
|
||||||
"tracing-appender",
|
"tracing-appender",
|
||||||
"tracing-core",
|
"tracing-core",
|
||||||
|
|
|
@ -38,8 +38,12 @@ zcash_proofs = "0.5"
|
||||||
ed25519-zebra = "2.0.0"
|
ed25519-zebra = "2.0.0"
|
||||||
|
|
||||||
# Metrics
|
# Metrics
|
||||||
|
hyper = { version = "0.14", default-features = false, features = ["server", "tcp", "http1"] }
|
||||||
|
ipnet = "2"
|
||||||
metrics = "0.14.2"
|
metrics = "0.14.2"
|
||||||
metrics-exporter-prometheus = "0.3"
|
metrics-exporter-prometheus = "0.3"
|
||||||
|
thiserror = "1"
|
||||||
|
tokio = { version = "1.0", features = ["rt", "net", "time", "macros"] }
|
||||||
|
|
||||||
# Temporary workaround for https://github.com/myrrlyn/funty/issues/3
|
# Temporary workaround for https://github.com/myrrlyn/funty/issues/3
|
||||||
funty = "=1.1.0"
|
funty = "=1.1.0"
|
||||||
|
|
|
@ -30,18 +30,12 @@ You can see what each method provides with `zcash-cli help METHOD_NAME`.
|
||||||
`zcashd` can optionally expose an HTTP server that acts as a Prometheus scrape
|
`zcashd` can optionally expose an HTTP server that acts as a Prometheus scrape
|
||||||
endpoint. The server will respond to `GET` requests on any request path.
|
endpoint. The server will respond to `GET` requests on any request path.
|
||||||
|
|
||||||
Note that HTTPS is not supported, and therefore connections to the endpoint are
|
To enable the endpoint, add `-prometheusport=:<port>` to your `zcashd`
|
||||||
not encrypted or authenticated. Access to the endpoint should be assumed to
|
configuration (either in `zcash.conf` or on the command line). After
|
||||||
compromise the privacy of node operations, by the provided metrics and/or by
|
|
||||||
timing side channels. Enabling the endpoint is **strongly discouraged** if the
|
|
||||||
node has a wallet holding live funds.
|
|
||||||
|
|
||||||
To enable the endpoint, add `-prometheusmetrics=<host_name>:<port>` to your
|
|
||||||
`zcashd` configuration (either in `zcash.conf` or on the command line). After
|
|
||||||
restarting `zcashd` you can then test the endpoint by querying it:
|
restarting `zcashd` you can then test the endpoint by querying it:
|
||||||
|
|
||||||
```
|
```
|
||||||
$ curl http://<host_name>:<port>
|
$ curl http://127.0.0.1:<port>
|
||||||
# TYPE peer_outbound_messages counter
|
# TYPE peer_outbound_messages counter
|
||||||
peer_outbound_messages 181
|
peer_outbound_messages 181
|
||||||
|
|
||||||
|
@ -59,6 +53,14 @@ block_verified_block_count 162
|
||||||
...
|
...
|
||||||
```
|
```
|
||||||
|
|
||||||
|
By default, access is restricted to localhost. This can be expanded with
|
||||||
|
`-metricsallowip=<ip>`, which can specify IPs or subnets. Note that HTTPS is not
|
||||||
|
supported, and therefore connections to the endpoint are not encrypted or
|
||||||
|
authenticated. Access to the endpoint should be assumed to compromise the
|
||||||
|
privacy of node operations, by the provided metrics and/or by timing side
|
||||||
|
channels. Non-localhost access is **strongly discouraged** if the node has a
|
||||||
|
wallet holding live funds.
|
||||||
|
|
||||||
### Example metrics collection with Docker
|
### Example metrics collection with Docker
|
||||||
|
|
||||||
The example instructions below were tested on Windows 10 using Docker Desktop
|
The example instructions below were tested on Windows 10 using Docker Desktop
|
||||||
|
|
|
@ -11,16 +11,18 @@ Prometheus metrics
|
||||||
a Prometheus scrape endpoint. The server will respond to `GET` requests on any
|
a Prometheus scrape endpoint. The server will respond to `GET` requests on any
|
||||||
request path.
|
request path.
|
||||||
|
|
||||||
Note that HTTPS is not supported, and therefore connections to the endpoint are
|
To enable the endpoint, add `-prometheusport=<port>` to your `zcashd`
|
||||||
not encrypted or authenticated. Access to the endpoint should be assumed to
|
configuration (either in `zcash.conf` or on the command line). After
|
||||||
compromise the privacy of node operations, by the provided metrics and/or by
|
|
||||||
timing side channels. Enabling the endpoint is **strongly discouraged** if the
|
|
||||||
node has a wallet holding live funds.
|
|
||||||
|
|
||||||
To enable the endpoint, add `-prometheusmetrics=<host_name>:<port>` to your
|
|
||||||
`zcashd` configuration (either in `zcash.conf` or on the command line). After
|
|
||||||
restarting `zcashd` you can then test the endpoint by querying it with e.g.
|
restarting `zcashd` you can then test the endpoint by querying it with e.g.
|
||||||
`curl http://<host_name>:<port>`.
|
`curl http://127.0.0.1:<port>`.
|
||||||
|
|
||||||
|
By default, access is restricted to localhost. This can be expanded with
|
||||||
|
`-metricsallowip=<ip>`, which can specify IPs or subnets. Note that HTTPS is not
|
||||||
|
supported, and therefore connections to the endpoint are not encrypted or
|
||||||
|
authenticated. Access to the endpoint should be assumed to compromise the
|
||||||
|
privacy of node operations, by the provided metrics and/or by timing side
|
||||||
|
channels. Non-localhost access is **strongly discouraged** if the node has a
|
||||||
|
wallet holding live funds.
|
||||||
|
|
||||||
The specific metrics names may change in subsequent releases, in particular to
|
The specific metrics names may change in subsequent releases, in particular to
|
||||||
improve interoperability with `zebrad`.
|
improve interoperability with `zebrad`.
|
||||||
|
|
32
src/init.cpp
32
src/init.cpp
|
@ -353,9 +353,6 @@ std::string HelpMessage(HelpMessageMode mode)
|
||||||
#ifndef WIN32
|
#ifndef WIN32
|
||||||
strUsage += HelpMessageOpt("-pid=<file>", strprintf(_("Specify pid file (default: %s)"), BITCOIN_PID_FILENAME));
|
strUsage += HelpMessageOpt("-pid=<file>", strprintf(_("Specify pid file (default: %s)"), BITCOIN_PID_FILENAME));
|
||||||
#endif
|
#endif
|
||||||
strUsage += HelpMessageOpt("-prometheusmetrics=<host_name>:<port>", _("Expose node metrics in the Prometheus exposition format. "
|
|
||||||
"An HTTP listener will be started on the configured hostname and port, which responds to GET requests on any request path. "
|
|
||||||
"SECURITY WARNING: this can potentially compromise privacy; read contrib/metrics/README.md before enabling."));
|
|
||||||
strUsage += HelpMessageOpt("-prune=<n>", strprintf(_("Reduce storage requirements by pruning (deleting) old blocks. This mode disables wallet support and is incompatible with -txindex. "
|
strUsage += HelpMessageOpt("-prune=<n>", strprintf(_("Reduce storage requirements by pruning (deleting) old blocks. This mode disables wallet support and is incompatible with -txindex. "
|
||||||
"Warning: Reverting this setting requires re-downloading the entire blockchain. "
|
"Warning: Reverting this setting requires re-downloading the entire blockchain. "
|
||||||
"(default: 0 = disable pruning blocks, >%u = target size in MiB to use for block files)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
|
"(default: 0 = disable pruning blocks, >%u = target size in MiB to use for block files)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
|
||||||
|
@ -415,6 +412,15 @@ std::string HelpMessage(HelpMessageMode mode)
|
||||||
strUsage += HelpMessageOpt("-zmqpubrawtx=<address>", _("Enable publish raw transaction in <address>"));
|
strUsage += HelpMessageOpt("-zmqpubrawtx=<address>", _("Enable publish raw transaction in <address>"));
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
strUsage += HelpMessageGroup(_("Monitoring options:"));
|
||||||
|
strUsage += HelpMessageOpt("-metricsallowip=<ip>", _("Allow metrics connections from specified source. "
|
||||||
|
"Valid for <ip> are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). "
|
||||||
|
"This option can be specified multiple times. (default: only localhost)"));
|
||||||
|
strUsage += HelpMessageOpt("-metricsbind=<addr>", _("Bind to given address to listen for metrics connections. (default: bind to all interfaces)"));
|
||||||
|
strUsage += HelpMessageOpt("-prometheusport=<port>", _("Expose node metrics in the Prometheus exposition format. "
|
||||||
|
"An HTTP listener will be started on <port>, which responds to GET requests on any request path. "
|
||||||
|
"Use -metricsallowip and -metricsbind to control access."));
|
||||||
|
|
||||||
strUsage += HelpMessageGroup(_("Debugging/Testing options:"));
|
strUsage += HelpMessageGroup(_("Debugging/Testing options:"));
|
||||||
if (showDebug)
|
if (showDebug)
|
||||||
{
|
{
|
||||||
|
@ -1226,13 +1232,25 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
|
||||||
// Count uptime
|
// Count uptime
|
||||||
MarkStartTime();
|
MarkStartTime();
|
||||||
|
|
||||||
std::string prometheusMetricsArg = GetArg("-prometheusmetrics", "");
|
int prometheusPort = GetArg("-prometheusport", -1);
|
||||||
if (prometheusMetricsArg != "") {
|
if (prometheusPort > 0) {
|
||||||
|
const std::vector<std::string>& vAllow = mapMultiArgs["-metricsallowip"];
|
||||||
|
std::vector<const char*> vAllowCstr;
|
||||||
|
for (const std::string& strAllow : vAllow) {
|
||||||
|
vAllowCstr.push_back(strAllow.c_str());
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string metricsBind = GetArg("-metricsbind", "");
|
||||||
|
const char* metricsBindCstr = nullptr;
|
||||||
|
if (!metricsBind.empty()) {
|
||||||
|
metricsBindCstr = metricsBind.c_str();
|
||||||
|
}
|
||||||
|
|
||||||
// Start up the metrics runtime. This spins off a Rust thread that runs
|
// Start up the metrics runtime. This spins off a Rust thread that runs
|
||||||
// the Prometheus exporter. We just let this thread die at process end.
|
// the Prometheus exporter. We just let this thread die at process end.
|
||||||
LogPrintf("metrics thread start");
|
LogPrintf("metrics thread start");
|
||||||
if (!metrics_run(prometheusMetricsArg.c_str())) {
|
if (!metrics_run(metricsBindCstr, vAllowCstr.data(), vAllowCstr.size(), prometheusPort)) {
|
||||||
return InitError(strprintf(_("Failed to start Prometheus metrics exporter on '%s'"), prometheusMetricsArg));
|
return InitError(strprintf(_("Failed to start Prometheus metrics exporter")));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -17,10 +17,14 @@ extern "C" {
|
||||||
/// Initializes the metrics runtime and runs the Prometheus exporter in a new
|
/// Initializes the metrics runtime and runs the Prometheus exporter in a new
|
||||||
/// thread.
|
/// thread.
|
||||||
///
|
///
|
||||||
/// listen_address is a string like <host_name>:<port>.
|
/// bind_address is an IP address to bind to, or empty to use the default.
|
||||||
///
|
///
|
||||||
/// Returns false if listen_address was not a valid SocketAddr.
|
/// Returns false on any error.
|
||||||
bool metrics_run(const char* listen_address);
|
bool metrics_run(
|
||||||
|
const char* bind_address,
|
||||||
|
const char* const* allow_ips,
|
||||||
|
size_t allow_ips_len,
|
||||||
|
uint16_t prometheus_port);
|
||||||
|
|
||||||
struct MetricsCallsite;
|
struct MetricsCallsite;
|
||||||
typedef struct MetricsCallsite MetricsCallsite;
|
typedef struct MetricsCallsite MetricsCallsite;
|
||||||
|
|
|
@ -2,39 +2,69 @@ use libc::{c_char, c_double};
|
||||||
use metrics::{try_recorder, GaugeValue, Key, KeyData, Label};
|
use metrics::{try_recorder, GaugeValue, Key, KeyData, Label};
|
||||||
use metrics_exporter_prometheus::PrometheusBuilder;
|
use metrics_exporter_prometheus::PrometheusBuilder;
|
||||||
use std::ffi::CStr;
|
use std::ffi::CStr;
|
||||||
use std::net::SocketAddr;
|
use std::net::{IpAddr, SocketAddr};
|
||||||
use std::ptr;
|
use std::ptr;
|
||||||
use std::slice;
|
use std::slice;
|
||||||
use tracing::error;
|
use tracing::error;
|
||||||
|
|
||||||
|
mod prometheus;
|
||||||
|
|
||||||
#[no_mangle]
|
#[no_mangle]
|
||||||
pub extern "C" fn metrics_run(listen_address: *const c_char) -> bool {
|
pub extern "C" fn metrics_run(
|
||||||
let listen_address = match unsafe { CStr::from_ptr(listen_address) }.to_str() {
|
bind_address: *const c_char,
|
||||||
Ok(addr) => addr,
|
allow_ips: *const *const c_char,
|
||||||
Err(_) => {
|
allow_ips_len: usize,
|
||||||
error!("-prometheusmetrics argument is not valid UTF-8");
|
prometheus_port: u16,
|
||||||
|
) -> bool {
|
||||||
|
// Parse any allowed IPs.
|
||||||
|
let allow_ips = unsafe { slice::from_raw_parts(allow_ips, allow_ips_len) };
|
||||||
|
let mut allow_ips: Vec<ipnet::IpNet> = match allow_ips
|
||||||
|
.iter()
|
||||||
|
.map(|&p| unsafe { CStr::from_ptr(p) })
|
||||||
|
.map(|s| {
|
||||||
|
s.to_str().ok().and_then(|s| {
|
||||||
|
s.parse()
|
||||||
|
.map_err(|e| {
|
||||||
|
error!("Invalid -metricsallowip argument '{}': {}", s, e);
|
||||||
|
})
|
||||||
|
.ok()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
{
|
||||||
|
Some(ips) => ips,
|
||||||
|
None => {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
listen_address
|
// We always allow localhost.
|
||||||
.parse::<SocketAddr>()
|
allow_ips.extend(&["127.0.0.0/8".parse().unwrap(), "::1/128".parse().unwrap()]);
|
||||||
.map_err(|e| {
|
|
||||||
error!(
|
// Parse the address to bind to.
|
||||||
"Invalid Prometheus metrics address '{}': {}",
|
let bind_address = SocketAddr::new(
|
||||||
listen_address, e
|
if allow_ips.is_empty() {
|
||||||
);
|
// Default to loopback if not allowing external IPs.
|
||||||
()
|
"127.0.0.1".parse::<IpAddr>().unwrap()
|
||||||
})
|
} else if bind_address.is_null() {
|
||||||
.and_then(|addr| {
|
// No specific bind address specified, bind to any.
|
||||||
PrometheusBuilder::new()
|
"0.0.0.0".parse::<IpAddr>().unwrap()
|
||||||
.listen_address(addr)
|
} else {
|
||||||
.install()
|
match unsafe { CStr::from_ptr(bind_address) }
|
||||||
.map_err(|e| {
|
.to_str()
|
||||||
error!("Failed to start Prometheus metrics exporter: {:?}", e);
|
.ok()
|
||||||
()
|
.and_then(|s| s.parse::<IpAddr>().ok())
|
||||||
})
|
{
|
||||||
})
|
Some(addr) => addr,
|
||||||
.is_ok()
|
None => {
|
||||||
|
error!("Invalid -metricsbind argument");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
prometheus_port,
|
||||||
|
);
|
||||||
|
|
||||||
|
prometheus::install(bind_address, PrometheusBuilder::new(), allow_ips).is_ok()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct FfiCallsite {
|
pub struct FfiCallsite {
|
||||||
|
|
|
@ -0,0 +1,113 @@
|
||||||
|
// This is mostly code copied from metrics_exporter_prometheus. It is licensed under the
|
||||||
|
// same license as the zcash codebase.
|
||||||
|
|
||||||
|
use hyper::{
|
||||||
|
server::{conn::AddrStream, Server},
|
||||||
|
service::{make_service_fn, service_fn},
|
||||||
|
{Body, Error as HyperError, Response, StatusCode},
|
||||||
|
};
|
||||||
|
use metrics::SetRecorderError;
|
||||||
|
use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusRecorder};
|
||||||
|
use std::future::Future;
|
||||||
|
use std::io;
|
||||||
|
use std::net::SocketAddr;
|
||||||
|
use std::thread;
|
||||||
|
use thiserror::Error as ThisError;
|
||||||
|
use tokio::{pin, runtime, select};
|
||||||
|
|
||||||
|
/// Errors that could occur while installing a Prometheus recorder/exporter.
|
||||||
|
#[derive(Debug, ThisError)]
|
||||||
|
pub enum InstallError {
|
||||||
|
/// Creating the networking event loop did not succeed.
|
||||||
|
#[error("failed to spawn Tokio runtime for endpoint: {0}")]
|
||||||
|
Io(#[from] io::Error),
|
||||||
|
|
||||||
|
/// Binding/listening to the given address did not succeed.
|
||||||
|
#[error("failed to bind to given listen address: {0}")]
|
||||||
|
Hyper(#[from] HyperError),
|
||||||
|
|
||||||
|
/// Installing the recorder did not succeed.
|
||||||
|
#[error("failed to install exporter as global recorder: {0}")]
|
||||||
|
Recorder(#[from] SetRecorderError),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A copy of `PrometheusBuilder::build_with_exporter` that adds support for an IP address
|
||||||
|
/// or subnet allowlist.
|
||||||
|
pub(super) fn build(
|
||||||
|
bind_address: SocketAddr,
|
||||||
|
builder: PrometheusBuilder,
|
||||||
|
allow_ips: Vec<ipnet::IpNet>,
|
||||||
|
) -> Result<
|
||||||
|
(
|
||||||
|
PrometheusRecorder,
|
||||||
|
impl Future<Output = Result<(), HyperError>> + Send + 'static,
|
||||||
|
),
|
||||||
|
InstallError,
|
||||||
|
> {
|
||||||
|
let recorder = builder.build();
|
||||||
|
let handle = recorder.handle();
|
||||||
|
|
||||||
|
let server = Server::try_bind(&bind_address)?;
|
||||||
|
|
||||||
|
let exporter = async move {
|
||||||
|
let make_svc = make_service_fn(move |socket: &AddrStream| {
|
||||||
|
let remote_addr = socket.remote_addr().ip();
|
||||||
|
let allowed = allow_ips.iter().any(|subnet| subnet.contains(&remote_addr));
|
||||||
|
let handle = handle.clone();
|
||||||
|
|
||||||
|
async move {
|
||||||
|
Ok::<_, HyperError>(service_fn(move |_| {
|
||||||
|
let handle = handle.clone();
|
||||||
|
|
||||||
|
async move {
|
||||||
|
if allowed {
|
||||||
|
let output = handle.render();
|
||||||
|
Ok(Response::new(Body::from(output)))
|
||||||
|
} else {
|
||||||
|
Response::builder()
|
||||||
|
.status(StatusCode::FORBIDDEN)
|
||||||
|
.body(Body::empty())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
server.serve(make_svc).await
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok((recorder, exporter))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A copy of `PrometheusBuilder::install` that adds support for an IP address or subnet
|
||||||
|
/// allowlist.
|
||||||
|
pub(super) fn install(
|
||||||
|
bind_address: SocketAddr,
|
||||||
|
builder: PrometheusBuilder,
|
||||||
|
allow_ips: Vec<ipnet::IpNet>,
|
||||||
|
) -> Result<(), InstallError> {
|
||||||
|
let runtime = runtime::Builder::new_current_thread()
|
||||||
|
.enable_all()
|
||||||
|
.build()?;
|
||||||
|
|
||||||
|
let (recorder, exporter) = {
|
||||||
|
let _guard = runtime.enter();
|
||||||
|
build(bind_address, builder, allow_ips)?
|
||||||
|
};
|
||||||
|
metrics::set_boxed_recorder(Box::new(recorder))?;
|
||||||
|
|
||||||
|
thread::Builder::new()
|
||||||
|
.name("zcash-prometheus".to_string())
|
||||||
|
.spawn(move || {
|
||||||
|
runtime.block_on(async move {
|
||||||
|
pin!(exporter);
|
||||||
|
loop {
|
||||||
|
select! {
|
||||||
|
_ = &mut exporter => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
})?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
|
@ -389,6 +389,7 @@ void ReadConfigFile(const std::string& confPath,
|
||||||
"externalip",
|
"externalip",
|
||||||
"fundingstream",
|
"fundingstream",
|
||||||
"loadblock",
|
"loadblock",
|
||||||
|
"metricsallowip",
|
||||||
"nuparams",
|
"nuparams",
|
||||||
"onlynet",
|
"onlynet",
|
||||||
"rpcallowip",
|
"rpcallowip",
|
||||||
|
|
Loading…
Reference in New Issue