metrics: Implement IP access control on Prometheus scrape endpoint
This commit is contained in:
parent
59da774f22
commit
d08cdbe5f7
|
@ -676,6 +676,12 @@ dependencies = [
|
|||
"cfg-if 1.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ipnet"
|
||||
version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "0.4.7"
|
||||
|
@ -719,12 +725,16 @@ dependencies = [
|
|||
"ed25519-zebra",
|
||||
"funty",
|
||||
"group",
|
||||
"hyper",
|
||||
"ipnet",
|
||||
"jubjub",
|
||||
"libc",
|
||||
"metrics",
|
||||
"metrics-exporter-prometheus",
|
||||
"rand_core",
|
||||
"subtle",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-appender",
|
||||
"tracing-core",
|
||||
|
|
|
@ -38,8 +38,12 @@ zcash_proofs = "0.5"
|
|||
ed25519-zebra = "2.0.0"
|
||||
|
||||
# Metrics
|
||||
hyper = { version = "0.14", default-features = false, features = ["server", "tcp", "http1"] }
|
||||
ipnet = "2"
|
||||
metrics = "0.14.2"
|
||||
metrics-exporter-prometheus = "0.3"
|
||||
thiserror = "1"
|
||||
tokio = { version = "1.0", features = ["rt", "net", "time", "macros"] }
|
||||
|
||||
# Temporary workaround for https://github.com/myrrlyn/funty/issues/3
|
||||
funty = "=1.1.0"
|
||||
|
|
|
@ -30,18 +30,12 @@ You can see what each method provides with `zcash-cli help METHOD_NAME`.
|
|||
`zcashd` can optionally expose an HTTP server that acts as a Prometheus scrape
|
||||
endpoint. The server will respond to `GET` requests on any request path.
|
||||
|
||||
Note that HTTPS is not supported, and therefore connections to the endpoint are
|
||||
not encrypted or authenticated. Access to the endpoint should be assumed to
|
||||
compromise the privacy of node operations, by the provided metrics and/or by
|
||||
timing side channels. Enabling the endpoint is **strongly discouraged** if the
|
||||
node has a wallet holding live funds.
|
||||
|
||||
To enable the endpoint, add `-prometheusmetrics=<host_name>:<port>` to your
|
||||
`zcashd` configuration (either in `zcash.conf` or on the command line). After
|
||||
To enable the endpoint, add `-prometheusport=:<port>` to your `zcashd`
|
||||
configuration (either in `zcash.conf` or on the command line). After
|
||||
restarting `zcashd` you can then test the endpoint by querying it:
|
||||
|
||||
```
|
||||
$ curl http://<host_name>:<port>
|
||||
$ curl http://127.0.0.1:<port>
|
||||
# TYPE peer_outbound_messages counter
|
||||
peer_outbound_messages 181
|
||||
|
||||
|
@ -59,6 +53,14 @@ block_verified_block_count 162
|
|||
...
|
||||
```
|
||||
|
||||
By default, access is restricted to localhost. This can be expanded with
|
||||
`-metricsallowip=<ip>`, which can specify IPs or subnets. Note that HTTPS is not
|
||||
supported, and therefore connections to the endpoint are not encrypted or
|
||||
authenticated. Access to the endpoint should be assumed to compromise the
|
||||
privacy of node operations, by the provided metrics and/or by timing side
|
||||
channels. Non-localhost access is **strongly discouraged** if the node has a
|
||||
wallet holding live funds.
|
||||
|
||||
### Example metrics collection with Docker
|
||||
|
||||
The example instructions below were tested on Windows 10 using Docker Desktop
|
||||
|
|
|
@ -11,16 +11,18 @@ Prometheus metrics
|
|||
a Prometheus scrape endpoint. The server will respond to `GET` requests on any
|
||||
request path.
|
||||
|
||||
Note that HTTPS is not supported, and therefore connections to the endpoint are
|
||||
not encrypted or authenticated. Access to the endpoint should be assumed to
|
||||
compromise the privacy of node operations, by the provided metrics and/or by
|
||||
timing side channels. Enabling the endpoint is **strongly discouraged** if the
|
||||
node has a wallet holding live funds.
|
||||
|
||||
To enable the endpoint, add `-prometheusmetrics=<host_name>:<port>` to your
|
||||
`zcashd` configuration (either in `zcash.conf` or on the command line). After
|
||||
To enable the endpoint, add `-prometheusport=<port>` to your `zcashd`
|
||||
configuration (either in `zcash.conf` or on the command line). After
|
||||
restarting `zcashd` you can then test the endpoint by querying it with e.g.
|
||||
`curl http://<host_name>:<port>`.
|
||||
`curl http://127.0.0.1:<port>`.
|
||||
|
||||
By default, access is restricted to localhost. This can be expanded with
|
||||
`-metricsallowip=<ip>`, which can specify IPs or subnets. Note that HTTPS is not
|
||||
supported, and therefore connections to the endpoint are not encrypted or
|
||||
authenticated. Access to the endpoint should be assumed to compromise the
|
||||
privacy of node operations, by the provided metrics and/or by timing side
|
||||
channels. Non-localhost access is **strongly discouraged** if the node has a
|
||||
wallet holding live funds.
|
||||
|
||||
The specific metrics names may change in subsequent releases, in particular to
|
||||
improve interoperability with `zebrad`.
|
||||
|
|
32
src/init.cpp
32
src/init.cpp
|
@ -353,9 +353,6 @@ std::string HelpMessage(HelpMessageMode mode)
|
|||
#ifndef WIN32
|
||||
strUsage += HelpMessageOpt("-pid=<file>", strprintf(_("Specify pid file (default: %s)"), BITCOIN_PID_FILENAME));
|
||||
#endif
|
||||
strUsage += HelpMessageOpt("-prometheusmetrics=<host_name>:<port>", _("Expose node metrics in the Prometheus exposition format. "
|
||||
"An HTTP listener will be started on the configured hostname and port, which responds to GET requests on any request path. "
|
||||
"SECURITY WARNING: this can potentially compromise privacy; read contrib/metrics/README.md before enabling."));
|
||||
strUsage += HelpMessageOpt("-prune=<n>", strprintf(_("Reduce storage requirements by pruning (deleting) old blocks. This mode disables wallet support and is incompatible with -txindex. "
|
||||
"Warning: Reverting this setting requires re-downloading the entire blockchain. "
|
||||
"(default: 0 = disable pruning blocks, >%u = target size in MiB to use for block files)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
|
||||
|
@ -415,6 +412,15 @@ std::string HelpMessage(HelpMessageMode mode)
|
|||
strUsage += HelpMessageOpt("-zmqpubrawtx=<address>", _("Enable publish raw transaction in <address>"));
|
||||
#endif
|
||||
|
||||
strUsage += HelpMessageGroup(_("Monitoring options:"));
|
||||
strUsage += HelpMessageOpt("-metricsallowip=<ip>", _("Allow metrics connections from specified source. "
|
||||
"Valid for <ip> are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). "
|
||||
"This option can be specified multiple times. (default: only localhost)"));
|
||||
strUsage += HelpMessageOpt("-metricsbind=<addr>", _("Bind to given address to listen for metrics connections. (default: bind to all interfaces)"));
|
||||
strUsage += HelpMessageOpt("-prometheusport=<port>", _("Expose node metrics in the Prometheus exposition format. "
|
||||
"An HTTP listener will be started on <port>, which responds to GET requests on any request path. "
|
||||
"Use -metricsallowip and -metricsbind to control access."));
|
||||
|
||||
strUsage += HelpMessageGroup(_("Debugging/Testing options:"));
|
||||
if (showDebug)
|
||||
{
|
||||
|
@ -1226,13 +1232,25 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
|
|||
// Count uptime
|
||||
MarkStartTime();
|
||||
|
||||
std::string prometheusMetricsArg = GetArg("-prometheusmetrics", "");
|
||||
if (prometheusMetricsArg != "") {
|
||||
int prometheusPort = GetArg("-prometheusport", -1);
|
||||
if (prometheusPort > 0) {
|
||||
const std::vector<std::string>& vAllow = mapMultiArgs["-metricsallowip"];
|
||||
std::vector<const char*> vAllowCstr;
|
||||
for (const std::string& strAllow : vAllow) {
|
||||
vAllowCstr.push_back(strAllow.c_str());
|
||||
}
|
||||
|
||||
std::string metricsBind = GetArg("-metricsbind", "");
|
||||
const char* metricsBindCstr = nullptr;
|
||||
if (!metricsBind.empty()) {
|
||||
metricsBindCstr = metricsBind.c_str();
|
||||
}
|
||||
|
||||
// Start up the metrics runtime. This spins off a Rust thread that runs
|
||||
// the Prometheus exporter. We just let this thread die at process end.
|
||||
LogPrintf("metrics thread start");
|
||||
if (!metrics_run(prometheusMetricsArg.c_str())) {
|
||||
return InitError(strprintf(_("Failed to start Prometheus metrics exporter on '%s'"), prometheusMetricsArg));
|
||||
if (!metrics_run(metricsBindCstr, vAllowCstr.data(), vAllowCstr.size(), prometheusPort)) {
|
||||
return InitError(strprintf(_("Failed to start Prometheus metrics exporter")));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -17,10 +17,14 @@ extern "C" {
|
|||
/// Initializes the metrics runtime and runs the Prometheus exporter in a new
|
||||
/// thread.
|
||||
///
|
||||
/// listen_address is a string like <host_name>:<port>.
|
||||
/// bind_address is an IP address to bind to, or empty to use the default.
|
||||
///
|
||||
/// Returns false if listen_address was not a valid SocketAddr.
|
||||
bool metrics_run(const char* listen_address);
|
||||
/// Returns false on any error.
|
||||
bool metrics_run(
|
||||
const char* bind_address,
|
||||
const char* const* allow_ips,
|
||||
size_t allow_ips_len,
|
||||
uint16_t prometheus_port);
|
||||
|
||||
struct MetricsCallsite;
|
||||
typedef struct MetricsCallsite MetricsCallsite;
|
||||
|
|
|
@ -2,39 +2,69 @@ use libc::{c_char, c_double};
|
|||
use metrics::{try_recorder, GaugeValue, Key, KeyData, Label};
|
||||
use metrics_exporter_prometheus::PrometheusBuilder;
|
||||
use std::ffi::CStr;
|
||||
use std::net::SocketAddr;
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::ptr;
|
||||
use std::slice;
|
||||
use tracing::error;
|
||||
|
||||
mod prometheus;
|
||||
|
||||
#[no_mangle]
|
||||
pub extern "C" fn metrics_run(listen_address: *const c_char) -> bool {
|
||||
let listen_address = match unsafe { CStr::from_ptr(listen_address) }.to_str() {
|
||||
Ok(addr) => addr,
|
||||
Err(_) => {
|
||||
error!("-prometheusmetrics argument is not valid UTF-8");
|
||||
pub extern "C" fn metrics_run(
|
||||
bind_address: *const c_char,
|
||||
allow_ips: *const *const c_char,
|
||||
allow_ips_len: usize,
|
||||
prometheus_port: u16,
|
||||
) -> bool {
|
||||
// Parse any allowed IPs.
|
||||
let allow_ips = unsafe { slice::from_raw_parts(allow_ips, allow_ips_len) };
|
||||
let mut allow_ips: Vec<ipnet::IpNet> = match allow_ips
|
||||
.iter()
|
||||
.map(|&p| unsafe { CStr::from_ptr(p) })
|
||||
.map(|s| {
|
||||
s.to_str().ok().and_then(|s| {
|
||||
s.parse()
|
||||
.map_err(|e| {
|
||||
error!("Invalid -metricsallowip argument '{}': {}", s, e);
|
||||
})
|
||||
.ok()
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
{
|
||||
Some(ips) => ips,
|
||||
None => {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
listen_address
|
||||
.parse::<SocketAddr>()
|
||||
.map_err(|e| {
|
||||
error!(
|
||||
"Invalid Prometheus metrics address '{}': {}",
|
||||
listen_address, e
|
||||
);
|
||||
()
|
||||
})
|
||||
.and_then(|addr| {
|
||||
PrometheusBuilder::new()
|
||||
.listen_address(addr)
|
||||
.install()
|
||||
.map_err(|e| {
|
||||
error!("Failed to start Prometheus metrics exporter: {:?}", e);
|
||||
()
|
||||
})
|
||||
})
|
||||
.is_ok()
|
||||
// We always allow localhost.
|
||||
allow_ips.extend(&["127.0.0.0/8".parse().unwrap(), "::1/128".parse().unwrap()]);
|
||||
|
||||
// Parse the address to bind to.
|
||||
let bind_address = SocketAddr::new(
|
||||
if allow_ips.is_empty() {
|
||||
// Default to loopback if not allowing external IPs.
|
||||
"127.0.0.1".parse::<IpAddr>().unwrap()
|
||||
} else if bind_address.is_null() {
|
||||
// No specific bind address specified, bind to any.
|
||||
"0.0.0.0".parse::<IpAddr>().unwrap()
|
||||
} else {
|
||||
match unsafe { CStr::from_ptr(bind_address) }
|
||||
.to_str()
|
||||
.ok()
|
||||
.and_then(|s| s.parse::<IpAddr>().ok())
|
||||
{
|
||||
Some(addr) => addr,
|
||||
None => {
|
||||
error!("Invalid -metricsbind argument");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
},
|
||||
prometheus_port,
|
||||
);
|
||||
|
||||
prometheus::install(bind_address, PrometheusBuilder::new(), allow_ips).is_ok()
|
||||
}
|
||||
|
||||
pub struct FfiCallsite {
|
||||
|
|
|
@ -0,0 +1,113 @@
|
|||
// This is mostly code copied from metrics_exporter_prometheus. It is licensed under the
|
||||
// same license as the zcash codebase.
|
||||
|
||||
use hyper::{
|
||||
server::{conn::AddrStream, Server},
|
||||
service::{make_service_fn, service_fn},
|
||||
{Body, Error as HyperError, Response, StatusCode},
|
||||
};
|
||||
use metrics::SetRecorderError;
|
||||
use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusRecorder};
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
use std::net::SocketAddr;
|
||||
use std::thread;
|
||||
use thiserror::Error as ThisError;
|
||||
use tokio::{pin, runtime, select};
|
||||
|
||||
/// Errors that could occur while installing a Prometheus recorder/exporter.
|
||||
#[derive(Debug, ThisError)]
|
||||
pub enum InstallError {
|
||||
/// Creating the networking event loop did not succeed.
|
||||
#[error("failed to spawn Tokio runtime for endpoint: {0}")]
|
||||
Io(#[from] io::Error),
|
||||
|
||||
/// Binding/listening to the given address did not succeed.
|
||||
#[error("failed to bind to given listen address: {0}")]
|
||||
Hyper(#[from] HyperError),
|
||||
|
||||
/// Installing the recorder did not succeed.
|
||||
#[error("failed to install exporter as global recorder: {0}")]
|
||||
Recorder(#[from] SetRecorderError),
|
||||
}
|
||||
|
||||
/// A copy of `PrometheusBuilder::build_with_exporter` that adds support for an IP address
|
||||
/// or subnet allowlist.
|
||||
pub(super) fn build(
|
||||
bind_address: SocketAddr,
|
||||
builder: PrometheusBuilder,
|
||||
allow_ips: Vec<ipnet::IpNet>,
|
||||
) -> Result<
|
||||
(
|
||||
PrometheusRecorder,
|
||||
impl Future<Output = Result<(), HyperError>> + Send + 'static,
|
||||
),
|
||||
InstallError,
|
||||
> {
|
||||
let recorder = builder.build();
|
||||
let handle = recorder.handle();
|
||||
|
||||
let server = Server::try_bind(&bind_address)?;
|
||||
|
||||
let exporter = async move {
|
||||
let make_svc = make_service_fn(move |socket: &AddrStream| {
|
||||
let remote_addr = socket.remote_addr().ip();
|
||||
let allowed = allow_ips.iter().any(|subnet| subnet.contains(&remote_addr));
|
||||
let handle = handle.clone();
|
||||
|
||||
async move {
|
||||
Ok::<_, HyperError>(service_fn(move |_| {
|
||||
let handle = handle.clone();
|
||||
|
||||
async move {
|
||||
if allowed {
|
||||
let output = handle.render();
|
||||
Ok(Response::new(Body::from(output)))
|
||||
} else {
|
||||
Response::builder()
|
||||
.status(StatusCode::FORBIDDEN)
|
||||
.body(Body::empty())
|
||||
}
|
||||
}
|
||||
}))
|
||||
}
|
||||
});
|
||||
|
||||
server.serve(make_svc).await
|
||||
};
|
||||
|
||||
Ok((recorder, exporter))
|
||||
}
|
||||
|
||||
/// A copy of `PrometheusBuilder::install` that adds support for an IP address or subnet
|
||||
/// allowlist.
|
||||
pub(super) fn install(
|
||||
bind_address: SocketAddr,
|
||||
builder: PrometheusBuilder,
|
||||
allow_ips: Vec<ipnet::IpNet>,
|
||||
) -> Result<(), InstallError> {
|
||||
let runtime = runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
|
||||
let (recorder, exporter) = {
|
||||
let _guard = runtime.enter();
|
||||
build(bind_address, builder, allow_ips)?
|
||||
};
|
||||
metrics::set_boxed_recorder(Box::new(recorder))?;
|
||||
|
||||
thread::Builder::new()
|
||||
.name("zcash-prometheus".to_string())
|
||||
.spawn(move || {
|
||||
runtime.block_on(async move {
|
||||
pin!(exporter);
|
||||
loop {
|
||||
select! {
|
||||
_ = &mut exporter => {}
|
||||
}
|
||||
}
|
||||
});
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
|
@ -389,6 +389,7 @@ void ReadConfigFile(const std::string& confPath,
|
|||
"externalip",
|
||||
"fundingstream",
|
||||
"loadblock",
|
||||
"metricsallowip",
|
||||
"nuparams",
|
||||
"onlynet",
|
||||
"rpcallowip",
|
||||
|
|
Loading…
Reference in New Issue