metrics: Implement IP access control on Prometheus scrape endpoint

Cargo.lock

@@ -676,6 +676,12 @@ dependencies = [
  "cfg-if 1.0.0",
 ]
 
+[[package]]
+name = "ipnet"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135"
+
 [[package]]
 name = "itoa"
 version = "0.4.7"
@@ -719,12 +725,16 @@ dependencies = [
  "ed25519-zebra",
  "funty",
  "group",
+ "hyper",
+ "ipnet",
  "jubjub",
  "libc",
  "metrics",
  "metrics-exporter-prometheus",
  "rand_core",
  "subtle",
+ "thiserror",
+ "tokio",
  "tracing",
  "tracing-appender",
  "tracing-core",

Cargo.toml

@@ -38,8 +38,12 @@ zcash_proofs = "0.5"
 ed25519-zebra = "2.0.0"
 
 # Metrics
+hyper = { version = "0.14", default-features = false, features = ["server", "tcp", "http1"] }
+ipnet = "2"
 metrics = "0.14.2"
 metrics-exporter-prometheus = "0.3"
+thiserror = "1"
+tokio = { version = "1.0", features = ["rt", "net", "time", "macros"] }
 
 # Temporary workaround for https://github.com/myrrlyn/funty/issues/3
 funty = "=1.1.0"

contrib/metrics/README.md

@@ -30,18 +30,12 @@ You can see what each method provides with `zcash-cli help METHOD_NAME`.
 `zcashd` can optionally expose an HTTP server that acts as a Prometheus scrape
 endpoint. The server will respond to `GET` requests on any request path.
 
-Note that HTTPS is not supported, and therefore connections to the endpoint are
-not encrypted or authenticated. Access to the endpoint should be assumed to
-compromise the privacy of node operations, by the provided metrics and/or by
-timing side channels. Enabling the endpoint is **strongly discouraged** if the
-node has a wallet holding live funds.
-
-To enable the endpoint, add `-prometheusmetrics=<host_name>:<port>` to your
-`zcashd` configuration (either in `zcash.conf` or on the command line). After
+To enable the endpoint, add `-prometheusport=:<port>` to your `zcashd`
+configuration (either in `zcash.conf` or on the command line). After
 restarting `zcashd` you can then test the endpoint by querying it:
 
 ```
-$ curl http://<host_name>:<port>
+$ curl http://127.0.0.1:<port>
 # TYPE peer_outbound_messages counter
 peer_outbound_messages 181
 
@@ -59,6 +53,14 @@ block_verified_block_count 162
 ...
 ```
 
+By default, access is restricted to localhost. This can be expanded with
+`-metricsallowip=<ip>`, which can specify IPs or subnets. Note that HTTPS is not
+supported, and therefore connections to the endpoint are not encrypted or
+authenticated. Access to the endpoint should be assumed to compromise the
+privacy of node operations, by the provided metrics and/or by timing side
+channels. Non-localhost access is **strongly discouraged** if the node has a
+wallet holding live funds.
+
 ### Example metrics collection with Docker
 
 The example instructions below were tested on Windows 10 using Docker Desktop

doc/release-notes.md

@@ -11,16 +11,18 @@ Prometheus metrics
 a Prometheus scrape endpoint. The server will respond to `GET` requests on any
 request path.
 
-Note that HTTPS is not supported, and therefore connections to the endpoint are
-not encrypted or authenticated. Access to the endpoint should be assumed to
-compromise the privacy of node operations, by the provided metrics and/or by
-timing side channels. Enabling the endpoint is **strongly discouraged** if the
-node has a wallet holding live funds.
-
-To enable the endpoint, add `-prometheusmetrics=<host_name>:<port>` to your
-`zcashd` configuration (either in `zcash.conf` or on the command line). After
+To enable the endpoint, add `-prometheusport=<port>` to your `zcashd`
+configuration (either in `zcash.conf` or on the command line). After
 restarting `zcashd` you can then test the endpoint by querying it with e.g.
-`curl http://<host_name>:<port>`.
+`curl http://127.0.0.1:<port>`.
+
+By default, access is restricted to localhost. This can be expanded with
+`-metricsallowip=<ip>`, which can specify IPs or subnets. Note that HTTPS is not
+supported, and therefore connections to the endpoint are not encrypted or
+authenticated. Access to the endpoint should be assumed to compromise the
+privacy of node operations, by the provided metrics and/or by timing side
+channels. Non-localhost access is **strongly discouraged** if the node has a
+wallet holding live funds.
 
 The specific metrics names may change in subsequent releases, in particular to
 improve interoperability with `zebrad`.

src/init.cpp

@@ -353,9 +353,6 @@ std::string HelpMessage(HelpMessageMode mode)
 #ifndef WIN32
     strUsage += HelpMessageOpt("-pid=<file>", strprintf(_("Specify pid file (default: %s)"), BITCOIN_PID_FILENAME));
 #endif
-    strUsage += HelpMessageOpt("-prometheusmetrics=<host_name>:<port>", _("Expose node metrics in the Prometheus exposition format. "
-            "An HTTP listener will be started on the configured hostname and port, which responds to GET requests on any request path. "
-            "SECURITY WARNING: this can potentially compromise privacy; read contrib/metrics/README.md before enabling."));
     strUsage += HelpMessageOpt("-prune=<n>", strprintf(_("Reduce storage requirements by pruning (deleting) old blocks. This mode disables wallet support and is incompatible with -txindex. "
             "Warning: Reverting this setting requires re-downloading the entire blockchain. "
             "(default: 0 = disable pruning blocks, >%u = target size in MiB to use for block files)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
@@ -415,6 +412,15 @@ std::string HelpMessage(HelpMessageMode mode)
     strUsage += HelpMessageOpt("-zmqpubrawtx=<address>", _("Enable publish raw transaction in <address>"));
 #endif
 
+    strUsage += HelpMessageGroup(_("Monitoring options:"));
+    strUsage += HelpMessageOpt("-metricsallowip=<ip>", _("Allow metrics connections from specified source. "
+            "Valid for <ip> are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). "
+            "This option can be specified multiple times. (default: only localhost)"));
+    strUsage += HelpMessageOpt("-metricsbind=<addr>", _("Bind to given address to listen for metrics connections. (default: bind to all interfaces)"));
+    strUsage += HelpMessageOpt("-prometheusport=<port>", _("Expose node metrics in the Prometheus exposition format. "
+            "An HTTP listener will be started on <port>, which responds to GET requests on any request path. "
+            "Use -metricsallowip and -metricsbind to control access."));
+
     strUsage += HelpMessageGroup(_("Debugging/Testing options:"));
     if (showDebug)
     {
@@ -1226,13 +1232,25 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
     // Count uptime
     MarkStartTime();
 
-    std::string prometheusMetricsArg = GetArg("-prometheusmetrics", "");
-    if (prometheusMetricsArg != "") {
+    int prometheusPort = GetArg("-prometheusport", -1);
+    if (prometheusPort > 0) {
+        const std::vector<std::string>& vAllow = mapMultiArgs["-metricsallowip"];
+        std::vector<const char*> vAllowCstr;
+        for (const std::string& strAllow : vAllow) {
+            vAllowCstr.push_back(strAllow.c_str());
+        }
+
+        std::string metricsBind = GetArg("-metricsbind", "");
+        const char* metricsBindCstr = nullptr;
+        if (!metricsBind.empty()) {
+            metricsBindCstr = metricsBind.c_str();
+        }
+
         // Start up the metrics runtime. This spins off a Rust thread that runs
         // the Prometheus exporter. We just let this thread die at process end.
         LogPrintf("metrics thread start");
-        if (!metrics_run(prometheusMetricsArg.c_str())) {
-            return InitError(strprintf(_("Failed to start Prometheus metrics exporter on '%s'"), prometheusMetricsArg));
+        if (!metrics_run(metricsBindCstr, vAllowCstr.data(), vAllowCstr.size(), prometheusPort)) {
+            return InitError(strprintf(_("Failed to start Prometheus metrics exporter")));
         }
     }
 

src/rust/include/rust/metrics.h

@@ -17,10 +17,14 @@ extern "C" {
 /// Initializes the metrics runtime and runs the Prometheus exporter in a new
 /// thread.
 ///
-/// listen_address is a string like <host_name>:<port>.
+/// bind_address is an IP address to bind to, or empty to use the default.
 ///
-/// Returns false if listen_address was not a valid SocketAddr.
-bool metrics_run(const char* listen_address);
+/// Returns false on any error.
+bool metrics_run(
+    const char* bind_address,
+    const char* const* allow_ips,
+    size_t allow_ips_len,
+    uint16_t prometheus_port);
 
 struct MetricsCallsite;
 typedef struct MetricsCallsite MetricsCallsite;

src/rust/src/metrics_ffi.rs

@@ -2,39 +2,69 @@ use libc::{c_char, c_double};
 use metrics::{try_recorder, GaugeValue, Key, KeyData, Label};
 use metrics_exporter_prometheus::PrometheusBuilder;
 use std::ffi::CStr;
-use std::net::SocketAddr;
+use std::net::{IpAddr, SocketAddr};
 use std::ptr;
 use std::slice;
 use tracing::error;
 
+mod prometheus;
+
 #[no_mangle]
-pub extern "C" fn metrics_run(listen_address: *const c_char) -> bool {
-    let listen_address = match unsafe { CStr::from_ptr(listen_address) }.to_str() {
-        Ok(addr) => addr,
-        Err(_) => {
-            error!("-prometheusmetrics argument is not valid UTF-8");
+pub extern "C" fn metrics_run(
+    bind_address: *const c_char,
+    allow_ips: *const *const c_char,
+    allow_ips_len: usize,
+    prometheus_port: u16,
+) -> bool {
+    // Parse any allowed IPs.
+    let allow_ips = unsafe { slice::from_raw_parts(allow_ips, allow_ips_len) };
+    let mut allow_ips: Vec<ipnet::IpNet> = match allow_ips
+        .iter()
+        .map(|&p| unsafe { CStr::from_ptr(p) })
+        .map(|s| {
+            s.to_str().ok().and_then(|s| {
+                s.parse()
+                    .map_err(|e| {
+                        error!("Invalid -metricsallowip argument '{}': {}", s, e);
+                    })
+                    .ok()
+            })
+        })
+        .collect()
+    {
+        Some(ips) => ips,
+        None => {
             return false;
         }
     };
-    listen_address
-        .parse::<SocketAddr>()
-        .map_err(|e| {
-            error!(
-                "Invalid Prometheus metrics address '{}': {}",
-                listen_address, e
-            );
-            ()
-        })
-        .and_then(|addr| {
-            PrometheusBuilder::new()
-                .listen_address(addr)
-                .install()
-                .map_err(|e| {
-                    error!("Failed to start Prometheus metrics exporter: {:?}", e);
-                    ()
-                })
-        })
-        .is_ok()
+    // We always allow localhost.
+    allow_ips.extend(&["127.0.0.0/8".parse().unwrap(), "::1/128".parse().unwrap()]);
+
+    // Parse the address to bind to.
+    let bind_address = SocketAddr::new(
+        if allow_ips.is_empty() {
+            // Default to loopback if not allowing external IPs.
+            "127.0.0.1".parse::<IpAddr>().unwrap()
+        } else if bind_address.is_null() {
+            // No specific bind address specified, bind to any.
+            "0.0.0.0".parse::<IpAddr>().unwrap()
+        } else {
+            match unsafe { CStr::from_ptr(bind_address) }
+                .to_str()
+                .ok()
+                .and_then(|s| s.parse::<IpAddr>().ok())
+            {
+                Some(addr) => addr,
+                None => {
+                    error!("Invalid -metricsbind argument");
+                    return false;
+                }
+            }
+        },
+        prometheus_port,
+    );
+
+    prometheus::install(bind_address, PrometheusBuilder::new(), allow_ips).is_ok()
 }
 
 pub struct FfiCallsite {

src/rust/src/metrics_ffi/prometheus.rs

@@ -0,0 +1,113 @@
+// This is mostly code copied from metrics_exporter_prometheus. It is licensed under the
+// same license as the zcash codebase.
+
+use hyper::{
+    server::{conn::AddrStream, Server},
+    service::{make_service_fn, service_fn},
+    {Body, Error as HyperError, Response, StatusCode},
+};
+use metrics::SetRecorderError;
+use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusRecorder};
+use std::future::Future;
+use std::io;
+use std::net::SocketAddr;
+use std::thread;
+use thiserror::Error as ThisError;
+use tokio::{pin, runtime, select};
+
+/// Errors that could occur while installing a Prometheus recorder/exporter.
+#[derive(Debug, ThisError)]
+pub enum InstallError {
+    /// Creating the networking event loop did not succeed.
+    #[error("failed to spawn Tokio runtime for endpoint: {0}")]
+    Io(#[from] io::Error),
+
+    /// Binding/listening to the given address did not succeed.
+    #[error("failed to bind to given listen address: {0}")]
+    Hyper(#[from] HyperError),
+
+    /// Installing the recorder did not succeed.
+    #[error("failed to install exporter as global recorder: {0}")]
+    Recorder(#[from] SetRecorderError),
+}
+
+/// A copy of `PrometheusBuilder::build_with_exporter` that adds support for an IP address
+/// or subnet allowlist.
+pub(super) fn build(
+    bind_address: SocketAddr,
+    builder: PrometheusBuilder,
+    allow_ips: Vec<ipnet::IpNet>,
+) -> Result<
+    (
+        PrometheusRecorder,
+        impl Future<Output = Result<(), HyperError>> + Send + 'static,
+    ),
+    InstallError,
+> {
+    let recorder = builder.build();
+    let handle = recorder.handle();
+
+    let server = Server::try_bind(&bind_address)?;
+
+    let exporter = async move {
+        let make_svc = make_service_fn(move |socket: &AddrStream| {
+            let remote_addr = socket.remote_addr().ip();
+            let allowed = allow_ips.iter().any(|subnet| subnet.contains(&remote_addr));
+            let handle = handle.clone();
+
+            async move {
+                Ok::<_, HyperError>(service_fn(move |_| {
+                    let handle = handle.clone();
+
+                    async move {
+                        if allowed {
+                            let output = handle.render();
+                            Ok(Response::new(Body::from(output)))
+                        } else {
+                            Response::builder()
+                                .status(StatusCode::FORBIDDEN)
+                                .body(Body::empty())
+                        }
+                    }
+                }))
+            }
+        });
+
+        server.serve(make_svc).await
+    };
+
+    Ok((recorder, exporter))
+}
+
+/// A copy of `PrometheusBuilder::install` that adds support for an IP address or subnet
+/// allowlist.
+pub(super) fn install(
+    bind_address: SocketAddr,
+    builder: PrometheusBuilder,
+    allow_ips: Vec<ipnet::IpNet>,
+) -> Result<(), InstallError> {
+    let runtime = runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()?;
+
+    let (recorder, exporter) = {
+        let _guard = runtime.enter();
+        build(bind_address, builder, allow_ips)?
+    };
+    metrics::set_boxed_recorder(Box::new(recorder))?;
+
+    thread::Builder::new()
+        .name("zcash-prometheus".to_string())
+        .spawn(move || {
+            runtime.block_on(async move {
+                pin!(exporter);
+                loop {
+                    select! {
+                        _ = &mut exporter => {}
+                    }
+                }
+            });
+        })?;
+
+    Ok(())
+}

src/util.cpp

@@ -389,6 +389,7 @@ void ReadConfigFile(const std::string& confPath,
         "externalip",
         "fundingstream",
         "loadblock",
+        "metricsallowip",
         "nuparams",
         "onlynet",
         "rpcallowip",