From 3d6c1e4426ba1d4e601327b6cbe88f9683518083 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 4 Jun 2021 23:21:06 +0100 Subject: [PATCH] metrics 0.16 and metrics-exporter-prometheus 0.5 Includes the allowlist logic that we upstreamed. --- Cargo.lock | 105 +++++++++++++++++++-- Cargo.toml | 4 +- contrib/debian/copyright | 6 -- src/rust/src/metrics_ffi.rs | 48 +++++----- src/rust/src/metrics_ffi/prometheus.rs | 124 ------------------------- 5 files changed, 120 insertions(+), 167 deletions(-) delete mode 100644 src/rust/src/metrics_ffi/prometheus.rs diff --git a/Cargo.lock b/Cargo.lock index f20b849fa..7ef44032e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,6 +31,17 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "ahash" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98" +dependencies = [ + "getrandom 0.2.3", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.18" @@ -453,6 +464,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + [[package]] name = "equihash" version = "0.1.0" @@ -612,6 +629,15 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +dependencies = [ + "ahash", +] + [[package]] name = "hermit-abi" version = "0.1.18" @@ -691,7 +717,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.9.1", ] [[package]] @@ -838,21 +864,23 @@ dependencies = [ [[package]] name = "metrics" -version = "0.14.2" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58e285601dcfb9f3a8f37a7093b9956a0df730b50848c8ac0117406aff06c851" +checksum = "9c0a7fa53d812d26e59d2baf7a6f77442041454ab32908eb304447f00f0dd4de" dependencies = [ "metrics-macros", "proc-macro-hack", + "t1ha", ] [[package]] name = "metrics-exporter-prometheus" -version = "0.3.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d23bb354bd7dd5d244f2e9a389a0c3249bb6b994aca71bd6c2f7cd6fa2657fc" +checksum = "423ac561fc3300388e62947ce7f944ba6d40468afe9916ce5b7774171d8b38b8" dependencies = [ "hyper", + "ipnet", "metrics", "metrics-util", "parking_lot", @@ -863,9 +891,9 @@ dependencies = [ [[package]] name = "metrics-macros" -version = "0.2.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11ac60cd4d3a869fd39d57baf0ed7f79fb677580d8d6655c4330d4f1126bc27b" +checksum = "caa72e4a3d157986dd2565c82ecbddcc23941513669a3766b938f6b72eb87f3f" dependencies = [ "lazy_static", "proc-macro-hack", @@ -877,21 +905,25 @@ dependencies = [ [[package]] name = "metrics-util" -version = "0.6.2" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ace44e2c64f785c3c37605ecf7504e5fc4efbb2936a49cc56c1084d79657a4d" +checksum = "c0c93306cf63ff153c57963151a011194af0f33c91fe30ec30faf98732350a1a" dependencies = [ "aho-corasick", "atomic-shim", "crossbeam-epoch 0.9.5", "crossbeam-utils 0.8.5", "dashmap", + "hashbrown 0.11.2", "indexmap", "metrics", + "num_cpus", "ordered-float", "parking_lot", "quanta", + "radix_trie", "sketches-ddsketch", + "t1ha", ] [[package]] @@ -916,6 +948,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + [[package]] name = "nonempty" version = "0.6.0" @@ -1144,6 +1185,16 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "643f8f41a8ebc4c5dc4515c82bb8abd397b527fc20fd681b7c011c2aee5d44fb" +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rand" version = "0.8.3" @@ -1264,12 +1315,36 @@ version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + [[package]] name = "scopeguard" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "serde" version = "1.0.126" @@ -1370,6 +1445,18 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "t1ha" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa44aa51ae1a544e2c35a38831ba54ae40591f21384816f531b84f3e984b9ccc" +dependencies = [ + "cfg-if 0.1.10", + "lazy_static", + "num-traits", + "rustc_version", +] + [[package]] name = "tap" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 4434ade99..6491301c2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,8 +40,8 @@ ed25519-zebra = "2.2.0" # Metrics hyper = { version = "=0.14.2", default-features = false, features = ["server", "tcp", "http1"] } ipnet = "2" -metrics = "0.14.2" -metrics-exporter-prometheus = "0.3" +metrics = "0.16" +metrics-exporter-prometheus = "0.5" thiserror = "1" tokio = { version = "1.0", features = ["rt", "net", "time", "macros"] } diff --git a/contrib/debian/copyright b/contrib/debian/copyright index 8b048579e..d2c2fbc58 100644 --- a/contrib/debian/copyright +++ b/contrib/debian/copyright @@ -137,12 +137,6 @@ Files: src/rust/include/tracing.h Copyright: Copyright (c) 2020 Jack Grigg License: Expat -Files: src/rust/src/metrics_ffi/prometheus.rs -Copyright: - 2020-2021 The contributors to the metrics project - 2021 Jack Grigg -License: Expat - Files: src/secp256k1/* Copyright: Copyright (c) 2013 Pieter Wuille License: Expat diff --git a/src/rust/src/metrics_ffi.rs b/src/rust/src/metrics_ffi.rs index 6acb51347..cdc4530b9 100644 --- a/src/rust/src/metrics_ffi.rs +++ b/src/rust/src/metrics_ffi.rs @@ -1,5 +1,5 @@ use libc::{c_char, c_double}; -use metrics::{try_recorder, GaugeValue, Key, KeyData, Label}; +use metrics::{try_recorder, GaugeValue, Key, Label}; use metrics_exporter_prometheus::PrometheusBuilder; use std::ffi::CStr; use std::net::{IpAddr, SocketAddr}; @@ -7,8 +7,6 @@ use std::ptr; use std::slice; use tracing::error; -mod prometheus; - #[no_mangle] pub extern "C" fn metrics_run( bind_address: *const c_char, @@ -64,11 +62,18 @@ pub extern "C" fn metrics_run( prometheus_port, ); - prometheus::install(bind_address, PrometheusBuilder::new(), allow_ips).is_ok() + allow_ips + .into_iter() + .fold( + PrometheusBuilder::new().listen_address(bind_address), + |builder, subnet| builder.add_allowed(subnet), + ) + .install() + .is_ok() } pub struct FfiCallsite { - key_data: KeyData, + key: Key, } #[no_mangle] @@ -98,7 +103,7 @@ pub extern "C" fn metrics_callsite( .collect(); Box::into_raw(Box::new(FfiCallsite { - key_data: KeyData::from_parts(name, labels), + key: Key::from_parts(name, labels), })) } @@ -139,7 +144,7 @@ pub extern "C" fn metrics_key( .collect(); Box::into_raw(Box::new(FfiKey { - inner: Key::Owned(KeyData::from_parts(name, labels)), + inner: Key::from_parts(name, labels), })) } } @@ -148,7 +153,7 @@ pub extern "C" fn metrics_key( pub extern "C" fn metrics_static_increment_counter(callsite: *const FfiCallsite, value: u64) { if let Some(recorder) = try_recorder() { let callsite = unsafe { callsite.as_ref().unwrap() }; - recorder.increment_counter(Key::Borrowed(&callsite.key_data), value); + recorder.increment_counter(&callsite.key, value); } } @@ -157,7 +162,7 @@ pub extern "C" fn metrics_increment_counter(key: *mut FfiKey, value: u64) { if let Some(recorder) = try_recorder() { if !key.is_null() { let key = unsafe { Box::from_raw(key) }; - recorder.increment_counter(key.inner, value); + recorder.increment_counter(&key.inner, value); } } } @@ -166,10 +171,7 @@ pub extern "C" fn metrics_increment_counter(key: *mut FfiKey, value: u64) { pub extern "C" fn metrics_static_update_gauge(callsite: *const FfiCallsite, value: c_double) { if let Some(recorder) = try_recorder() { let callsite = unsafe { callsite.as_ref().unwrap() }; - recorder.update_gauge( - Key::Borrowed(&callsite.key_data), - GaugeValue::Absolute(value), - ); + recorder.update_gauge(&callsite.key, GaugeValue::Absolute(value)); } } @@ -178,7 +180,7 @@ pub extern "C" fn metrics_update_gauge(key: *mut FfiKey, value: c_double) { if let Some(recorder) = try_recorder() { if !key.is_null() { let key = unsafe { Box::from_raw(key) }; - recorder.update_gauge(key.inner, GaugeValue::Absolute(value)); + recorder.update_gauge(&key.inner, GaugeValue::Absolute(value)); } } } @@ -187,10 +189,7 @@ pub extern "C" fn metrics_update_gauge(key: *mut FfiKey, value: c_double) { pub extern "C" fn metrics_static_increment_gauge(callsite: *const FfiCallsite, value: c_double) { if let Some(recorder) = try_recorder() { let callsite = unsafe { callsite.as_ref().unwrap() }; - recorder.update_gauge( - Key::Borrowed(&callsite.key_data), - GaugeValue::Increment(value), - ); + recorder.update_gauge(&callsite.key, GaugeValue::Increment(value)); } } @@ -199,7 +198,7 @@ pub extern "C" fn metrics_increment_gauge(key: *mut FfiKey, value: c_double) { if let Some(recorder) = try_recorder() { if !key.is_null() { let key = unsafe { Box::from_raw(key) }; - recorder.update_gauge(key.inner, GaugeValue::Increment(value)); + recorder.update_gauge(&key.inner, GaugeValue::Increment(value)); } } } @@ -208,10 +207,7 @@ pub extern "C" fn metrics_increment_gauge(key: *mut FfiKey, value: c_double) { pub extern "C" fn metrics_static_decrement_gauge(callsite: *const FfiCallsite, value: c_double) { if let Some(recorder) = try_recorder() { let callsite = unsafe { callsite.as_ref().unwrap() }; - recorder.update_gauge( - Key::Borrowed(&callsite.key_data), - GaugeValue::Decrement(value), - ); + recorder.update_gauge(&callsite.key, GaugeValue::Decrement(value)); } } @@ -220,7 +216,7 @@ pub extern "C" fn metrics_decrement_gauge(key: *mut FfiKey, value: c_double) { if let Some(recorder) = try_recorder() { if !key.is_null() { let key = unsafe { Box::from_raw(key) }; - recorder.update_gauge(key.inner, GaugeValue::Decrement(value)); + recorder.update_gauge(&key.inner, GaugeValue::Decrement(value)); } } } @@ -229,7 +225,7 @@ pub extern "C" fn metrics_decrement_gauge(key: *mut FfiKey, value: c_double) { pub extern "C" fn metrics_static_record_histogram(callsite: *const FfiCallsite, value: c_double) { if let Some(recorder) = try_recorder() { let callsite = unsafe { callsite.as_ref().unwrap() }; - recorder.record_histogram(Key::Borrowed(&callsite.key_data), value); + recorder.record_histogram(&callsite.key, value); } } @@ -238,7 +234,7 @@ pub extern "C" fn metrics_record_histogram(key: *mut FfiKey, value: c_double) { if let Some(recorder) = try_recorder() { if !key.is_null() { let key = unsafe { Box::from_raw(key) }; - recorder.record_histogram(key.inner, value); + recorder.record_histogram(&key.inner, value); } } } diff --git a/src/rust/src/metrics_ffi/prometheus.rs b/src/rust/src/metrics_ffi/prometheus.rs deleted file mode 100644 index da11f5f6b..000000000 --- a/src/rust/src/metrics_ffi/prometheus.rs +++ /dev/null @@ -1,124 +0,0 @@ -// This is mostly code copied from metrics_exporter_prometheus. The copied portions are -// licensed under the same terms as the zcash codebase (reproduced below from -// https://github.com/metrics-rs/metrics/blob/main/metrics-exporter-prometheus/LICENSE): -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -use hyper::{ - server::{conn::AddrStream, Server}, - service::{make_service_fn, service_fn}, - {Body, Error as HyperError, Response, StatusCode}, -}; -use metrics::SetRecorderError; -use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusRecorder}; -use std::future::Future; -use std::io; -use std::net::SocketAddr; -use std::thread; -use thiserror::Error as ThisError; -use tokio::{pin, runtime, select}; - -/// Errors that could occur while installing a Prometheus recorder/exporter. -#[derive(Debug, ThisError)] -pub enum InstallError { - /// Creating the networking event loop did not succeed. - #[error("failed to spawn Tokio runtime for endpoint: {0}")] - Io(#[from] io::Error), - - /// Binding/listening to the given address did not succeed. - #[error("failed to bind to given listen address: {0}")] - Hyper(#[from] HyperError), - - /// Installing the recorder did not succeed. - #[error("failed to install exporter as global recorder: {0}")] - Recorder(#[from] SetRecorderError), -} - -/// A copy of `PrometheusBuilder::build_with_exporter` that adds support for an IP address -/// or subnet allowlist. -pub(super) fn build( - bind_address: SocketAddr, - builder: PrometheusBuilder, - allow_ips: Vec, -) -> Result< - ( - PrometheusRecorder, - impl Future> + Send + 'static, - ), - InstallError, -> { - let recorder = builder.build(); - let handle = recorder.handle(); - - let server = Server::try_bind(&bind_address)?; - - let exporter = async move { - let make_svc = make_service_fn(move |socket: &AddrStream| { - let remote_addr = socket.remote_addr().ip(); - let allowed = allow_ips.iter().any(|subnet| subnet.contains(&remote_addr)); - let handle = handle.clone(); - - async move { - Ok::<_, HyperError>(service_fn(move |_| { - let handle = handle.clone(); - - async move { - if allowed { - let output = handle.render(); - Ok(Response::new(Body::from(output))) - } else { - Response::builder() - .status(StatusCode::FORBIDDEN) - .body(Body::empty()) - } - } - })) - } - }); - - server.serve(make_svc).await - }; - - Ok((recorder, exporter)) -} - -/// A copy of `PrometheusBuilder::install` that adds support for an IP address or subnet -/// allowlist. -pub(super) fn install( - bind_address: SocketAddr, - builder: PrometheusBuilder, - allow_ips: Vec, -) -> Result<(), InstallError> { - let runtime = runtime::Builder::new_current_thread() - .enable_all() - .build()?; - - let (recorder, exporter) = { - let _guard = runtime.enter(); - build(bind_address, builder, allow_ips)? - }; - metrics::set_boxed_recorder(Box::new(recorder))?; - - thread::Builder::new() - .name("zcash-prometheus".to_string()) - .spawn(move || { - runtime.block_on(async move { - pin!(exporter); - loop { - select! { - _ = &mut exporter => {} - } - } - }); - })?; - - Ok(()) -}