From 29e73b3f3ea296aff4ab08c892bd5121d12b8231 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 17 Jun 2022 05:56:40 +1000 Subject: [PATCH] breaking(diagnostics): make zebrad diagnostics into optional compile-time features (#4539) * Disable the flamegraph feature by default at compile time * Disable the journald feature by default at compile time * Also disable inferno dependency, and rearrange features * Disable the prometheus feature by default at compile time * Disable the tracing filter reload feature by default at compile time * Disable tests when corresponding features are disabled * Add compile-time tracing features to user docs * Add compile-time features to the metrics user docs * Document diagnostics as part of the start command tasks and services * breaking(diagnostics): rename "enable-sentry" feature to "sentry" (#4623) * Also skip conflict tests when those ports are disabled * breaking(diagnostics): rename "enable-sentry" feature to "sentry" This is mostly: ```sh fastmod enable-sentry sentry ``` Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- book/src/user/metrics.md | 3 +- book/src/user/requirements.md | 2 +- book/src/user/tracing.md | 26 ++++++--- docker/Dockerfile | 6 +- docker/zcash-params/Dockerfile | 4 +- zebrad/Cargo.toml | 33 ++++++++--- zebrad/src/application.rs | 4 +- zebrad/src/commands/start.rs | 6 ++ zebrad/src/components/metrics.rs | 16 ++++++ zebrad/src/components/tracing.rs | 4 ++ zebrad/src/components/tracing/component.rs | 64 ++++++++++++++++++---- zebrad/src/components/tracing/endpoint.rs | 33 +++++++++-- zebrad/src/lib.rs | 2 +- zebrad/tests/acceptance.rs | 6 +- 14 files changed, 168 insertions(+), 41 deletions(-) diff --git a/book/src/user/metrics.md b/book/src/user/metrics.md index 5fc69e930..c2fb19ed7 100644 --- a/book/src/user/metrics.md +++ b/book/src/user/metrics.md @@ -1,6 +1,7 @@ # Zebra Metrics -Zebra has support for Prometheus, configured using the [`MetricsSection`][metrics_section]. +Zebra has support for Prometheus, configured using the `prometheus` compile-time feature, +and the [`MetricsSection`][metrics_section] runtime configuration. This requires supporting infrastructure to collect and visualize metrics, for example: diff --git a/book/src/user/requirements.md b/book/src/user/requirements.md index f8421eb39..44adf0be3 100644 --- a/book/src/user/requirements.md +++ b/book/src/user/requirements.md @@ -26,7 +26,7 @@ tested its exact limits yet. ## Sentry Production Monitoring -Compile Zebra with `--features enable-sentry` to monitor it using Sentry in production. +Compile Zebra with `--features sentry` to monitor it using Sentry in production. ## Lightwalletd Test Requirements diff --git a/book/src/user/tracing.md b/book/src/user/tracing.md index 620ef4065..97d5b4e86 100644 --- a/book/src/user/tracing.md +++ b/book/src/user/tracing.md @@ -1,7 +1,12 @@ # Tracing Zebra +## Dynamic Tracing + Zebra supports dynamic tracing, configured using the config's -[`TracingSection`][tracing_section] and (optionally) an HTTP RPC endpoint. +[`TracingSection`][tracing_section] and an HTTP RPC endpoint. + +Activate this feature using the `filter-reload` compile-time feature, +and the [`filter`][filter] and `endpoint_addr` runtime config options. If the `endpoint_addr` is specified, `zebrad` will open an HTTP endpoint allowing dynamic runtime configuration of the tracing filter. For instance, @@ -12,13 +17,20 @@ if the config had `endpoint_addr = '127.0.0.1:3000'`, then See the [`filter`][filter] documentation for more details. -Zebra also has support for: +## `journald` Logging -* Generating [flamegraphs] of tracing spans, configured using the -[`flamegraph`][flamegraph] option. -* Sending tracing spans and events to [systemd-journald][systemd_journald], -on Linux distributions that use `systemd`. Configured using the -[`use_journald`][use_journald] option. +Zebra can send tracing spans and events to [systemd-journald][systemd_journald], +on Linux distributions that use `systemd`. + +Activate `journald` logging using the `journald` compile-time feature, +and the [`use_journald`][use_journald] runtime config option. + +## Flamegraphs + +Zebra can generate [flamegraphs] of tracing spans. + +Activate flamegraphs using the `flamegraph` compile-time feature, +and the [`flamegraph`][flamegraph] runtime config option. [tracing_section]: https://doc.zebra.zfnd.org/zebrad/config/struct.TracingSection.html [filter]: https://doc.zebra.zfnd.org/zebrad/config/struct.TracingSection.html#structfield.filter diff --git a/docker/Dockerfile b/docker/Dockerfile index a564599e5..5bb39b963 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -100,7 +100,7 @@ COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/lightwalletd /lightwalletd /us # This is the caching Docker layer for Rust! # # TODO: is it faster to use --tests here? -RUN cargo chef cook --release --features enable-sentry,lightwalletd-grpc-tests --workspace --recipe-path recipe.json +RUN cargo chef cook --release --features sentry,lightwalletd-grpc-tests --workspace --recipe-path recipe.json COPY . . RUN cargo test --locked --release --features lightwalletd-grpc-tests --workspace --no-run @@ -118,11 +118,11 @@ CMD [ "cargo"] # `test` stage. This step is a dependency for the `runtime` stage, which uses the resulting # zebrad binary from this step. FROM deps AS release -RUN cargo chef cook --release --features enable-sentry --recipe-path recipe.json +RUN cargo chef cook --release --features sentry --recipe-path recipe.json COPY . . # Build zebra -RUN cargo build --locked --release --features enable-sentry --package zebrad --bin zebrad +RUN cargo build --locked --release --features sentry --package zebrad --bin zebrad # This stage is only used when deploying nodes or when only the resulting zebrad binary is needed # diff --git a/docker/zcash-params/Dockerfile b/docker/zcash-params/Dockerfile index 65da3839c..c441e1719 100644 --- a/docker/zcash-params/Dockerfile +++ b/docker/zcash-params/Dockerfile @@ -23,7 +23,7 @@ RUN apt-get -qq update && \ ENV CARGO_HOME /app/.cargo/ # Build dependencies - this is the caching Docker layer! -RUN cargo chef cook --release --features enable-sentry --package zebrad --recipe-path recipe.json +RUN cargo chef cook --release --features sentry --package zebrad --recipe-path recipe.json ARG RUST_BACKTRACE=0 ENV RUST_BACKTRACE ${RUST_BACKTRACE} @@ -36,4 +36,4 @@ ENV COLORBT_SHOW_HIDDEN ${COLORBT_SHOW_HIDDEN} COPY . . # Pre-download Zcash Sprout and Sapling parameters -RUN cargo run --locked --release --features enable-sentry --package zebrad --bin zebrad download +RUN cargo run --locked --release --features sentry --package zebrad --bin zebrad download diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index 1c4c3a29c..bc81c110a 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -17,7 +17,13 @@ default-run = "zebrad" default = ["release_max_level_info"] # Production features that activate extra dependencies -enable-sentry = ["sentry", "sentry-tracing"] + +sentry = ["dep:sentry", "sentry-tracing"] +flamegraph = ["tracing-flame", "inferno"] +journald = ["tracing-journald"] +filter-reload = ["hyper"] + +prometheus = ["metrics-exporter-prometheus"] # Production features that modify dependency behaviour @@ -78,7 +84,6 @@ lazy_static = "1.4.0" serde = { version = "1.0.137", features = ["serde_derive"] } toml = "0.5.9" -hyper = { version = "0.14.19", features = ["full"] } futures = "0.3.21" tokio = { version = "1.19.2", features = ["time", "rt-multi-thread", "macros", "tracing", "signal"] } tower = { version = "0.4.12", features = ["hedge", "limit"] } @@ -87,27 +92,36 @@ pin-project = "1.0.10" color-eyre = { version = "0.6.1", default_features = false, features = ["issue-url"] } thiserror = "1.0.31" -tracing-flame = "0.2.0" -tracing-journald = "0.3.0" tracing-subscriber = { version = "0.3.11", features = ["env-filter"] } tracing-error = "0.2.0" tracing-futures = "0.2.5" tracing = "0.1.31" metrics = "0.18.1" -metrics-exporter-prometheus = { version = "0.9.0", default-features = false, features = ["http-listener"] } dirs = "4.0.0" -inferno = { version = "0.11.4", default-features = false } atty = "0.2.14" num-integer = "0.1.45" rand = { version = "0.8.5", package = "rand" } -# prod feature enable-sentry +# prod feature sentry sentry-tracing = { version = "0.26.0", optional = true } sentry = { version = "0.26.0", default-features = false, features = ["backtrace", "contexts", "reqwest", "rustls"], optional = true } +# prod feature flamegraph +tracing-flame = { version = "0.2.0", optional = true } +inferno = { version = "0.11.4", default-features = false, optional = true } + +# prod feature journald +tracing-journald = { version = "0.3.0", optional = true } + +# prod feature filter-reload +hyper = { version = "0.14.19", features = ["http1", "http2", "server"], optional = true } + +# prod feature prometheus +metrics-exporter-prometheus = { version = "0.9.0", default-features = false, features = ["http-listener"], optional = true } + # prod feature release_max_level_info # # zebrad uses tracing for logging, @@ -132,12 +146,15 @@ abscissa_core = { version = "0.5", features = ["testing"] } hex = "0.4.3" once_cell = "1.12.0" regex = "1.5.6" -reqwest = "0.11.11" semver = "1.0.10" + # zebra-rpc needs the preserve_order feature, it also makes test results more stable serde_json = { version = "1.0.81", features = ["preserve_order"] } tempfile = "3.3.0" +hyper = { version = "0.14.18", features = ["http1", "http2", "server"]} +reqwest = "0.11.11" + tokio = { version = "1.19.2", features = ["full", "tracing", "test-util"] } tokio-stream = "0.1.9" diff --git a/zebrad/src/application.rs b/zebrad/src/application.rs index e78bee43c..494367216 100644 --- a/zebrad/src/application.rs +++ b/zebrad/src/application.rs @@ -314,7 +314,7 @@ impl Application for ZebradApp { // The Sentry default config pulls in the DSN from the `SENTRY_DSN` // environment variable. - #[cfg(feature = "enable-sentry")] + #[cfg(feature = "sentry")] let guard = sentry::init(sentry::ClientOptions { debug: true, release: Some(app_version().to_string().into()), @@ -325,7 +325,7 @@ impl Application for ZebradApp { let panic_report = panic_hook.panic_report(panic_info); eprintln!("{}", panic_report); - #[cfg(feature = "enable-sentry")] + #[cfg(feature = "sentry")] { let event = crate::sentry::panic_event_from(panic_report); sentry::capture_event(event); diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index 05b4d726f..9a1bce4b3 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -58,6 +58,12 @@ //! * JSON-RPC Service //! * answers RPC client requests using the State Service and Mempool Service //! * submits client transactions to the node's mempool +//! +//! Zebra also has diagnostic support +//! * [metrics](https://github.com/ZcashFoundation/zebra/blob/main/book/src/user/metrics.md) +//! * [tracing](https://github.com/ZcashFoundation/zebra/blob/main/book/src/user/tracing.md) +//! +//! Some of the diagnostic features are optional, and need to be enabled at compile-time. use std::{cmp::max, ops::Add, time::Duration}; diff --git a/zebrad/src/components/metrics.rs b/zebrad/src/components/metrics.rs index 157e4646f..369756147 100644 --- a/zebrad/src/components/metrics.rs +++ b/zebrad/src/components/metrics.rs @@ -10,6 +10,7 @@ pub struct MetricsEndpoint {} impl MetricsEndpoint { /// Create the component. + #[cfg(feature = "prometheus")] pub fn new(config: &ZebradConfig) -> Result { if let Some(addr) = config.metrics.endpoint_addr { info!("Trying to open metrics endpoint at {}...", addr); @@ -38,6 +39,21 @@ impl MetricsEndpoint { ), } } + + Ok(Self {}) + } + + /// Create the component. + #[cfg(not(feature = "prometheus"))] + pub fn new(config: &ZebradConfig) -> Result { + if let Some(addr) = config.metrics.endpoint_addr { + warn!( + ?addr, + "unable to activate configured metrics endpoint: \ + enable the 'prometheus' feature when compiling zebrad", + ); + } + Ok(Self {}) } } diff --git a/zebrad/src/components/tracing.rs b/zebrad/src/components/tracing.rs index 804b54739..d78739cba 100644 --- a/zebrad/src/components/tracing.rs +++ b/zebrad/src/components/tracing.rs @@ -2,8 +2,12 @@ mod component; mod endpoint; + +#[cfg(feature = "flamegraph")] mod flame; pub use component::Tracing; pub use endpoint::TracingEndpoint; + +#[cfg(feature = "flamegraph")] pub use flame::{layer, Grapher}; diff --git a/zebrad/src/components/tracing/component.rs b/zebrad/src/components/tracing/component.rs index fd23b3cdd..bab5675d1 100644 --- a/zebrad/src/components/tracing/component.rs +++ b/zebrad/src/components/tracing/component.rs @@ -1,6 +1,6 @@ //! The Abscissa component for Zebra's `tracing` implementation. -use abscissa_core::{Component, FrameworkError, FrameworkErrorKind, Shutdown}; +use abscissa_core::{Component, FrameworkError, Shutdown}; use tracing_error::ErrorLayer; use tracing_subscriber::{ fmt::Formatter, layer::SubscriberExt, reload::Handle, util::SubscriberInitExt, EnvFilter, @@ -8,6 +8,7 @@ use tracing_subscriber::{ use crate::{application::app_version, config::TracingSection}; +#[cfg(feature = "flamegraph")] use super::flame; /// Abscissa component for initializing the `tracing` subsystem @@ -21,6 +22,7 @@ pub struct Tracing { initial_filter: String, /// The installed flame graph collector, if enabled. + #[cfg(feature = "flamegraph")] flamegrapher: Option, } @@ -35,7 +37,9 @@ impl Tracing { let use_color = config.force_use_color || (config.use_color && atty::is(atty::Stream::Stdout)); - // Construct a format subscriber with the supplied global logging filter, and enable reloading. + // Construct a format subscriber with the supplied global logging filter, + // and optionally enable reloading. + // // TODO: when fmt::Subscriber supports per-layer filtering, always enable this code #[cfg(not(all(feature = "tokio-console", tokio_unstable)))] let (subscriber, filter_handle) = { @@ -43,13 +47,22 @@ impl Tracing { let logger = FmtSubscriber::builder() .with_ansi(use_color) - .with_env_filter(&filter) - .with_filter_reloading(); + .with_env_filter(&filter); + + // Enable reloading if that feature is selected. + #[cfg(feature = "filter-reload")] + let (filter_handle, logger) = { + let logger = logger.with_filter_reloading(); + + (Some(logger.reload_handle()), logger) + }; + + #[cfg(not(feature = "filter-reload"))] + let filter_handle = None; - let filter_handle = logger.reload_handle(); let subscriber = logger.finish().with(ErrorLayer::default()); - (subscriber, Some(filter_handle)) + (subscriber, filter_handle) }; // Construct a tracing registry with the supplied per-layer logging filter, @@ -82,6 +95,7 @@ impl Tracing { // Add optional layers based on dynamic and compile-time configs // Add a flamegraph + #[cfg(feature = "flamegraph")] let (flamelayer, flamegrapher) = if let Some(path) = flame_root { let (flamelayer, flamegrapher) = flame::layer(path); @@ -89,9 +103,13 @@ impl Tracing { } else { (None, None) }; + #[cfg(feature = "flamegraph")] let subscriber = subscriber.with(flamelayer); + #[cfg(feature = "journald")] let journaldlayer = if config.use_journald { + use abscissa_core::FrameworkErrorKind; + let layer = tracing_journald::layer() .map_err(|e| FrameworkErrorKind::ComponentError.context(e))?; @@ -107,9 +125,10 @@ impl Tracing { } else { None }; + #[cfg(feature = "journald")] let subscriber = subscriber.with(journaldlayer); - #[cfg(feature = "enable-sentry")] + #[cfg(feature = "sentry")] let subscriber = subscriber.with(sentry_tracing::layer()); // spawn the console server in the background, and apply the console layer @@ -127,14 +146,33 @@ impl Tracing { LOG_STATIC_MAX_LEVEL = ?log::STATIC_MAX_LEVEL, "started tracing component", ); + if flame_root.is_some() { - info!("installed flamegraph tracing layer"); + if cfg!(feature = "flamegraph") { + info!(flamegraph = ?flame_root, "installed flamegraph tracing layer"); + } else { + warn!( + flamegraph = ?flame_root, + "unable to activate configured flamegraph: \ + enable the 'flamegraph' feature when compiling zebrad", + ); + } } + if config.use_journald { - info!(?filter, "installed journald tracing layer"); + if cfg!(feature = "journald") { + info!("installed journald tracing layer"); + } else { + warn!( + "unable to activate configured journald tracing: \ + enable the 'journald' feature when compiling zebrad", + ); + } } - #[cfg(feature = "enable-sentry")] + + #[cfg(feature = "sentry")] info!("installed sentry tracing layer"); + #[cfg(all(feature = "tokio-console", tokio_unstable))] info!( TRACING_STATIC_MAX_LEVEL = ?tracing::level_filters::STATIC_MAX_LEVEL, @@ -145,6 +183,7 @@ impl Tracing { Ok(Self { filter_handle, initial_filter: filter, + #[cfg(feature = "flamegraph")] flamegrapher, }) } @@ -204,12 +243,17 @@ impl Component for Tracing { } fn before_shutdown(&self, _kind: Shutdown) -> Result<(), FrameworkError> { + #[cfg(feature = "flamegraph")] if let Some(ref grapher) = self.flamegrapher { + use abscissa_core::FrameworkErrorKind; + info!("writing flamegraph"); + grapher .write_flamegraph() .map_err(|e| FrameworkErrorKind::ComponentError.context(e))? } + Ok(()) } } diff --git a/zebrad/src/components/tracing/endpoint.rs b/zebrad/src/components/tracing/endpoint.rs index 706a5fb73..56b5ef828 100644 --- a/zebrad/src/components/tracing/endpoint.rs +++ b/zebrad/src/components/tracing/endpoint.rs @@ -3,20 +3,27 @@ use std::net::SocketAddr; use abscissa_core::{Component, FrameworkError}; -use hyper::service::{make_service_fn, service_fn}; -use hyper::{Body, Request, Response, Server}; -use crate::{components::tokio::TokioComponent, config::ZebradConfig, prelude::*}; +use crate::config::ZebradConfig; -use super::Tracing; +#[cfg(feature = "filter-reload")] +use hyper::{Body, Request, Response}; + +#[cfg(feature = "filter-reload")] +use crate::{components::tokio::TokioComponent, prelude::*}; /// Abscissa component which runs a tracing filter endpoint. #[derive(Debug, Component)] -#[component(inject = "init_tokio(zebrad::components::tokio::TokioComponent)")] +#[cfg_attr( + feature = "filter-reload", + component(inject = "init_tokio(zebrad::components::tokio::TokioComponent)") +)] pub struct TracingEndpoint { + #[allow(dead_code)] addr: Option, } +#[cfg(feature = "filter-reload")] async fn read_filter(req: Request) -> Result { std::str::from_utf8( &hyper::body::to_bytes(req.into_body()) @@ -30,12 +37,25 @@ async fn read_filter(req: Request) -> Result { impl TracingEndpoint { /// Create the component. pub fn new(config: &ZebradConfig) -> Result { + if !cfg!(feature = "filter-reload") { + warn!(addr = ?config.tracing.endpoint_addr, + "unable to activate configured tracing filter endpoint: \ + enable the 'filter-reload' feature when compiling zebrad", + ); + } + Ok(Self { addr: config.tracing.endpoint_addr, }) } + #[cfg(feature = "filter-reload")] pub fn init_tokio(&mut self, tokio_component: &TokioComponent) -> Result<(), FrameworkError> { + use hyper::{ + service::{make_service_fn, service_fn}, + Server, + }; + let addr = if let Some(addr) = self.addr { addr } else { @@ -75,10 +95,13 @@ impl TracingEndpoint { } } +#[cfg(feature = "filter-reload")] #[instrument] async fn request_handler(req: Request) -> Result, hyper::Error> { use hyper::{Method, StatusCode}; + use super::Tracing; + let rsp = match (req.method(), req.uri().path()) { (&Method::GET, "/") => Response::new(Body::from( r#" diff --git a/zebrad/src/lib.rs b/zebrad/src/lib.rs index b69761257..2ce4ed1e6 100644 --- a/zebrad/src/lib.rs +++ b/zebrad/src/lib.rs @@ -37,5 +37,5 @@ pub mod components; pub mod config; pub mod prelude; -#[cfg(feature = "enable-sentry")] +#[cfg(feature = "sentry")] pub mod sentry; diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 7c4e9022d..e3eaf37ee 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -103,7 +103,7 @@ //! //! Please refer to the documentation of each test for more information. -use std::{collections::HashSet, convert::TryInto, env, path::PathBuf}; +use std::{collections::HashSet, env, path::PathBuf}; use color_eyre::{ eyre::{Result, WrapErr}, @@ -866,6 +866,7 @@ fn full_sync_testnet() -> Result<()> { full_sync_test(Testnet, "FULL_SYNC_TESTNET_TIMEOUT_MINUTES") } +#[cfg(feature = "prometheus")] #[tokio::test] async fn metrics_endpoint() -> Result<()> { use hyper::Client; @@ -921,6 +922,7 @@ async fn metrics_endpoint() -> Result<()> { Ok(()) } +#[cfg(feature = "filter-reload")] #[tokio::test] async fn tracing_endpoint() -> Result<()> { use hyper::{Body, Client, Request}; @@ -1386,6 +1388,7 @@ fn zebra_zcash_listener_conflict() -> Result<()> { /// exclusive use of the port. The second node will panic with the Zcash metrics /// conflict hint added in #1535. #[test] +#[cfg(feature = "prometheus")] fn zebra_metrics_conflict() -> Result<()> { zebra_test::init(); @@ -1414,6 +1417,7 @@ fn zebra_metrics_conflict() -> Result<()> { /// exclusive use of the port. The second node will panic with the Zcash tracing /// conflict hint added in #1535. #[test] +#[cfg(feature = "filter-reload")] fn zebra_tracing_conflict() -> Result<()> { zebra_test::init();