Setup tracing-flame for use profiling zebrad (#436)

* Setup tracing-flame for use profiling zebrad

* start work on conditional flamegraph generation

* review time!

* update comments

* Update Cargo.toml

* disable default features for inferno

* reorganize

* missing one trait

* Apply suggestions from code review

* graceful shutdown!

* remove special case handling on ctrlc for cleanup

* rename signal fn to better represent its responsibility

* remove unused global hook for flushing flamegraph

* move tracing logic to the right file

* just copy linkerd's signal handling logic

* update book

* make zebrad app drop on shutdown normally

* Update zebrad/src/components/tokio.rs

Co-authored-by: teor <teor@riseup.net>

* Update zebrad/src/application.rs

Co-authored-by: teor <teor@riseup.net>

* Apply suggestions from code review

Co-authored-by: teor <teor@riseup.net>

* cleanup a little

* ooh yea there's an API for that

* setup env-filter for backup subscriber

* document env filter

* document return codes

* forgot to save

* Update book/src/applications/zebrad.md

Co-authored-by: teor <teor@riseup.net>

Co-authored-by: teor <teor@riseup.net>
This commit is contained in:
Jane Lusby 2020-08-05 16:35:56 -07:00 committed by GitHub
parent ded273413a
commit 867dd0b475
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 438 additions and 103 deletions

196
Cargo.lock generated
View File

@ -36,7 +36,7 @@ dependencies = [
"ident_case",
"proc-macro2 1.0.19",
"quote 1.0.7",
"syn 1.0.35",
"syn 1.0.37",
"synstructure",
]
@ -55,6 +55,15 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
[[package]]
name = "ahash"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217"
dependencies = [
"const-random",
]
[[package]]
name = "aho-corasick"
version = "0.7.13"
@ -94,6 +103,15 @@ version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
[[package]]
name = "arrayvec"
version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9"
dependencies = [
"nodrop",
]
[[package]]
name = "arrayvec"
version = "0.5.1"
@ -209,7 +227,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a"
dependencies = [
"arrayref",
"arrayvec",
"arrayvec 0.5.1",
"constant_time_eq",
]
@ -220,7 +238,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab9e07352b829279624ceb7c64adb4f585dacdb81d35cafae81139ccd617cf44"
dependencies = [
"arrayref",
"arrayvec",
"arrayvec 0.5.1",
"constant_time_eq",
]
@ -275,6 +293,12 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
[[package]]
name = "bytemuck"
version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db7a1029718df60331e557c9e83a55523c955e5dd2a7bfeffad6bbd50b538ae9"
[[package]]
name = "byteorder"
version = "1.3.4"
@ -387,6 +411,26 @@ dependencies = [
"tracing-error",
]
[[package]]
name = "const-random"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f1af9ac737b2dd2d577701e59fd09ba34822f6f2ebdb30a7647405d9e55e16a"
dependencies = [
"const-random-macro",
"proc-macro-hack",
]
[[package]]
name = "const-random-macro"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25e4c606eb459dd29f7c57b2e0879f2b6f14ee130918c2b78ccb58a9624e6c7a"
dependencies = [
"getrandom",
"proc-macro-hack",
]
[[package]]
name = "constant_time_eq"
version = "0.1.5"
@ -487,7 +531,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39858aa5bac06462d4dd4b9164848eb81ffc4aa5c479746393598fd193afa227"
dependencies = [
"quote 1.0.7",
"syn 1.0.35",
"syn 1.0.37",
]
[[package]]
@ -525,7 +569,7 @@ dependencies = [
"proc-macro2 1.0.19",
"quote 1.0.7",
"strsim 0.9.3",
"syn 1.0.35",
"syn 1.0.37",
]
[[package]]
@ -536,7 +580,7 @@ checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72"
dependencies = [
"darling_core",
"quote 1.0.7",
"syn 1.0.35",
"syn 1.0.37",
]
[[package]]
@ -585,7 +629,7 @@ checksum = "adc2ab4d5a16117f9029e9a6b5e4e79f4c67f6519bc134210d4d4a04ba31f41b"
dependencies = [
"proc-macro2 1.0.19",
"quote 1.0.7",
"syn 1.0.35",
"syn 1.0.37",
]
[[package]]
@ -610,9 +654,9 @@ dependencies = [
[[package]]
name = "ed25519-zebra"
version = "2.1.0"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a045d3ca7d15222d578515dc6b54fea6c3591763b8fe2f67a45bbd56d5f1989b"
checksum = "833d5de20d6c876d03b23d13d9caa75fb682d8939d7d418938699a35ee556491"
dependencies = [
"curve25519-dalek",
"hex",
@ -761,7 +805,7 @@ dependencies = [
"proc-macro-hack",
"proc-macro2 1.0.19",
"quote 1.0.7",
"syn 1.0.35",
"syn 1.0.37",
]
[[package]]
@ -870,7 +914,7 @@ checksum = "90454ce4de40b7ca6a8968b5ef367bdab48413962588d0d2b1638d60090c35d7"
dependencies = [
"proc-macro2 1.0.19",
"quote 1.0.7",
"syn 1.0.35",
"syn 1.0.37",
]
[[package]]
@ -1031,6 +1075,22 @@ dependencies = [
"hashbrown",
]
[[package]]
name = "inferno"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4eb1402c92d29c8b44e090b9b0fc25f5714253f959c9f42e378b91cff4d952f"
dependencies = [
"ahash",
"itoa",
"lazy_static",
"log",
"num-format",
"quick-xml",
"rgb",
"str_stack",
]
[[package]]
name = "instant"
version = "0.1.6"
@ -1080,9 +1140,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.73"
version = "0.2.74"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd7d4bd64732af4bf3a67f367c27df8520ad7e230c5817b8ff485864d80242b9"
checksum = "a2f02823cf78b754822df5f7f268fb59822e7296276d3e069d8e8cb26a14bd10"
[[package]]
name = "linked-hash-map"
@ -1335,6 +1395,22 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "nodrop"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
[[package]]
name = "num-format"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465"
dependencies = [
"arrayvec 0.4.12",
"itoa",
]
[[package]]
name = "num-integer"
version = "0.1.43"
@ -1388,7 +1464,7 @@ version = "1.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34d38aeaffc032ec69faa476b3caaca8d4dd7f3f798137ff30359e5c7869ceb6"
dependencies = [
"arrayvec",
"arrayvec 0.5.1",
"bitvec",
"byte-slice-cast",
"serde",
@ -1461,7 +1537,7 @@ checksum = "2c0e815c3ee9a031fdf5af21c10aa17c573c9c6a566328d99e3936c34e36461f"
dependencies = [
"proc-macro2 1.0.19",
"quote 1.0.7",
"syn 1.0.35",
"syn 1.0.37",
]
[[package]]
@ -1495,35 +1571,33 @@ dependencies = [
[[package]]
name = "proc-macro-error"
version = "1.0.3"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc175e9777c3116627248584e8f8b3e2987405cabe1c0adf7d1dd28f09dc7880"
checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
dependencies = [
"proc-macro-error-attr",
"proc-macro2 1.0.19",
"quote 1.0.7",
"syn 1.0.35",
"syn 1.0.37",
"version_check",
]
[[package]]
name = "proc-macro-error-attr"
version = "1.0.3"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3cc9795ca17eb581285ec44936da7fc2335a3f34f2ddd13118b6f4d515435c50"
checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
dependencies = [
"proc-macro2 1.0.19",
"quote 1.0.7",
"syn 1.0.35",
"syn-mid",
"version_check",
]
[[package]]
name = "proc-macro-hack"
version = "0.5.16"
version = "0.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4"
checksum = "99c605b9a0adc77b7211c6b1f722dcb613d68d66859a44f3d485a6da332b0598"
[[package]]
name = "proc-macro-nested"
@ -1598,6 +1672,15 @@ version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
[[package]]
name = "quick-xml"
version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3cc440ee4802a86e357165021e3e255a9143724da31db1e2ea540214c96a0f82"
dependencies = [
"memchr",
]
[[package]]
name = "quote"
version = "0.6.13"
@ -1787,6 +1870,15 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "rgb"
version = "0.8.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90ef54b45ae131327a88597e2463fee4098ad6c88ba7b6af4b3987db8aad4098"
dependencies = [
"bytemuck",
]
[[package]]
name = "ripemd160"
version = "0.8.0"
@ -1918,7 +2010,7 @@ checksum = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e"
dependencies = [
"proc-macro2 1.0.19",
"quote 1.0.7",
"syn 1.0.35",
"syn 1.0.37",
]
[[package]]
@ -2054,7 +2146,7 @@ checksum = "5254766110c377a921c002ca0775d4e384ba69af951fc4329d9dd77af2c25763"
dependencies = [
"proc-macro2 1.0.19",
"quote 1.0.7",
"syn 1.0.35",
"syn 1.0.37",
]
[[package]]
@ -2063,6 +2155,12 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "str_stack"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb"
[[package]]
name = "strsim"
version = "0.8.0"
@ -2096,7 +2194,7 @@ dependencies = [
"proc-macro-error",
"proc-macro2 1.0.19",
"quote 1.0.7",
"syn 1.0.35",
"syn 1.0.37",
]
[[package]]
@ -2118,26 +2216,15 @@ dependencies = [
[[package]]
name = "syn"
version = "1.0.35"
version = "1.0.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb7f4c519df8c117855e19dd8cc851e89eb746fe7a73f0157e0d95fdec5369b0"
checksum = "239f255b9e3429350f188c27b807fc9920a15eb9145230ff1a7d054c08fec319"
dependencies = [
"proc-macro2 1.0.19",
"quote 1.0.7",
"unicode-xid 0.2.1",
]
[[package]]
name = "syn-mid"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a"
dependencies = [
"proc-macro2 1.0.19",
"quote 1.0.7",
"syn 1.0.35",
]
[[package]]
name = "synstructure"
version = "0.12.4"
@ -2146,7 +2233,7 @@ checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701"
dependencies = [
"proc-macro2 1.0.19",
"quote 1.0.7",
"syn 1.0.35",
"syn 1.0.37",
"unicode-xid 0.2.1",
]
@ -2209,7 +2296,7 @@ checksum = "bd80fc12f73063ac132ac92aceea36734f04a1d93c1240c6944e23a3b8841793"
dependencies = [
"proc-macro2 1.0.19",
"quote 1.0.7",
"syn 1.0.35",
"syn 1.0.37",
]
[[package]]
@ -2264,7 +2351,7 @@ checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389"
dependencies = [
"proc-macro2 1.0.19",
"quote 1.0.7",
"syn 1.0.35",
"syn 1.0.37",
]
[[package]]
@ -2327,7 +2414,7 @@ name = "tower-batch"
version = "0.1.0"
dependencies = [
"color-eyre",
"ed25519-zebra 2.1.0",
"ed25519-zebra 2.1.1",
"futures",
"futures-core",
"pin-project",
@ -2486,7 +2573,7 @@ checksum = "f0693bf8d6f2bf22c690fc61a9d21ac69efdbb894a17ed596b9af0f01e64b84b"
dependencies = [
"proc-macro2 1.0.19",
"quote 1.0.7",
"syn 1.0.35",
"syn 1.0.37",
]
[[package]]
@ -2508,6 +2595,17 @@ dependencies = [
"tracing-subscriber",
]
[[package]]
name = "tracing-flame"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd520fe41c667b437952383f3a1ec14f1fa45d653f719a77eedd6e6a02d8fa54"
dependencies = [
"lazy_static",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "tracing-futures"
version = "0.2.4"
@ -2573,9 +2671,9 @@ checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33"
[[package]]
name = "uint"
version = "0.8.3"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "173cd16430c206dc1a430af8a89a0e9c076cf15cb42b4aedb10e8cc8fee73681"
checksum = "429ffcad8c8c15f874578c7337d156a3727eb4a1c2374c0ae937ad9a9b748c80"
dependencies = [
"byteorder",
"crunchy",
@ -2890,6 +2988,7 @@ dependencies = [
"futures",
"gumdrop",
"hyper",
"inferno",
"metrics",
"metrics-runtime",
"once_cell",
@ -2901,6 +3000,7 @@ dependencies = [
"tower",
"tracing",
"tracing-error",
"tracing-flame",
"tracing-futures",
"tracing-log",
"tracing-subscriber",
@ -2928,6 +3028,6 @@ checksum = "de251eec69fc7c1bc3923403d18ececb929380e016afe103da75f396704f8ca2"
dependencies = [
"proc-macro2 1.0.19",
"quote 1.0.7",
"syn 1.0.35",
"syn 1.0.37",
"synstructure",
]

View File

@ -1,3 +1,8 @@
# zebrad
## Return Codes
- 0 => Application exited successfully
- 1 => Application exited unsuccessfully
- 2 => Application crashed
- zebrad may also return platform-dependent codes

View File

@ -166,15 +166,16 @@ impl TestChild {
#[spandoc::spandoc]
pub fn wait_with_output(self) -> Result<TestOutput> {
let cmd = format!("{:?}", self);
/// SPANDOC: waiting for command to exit
let output = self.child.wait_with_output().with_section({
let cmd = cmd.clone();
let cmd = self.cmd.clone();
|| cmd.header("Command:")
})?;
Ok(TestOutput { output, cmd })
Ok(TestOutput {
output,
cmd: self.cmd,
})
}
}

View File

@ -21,7 +21,7 @@ rand = "0.7"
hyper = "0.13.7"
futures = "0.3"
tokio = { version = "0.2.22", features = ["time", "rt-threaded", "stream", "macros", "tracing"] }
tokio = { version = "0.2.22", features = ["time", "rt-threaded", "stream", "macros", "tracing", "signal"] }
tower = "0.3"
color-eyre = "0.5"
@ -35,6 +35,8 @@ tracing-error = "0.1.2"
metrics-runtime = "0.13"
metrics = "0.12"
dirs = "3.0.1"
tracing-flame = "0.1.0"
inferno = { version = "0.10.0", default-features = false }
[dev-dependencies]
abscissa_core = { version = "0.5", features = ["testing"] }

View File

@ -1,14 +1,16 @@
//! Zebrad Abscissa Application
use crate::{commands::ZebradCmd, config::ZebradConfig};
use crate::{commands::ZebradCmd, components::tracing::FlameGrapher, config::ZebradConfig};
use abscissa_core::{
application::{self, AppCell},
config,
config::Configurable,
terminal::component::Terminal,
trace::Tracing,
Application, Component, EntryPoint, FrameworkError, StandardPaths,
Application, Component, EntryPoint, FrameworkError, Shutdown, StandardPaths,
};
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
use application::fatal_error;
use std::{fmt, process};
/// Application state
pub static APPLICATION: AppCell<ZebradApp> = AppCell::new();
@ -33,11 +35,14 @@ pub fn app_config() -> config::Reader<ZebradApp> {
}
/// Zebrad Application
#[derive(Debug)]
pub struct ZebradApp {
/// Application configuration.
config: Option<ZebradConfig>,
/// drop handle for tracing-flame layer to ensure it flushes its buffer when
/// the application exits
flame_guard: Option<FlameGrapher>,
/// Application state.
state: application::State<Self>,
}
@ -50,11 +55,21 @@ impl Default for ZebradApp {
fn default() -> Self {
Self {
config: None,
flame_guard: None,
state: application::State::default(),
}
}
}
impl fmt::Debug for ZebradApp {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ZebraApp")
.field("config", &self.config)
.field("state", &self.state)
.finish()
}
}
impl Application for ZebradApp {
/// Entrypoint command for this application.
type Cmd = EntryPoint<ZebradCmd>;
@ -95,7 +110,7 @@ impl Application for ZebradApp {
let tracing = self.tracing_component();
Ok(vec![Box::new(terminal), Box::new(tracing)])
} else {
init_tracing_backup();
crate::components::tracing::init_backup(&self.config().tracing);
Ok(vec![Box::new(terminal)])
}
}
@ -121,6 +136,32 @@ impl Application for ZebradApp {
self.state.components.register(components)
}
/// Load this application's configuration and initialize its components.
fn init(&mut self, command: &Self::Cmd) -> Result<(), FrameworkError> {
// Load configuration
let config = command
.config_path()
.map(|path| self.load_config(&path))
.transpose()?
.unwrap_or_default();
let config = command.process_config(config)?;
self.config = Some(config);
// Create and register components with the application.
// We do this first to calculate a proper dependency ordering before
// application configuration is processed
self.register_components(command)?;
let config = self.config.take().unwrap();
// Fire callback regardless of whether any config was loaded to
// in order to signal state in the application lifecycle
self.after_config(config, command)?;
Ok(())
}
/// Post-configuration lifecycle callback.
///
/// Called regardless of whether config is loaded to indicate this is the
@ -175,23 +216,29 @@ impl Application for ZebradApp {
Ok(())
}
fn shutdown(&mut self, shutdown: Shutdown) -> ! {
if let Err(e) = self.state().components.shutdown(self, shutdown) {
fatal_error(self, &e)
}
// Swap out a fake app so we can trigger the destructor on the original
let _ = std::mem::take(self);
match shutdown {
Shutdown::Graceful => process::exit(0),
Shutdown::Forced => process::exit(1),
Shutdown::Crash => process::exit(2),
}
}
}
impl ZebradApp {
fn tracing_component(&self) -> Tracing {
// Construct a tracing subscriber with the supplied filter and enable reloading.
let builder = tracing_subscriber::FmtSubscriber::builder()
// Set the filter to warn initially, then reset it in after_config.
.with_env_filter("warn")
.with_filter_reloading();
let filter_handle = builder.reload_handle();
builder
.finish()
.with(tracing_error::ErrorLayer::default())
.init();
filter_handle.into()
fn tracing_component(&mut self) -> Tracing {
let config = &self.config().tracing;
let (component, guard) = crate::components::tracing::init(config);
self.flame_guard = guard;
component
}
/// Returns true if command is a server command.
@ -206,9 +253,3 @@ impl ZebradApp {
}
}
}
fn init_tracing_backup() {
tracing_subscriber::Registry::default()
.with(tracing_error::ErrorLayer::default())
.init();
}

View File

@ -13,6 +13,7 @@ use tower::{buffer::Buffer, Service, ServiceExt};
use zebra_network::{AddressBook, BoxedStdError, Request, Response};
use crate::components::tokio::RuntimeRun;
use crate::prelude::*;
use color_eyre::eyre::{eyre, Report};
@ -121,9 +122,7 @@ impl Runnable for SeedCmd {
.take();
rt.expect("runtime should not already be taken")
.block_on(self.seed())
// Surface any error that occurred executing the future.
.unwrap();
.run(self.seed());
}
}

View File

@ -19,6 +19,7 @@
//! * This task runs in the background and continuously queries the network for
//! new blocks to be verified and added to the local state
use crate::components::tokio::RuntimeRun;
use crate::config::ZebradConfig;
use crate::{components::tokio::TokioComponent, prelude::*};
@ -72,17 +73,8 @@ impl Runnable for StartCmd {
.rt
.take();
let result = rt
.expect("runtime should not already be taken")
.block_on(self.start());
match result {
Ok(()) => {}
Err(e) => {
eprintln!("Error: {:?}", e);
std::process::exit(1);
}
}
rt.expect("runtime should not already be taken")
.run(self.start());
}
}

View File

@ -1,7 +1,9 @@
//! A component owning the Tokio runtime.
use abscissa_core::{Component, FrameworkError};
use crate::prelude::*;
use abscissa_core::{Application, Component, FrameworkError, Shutdown};
use color_eyre::Report;
use std::future::Future;
use tokio::runtime::Runtime;
/// An Abscissa component which owns a Tokio runtime.
@ -23,3 +25,84 @@ impl TokioComponent {
})
}
}
/// Zebrad's graceful shutdown function, blocks until one of the supported
/// shutdown signals is received.
async fn shutdown() {
imp::shutdown().await;
}
/// Extension trait to centralize entry point for runnable subcommands that
/// depend on tokio
pub(crate) trait RuntimeRun {
fn run(&mut self, fut: impl Future<Output = Result<(), Report>>);
}
impl RuntimeRun for Runtime {
fn run(&mut self, fut: impl Future<Output = Result<(), Report>>) {
let result = self.block_on(async move {
tokio::select! {
result = fut => result,
_ = shutdown() => Ok(()),
}
});
match result {
Ok(()) => {}
Err(e) => {
eprintln!("Error: {:?}", e);
app_writer().shutdown(Shutdown::Forced);
}
}
}
}
#[cfg(unix)]
mod imp {
use tokio::signal::unix::{signal, SignalKind};
use tracing::info;
pub(super) async fn shutdown() {
tokio::select! {
// SIGINT - Terminal interrupt signal. Typically generated by shells in response to Ctrl-C.
() = sig(SignalKind::interrupt(), "SIGINT") => {}
// SIGTERM - Standard shutdown signal used by process launchers.
() = sig(SignalKind::terminate(), "SIGTERM") => {}
};
}
async fn sig(kind: SignalKind, name: &'static str) {
// Create a Future that completes the first
// time the process receives 'sig'.
signal(kind)
.expect("Failed to register signal handler")
.recv()
.await;
info!(
// use target to remove 'imp' from output
target: "zebrad::signal",
"received {}, starting shutdown",
name,
);
}
}
#[cfg(not(unix))]
mod imp {
use tracing::info;
pub(super) async fn shutdown() {
// Wait for Ctrl-C in Windows terminals.
// (Zebra doesn't support NT Service control messages. Use a service wrapper for long-running instances.)
tokio::signal::ctrl_c()
.await
.expect("listening for ctrl-c signal should never fail");
info!(
// use target to remove 'imp' from output
target: "zebrad::signal",
"received Ctrl-C, starting shutdown",
);
}
}

View File

@ -1,11 +1,17 @@
//! An HTTP endpoint for dynamically setting tracing filters.
use crate::{components::tokio::TokioComponent, config::TracingSection, prelude::*};
use abscissa_core::{Component, FrameworkError};
use abscissa_core::{trace::Tracing, Component, FrameworkError};
use color_eyre::eyre::Report;
use hyper::service::{make_service_fn, service_fn};
use hyper::{Body, Request, Response, Server};
use std::{
fs::File,
io::{BufReader, BufWriter},
path::PathBuf,
sync::Arc,
};
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
/// Abscissa component which runs a tracing filter endpoint.
#[derive(Debug, Component)]
@ -125,3 +131,75 @@ To set the filter, POST the new filter string to /filter:
};
Ok(rsp)
}
#[derive(Clone)]
pub(crate) struct FlameGrapher {
guard: Arc<tracing_flame::FlushGuard<BufWriter<File>>>,
path: PathBuf,
}
impl FlameGrapher {
fn make_flamegraph(&self) -> Result<(), Report> {
self.guard.flush()?;
let out_path = self.path.with_extension("svg");
let inf = File::open(&self.path)?;
let reader = BufReader::new(inf);
let out = File::create(out_path)?;
let writer = BufWriter::new(out);
let mut opts = inferno::flamegraph::Options::default();
info!("writing flamegraph to disk...");
inferno::flamegraph::from_reader(&mut opts, reader, writer)?;
Ok(())
}
}
impl Drop for FlameGrapher {
fn drop(&mut self) {
match self.make_flamegraph() {
Ok(()) => {}
Err(report) => {
warn!(
"Error while constructing flamegraph during shutdown: {:?}",
report
);
}
}
}
}
pub(crate) fn init(config: &TracingSection) -> (Tracing, Option<FlameGrapher>) {
// Construct a tracing subscriber with the supplied filter and enable reloading.
let builder = tracing_subscriber::FmtSubscriber::builder()
.with_env_filter(config.env_filter())
.with_filter_reloading();
let filter_handle = builder.reload_handle();
let subscriber = builder.finish().with(tracing_error::ErrorLayer::default());
let guard = if let Some(flamegraph_path) = config.flamegraph.as_deref() {
let flamegraph_path = flamegraph_path.with_extension("folded");
let (flame_layer, guard) = tracing_flame::FlameLayer::with_file(&flamegraph_path).unwrap();
let flame_layer = flame_layer
.with_empty_samples(false)
.with_threads_collapsed(true);
subscriber.with(flame_layer).init();
Some(FlameGrapher {
guard: Arc::new(guard),
path: flamegraph_path,
})
} else {
subscriber.init();
None
};
(filter_handle.into(), guard)
}
pub(crate) fn init_backup(config: &TracingSection) {
tracing_subscriber::Registry::default()
.with(config.env_filter())
.with(tracing_error::ErrorLayer::default())
.init();
}

View File

@ -4,10 +4,11 @@
//! application's configuration file and/or command-line options
//! for specifying it.
use std::net::SocketAddr;
use std::{net::SocketAddr, path::PathBuf};
use serde::{Deserialize, Serialize};
use tracing_subscriber::EnvFilter;
use zebra_network::Config as NetworkSection;
use zebra_state::Config as StateSection;
@ -59,12 +60,28 @@ pub struct TracingSection {
/// The endpoint address used for tracing.
pub endpoint_addr: SocketAddr,
}
impl Default for TracingSection {
fn default() -> Self {
Self::populated()
}
/// The path to write a flamegraph of tracing spans too.
///
/// This path is not used verbatim when writing out the flamegraph. This is
/// because the flamegraph is written out as two parts. First the flamegraph
/// is constantly persisted to the disk in a "folded" representation that
/// records collapsed stack traces of the tracing spans that are active.
/// Then, when the application is finished running the destructor will flush
/// the flamegraph output to the folded file and then read that file and
/// generate the final flamegraph from it as an SVG.
///
/// The need to create two files means that we will slightly manipulate the
/// path given to us to create the two representations.
///
/// # Example
///
/// Given `flamegraph = "flamegraph"` we will generate a `flamegraph.svg`
/// and a `flamegraph.folded` file in the current directory.
///
/// If you provide a path with an extension the extension will be ignored and
/// replaced with `.folded` and `.svg` for the respective files.
pub flamegraph: Option<PathBuf>,
}
impl TracingSection {
@ -72,8 +89,25 @@ impl TracingSection {
Self {
filter: Some("info".to_owned()),
endpoint_addr: "0.0.0.0:3000".parse().unwrap(),
flamegraph: None,
}
}
/// Constructs an EnvFilter for use in our tracing subscriber.
///
/// The env filter controls filtering of spans and events, but not how
/// they're emitted. Creating an env filter alone doesn't enable logging, it
/// needs to be used in conjunction with other layers like a fmt subscriber,
/// for logs, or an error layer, for SpanTraces.
pub fn env_filter(&self) -> EnvFilter {
self.filter.as_deref().unwrap_or("info").into()
}
}
impl Default for TracingSection {
fn default() -> Self {
Self::populated()
}
}
/// Metrics configuration section.