Setup tracing-flame for use profiling zebrad (#436)

* Setup tracing-flame for use profiling zebrad

* start work on conditional flamegraph generation

* review time!

* update comments

* Update Cargo.toml

* disable default features for inferno

* reorganize

* missing one trait

* Apply suggestions from code review

* graceful shutdown!

* remove special case handling on ctrlc for cleanup

* rename signal fn to better represent its responsibility

* remove unused global hook for flushing flamegraph

* move tracing logic to the right file

* just copy linkerd's signal handling logic

* update book

* make zebrad app drop on shutdown normally

* Update zebrad/src/components/tokio.rs

Co-authored-by: teor <teor@riseup.net>

* Update zebrad/src/application.rs

Co-authored-by: teor <teor@riseup.net>

* Apply suggestions from code review

Co-authored-by: teor <teor@riseup.net>

* cleanup a little

* ooh yea there's an API for that

* setup env-filter for backup subscriber

* document env filter

* document return codes

* forgot to save

* Update book/src/applications/zebrad.md

Co-authored-by: teor <teor@riseup.net>

Co-authored-by: teor <teor@riseup.net>

Cargo.lock

@@ -36,7 +36,7 @@ dependencies = [
  "ident_case",
  "proc-macro2 1.0.19",
  "quote 1.0.7",
- "syn 1.0.35",
+ "syn 1.0.37",
  "synstructure",
 ]
 
@@ -55,6 +55,15 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
 
+[[package]]
+name = "ahash"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217"
+dependencies = [
+ "const-random",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "0.7.13"
@@ -94,6 +103,15 @@ version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
 
+[[package]]
+name = "arrayvec"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9"
+dependencies = [
+ "nodrop",
+]
+
 [[package]]
 name = "arrayvec"
 version = "0.5.1"
@@ -209,7 +227,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a"
 dependencies = [
  "arrayref",
- "arrayvec",
+ "arrayvec 0.5.1",
  "constant_time_eq",
 ]
 
@@ -220,7 +238,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab9e07352b829279624ceb7c64adb4f585dacdb81d35cafae81139ccd617cf44"
 dependencies = [
  "arrayref",
- "arrayvec",
+ "arrayvec 0.5.1",
  "constant_time_eq",
 ]
 
@@ -275,6 +293,12 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
 
+[[package]]
+name = "bytemuck"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db7a1029718df60331e557c9e83a55523c955e5dd2a7bfeffad6bbd50b538ae9"
+
 [[package]]
 name = "byteorder"
 version = "1.3.4"
@@ -387,6 +411,26 @@ dependencies = [
  "tracing-error",
 ]
 
+[[package]]
+name = "const-random"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f1af9ac737b2dd2d577701e59fd09ba34822f6f2ebdb30a7647405d9e55e16a"
+dependencies = [
+ "const-random-macro",
+ "proc-macro-hack",
+]
+
+[[package]]
+name = "const-random-macro"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25e4c606eb459dd29f7c57b2e0879f2b6f14ee130918c2b78ccb58a9624e6c7a"
+dependencies = [
+ "getrandom",
+ "proc-macro-hack",
+]
+
 [[package]]
 name = "constant_time_eq"
 version = "0.1.5"
@@ -487,7 +531,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39858aa5bac06462d4dd4b9164848eb81ffc4aa5c479746393598fd193afa227"
 dependencies = [
  "quote 1.0.7",
- "syn 1.0.35",
+ "syn 1.0.37",
 ]
 
 [[package]]
@@ -525,7 +569,7 @@ dependencies = [
  "proc-macro2 1.0.19",
  "quote 1.0.7",
  "strsim 0.9.3",
- "syn 1.0.35",
+ "syn 1.0.37",
 ]
 
 [[package]]
@@ -536,7 +580,7 @@ checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72"
 dependencies = [
  "darling_core",
  "quote 1.0.7",
- "syn 1.0.35",
+ "syn 1.0.37",
 ]
 
 [[package]]
@@ -585,7 +629,7 @@ checksum = "adc2ab4d5a16117f9029e9a6b5e4e79f4c67f6519bc134210d4d4a04ba31f41b"
 dependencies = [
  "proc-macro2 1.0.19",
  "quote 1.0.7",
- "syn 1.0.35",
+ "syn 1.0.37",
 ]
 
 [[package]]
@@ -610,9 +654,9 @@ dependencies = [
 
 [[package]]
 name = "ed25519-zebra"
-version = "2.1.0"
+version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a045d3ca7d15222d578515dc6b54fea6c3591763b8fe2f67a45bbd56d5f1989b"
+checksum = "833d5de20d6c876d03b23d13d9caa75fb682d8939d7d418938699a35ee556491"
 dependencies = [
  "curve25519-dalek",
  "hex",
@@ -761,7 +805,7 @@ dependencies = [
  "proc-macro-hack",
  "proc-macro2 1.0.19",
  "quote 1.0.7",
- "syn 1.0.35",
+ "syn 1.0.37",
 ]
 
 [[package]]
@@ -870,7 +914,7 @@ checksum = "90454ce4de40b7ca6a8968b5ef367bdab48413962588d0d2b1638d60090c35d7"
 dependencies = [
  "proc-macro2 1.0.19",
  "quote 1.0.7",
- "syn 1.0.35",
+ "syn 1.0.37",
 ]
 
 [[package]]
@@ -1031,6 +1075,22 @@ dependencies = [
  "hashbrown",
 ]
 
+[[package]]
+name = "inferno"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4eb1402c92d29c8b44e090b9b0fc25f5714253f959c9f42e378b91cff4d952f"
+dependencies = [
+ "ahash",
+ "itoa",
+ "lazy_static",
+ "log",
+ "num-format",
+ "quick-xml",
+ "rgb",
+ "str_stack",
+]
+
 [[package]]
 name = "instant"
 version = "0.1.6"
@@ -1080,9 +1140,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.73"
+version = "0.2.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd7d4bd64732af4bf3a67f367c27df8520ad7e230c5817b8ff485864d80242b9"
+checksum = "a2f02823cf78b754822df5f7f268fb59822e7296276d3e069d8e8cb26a14bd10"
 
 [[package]]
 name = "linked-hash-map"
@@ -1335,6 +1395,22 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "nodrop"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
+
+[[package]]
+name = "num-format"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465"
+dependencies = [
+ "arrayvec 0.4.12",
+ "itoa",
+]
+
 [[package]]
 name = "num-integer"
 version = "0.1.43"
@@ -1388,7 +1464,7 @@ version = "1.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34d38aeaffc032ec69faa476b3caaca8d4dd7f3f798137ff30359e5c7869ceb6"
 dependencies = [
- "arrayvec",
+ "arrayvec 0.5.1",
  "bitvec",
  "byte-slice-cast",
  "serde",
@@ -1461,7 +1537,7 @@ checksum = "2c0e815c3ee9a031fdf5af21c10aa17c573c9c6a566328d99e3936c34e36461f"
 dependencies = [
  "proc-macro2 1.0.19",
  "quote 1.0.7",
- "syn 1.0.35",
+ "syn 1.0.37",
 ]
 
 [[package]]
@@ -1495,35 +1571,33 @@ dependencies = [
 
 [[package]]
 name = "proc-macro-error"
-version = "1.0.3"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc175e9777c3116627248584e8f8b3e2987405cabe1c0adf7d1dd28f09dc7880"
+checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
 dependencies = [
  "proc-macro-error-attr",
  "proc-macro2 1.0.19",
  "quote 1.0.7",
- "syn 1.0.35",
+ "syn 1.0.37",
  "version_check",
 ]
 
 [[package]]
 name = "proc-macro-error-attr"
-version = "1.0.3"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3cc9795ca17eb581285ec44936da7fc2335a3f34f2ddd13118b6f4d515435c50"
+checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
 dependencies = [
  "proc-macro2 1.0.19",
  "quote 1.0.7",
- "syn 1.0.35",
- "syn-mid",
  "version_check",
 ]
 
 [[package]]
 name = "proc-macro-hack"
-version = "0.5.16"
+version = "0.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4"
+checksum = "99c605b9a0adc77b7211c6b1f722dcb613d68d66859a44f3d485a6da332b0598"
 
 [[package]]
 name = "proc-macro-nested"
@@ -1598,6 +1672,15 @@ version = "1.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
 
+[[package]]
+name = "quick-xml"
+version = "0.18.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3cc440ee4802a86e357165021e3e255a9143724da31db1e2ea540214c96a0f82"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "quote"
 version = "0.6.13"
@@ -1787,6 +1870,15 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "rgb"
+version = "0.8.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90ef54b45ae131327a88597e2463fee4098ad6c88ba7b6af4b3987db8aad4098"
+dependencies = [
+ "bytemuck",
+]
+
 [[package]]
 name = "ripemd160"
 version = "0.8.0"
@@ -1918,7 +2010,7 @@ checksum = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e"
 dependencies = [
  "proc-macro2 1.0.19",
  "quote 1.0.7",
- "syn 1.0.35",
+ "syn 1.0.37",
 ]
 
 [[package]]
@@ -2054,7 +2146,7 @@ checksum = "5254766110c377a921c002ca0775d4e384ba69af951fc4329d9dd77af2c25763"
 dependencies = [
  "proc-macro2 1.0.19",
  "quote 1.0.7",
- "syn 1.0.35",
+ "syn 1.0.37",
 ]
 
 [[package]]
@@ -2063,6 +2155,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
 
+[[package]]
+name = "str_stack"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb"
+
 [[package]]
 name = "strsim"
 version = "0.8.0"
@@ -2096,7 +2194,7 @@ dependencies = [
  "proc-macro-error",
  "proc-macro2 1.0.19",
  "quote 1.0.7",
- "syn 1.0.35",
+ "syn 1.0.37",
 ]
 
 [[package]]
@@ -2118,26 +2216,15 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "1.0.35"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb7f4c519df8c117855e19dd8cc851e89eb746fe7a73f0157e0d95fdec5369b0"
+checksum = "239f255b9e3429350f188c27b807fc9920a15eb9145230ff1a7d054c08fec319"
 dependencies = [
  "proc-macro2 1.0.19",
  "quote 1.0.7",
  "unicode-xid 0.2.1",
 ]
 
-[[package]]
-name = "syn-mid"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a"
-dependencies = [
- "proc-macro2 1.0.19",
- "quote 1.0.7",
- "syn 1.0.35",
-]
-
 [[package]]
 name = "synstructure"
 version = "0.12.4"
@@ -2146,7 +2233,7 @@ checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701"
 dependencies = [
  "proc-macro2 1.0.19",
  "quote 1.0.7",
- "syn 1.0.35",
+ "syn 1.0.37",
  "unicode-xid 0.2.1",
 ]
 
@@ -2209,7 +2296,7 @@ checksum = "bd80fc12f73063ac132ac92aceea36734f04a1d93c1240c6944e23a3b8841793"
 dependencies = [
  "proc-macro2 1.0.19",
  "quote 1.0.7",
- "syn 1.0.35",
+ "syn 1.0.37",
 ]
 
 [[package]]
@@ -2264,7 +2351,7 @@ checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389"
 dependencies = [
  "proc-macro2 1.0.19",
  "quote 1.0.7",
- "syn 1.0.35",
+ "syn 1.0.37",
 ]
 
 [[package]]
@@ -2327,7 +2414,7 @@ name = "tower-batch"
 version = "0.1.0"
 dependencies = [
  "color-eyre",
- "ed25519-zebra 2.1.0",
+ "ed25519-zebra 2.1.1",
  "futures",
  "futures-core",
  "pin-project",
@@ -2486,7 +2573,7 @@ checksum = "f0693bf8d6f2bf22c690fc61a9d21ac69efdbb894a17ed596b9af0f01e64b84b"
 dependencies = [
  "proc-macro2 1.0.19",
  "quote 1.0.7",
- "syn 1.0.35",
+ "syn 1.0.37",
 ]
 
 [[package]]
@@ -2508,6 +2595,17 @@ dependencies = [
  "tracing-subscriber",
 ]
 
+[[package]]
+name = "tracing-flame"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd520fe41c667b437952383f3a1ec14f1fa45d653f719a77eedd6e6a02d8fa54"
+dependencies = [
+ "lazy_static",
+ "tracing",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "tracing-futures"
 version = "0.2.4"
@@ -2573,9 +2671,9 @@ checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33"
 
 [[package]]
 name = "uint"
-version = "0.8.3"
+version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "173cd16430c206dc1a430af8a89a0e9c076cf15cb42b4aedb10e8cc8fee73681"
+checksum = "429ffcad8c8c15f874578c7337d156a3727eb4a1c2374c0ae937ad9a9b748c80"
 dependencies = [
  "byteorder",
  "crunchy",
@@ -2890,6 +2988,7 @@ dependencies = [
  "futures",
  "gumdrop",
  "hyper",
+ "inferno",
  "metrics",
  "metrics-runtime",
  "once_cell",
@@ -2901,6 +3000,7 @@ dependencies = [
  "tower",
  "tracing",
  "tracing-error",
+ "tracing-flame",
  "tracing-futures",
  "tracing-log",
  "tracing-subscriber",
@@ -2928,6 +3028,6 @@ checksum = "de251eec69fc7c1bc3923403d18ececb929380e016afe103da75f396704f8ca2"
 dependencies = [
  "proc-macro2 1.0.19",
  "quote 1.0.7",
- "syn 1.0.35",
+ "syn 1.0.37",
  "synstructure",
 ]

book/src/applications/zebrad.md

@@ -1,3 +1,8 @@
 # zebrad
 
+## Return Codes
 
+- 0 => Application exited successfully
+- 1 => Application exited unsuccessfully
+- 2 => Application crashed
+- zebrad may also return platform-dependent codes

zebra-test/src/command.rs

@@ -166,15 +166,16 @@ impl TestChild {
 
     #[spandoc::spandoc]
     pub fn wait_with_output(self) -> Result<TestOutput> {
-        let cmd = format!("{:?}", self);
-
         /// SPANDOC: waiting for command to exit
         let output = self.child.wait_with_output().with_section({
-            let cmd = cmd.clone();
+            let cmd = self.cmd.clone();
             || cmd.header("Command:")
         })?;
 
-        Ok(TestOutput { output, cmd })
+        Ok(TestOutput {
+            output,
+            cmd: self.cmd,
+        })
     }
 }
 

zebrad/Cargo.toml

@@ -21,7 +21,7 @@ rand = "0.7"
 
 hyper = "0.13.7"
 futures = "0.3"
-tokio = { version = "0.2.22", features = ["time", "rt-threaded", "stream", "macros", "tracing"] }
+tokio = { version = "0.2.22", features = ["time", "rt-threaded", "stream", "macros", "tracing", "signal"] }
 tower = "0.3"
 
 color-eyre = "0.5"
@@ -35,6 +35,8 @@ tracing-error = "0.1.2"
 metrics-runtime = "0.13"
 metrics = "0.12"
 dirs = "3.0.1"
+tracing-flame = "0.1.0"
+inferno = { version = "0.10.0", default-features = false }
 
 [dev-dependencies]
 abscissa_core = { version = "0.5", features = ["testing"] }

zebrad/src/application.rs

@@ -1,14 +1,16 @@
 //! Zebrad Abscissa Application
 
-use crate::{commands::ZebradCmd, config::ZebradConfig};
+use crate::{commands::ZebradCmd, components::tracing::FlameGrapher, config::ZebradConfig};
 use abscissa_core::{
     application::{self, AppCell},
     config,
+    config::Configurable,
     terminal::component::Terminal,
     trace::Tracing,
-    Application, Component, EntryPoint, FrameworkError, StandardPaths,
+    Application, Component, EntryPoint, FrameworkError, Shutdown, StandardPaths,
 };
-use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
+use application::fatal_error;
+use std::{fmt, process};
 
 /// Application state
 pub static APPLICATION: AppCell<ZebradApp> = AppCell::new();
@@ -33,11 +35,14 @@ pub fn app_config() -> config::Reader<ZebradApp> {
 }
 
 /// Zebrad Application
-#[derive(Debug)]
 pub struct ZebradApp {
     /// Application configuration.
     config: Option<ZebradConfig>,
 
+    /// drop handle for tracing-flame layer to ensure it flushes its buffer when
+    /// the application exits
+    flame_guard: Option<FlameGrapher>,
+
     /// Application state.
     state: application::State<Self>,
 }
@@ -50,11 +55,21 @@ impl Default for ZebradApp {
     fn default() -> Self {
         Self {
             config: None,
+            flame_guard: None,
             state: application::State::default(),
         }
     }
 }
 
+impl fmt::Debug for ZebradApp {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("ZebraApp")
+            .field("config", &self.config)
+            .field("state", &self.state)
+            .finish()
+    }
+}
+
 impl Application for ZebradApp {
     /// Entrypoint command for this application.
     type Cmd = EntryPoint<ZebradCmd>;
@@ -95,7 +110,7 @@ impl Application for ZebradApp {
             let tracing = self.tracing_component();
             Ok(vec![Box::new(terminal), Box::new(tracing)])
         } else {
-            init_tracing_backup();
+            crate::components::tracing::init_backup(&self.config().tracing);
             Ok(vec![Box::new(terminal)])
         }
     }
@@ -121,6 +136,32 @@ impl Application for ZebradApp {
         self.state.components.register(components)
     }
 
+    /// Load this application's configuration and initialize its components.
+    fn init(&mut self, command: &Self::Cmd) -> Result<(), FrameworkError> {
+        // Load configuration
+        let config = command
+            .config_path()
+            .map(|path| self.load_config(&path))
+            .transpose()?
+            .unwrap_or_default();
+
+        let config = command.process_config(config)?;
+        self.config = Some(config);
+
+        // Create and register components with the application.
+        // We do this first to calculate a proper dependency ordering before
+        // application configuration is processed
+        self.register_components(command)?;
+
+        let config = self.config.take().unwrap();
+
+        // Fire callback regardless of whether any config was loaded to
+        // in order to signal state in the application lifecycle
+        self.after_config(config, command)?;
+
+        Ok(())
+    }
+
     /// Post-configuration lifecycle callback.
     ///
     /// Called regardless of whether config is loaded to indicate this is the
@@ -175,23 +216,29 @@ impl Application for ZebradApp {
 
         Ok(())
     }
+
+    fn shutdown(&mut self, shutdown: Shutdown) -> ! {
+        if let Err(e) = self.state().components.shutdown(self, shutdown) {
+            fatal_error(self, &e)
+        }
+
+        // Swap out a fake app so we can trigger the destructor on the original
+        let _ = std::mem::take(self);
+
+        match shutdown {
+            Shutdown::Graceful => process::exit(0),
+            Shutdown::Forced => process::exit(1),
+            Shutdown::Crash => process::exit(2),
+        }
+    }
 }
 
 impl ZebradApp {
-    fn tracing_component(&self) -> Tracing {
+    fn tracing_component(&mut self) -> Tracing {
-        // Construct a tracing subscriber with the supplied filter and enable reloading.
+        let config = &self.config().tracing;
-        let builder = tracing_subscriber::FmtSubscriber::builder()
+        let (component, guard) = crate::components::tracing::init(config);
-            // Set the filter to warn initially, then reset it in after_config.
+        self.flame_guard = guard;
-            .with_env_filter("warn")
+        component
-            .with_filter_reloading();
-        let filter_handle = builder.reload_handle();
-
-        builder
-            .finish()
-            .with(tracing_error::ErrorLayer::default())
-            .init();
-
-        filter_handle.into()
     }
 
     /// Returns true if command is a server command.
@@ -206,9 +253,3 @@ impl ZebradApp {
         }
     }
 }
-
-fn init_tracing_backup() {
-    tracing_subscriber::Registry::default()
-        .with(tracing_error::ErrorLayer::default())
-        .init();
-}

zebrad/src/commands/seed.rs

@@ -13,6 +13,7 @@ use tower::{buffer::Buffer, Service, ServiceExt};
 
 use zebra_network::{AddressBook, BoxedStdError, Request, Response};
 
+use crate::components::tokio::RuntimeRun;
 use crate::prelude::*;
 use color_eyre::eyre::{eyre, Report};
 
@@ -121,9 +122,7 @@ impl Runnable for SeedCmd {
             .take();
 
         rt.expect("runtime should not already be taken")
-            .block_on(self.seed())
+            .run(self.seed());
-            // Surface any error that occurred executing the future.
-            .unwrap();
     }
 }
 

zebrad/src/commands/start.rs

@@ -19,6 +19,7 @@
 //!    * This task runs in the background and continuously queries the network for
 //!    new blocks to be verified and added to the local state
 
+use crate::components::tokio::RuntimeRun;
 use crate::config::ZebradConfig;
 use crate::{components::tokio::TokioComponent, prelude::*};
 
@@ -72,17 +73,8 @@ impl Runnable for StartCmd {
             .rt
             .take();
 
-        let result = rt
+        rt.expect("runtime should not already be taken")
-            .expect("runtime should not already be taken")
+            .run(self.start());
-            .block_on(self.start());
-
-        match result {
-            Ok(()) => {}
-            Err(e) => {
-                eprintln!("Error: {:?}", e);
-                std::process::exit(1);
-            }
-        }
     }
 }
 

zebrad/src/components/tokio.rs

@@ -1,7 +1,9 @@
 //! A component owning the Tokio runtime.
 
-use abscissa_core::{Component, FrameworkError};
+use crate::prelude::*;
-
+use abscissa_core::{Application, Component, FrameworkError, Shutdown};
+use color_eyre::Report;
+use std::future::Future;
 use tokio::runtime::Runtime;
 
 /// An Abscissa component which owns a Tokio runtime.
@@ -23,3 +25,84 @@ impl TokioComponent {
         })
     }
 }
+
+/// Zebrad's graceful shutdown function, blocks until one of the supported
+/// shutdown signals is received.
+async fn shutdown() {
+    imp::shutdown().await;
+}
+
+/// Extension trait to centralize entry point for runnable subcommands that
+/// depend on tokio
+pub(crate) trait RuntimeRun {
+    fn run(&mut self, fut: impl Future<Output = Result<(), Report>>);
+}
+
+impl RuntimeRun for Runtime {
+    fn run(&mut self, fut: impl Future<Output = Result<(), Report>>) {
+        let result = self.block_on(async move {
+            tokio::select! {
+                result = fut => result,
+                _ = shutdown() => Ok(()),
+            }
+        });
+
+        match result {
+            Ok(()) => {}
+            Err(e) => {
+                eprintln!("Error: {:?}", e);
+                app_writer().shutdown(Shutdown::Forced);
+            }
+        }
+    }
+}
+
+#[cfg(unix)]
+mod imp {
+    use tokio::signal::unix::{signal, SignalKind};
+    use tracing::info;
+
+    pub(super) async fn shutdown() {
+        tokio::select! {
+            // SIGINT  - Terminal interrupt signal. Typically generated by shells in response to Ctrl-C.
+            () = sig(SignalKind::interrupt(), "SIGINT") => {}
+            // SIGTERM - Standard shutdown signal used by process launchers.
+            () = sig(SignalKind::terminate(), "SIGTERM") => {}
+        };
+    }
+
+    async fn sig(kind: SignalKind, name: &'static str) {
+        // Create a Future that completes the first
+        // time the process receives 'sig'.
+        signal(kind)
+            .expect("Failed to register signal handler")
+            .recv()
+            .await;
+
+        info!(
+            // use target to remove 'imp' from output
+            target: "zebrad::signal",
+            "received {}, starting shutdown",
+            name,
+        );
+    }
+}
+
+#[cfg(not(unix))]
+mod imp {
+    use tracing::info;
+
+    pub(super) async fn shutdown() {
+        //  Wait for Ctrl-C in Windows terminals.
+        // (Zebra doesn't support NT Service control messages. Use a service wrapper for long-running instances.)
+        tokio::signal::ctrl_c()
+            .await
+            .expect("listening for ctrl-c signal should never fail");
+
+        info!(
+            // use target to remove 'imp' from output
+            target: "zebrad::signal",
+            "received Ctrl-C, starting shutdown",
+        );
+    }
+}

zebrad/src/components/tracing.rs

@@ -1,11 +1,17 @@
 //! An HTTP endpoint for dynamically setting tracing filters.
 
 use crate::{components::tokio::TokioComponent, config::TracingSection, prelude::*};
-
+use abscissa_core::{trace::Tracing, Component, FrameworkError};
-use abscissa_core::{Component, FrameworkError};
+use color_eyre::eyre::Report;
-
 use hyper::service::{make_service_fn, service_fn};
 use hyper::{Body, Request, Response, Server};
+use std::{
+    fs::File,
+    io::{BufReader, BufWriter},
+    path::PathBuf,
+    sync::Arc,
+};
+use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
 
 /// Abscissa component which runs a tracing filter endpoint.
 #[derive(Debug, Component)]
@@ -125,3 +131,75 @@ To set the filter, POST the new filter string to /filter:
     };
     Ok(rsp)
 }
+
+#[derive(Clone)]
+pub(crate) struct FlameGrapher {
+    guard: Arc<tracing_flame::FlushGuard<BufWriter<File>>>,
+    path: PathBuf,
+}
+
+impl FlameGrapher {
+    fn make_flamegraph(&self) -> Result<(), Report> {
+        self.guard.flush()?;
+        let out_path = self.path.with_extension("svg");
+        let inf = File::open(&self.path)?;
+        let reader = BufReader::new(inf);
+
+        let out = File::create(out_path)?;
+        let writer = BufWriter::new(out);
+
+        let mut opts = inferno::flamegraph::Options::default();
+        info!("writing flamegraph to disk...");
+        inferno::flamegraph::from_reader(&mut opts, reader, writer)?;
+
+        Ok(())
+    }
+}
+
+impl Drop for FlameGrapher {
+    fn drop(&mut self) {
+        match self.make_flamegraph() {
+            Ok(()) => {}
+            Err(report) => {
+                warn!(
+                    "Error while constructing flamegraph during shutdown: {:?}",
+                    report
+                );
+            }
+        }
+    }
+}
+
+pub(crate) fn init(config: &TracingSection) -> (Tracing, Option<FlameGrapher>) {
+    // Construct a tracing subscriber with the supplied filter and enable reloading.
+    let builder = tracing_subscriber::FmtSubscriber::builder()
+        .with_env_filter(config.env_filter())
+        .with_filter_reloading();
+    let filter_handle = builder.reload_handle();
+    let subscriber = builder.finish().with(tracing_error::ErrorLayer::default());
+
+    let guard = if let Some(flamegraph_path) = config.flamegraph.as_deref() {
+        let flamegraph_path = flamegraph_path.with_extension("folded");
+        let (flame_layer, guard) = tracing_flame::FlameLayer::with_file(&flamegraph_path).unwrap();
+        let flame_layer = flame_layer
+            .with_empty_samples(false)
+            .with_threads_collapsed(true);
+        subscriber.with(flame_layer).init();
+        Some(FlameGrapher {
+            guard: Arc::new(guard),
+            path: flamegraph_path,
+        })
+    } else {
+        subscriber.init();
+        None
+    };
+
+    (filter_handle.into(), guard)
+}
+
+pub(crate) fn init_backup(config: &TracingSection) {
+    tracing_subscriber::Registry::default()
+        .with(config.env_filter())
+        .with(tracing_error::ErrorLayer::default())
+        .init();
+}

zebrad/src/config.rs

@@ -4,10 +4,11 @@
 //! application's configuration file and/or command-line options
 //! for specifying it.
 
-use std::net::SocketAddr;
+use std::{net::SocketAddr, path::PathBuf};
 
 use serde::{Deserialize, Serialize};
 
+use tracing_subscriber::EnvFilter;
 use zebra_network::Config as NetworkSection;
 use zebra_state::Config as StateSection;
 
@@ -59,12 +60,28 @@ pub struct TracingSection {
 
     /// The endpoint address used for tracing.
     pub endpoint_addr: SocketAddr,
-}
 
-impl Default for TracingSection {
+    /// The path to write a flamegraph of tracing spans too.
-    fn default() -> Self {
+    ///
-        Self::populated()
+    /// This path is not used verbatim when writing out the flamegraph. This is
-    }
+    /// because the flamegraph is written out as two parts. First the flamegraph
+    /// is constantly persisted to the disk in a "folded" representation that
+    /// records collapsed stack traces of the tracing spans that are active.
+    /// Then, when the application is finished running the destructor will flush
+    /// the flamegraph output to the folded file and then read that file and
+    /// generate the final flamegraph from it as an SVG.
+    ///
+    /// The need to create two files means that we will slightly manipulate the
+    /// path given to us to create the two representations.
+    ///
+    /// # Example
+    ///
+    /// Given `flamegraph = "flamegraph"` we will generate a `flamegraph.svg`
+    /// and a `flamegraph.folded` file in the current directory.
+    ///
+    /// If you provide a path with an extension the extension will be ignored and
+    /// replaced with `.folded` and `.svg` for the respective files.
+    pub flamegraph: Option<PathBuf>,
 }
 
 impl TracingSection {
@@ -72,8 +89,25 @@ impl TracingSection {
         Self {
             filter: Some("info".to_owned()),
             endpoint_addr: "0.0.0.0:3000".parse().unwrap(),
+            flamegraph: None,
         }
     }
+
+    /// Constructs an EnvFilter for use in our tracing subscriber.
+    ///
+    /// The env filter controls filtering of spans and events, but not how
+    /// they're emitted. Creating an env filter alone doesn't enable logging, it
+    /// needs to be used in conjunction with other layers like a fmt subscriber,
+    /// for logs, or an error layer, for SpanTraces.
+    pub fn env_filter(&self) -> EnvFilter {
+        self.filter.as_deref().unwrap_or("info").into()
+    }
+}
+
+impl Default for TracingSection {
+    fn default() -> Self {
+        Self::populated()
+    }
 }
 
 /// Metrics configuration section.