zecfoundation
/
zebra
mirror of https://github.com/ZcashFoundation/zebra.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Metrics MVP: add two metrics and export them to Prometheus. 

Co-authored-by: Deirdre Connolly <deirdre@zfnd.org>
This commit is contained in:
Henry de Valence 2020-02-14 13:38:33 -08:00 committed by Deirdre Connolly
parent b443d7a4be
commit 75d3d44fb3
9 changed files with 385 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

298
Cargo.lock generated
View File

@ -60,6 +60,12 @@ dependencies = [
"winapi 0.3.8",
]
[[package]]
name = "arc-swap"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc4662175ead9cd84451d5c35070517777949a2ed84551764129cedb88384841"
[[package]]
name = "arc-swap"
version = "0.4.4"
@ -268,6 +274,41 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
[[package]]
name = "crossbeam-epoch"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac"
dependencies = [
"autocfg 0.1.7",
"cfg-if",
"crossbeam-utils 0.7.0",
"lazy_static",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
dependencies = [
"cfg-if",
"lazy_static",
]
[[package]]
name = "crossbeam-utils"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4"
dependencies = [
"autocfg 0.1.7",
"cfg-if",
"lazy_static",
]
[[package]]
name = "curve25519-dalek"
version = "2.0.0"
@ -325,6 +366,12 @@ dependencies = [
"generic-array",
]
[[package]]
name = "dtoa"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4358a9e11b9a09cf52383b451b49a169e8d797b68aa02301ff586d70d9661ea3"
[[package]]
name = "ed25519-zebra"
version = "0.2.0"
@ -531,6 +578,16 @@ dependencies = [
"tokio-util",
]
[[package]]
name = "hdrhistogram"
version = "6.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d331ebcdbca4acbefe5da8c3299b2e246f198a8294cc5163354e743398b89d"
dependencies = [
"byteorder",
"num-traits",
]
[[package]]
name = "hermit-abi"
version = "0.1.6"
@ -603,6 +660,17 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]]
name = "im"
version = "12.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de38d1511a0ce7677538acb1e31b5df605147c458e061b2cdb89858afb1cd182"
dependencies = [
"rustc_version",
"sized-chunks",
"typenum",
]
[[package]]
name = "indexmap"
version = "1.3.2"
@ -659,6 +727,21 @@ version = "0.2.66"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558"
[[package]]
name = "linked-hash-map"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83"
[[package]]
name = "lock_api"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79b2de95ecb4691949fea4716ca53cdbcfccb2c612e19644a8bad05edcf9f47b"
dependencies = [
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.8"
@ -689,6 +772,118 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3197e20c7edb283f87c071ddfc7a2cca8f8e0b888c242959846a6fce03c72223"
[[package]]
name = "memoffset"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9"
dependencies = [
"rustc_version",
]
[[package]]
name = "metrics"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51b70227ece8711a1aa2f99655efd795d0cff297a5b9fe39645a93aacf6ad39d"
dependencies = [
"metrics-core",
]
[[package]]
name = "metrics-core"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c064b3a1ff41f4bf6c91185c8a0caeccf8a8a27e9d0f92cc54cf3dbec812f48"
[[package]]
name = "metrics-exporter-http"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e14017d204ae062dc5c68a321e3dbdcd9b30181305cb6b067932f7f03f754e27"
dependencies = [
"hyper",
"log",
"metrics-core",
]
[[package]]
name = "metrics-exporter-log"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3fc63816bd5f8bde5eb31ce471f9633adc69ba1c55b44191b4d5fc7e263e8ab"
dependencies = [
"log",
"metrics-core",
"tokio",
]
[[package]]
name = "metrics-observer-json"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbe930460a6c336b8f873dcfb28da3f805fd0dbadbea7beaf3042c7fb1d9fcd3"
dependencies = [
"hdrhistogram",
"metrics-core",
"metrics-util",
"serde_json",
]
[[package]]
name = "metrics-observer-prometheus"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f9bb94f40e189c87cf70ef1c78815b949ab9d28fe76ebb81f15f79bd19a33d6"
dependencies = [
"hdrhistogram",
"metrics-core",
"metrics-util",
]
[[package]]
name = "metrics-observer-yaml"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83f66811013592560efc75d75a92d6e2f415a11b52f085e51d9fb4d1edec6335"
dependencies = [
"hdrhistogram",
"metrics-core",
"metrics-util",
"serde_yaml",
]
[[package]]
name = "metrics-runtime"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "beb3035626782c533953bcc1a349467543b7f0e9d7b0c92edc61ee2bda7033d6"
dependencies = [
"arc-swap 0.3.11",
"crossbeam-utils 0.6.6",
"im",
"metrics",
"metrics-core",
"metrics-exporter-http",
"metrics-exporter-log",
"metrics-observer-json",
"metrics-observer-prometheus",
"metrics-observer-yaml",
"metrics-util",
"parking_lot",
"quanta",
]
[[package]]
name = "metrics-util"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d11f8090a8886339f9468a04eeea0711e4cf27538b134014664308041307a1c5"
dependencies = [
"crossbeam-epoch",
"serde",
]
[[package]]
name = "mio"
version = "0.6.21"
@ -792,6 +987,32 @@ dependencies = [
"stable_deref_trait",
]
[[package]]
name = "parking_lot"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252"
dependencies = [
"lock_api",
"parking_lot_core",
"rustc_version",
]
[[package]]
name = "parking_lot_core"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b"
dependencies = [
"cfg-if",
"cloudabi",
"libc",
"redox_syscall",
"rustc_version",
"smallvec",
"winapi 0.3.8",
]
[[package]]
name = "pin-project"
version = "0.4.8"
@ -896,6 +1117,16 @@ dependencies = [
"syn 0.15.44",
]
[[package]]
name = "quanta"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4f7a1905379198075914bc93d32a5465c40474f90a078bb13439cb00c547bcc"
dependencies = [
"libc",
"winapi 0.3.8",
]
[[package]]
name = "quick-error"
version = "1.2.3"
@ -1139,6 +1370,15 @@ version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
[[package]]
name = "rustc_version"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
dependencies = [
"semver",
]
[[package]]
name = "rusty-fork"
version = "0.2.2"
@ -1151,6 +1391,18 @@ dependencies = [
"wait-timeout",
]
[[package]]
name = "ryu"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8"
[[package]]
name = "scopeguard"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d"
[[package]]
name = "secrecy"
version = "0.6.0"
@ -1197,6 +1449,29 @@ dependencies = [
"syn 1.0.13",
]
[[package]]
name = "serde_json"
version = "1.0.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9371ade75d4c2d6cb154141b9752cf3781ec9c05e0e5cf35060e1e70ee7b9c25"
dependencies = [
"itoa",
"ryu",
"serde",
]
[[package]]
name = "serde_yaml"
version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "691b17f19fc1ec9d94ec0b5864859290dff279dbd7b03f017afda54eb36c3c35"
dependencies = [
"dtoa",
"linked-hash-map",
"serde",
"yaml-rust",
]
[[package]]
name = "sha2"
version = "0.8.1"
@ -1225,10 +1500,19 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94f478ede9f64724c5d173d7bb56099ec3e2d9fc2774aac65d34b8b890405f41"
dependencies = [
"arc-swap",
"arc-swap 0.4.4",
"libc",
]
[[package]]
name = "sized-chunks"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d3e7f23bad2d6694e0f46f5e470ec27eb07b8f3e8b309a4b0dc17501928b9f2"
dependencies = [
"typenum",
]
[[package]]
name = "slab"
version = "0.4.2"
@ -1710,6 +1994,15 @@ dependencies = [
"winapi-build",
]
[[package]]
name = "yaml-rust"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65923dd1784f44da1d2c3dbbc5e822045628c590ba72123e1c73d3c230c4434d"
dependencies = [
"linked-hash-map",
]
[[package]]
name = "zebra-chain"
version = "0.1.0"
@ -1746,6 +2039,7 @@ dependencies = [
"futures",
"hex",
"indexmap",
"metrics",
"pin-project",
"proptest",
"proptest-derive",
@ -1782,6 +2076,8 @@ dependencies = [
"futures",
"gumdrop",
"hyper",
"metrics",
"metrics-runtime",
"once_cell",
"rand 0.7.3",
"serde",

2
zebra-network/Cargo.toml
View File

@ -33,4 +33,6 @@ tracing-futures = "0.2"
tower = "0.3"
tower-load = "0.3"
metrics = "0.12"
zebra-chain = { path = "../zebra-chain" }

9
zebra-network/src/peer_set/set.rs
View File

@ -1,5 +1,6 @@
use std::{
collections::HashMap,
convert::TryInto,
fmt::Debug,
future::Future,
marker::PhantomData,
@ -230,6 +231,14 @@ where
num_ready = self.ready_services.len(),
num_unready = self.unready_services.len(),
);
metrics::gauge!(
"pool.num_ready",
self.ready_services.len().try_into().unwrap()
);
metrics::gauge!(
"pool.num_unready",
self.unready_services.len().try_into().unwrap()
);
loop {
// Re-check that the pre-selected service is ready, in case

3
zebrad/Cargo.toml
View File

@ -25,6 +25,9 @@ hyper = "0.13.2"
tower = "0.3"
metrics-runtime = "0.13"
metrics = "0.12"
zebra-chain = { path = "../zebra-chain" }
zebra-network = { path = "../zebra-network" }

5
zebrad/src/application.rs
View File

@ -82,11 +82,14 @@ impl Application for ZebradApp {
/// beyond the default ones provided by the framework, this is the place
/// to do so.
fn register_components(&mut self, command: &Self::Cmd) -> Result<(), FrameworkError> {
use crate::components::{tokio::TokioComponent, tracing::TracingEndpoint};
use crate::components::{
metrics::MetricsEndpoint, tokio::TokioComponent, tracing::TracingEndpoint,
};
let mut components = self.framework_components(command)?;
components.push(Box::new(TokioComponent::new()?));
components.push(Box::new(TracingEndpoint::new()?));
components.push(Box::new(MetricsEndpoint::new()?));
self.state.components.register(components)
}

1
zebrad/src/components.rs
View File

@ -1,2 +1,3 @@
pub mod metrics;
pub mod tokio;
pub mod tracing;

48
zebrad/src/components/metrics.rs Normal file
View File

@ -0,0 +1,48 @@
//! An HTTP endpoint for metrics collection.
use metrics_runtime::{exporters::HttpExporter, observers::PrometheusBuilder, Receiver};
use abscissa_core::{Component, FrameworkError};
use crate::{components::tokio::TokioComponent, prelude::*};
/// Abscissa component which runs a metrics endpoint.
#[derive(Debug, Component)]
#[component(inject = "init_tokio(zebrad::components::tokio::TokioComponent)")]
pub struct MetricsEndpoint {}
impl MetricsEndpoint {
/// Create the component.
pub fn new() -> Result<Self, FrameworkError> {
Ok(Self {})
}
/// Do setup after receiving a tokio runtime.
pub fn init_tokio(&mut self, tokio_component: &TokioComponent) -> Result<(), FrameworkError> {
info!("Initializing metrics endpoint");
// XXX load metrics addr from config
let addr = "127.0.0.1:9999"
.parse()
.expect("Hardcoded address should be parseable");
// XXX do we need to hold on to the receiver?
let receiver = Receiver::builder()
.build()
.expect("Receiver config should be valid");
// XXX ???? connect this ???
let _sink = receiver.sink();
let endpoint = HttpExporter::new(receiver.controller(), PrometheusBuilder::new(), addr);
tokio_component
.rt
.as_ref()
.expect("runtime should not be taken")
.spawn(endpoint.async_run());
metrics::set_boxed_recorder(Box::new(receiver)).expect("XXX FIXME ERROR CONVERSION");
Ok(())
}
}

9
zebrad/src/components/tracing.rs
View File

@ -8,17 +8,10 @@ use hyper::service::{make_service_fn, service_fn};
use hyper::{Body, Request, Response, Server};
/// Abscissa component which runs a tracing filter endpoint.
#[derive(Component)]
#[derive(Debug, Component)]
#[component(inject = "init_tokio(zebrad::components::tokio::TokioComponent)")]
pub struct TracingEndpoint {}
impl ::std::fmt::Debug for TracingEndpoint {
fn fmt(&self, f: &mut ::std::fmt::Formatter) -> Result<(), ::std::fmt::Error> {
// Debug is required by Component, can't be derived as a Handle is not Debug
write!(f, "TracingEndpoint")
}
}
async fn read_filter(req: Request<Body>) -> Result<String, String> {
std::str::from_utf8(
&hyper::body::to_bytes(req.into_body())

21
zebrad/src/config.rs
View File

@ -4,6 +4,8 @@
//! application's configuration file and/or command-line options
//! for specifying it.
use std::net::SocketAddr;
use serde::{Deserialize, Serialize};
use zebra_network::Config as NetworkSection;
@ -14,8 +16,10 @@ use zebra_network::Config as NetworkSection;
pub struct ZebradConfig {
/// Tracing configuration
pub tracing: TracingSection,
/// Networking configuration,
/// Networking configuration
pub network: NetworkSection,
/// Metrics configuration
pub metrics: MetricsSection,
}
/// Tracing configuration section.
@ -33,3 +37,18 @@ impl Default for TracingSection {
}
}
}
/// Metrics configuration section.
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct MetricsSection {
pub endpoint_addr: SocketAddr,
}
impl Default for MetricsSection {
fn default() -> Self {
Self {
endpoint_addr: "127.0.0.1:9999".parse().unwrap(),
}
}
}