2021-11-08 11:15:46 -08:00
|
|
|
use {
|
2022-10-07 03:44:53 -07:00
|
|
|
crate::MetricsConfig,
|
2021-11-08 11:15:46 -08:00
|
|
|
log::*,
|
|
|
|
std::collections::HashMap,
|
2022-10-07 03:46:19 -07:00
|
|
|
std::fmt,
|
2021-11-08 11:15:46 -08:00
|
|
|
std::sync::{atomic, Arc, Mutex, RwLock},
|
|
|
|
tokio::time,
|
2022-10-07 03:44:53 -07:00
|
|
|
warp::{Filter, Rejection, Reply},
|
2021-11-08 11:15:46 -08:00
|
|
|
};
|
|
|
|
|
|
|
|
#[derive(Debug)]
|
|
|
|
enum Value {
|
2022-10-07 03:46:19 -07:00
|
|
|
U64 {
|
|
|
|
value: Arc<atomic::AtomicU64>,
|
|
|
|
metric_type: MetricType,
|
|
|
|
},
|
|
|
|
I64 {
|
|
|
|
value: Arc<atomic::AtomicI64>,
|
|
|
|
metric_type: MetricType,
|
|
|
|
},
|
|
|
|
Bool {
|
|
|
|
value: Arc<Mutex<bool>>,
|
|
|
|
metric_type: MetricType,
|
|
|
|
},
|
2022-10-07 03:44:53 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Debug, Clone)]
|
|
|
|
pub enum MetricType {
|
|
|
|
Counter,
|
|
|
|
Gauge,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl fmt::Display for MetricType {
|
|
|
|
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
|
|
|
match self {
|
|
|
|
MetricType::Counter => {
|
|
|
|
write!(f, "counter")
|
2022-10-07 03:46:19 -07:00
|
|
|
}
|
2022-10-07 03:44:53 -07:00
|
|
|
MetricType::Gauge => {
|
|
|
|
write!(f, "gauge")
|
2022-10-07 03:46:19 -07:00
|
|
|
}
|
2022-10-07 03:44:53 -07:00
|
|
|
}
|
|
|
|
}
|
2021-11-09 05:23:42 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Debug)]
|
|
|
|
enum PrevValue {
|
|
|
|
U64(u64),
|
|
|
|
I64(i64),
|
2022-10-07 03:44:53 -07:00
|
|
|
Bool(bool),
|
2021-11-08 11:15:46 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Clone)]
|
2021-11-09 05:23:42 -08:00
|
|
|
pub struct MetricU64 {
|
|
|
|
value: Arc<atomic::AtomicU64>,
|
2021-11-08 11:15:46 -08:00
|
|
|
}
|
2021-11-09 05:23:42 -08:00
|
|
|
impl MetricU64 {
|
2022-01-06 03:54:35 -08:00
|
|
|
pub fn value(&self) -> u64 {
|
|
|
|
self.value.load(atomic::Ordering::Acquire)
|
|
|
|
}
|
|
|
|
|
2021-11-09 05:23:42 -08:00
|
|
|
pub fn set(&mut self, value: u64) {
|
2021-11-08 11:15:46 -08:00
|
|
|
self.value.store(value, atomic::Ordering::Release);
|
|
|
|
}
|
|
|
|
|
2022-01-06 03:54:35 -08:00
|
|
|
pub fn set_max(&mut self, value: u64) {
|
|
|
|
self.value.fetch_max(value, atomic::Ordering::AcqRel);
|
|
|
|
}
|
|
|
|
|
2022-01-04 01:15:22 -08:00
|
|
|
pub fn add(&mut self, value: u64) {
|
|
|
|
self.value.fetch_add(value, atomic::Ordering::AcqRel);
|
|
|
|
}
|
|
|
|
|
2021-11-08 11:15:46 -08:00
|
|
|
pub fn increment(&mut self) {
|
|
|
|
self.value.fetch_add(1, atomic::Ordering::AcqRel);
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn decrement(&mut self) {
|
|
|
|
self.value.fetch_sub(1, atomic::Ordering::AcqRel);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Clone)]
|
2021-11-09 05:23:42 -08:00
|
|
|
pub struct MetricI64 {
|
|
|
|
value: Arc<atomic::AtomicI64>,
|
2021-11-08 11:15:46 -08:00
|
|
|
}
|
2021-11-09 05:23:42 -08:00
|
|
|
impl MetricI64 {
|
|
|
|
pub fn set(&mut self, value: i64) {
|
2021-11-08 11:15:46 -08:00
|
|
|
self.value.store(value, atomic::Ordering::Release);
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn increment(&mut self) {
|
|
|
|
self.value.fetch_add(1, atomic::Ordering::AcqRel);
|
|
|
|
}
|
2021-11-09 05:23:42 -08:00
|
|
|
|
|
|
|
pub fn decrement(&mut self) {
|
|
|
|
self.value.fetch_sub(1, atomic::Ordering::AcqRel);
|
|
|
|
}
|
2021-11-08 11:15:46 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Clone)]
|
2022-10-07 03:44:53 -07:00
|
|
|
pub struct MetricBool {
|
|
|
|
value: Arc<Mutex<bool>>,
|
2021-11-08 11:15:46 -08:00
|
|
|
}
|
|
|
|
|
2022-10-07 03:44:53 -07:00
|
|
|
impl MetricBool {
|
|
|
|
pub fn set(&self, value: bool) {
|
2021-11-08 11:15:46 -08:00
|
|
|
*self.value.lock().unwrap() = value;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Clone)]
|
|
|
|
pub struct Metrics {
|
|
|
|
registry: Arc<RwLock<HashMap<String, Value>>>,
|
2022-10-08 04:57:47 -07:00
|
|
|
labels: HashMap<String, String>,
|
2021-11-08 11:15:46 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
impl Metrics {
|
2022-10-07 03:44:53 -07:00
|
|
|
pub fn register_u64(&self, name: String, metric_type: MetricType) -> MetricU64 {
|
2022-01-06 03:54:35 -08:00
|
|
|
let mut registry = self.registry.write().unwrap();
|
2022-10-07 03:46:19 -07:00
|
|
|
let value = registry.entry(name).or_insert(Value::U64 {
|
|
|
|
value: Arc::new(atomic::AtomicU64::new(0)),
|
|
|
|
metric_type: metric_type,
|
|
|
|
});
|
2022-01-06 03:54:35 -08:00
|
|
|
MetricU64 {
|
|
|
|
value: match value {
|
2022-10-07 03:46:19 -07:00
|
|
|
Value::U64 {
|
|
|
|
value: v,
|
|
|
|
metric_type: _,
|
|
|
|
} => v.clone(),
|
2022-01-06 03:54:35 -08:00
|
|
|
_ => panic!("bad metric type"),
|
|
|
|
},
|
|
|
|
}
|
2021-11-08 11:15:46 -08:00
|
|
|
}
|
|
|
|
|
2022-10-07 03:44:53 -07:00
|
|
|
pub fn register_i64(&self, name: String, metric_type: MetricType) -> MetricI64 {
|
2022-01-06 03:54:35 -08:00
|
|
|
let mut registry = self.registry.write().unwrap();
|
2022-10-07 03:46:19 -07:00
|
|
|
let value = registry.entry(name).or_insert(Value::I64 {
|
|
|
|
value: Arc::new(atomic::AtomicI64::new(0)),
|
|
|
|
metric_type: metric_type,
|
|
|
|
});
|
2022-01-06 03:54:35 -08:00
|
|
|
MetricI64 {
|
|
|
|
value: match value {
|
2022-10-07 03:46:19 -07:00
|
|
|
Value::I64 {
|
|
|
|
value: v,
|
|
|
|
metric_type: _,
|
|
|
|
} => v.clone(),
|
2022-01-06 03:54:35 -08:00
|
|
|
_ => panic!("bad metric type"),
|
|
|
|
},
|
|
|
|
}
|
2021-11-08 11:15:46 -08:00
|
|
|
}
|
|
|
|
|
2022-10-07 03:44:53 -07:00
|
|
|
pub fn register_bool(&self, name: String) -> MetricBool {
|
2022-01-06 03:54:35 -08:00
|
|
|
let mut registry = self.registry.write().unwrap();
|
2022-10-07 03:46:19 -07:00
|
|
|
let value = registry.entry(name).or_insert(Value::Bool {
|
|
|
|
value: Arc::new(Mutex::new(false)),
|
|
|
|
metric_type: MetricType::Gauge,
|
|
|
|
});
|
2022-10-07 03:44:53 -07:00
|
|
|
MetricBool {
|
2022-01-06 03:54:35 -08:00
|
|
|
value: match value {
|
2022-10-07 03:46:19 -07:00
|
|
|
Value::Bool {
|
|
|
|
value: v,
|
|
|
|
metric_type: _,
|
|
|
|
} => v.clone(),
|
2022-01-06 03:54:35 -08:00
|
|
|
_ => panic!("bad metric type"),
|
|
|
|
},
|
|
|
|
}
|
2021-11-08 11:15:46 -08:00
|
|
|
}
|
2022-10-05 15:50:21 -07:00
|
|
|
|
2022-10-07 03:44:53 -07:00
|
|
|
pub fn get_registry_vec(&self) -> Vec<(String, String, String)> {
|
|
|
|
let mut vec: Vec<(String, String, String)> = Vec::new();
|
2022-10-05 15:50:21 -07:00
|
|
|
let metrics = self.registry.read().unwrap();
|
|
|
|
for (name, value) in metrics.iter() {
|
2022-10-07 03:44:53 -07:00
|
|
|
let (value_str, type_str) = match value {
|
2022-10-07 03:46:19 -07:00
|
|
|
Value::U64 {
|
|
|
|
value: v,
|
|
|
|
metric_type: t,
|
|
|
|
} => (
|
|
|
|
format!("{}", v.load(atomic::Ordering::Acquire)),
|
|
|
|
t.to_string(),
|
|
|
|
),
|
|
|
|
Value::I64 {
|
|
|
|
value: v,
|
|
|
|
metric_type: t,
|
|
|
|
} => (
|
|
|
|
format!("{}", v.load(atomic::Ordering::Acquire)),
|
|
|
|
t.to_string(),
|
|
|
|
),
|
|
|
|
Value::Bool {
|
|
|
|
value: v,
|
|
|
|
metric_type: t,
|
|
|
|
} => {
|
|
|
|
let bool_to_int = if *v.lock().unwrap() { 1 } else { 0 };
|
2022-10-07 03:44:53 -07:00
|
|
|
(format!("{}", bool_to_int), t.to_string())
|
2022-10-05 15:50:21 -07:00
|
|
|
}
|
|
|
|
};
|
2022-10-07 03:44:53 -07:00
|
|
|
vec.push((name.clone(), value_str, type_str));
|
2022-10-05 15:50:21 -07:00
|
|
|
}
|
|
|
|
vec
|
|
|
|
}
|
2021-11-08 11:15:46 -08:00
|
|
|
}
|
|
|
|
|
2022-10-07 03:44:53 -07:00
|
|
|
async fn handle_prometheus_poll(metrics: Metrics) -> Result<impl Reply, Rejection> {
|
|
|
|
debug!("handle_prometheus_poll");
|
2022-10-08 04:57:47 -07:00
|
|
|
let label_strings_vec: Vec<String> = metrics
|
|
|
|
.labels
|
2022-10-07 03:44:53 -07:00
|
|
|
.iter()
|
|
|
|
.map(|(name, value)| format!("{}=\"{}\"", name, value))
|
|
|
|
.collect();
|
|
|
|
let lines: Vec<String> = metrics
|
|
|
|
.get_registry_vec()
|
|
|
|
.iter()
|
|
|
|
.map(|(name, value, type_name)| {
|
2022-10-07 05:15:27 -07:00
|
|
|
let sanitized_name = str::replace(name, "-", "_");
|
2022-10-07 03:44:53 -07:00
|
|
|
format!(
|
2022-10-07 05:15:27 -07:00
|
|
|
"# HELP {} \n# TYPE {} {}\n{}{{{}}} {}",
|
|
|
|
sanitized_name,
|
|
|
|
sanitized_name,
|
2022-10-07 03:44:53 -07:00
|
|
|
type_name,
|
2022-10-07 05:15:27 -07:00
|
|
|
sanitized_name,
|
2022-10-07 03:44:53 -07:00
|
|
|
label_strings_vec.join(","),
|
|
|
|
value
|
|
|
|
)
|
|
|
|
})
|
|
|
|
.collect();
|
|
|
|
Ok(format!("{}\n", lines.join("\n")))
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn with_metrics(
|
|
|
|
metrics: Metrics,
|
|
|
|
) -> impl Filter<Extract = (Metrics,), Error = std::convert::Infallible> + Clone {
|
|
|
|
warp::any().map(move || metrics.clone())
|
|
|
|
}
|
|
|
|
|
2022-10-08 04:57:47 -07:00
|
|
|
pub fn start(config: MetricsConfig, process_name: String) -> Metrics {
|
2021-11-08 11:15:46 -08:00
|
|
|
let mut write_interval = time::interval(time::Duration::from_secs(60));
|
|
|
|
|
|
|
|
let registry = Arc::new(RwLock::new(HashMap::<String, Value>::new()));
|
|
|
|
let registry_c = Arc::clone(®istry);
|
2022-10-08 04:57:47 -07:00
|
|
|
let labels = HashMap::from([(String::from("process"), process_name)]);
|
|
|
|
let metrics_tx = Metrics { registry, labels };
|
2022-10-07 03:44:53 -07:00
|
|
|
let metrics_route = warp::path!("metrics")
|
|
|
|
.and(with_metrics(metrics_tx.clone()))
|
|
|
|
.and_then(handle_prometheus_poll);
|
|
|
|
|
|
|
|
if config.output_http {
|
|
|
|
// serve prometheus metrics endpoint
|
|
|
|
tokio::spawn(async move {
|
|
|
|
warp::serve(metrics_route).run(([0, 0, 0, 0], 9091)).await;
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
if config.output_stdout {
|
|
|
|
// periodically log to stdout
|
|
|
|
tokio::spawn(async move {
|
|
|
|
let mut previous_values = HashMap::<String, PrevValue>::new();
|
|
|
|
loop {
|
|
|
|
write_interval.tick().await;
|
|
|
|
|
|
|
|
// Nested locking! Safe because the only other user locks registry for writing and doesn't
|
|
|
|
// acquire any interior locks.
|
|
|
|
let metrics = registry_c.read().unwrap();
|
|
|
|
for (name, value) in metrics.iter() {
|
|
|
|
let previous_value = previous_values.get_mut(name);
|
|
|
|
match value {
|
2022-10-07 03:46:19 -07:00
|
|
|
Value::U64 {
|
|
|
|
value: v,
|
|
|
|
metric_type: _,
|
|
|
|
} => {
|
2022-10-07 03:44:53 -07:00
|
|
|
let new_value = v.load(atomic::Ordering::Acquire);
|
|
|
|
let previous_value = if let Some(PrevValue::U64(v)) = previous_value {
|
|
|
|
let prev = *v;
|
|
|
|
*v = new_value;
|
|
|
|
prev
|
|
|
|
} else {
|
|
|
|
previous_values.insert(name.clone(), PrevValue::U64(new_value));
|
|
|
|
0
|
|
|
|
};
|
|
|
|
let diff = new_value.wrapping_sub(previous_value) as i64;
|
|
|
|
info!("metric: {}: {} ({:+})", name, new_value, diff);
|
|
|
|
}
|
2022-10-07 03:46:19 -07:00
|
|
|
Value::I64 {
|
|
|
|
value: v,
|
|
|
|
metric_type: _,
|
|
|
|
} => {
|
2022-10-07 03:44:53 -07:00
|
|
|
let new_value = v.load(atomic::Ordering::Acquire);
|
|
|
|
let previous_value = if let Some(PrevValue::I64(v)) = previous_value {
|
|
|
|
let prev = *v;
|
|
|
|
*v = new_value;
|
|
|
|
prev
|
|
|
|
} else {
|
|
|
|
previous_values.insert(name.clone(), PrevValue::I64(new_value));
|
|
|
|
0
|
|
|
|
};
|
|
|
|
let diff = new_value - previous_value;
|
|
|
|
info!("metric: {}: {} ({:+})", name, new_value, diff);
|
|
|
|
}
|
2022-10-07 03:46:19 -07:00
|
|
|
Value::Bool {
|
|
|
|
value: v,
|
|
|
|
metric_type: _,
|
|
|
|
} => {
|
2022-10-07 03:44:53 -07:00
|
|
|
let new_value = v.lock().unwrap();
|
2022-10-07 03:46:19 -07:00
|
|
|
let previous_value = if let Some(PrevValue::Bool(v)) = previous_value {
|
2022-10-07 03:44:53 -07:00
|
|
|
let mut prev = new_value.clone();
|
|
|
|
std::mem::swap(&mut prev, v);
|
|
|
|
prev
|
|
|
|
} else {
|
|
|
|
previous_values
|
|
|
|
.insert(name.clone(), PrevValue::Bool(new_value.clone()));
|
|
|
|
false
|
|
|
|
};
|
|
|
|
if *new_value == previous_value {
|
|
|
|
info!("metric: {}: {} (unchanged)", name, &*new_value);
|
|
|
|
} else {
|
|
|
|
info!(
|
|
|
|
"metric: {}: {} (before: {})",
|
|
|
|
name, &*new_value, previous_value
|
|
|
|
);
|
|
|
|
}
|
2021-11-09 05:23:42 -08:00
|
|
|
}
|
2021-11-08 11:15:46 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2022-10-07 03:44:53 -07:00
|
|
|
});
|
|
|
|
}
|
2021-11-08 11:15:46 -08:00
|
|
|
|
2022-10-07 03:44:53 -07:00
|
|
|
metrics_tx
|
2021-11-08 11:15:46 -08:00
|
|
|
}
|