Add prometheus metrics to crank (#517)
* Add prometheus metrics to keeper
This commit is contained in:
parent
30984bbef1
commit
6be7810c56
|
@ -3116,14 +3116,17 @@ dependencies = [
|
||||||
"fixed",
|
"fixed",
|
||||||
"futures 0.3.25",
|
"futures 0.3.25",
|
||||||
"itertools 0.10.5",
|
"itertools 0.10.5",
|
||||||
|
"lazy_static",
|
||||||
"log 0.4.17",
|
"log 0.4.17",
|
||||||
"mango-v4",
|
"mango-v4",
|
||||||
"mango-v4-client",
|
"mango-v4-client",
|
||||||
|
"prometheus",
|
||||||
"pyth-sdk-solana",
|
"pyth-sdk-solana",
|
||||||
"serum_dex 0.5.10",
|
"serum_dex 0.5.10",
|
||||||
"solana-client",
|
"solana-client",
|
||||||
"solana-sdk",
|
"solana-sdk",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"warp",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
@ -4153,6 +4156,21 @@ dependencies = [
|
||||||
"yansi",
|
"yansi",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "prometheus"
|
||||||
|
version = "0.13.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if 1.0.0",
|
||||||
|
"fnv",
|
||||||
|
"lazy_static",
|
||||||
|
"memchr",
|
||||||
|
"parking_lot 0.12.1",
|
||||||
|
"protobuf",
|
||||||
|
"thiserror",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "prost"
|
name = "prost"
|
||||||
version = "0.11.3"
|
version = "0.11.3"
|
||||||
|
@ -4208,6 +4226,12 @@ dependencies = [
|
||||||
"prost",
|
"prost",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "protobuf"
|
||||||
|
version = "2.28.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "protobuf-src"
|
name = "protobuf-src"
|
||||||
version = "1.1.0+21.5"
|
version = "1.1.0+21.5"
|
||||||
|
|
|
@ -26,3 +26,6 @@ serum_dex = { git = "https://github.com/openbook-dex/program.git", default-featu
|
||||||
solana-client = "~1.14.9"
|
solana-client = "~1.14.9"
|
||||||
solana-sdk = "~1.14.9"
|
solana-sdk = "~1.14.9"
|
||||||
tokio = { version = "1.14.1", features = ["rt-multi-thread", "time", "macros", "sync"] }
|
tokio = { version = "1.14.1", features = ["rt-multi-thread", "time", "macros", "sync"] }
|
||||||
|
prometheus = "0.13.3"
|
||||||
|
warp = "0.3.3"
|
||||||
|
lazy_static = "1.4.0"
|
||||||
|
|
|
@ -6,14 +6,72 @@ use itertools::Itertools;
|
||||||
use anchor_lang::{__private::bytemuck::cast_ref, solana_program};
|
use anchor_lang::{__private::bytemuck::cast_ref, solana_program};
|
||||||
use futures::Future;
|
use futures::Future;
|
||||||
use mango_v4::state::{EventQueue, EventType, FillEvent, OutEvent, PerpMarket, TokenIndex};
|
use mango_v4::state::{EventQueue, EventType, FillEvent, OutEvent, PerpMarket, TokenIndex};
|
||||||
|
use prometheus::{register_histogram, Encoder, Histogram, IntCounter, Registry};
|
||||||
use solana_sdk::{
|
use solana_sdk::{
|
||||||
instruction::{AccountMeta, Instruction},
|
instruction::{AccountMeta, Instruction},
|
||||||
pubkey::Pubkey,
|
pubkey::Pubkey,
|
||||||
};
|
};
|
||||||
use tokio::time;
|
use tokio::time;
|
||||||
|
use warp::Filter;
|
||||||
|
|
||||||
|
lazy_static::lazy_static! {
|
||||||
|
pub static ref METRICS_REGISTRY: Registry = Registry::new_custom(Some("keeper".to_string()), None).unwrap();
|
||||||
|
pub static ref METRIC_UPDATE_TOKENS_SUCCESS: IntCounter =
|
||||||
|
IntCounter::new("update_tokens_success", "Successful update token transactions").unwrap();
|
||||||
|
pub static ref METRIC_UPDATE_TOKENS_FAILURE: IntCounter =
|
||||||
|
IntCounter::new("update_tokens_failure", "Failed update token transactions").unwrap();
|
||||||
|
pub static ref METRIC_CONSUME_EVENTS_SUCCESS: IntCounter =
|
||||||
|
IntCounter::new("consume_events_success", "Successful consume events transactions").unwrap();
|
||||||
|
pub static ref METRIC_CONSUME_EVENTS_FAILURE: IntCounter =
|
||||||
|
IntCounter::new("consume_events_failure", "Failed consume events transactions").unwrap();
|
||||||
|
pub static ref METRIC_UPDATE_FUNDING_SUCCESS: IntCounter =
|
||||||
|
IntCounter::new("update_funding_success", "Successful update funding transactions").unwrap();
|
||||||
|
pub static ref METRIC_UPDATE_FUNDING_FAILURE: IntCounter =
|
||||||
|
IntCounter::new("update_funding_failure", "Failed update funding transactions").unwrap();
|
||||||
|
pub static ref METRIC_CONFIRMATION_TIMES: Histogram = register_histogram!(
|
||||||
|
"confirmation_times", "Transaction confirmation times",
|
||||||
|
vec![1000.0, 3000.0, 5000.0, 7000.0, 10000.0, 15000.0, 20000.0, 30000.0, 40000.0, 50000.0, 60000.0]
|
||||||
|
).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
// TODO: move instructions into the client proper
|
// TODO: move instructions into the client proper
|
||||||
|
|
||||||
|
async fn serve_metrics() {
|
||||||
|
METRICS_REGISTRY
|
||||||
|
.register(Box::new(METRIC_UPDATE_TOKENS_SUCCESS.clone()))
|
||||||
|
.unwrap();
|
||||||
|
METRICS_REGISTRY
|
||||||
|
.register(Box::new(METRIC_UPDATE_TOKENS_FAILURE.clone()))
|
||||||
|
.unwrap();
|
||||||
|
METRICS_REGISTRY
|
||||||
|
.register(Box::new(METRIC_CONSUME_EVENTS_SUCCESS.clone()))
|
||||||
|
.unwrap();
|
||||||
|
METRICS_REGISTRY
|
||||||
|
.register(Box::new(METRIC_CONSUME_EVENTS_FAILURE.clone()))
|
||||||
|
.unwrap();
|
||||||
|
METRICS_REGISTRY
|
||||||
|
.register(Box::new(METRIC_UPDATE_FUNDING_SUCCESS.clone()))
|
||||||
|
.unwrap();
|
||||||
|
METRICS_REGISTRY
|
||||||
|
.register(Box::new(METRIC_UPDATE_FUNDING_FAILURE.clone()))
|
||||||
|
.unwrap();
|
||||||
|
METRICS_REGISTRY
|
||||||
|
.register(Box::new(METRIC_CONFIRMATION_TIMES.clone()))
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let metrics_route = warp::path!("metrics").map(|| {
|
||||||
|
let mut buffer = Vec::<u8>::new();
|
||||||
|
let encoder = prometheus::TextEncoder::new();
|
||||||
|
encoder
|
||||||
|
.encode(&METRICS_REGISTRY.gather(), &mut buffer)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
String::from_utf8(buffer.clone()).unwrap()
|
||||||
|
});
|
||||||
|
println!("Metrics server starting on port 9091");
|
||||||
|
warp::serve(metrics_route).run(([0, 0, 0, 0], 9091)).await;
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn runner(
|
pub async fn runner(
|
||||||
mango_client: Arc<MangoClient>,
|
mango_client: Arc<MangoClient>,
|
||||||
debugging_handle: impl Future,
|
debugging_handle: impl Future,
|
||||||
|
@ -83,7 +141,8 @@ pub async fn runner(
|
||||||
mango_client.clone(),
|
mango_client.clone(),
|
||||||
interval_check_new_listings_and_abort
|
interval_check_new_listings_and_abort
|
||||||
),
|
),
|
||||||
debugging_handle
|
serve_metrics(),
|
||||||
|
debugging_handle,
|
||||||
);
|
);
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
|
@ -165,19 +224,24 @@ pub async fn loop_update_index_and_rate(
|
||||||
.send_and_confirm_permissionless_tx(instructions)
|
.send_and_confirm_permissionless_tx(instructions)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
|
let confirmation_time = pre.elapsed().as_millis();
|
||||||
|
METRIC_CONFIRMATION_TIMES.observe(confirmation_time as f64);
|
||||||
|
|
||||||
if let Err(e) = sig_result {
|
if let Err(e) = sig_result {
|
||||||
|
METRIC_UPDATE_TOKENS_FAILURE.inc();
|
||||||
log::info!(
|
log::info!(
|
||||||
"metricName=UpdateTokensV4Failure tokens={} durationMs={} error={}",
|
"metricName=UpdateTokensV4Failure tokens={} durationMs={} error={}",
|
||||||
token_names,
|
token_names,
|
||||||
pre.elapsed().as_millis(),
|
confirmation_time,
|
||||||
e
|
e
|
||||||
);
|
);
|
||||||
log::error!("{:?}", e)
|
log::error!("{:?}", e)
|
||||||
} else {
|
} else {
|
||||||
|
METRIC_UPDATE_TOKENS_SUCCESS.inc();
|
||||||
log::info!(
|
log::info!(
|
||||||
"metricName=UpdateTokensV4Success tokens={} durationMs={}",
|
"metricName=UpdateTokensV4Success tokens={} durationMs={}",
|
||||||
token_names,
|
token_names,
|
||||||
pre.elapsed().as_millis(),
|
confirmation_time,
|
||||||
);
|
);
|
||||||
log::info!("{:?}", sig_result);
|
log::info!("{:?}", sig_result);
|
||||||
}
|
}
|
||||||
|
@ -278,20 +342,25 @@ pub async fn loop_consume_events(
|
||||||
|
|
||||||
let sig_result = client.send_and_confirm_permissionless_tx(vec![ix]).await;
|
let sig_result = client.send_and_confirm_permissionless_tx(vec![ix]).await;
|
||||||
|
|
||||||
|
let confirmation_time = pre.elapsed().as_millis();
|
||||||
|
METRIC_CONFIRMATION_TIMES.observe(confirmation_time as f64);
|
||||||
|
|
||||||
if let Err(e) = sig_result {
|
if let Err(e) = sig_result {
|
||||||
|
METRIC_CONSUME_EVENTS_FAILURE.inc();
|
||||||
log::info!(
|
log::info!(
|
||||||
"metricName=ConsumeEventsV4Failure market={} durationMs={} consumed={} error={}",
|
"metricName=ConsumeEventsV4Failure market={} durationMs={} consumed={} error={}",
|
||||||
perp_market.name(),
|
perp_market.name(),
|
||||||
pre.elapsed().as_millis(),
|
confirmation_time,
|
||||||
num_of_events,
|
num_of_events,
|
||||||
e.to_string()
|
e.to_string()
|
||||||
);
|
);
|
||||||
log::error!("{:?}", e)
|
log::error!("{:?}", e)
|
||||||
} else {
|
} else {
|
||||||
|
METRIC_CONSUME_EVENTS_SUCCESS.inc();
|
||||||
log::info!(
|
log::info!(
|
||||||
"metricName=ConsumeEventsV4Success market={} durationMs={} consumed={}",
|
"metricName=ConsumeEventsV4Success market={} durationMs={} consumed={}",
|
||||||
perp_market.name(),
|
perp_market.name(),
|
||||||
pre.elapsed().as_millis(),
|
confirmation_time,
|
||||||
num_of_events,
|
num_of_events,
|
||||||
);
|
);
|
||||||
log::info!("{:?}", sig_result);
|
log::info!("{:?}", sig_result);
|
||||||
|
@ -328,19 +397,24 @@ pub async fn loop_update_funding(
|
||||||
};
|
};
|
||||||
let sig_result = client.send_and_confirm_permissionless_tx(vec![ix]).await;
|
let sig_result = client.send_and_confirm_permissionless_tx(vec![ix]).await;
|
||||||
|
|
||||||
|
let confirmation_time = pre.elapsed().as_millis();
|
||||||
|
METRIC_CONFIRMATION_TIMES.observe(confirmation_time as f64);
|
||||||
|
|
||||||
if let Err(e) = sig_result {
|
if let Err(e) = sig_result {
|
||||||
|
METRIC_UPDATE_FUNDING_FAILURE.inc();
|
||||||
log::error!(
|
log::error!(
|
||||||
"metricName=UpdateFundingV4Error market={} durationMs={} error={}",
|
"metricName=UpdateFundingV4Error market={} durationMs={} error={}",
|
||||||
perp_market.name(),
|
perp_market.name(),
|
||||||
pre.elapsed().as_millis(),
|
confirmation_time,
|
||||||
e.to_string()
|
e.to_string()
|
||||||
);
|
);
|
||||||
log::error!("{:?}", e)
|
log::error!("{:?}", e)
|
||||||
} else {
|
} else {
|
||||||
|
METRIC_UPDATE_FUNDING_SUCCESS.inc();
|
||||||
log::info!(
|
log::info!(
|
||||||
"metricName=UpdateFundingV4Success market={} durationMs={}",
|
"metricName=UpdateFundingV4Success market={} durationMs={}",
|
||||||
perp_market.name(),
|
perp_market.name(),
|
||||||
pre.elapsed().as_millis(),
|
confirmation_time,
|
||||||
);
|
);
|
||||||
log::info!("{:?}", sig_result);
|
log::info!("{:?}", sig_result);
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,3 +4,7 @@ kill_timeout = 30
|
||||||
|
|
||||||
[build]
|
[build]
|
||||||
dockerfile = "../bin/keeper/Dockerfile.keeper"
|
dockerfile = "../bin/keeper/Dockerfile.keeper"
|
||||||
|
|
||||||
|
[metrics]
|
||||||
|
port = 9091
|
||||||
|
path = "/metrics"
|
Loading…
Reference in New Issue