blockworks-foundation
/
mango-v4
mirror of https://github.com/blockworks-foundation/mango-v4.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add prometheus metrics to crank (#517) 

* Add prometheus metrics to keeper
This commit is contained in:
riordanp 2023-03-29 15:51:32 +01:00 committed by GitHub
parent 30984bbef1
commit 6be7810c56
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 112 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
Cargo.lock generated
View File

@ -3116,14 +3116,17 @@ dependencies = [
"fixed",
"futures 0.3.25",
"itertools 0.10.5",
"lazy_static",
"log 0.4.17",
"mango-v4",
"mango-v4-client",
"prometheus",
"pyth-sdk-solana",
"serum_dex 0.5.10",
"solana-client",
"solana-sdk",
"tokio",
"warp",
]
[[package]]
@ -4153,6 +4156,21 @@ dependencies = [
"yansi",
]
[[package]]
name = "prometheus"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c"
dependencies = [
"cfg-if 1.0.0",
"fnv",
"lazy_static",
"memchr",
"parking_lot 0.12.1",
"protobuf",
"thiserror",
]
[[package]]
name = "prost"
version = "0.11.3"
@ -4208,6 +4226,12 @@ dependencies = [
"prost",
]
[[package]]
name = "protobuf"
version = "2.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
[[package]]
name = "protobuf-src"
version = "1.1.0+21.5"

3
bin/keeper/Cargo.toml
View File

@ -26,3 +26,6 @@ serum_dex = { git = "https://github.com/openbook-dex/program.git", default-featu
solana-client = "~1.14.9"
solana-sdk = "~1.14.9"
tokio = { version = "1.14.1", features = ["rt-multi-thread", "time", "macros", "sync"] }
prometheus = "0.13.3"
warp = "0.3.3"
lazy_static = "1.4.0"

88
bin/keeper/src/crank.rs
View File

@ -6,14 +6,72 @@ use itertools::Itertools;
use anchor_lang::{__private::bytemuck::cast_ref, solana_program};
use futures::Future;
use mango_v4::state::{EventQueue, EventType, FillEvent, OutEvent, PerpMarket, TokenIndex};
use prometheus::{register_histogram, Encoder, Histogram, IntCounter, Registry};
use solana_sdk::{
instruction::{AccountMeta, Instruction},
pubkey::Pubkey,
};
use tokio::time;
use warp::Filter;
lazy_static::lazy_static! {
pub static ref METRICS_REGISTRY: Registry = Registry::new_custom(Some("keeper".to_string()), None).unwrap();
pub static ref METRIC_UPDATE_TOKENS_SUCCESS: IntCounter =
IntCounter::new("update_tokens_success", "Successful update token transactions").unwrap();
pub static ref METRIC_UPDATE_TOKENS_FAILURE: IntCounter =
IntCounter::new("update_tokens_failure", "Failed update token transactions").unwrap();
pub static ref METRIC_CONSUME_EVENTS_SUCCESS: IntCounter =
IntCounter::new("consume_events_success", "Successful consume events transactions").unwrap();
pub static ref METRIC_CONSUME_EVENTS_FAILURE: IntCounter =
IntCounter::new("consume_events_failure", "Failed consume events transactions").unwrap();
pub static ref METRIC_UPDATE_FUNDING_SUCCESS: IntCounter =
IntCounter::new("update_funding_success", "Successful update funding transactions").unwrap();
pub static ref METRIC_UPDATE_FUNDING_FAILURE: IntCounter =
IntCounter::new("update_funding_failure", "Failed update funding transactions").unwrap();
pub static ref METRIC_CONFIRMATION_TIMES: Histogram = register_histogram!(
"confirmation_times", "Transaction confirmation times",
vec![1000.0, 3000.0, 5000.0, 7000.0, 10000.0, 15000.0, 20000.0, 30000.0, 40000.0, 50000.0, 60000.0]
).unwrap();
}
// TODO: move instructions into the client proper
async fn serve_metrics() {
METRICS_REGISTRY
.register(Box::new(METRIC_UPDATE_TOKENS_SUCCESS.clone()))
.unwrap();
METRICS_REGISTRY
.register(Box::new(METRIC_UPDATE_TOKENS_FAILURE.clone()))
.unwrap();
METRICS_REGISTRY
.register(Box::new(METRIC_CONSUME_EVENTS_SUCCESS.clone()))
.unwrap();
METRICS_REGISTRY
.register(Box::new(METRIC_CONSUME_EVENTS_FAILURE.clone()))
.unwrap();
METRICS_REGISTRY
.register(Box::new(METRIC_UPDATE_FUNDING_SUCCESS.clone()))
.unwrap();
METRICS_REGISTRY
.register(Box::new(METRIC_UPDATE_FUNDING_FAILURE.clone()))
.unwrap();
METRICS_REGISTRY
.register(Box::new(METRIC_CONFIRMATION_TIMES.clone()))
.unwrap();
let metrics_route = warp::path!("metrics").map(|| {
let mut buffer = Vec::<u8>::new();
let encoder = prometheus::TextEncoder::new();
encoder
.encode(&METRICS_REGISTRY.gather(), &mut buffer)
.unwrap();
String::from_utf8(buffer.clone()).unwrap()
});
println!("Metrics server starting on port 9091");
warp::serve(metrics_route).run(([0, 0, 0, 0], 9091)).await;
}
pub async fn runner(
mango_client: Arc<MangoClient>,
debugging_handle: impl Future,
@ -83,7 +141,8 @@ pub async fn runner(
mango_client.clone(),
interval_check_new_listings_and_abort
),
debugging_handle
serve_metrics(),
debugging_handle,
);
Ok(())
@ -165,19 +224,24 @@ pub async fn loop_update_index_and_rate(
.send_and_confirm_permissionless_tx(instructions)
.await;
let confirmation_time = pre.elapsed().as_millis();
METRIC_CONFIRMATION_TIMES.observe(confirmation_time as f64);
if let Err(e) = sig_result {
METRIC_UPDATE_TOKENS_FAILURE.inc();
log::info!(
"metricName=UpdateTokensV4Failure tokens={} durationMs={} error={}",
token_names,
pre.elapsed().as_millis(),
confirmation_time,
e
);
log::error!("{:?}", e)
} else {
METRIC_UPDATE_TOKENS_SUCCESS.inc();
log::info!(
"metricName=UpdateTokensV4Success tokens={} durationMs={}",
token_names,
pre.elapsed().as_millis(),
confirmation_time,
);
log::info!("{:?}", sig_result);
}
@ -278,20 +342,25 @@ pub async fn loop_consume_events(
let sig_result = client.send_and_confirm_permissionless_tx(vec![ix]).await;
let confirmation_time = pre.elapsed().as_millis();
METRIC_CONFIRMATION_TIMES.observe(confirmation_time as f64);
if let Err(e) = sig_result {
METRIC_CONSUME_EVENTS_FAILURE.inc();
log::info!(
"metricName=ConsumeEventsV4Failure market={} durationMs={} consumed={} error={}",
perp_market.name(),
pre.elapsed().as_millis(),
confirmation_time,
num_of_events,
e.to_string()
);
log::error!("{:?}", e)
} else {
METRIC_CONSUME_EVENTS_SUCCESS.inc();
log::info!(
"metricName=ConsumeEventsV4Success market={} durationMs={} consumed={}",
perp_market.name(),
pre.elapsed().as_millis(),
confirmation_time,
num_of_events,
);
log::info!("{:?}", sig_result);
@ -328,19 +397,24 @@ pub async fn loop_update_funding(
};
let sig_result = client.send_and_confirm_permissionless_tx(vec![ix]).await;
let confirmation_time = pre.elapsed().as_millis();
METRIC_CONFIRMATION_TIMES.observe(confirmation_time as f64);
if let Err(e) = sig_result {
METRIC_UPDATE_FUNDING_FAILURE.inc();
log::error!(
"metricName=UpdateFundingV4Error market={} durationMs={} error={}",
perp_market.name(),
pre.elapsed().as_millis(),
confirmation_time,
e.to_string()
);
log::error!("{:?}", e)
} else {
METRIC_UPDATE_FUNDING_SUCCESS.inc();
log::info!(
"metricName=UpdateFundingV4Success market={} durationMs={}",
perp_market.name(),
pre.elapsed().as_millis(),
confirmation_time,
);
log::info!("{:?}", sig_result);
}

4
cd/keeper.toml
View File

@ -4,3 +4,7 @@ kill_timeout = 30
[build]
dockerfile = "../bin/keeper/Dockerfile.keeper"
[metrics]
port = 9091
path = "/metrics"