diff --git a/Cargo.lock b/Cargo.lock index 89f175160..6782c6272 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3116,14 +3116,17 @@ dependencies = [ "fixed", "futures 0.3.25", "itertools 0.10.5", + "lazy_static", "log 0.4.17", "mango-v4", "mango-v4-client", + "prometheus", "pyth-sdk-solana", "serum_dex 0.5.10", "solana-client", "solana-sdk", "tokio", + "warp", ] [[package]] @@ -4153,6 +4156,21 @@ dependencies = [ "yansi", ] +[[package]] +name = "prometheus" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" +dependencies = [ + "cfg-if 1.0.0", + "fnv", + "lazy_static", + "memchr", + "parking_lot 0.12.1", + "protobuf", + "thiserror", +] + [[package]] name = "prost" version = "0.11.3" @@ -4208,6 +4226,12 @@ dependencies = [ "prost", ] +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + [[package]] name = "protobuf-src" version = "1.1.0+21.5" diff --git a/bin/keeper/Cargo.toml b/bin/keeper/Cargo.toml index 6c1a23294..7a260a365 100644 --- a/bin/keeper/Cargo.toml +++ b/bin/keeper/Cargo.toml @@ -26,3 +26,6 @@ serum_dex = { git = "https://github.com/openbook-dex/program.git", default-featu solana-client = "~1.14.9" solana-sdk = "~1.14.9" tokio = { version = "1.14.1", features = ["rt-multi-thread", "time", "macros", "sync"] } +prometheus = "0.13.3" +warp = "0.3.3" +lazy_static = "1.4.0" diff --git a/bin/keeper/src/crank.rs b/bin/keeper/src/crank.rs index f4c7f5ca5..572a03721 100644 --- a/bin/keeper/src/crank.rs +++ b/bin/keeper/src/crank.rs @@ -6,14 +6,72 @@ use itertools::Itertools; use anchor_lang::{__private::bytemuck::cast_ref, solana_program}; use futures::Future; use mango_v4::state::{EventQueue, EventType, FillEvent, OutEvent, PerpMarket, TokenIndex}; +use prometheus::{register_histogram, Encoder, Histogram, IntCounter, Registry}; use solana_sdk::{ instruction::{AccountMeta, Instruction}, pubkey::Pubkey, }; use tokio::time; +use warp::Filter; + +lazy_static::lazy_static! { + pub static ref METRICS_REGISTRY: Registry = Registry::new_custom(Some("keeper".to_string()), None).unwrap(); + pub static ref METRIC_UPDATE_TOKENS_SUCCESS: IntCounter = + IntCounter::new("update_tokens_success", "Successful update token transactions").unwrap(); + pub static ref METRIC_UPDATE_TOKENS_FAILURE: IntCounter = + IntCounter::new("update_tokens_failure", "Failed update token transactions").unwrap(); + pub static ref METRIC_CONSUME_EVENTS_SUCCESS: IntCounter = + IntCounter::new("consume_events_success", "Successful consume events transactions").unwrap(); + pub static ref METRIC_CONSUME_EVENTS_FAILURE: IntCounter = + IntCounter::new("consume_events_failure", "Failed consume events transactions").unwrap(); + pub static ref METRIC_UPDATE_FUNDING_SUCCESS: IntCounter = + IntCounter::new("update_funding_success", "Successful update funding transactions").unwrap(); + pub static ref METRIC_UPDATE_FUNDING_FAILURE: IntCounter = + IntCounter::new("update_funding_failure", "Failed update funding transactions").unwrap(); + pub static ref METRIC_CONFIRMATION_TIMES: Histogram = register_histogram!( + "confirmation_times", "Transaction confirmation times", + vec![1000.0, 3000.0, 5000.0, 7000.0, 10000.0, 15000.0, 20000.0, 30000.0, 40000.0, 50000.0, 60000.0] + ).unwrap(); +} // TODO: move instructions into the client proper +async fn serve_metrics() { + METRICS_REGISTRY + .register(Box::new(METRIC_UPDATE_TOKENS_SUCCESS.clone())) + .unwrap(); + METRICS_REGISTRY + .register(Box::new(METRIC_UPDATE_TOKENS_FAILURE.clone())) + .unwrap(); + METRICS_REGISTRY + .register(Box::new(METRIC_CONSUME_EVENTS_SUCCESS.clone())) + .unwrap(); + METRICS_REGISTRY + .register(Box::new(METRIC_CONSUME_EVENTS_FAILURE.clone())) + .unwrap(); + METRICS_REGISTRY + .register(Box::new(METRIC_UPDATE_FUNDING_SUCCESS.clone())) + .unwrap(); + METRICS_REGISTRY + .register(Box::new(METRIC_UPDATE_FUNDING_FAILURE.clone())) + .unwrap(); + METRICS_REGISTRY + .register(Box::new(METRIC_CONFIRMATION_TIMES.clone())) + .unwrap(); + + let metrics_route = warp::path!("metrics").map(|| { + let mut buffer = Vec::::new(); + let encoder = prometheus::TextEncoder::new(); + encoder + .encode(&METRICS_REGISTRY.gather(), &mut buffer) + .unwrap(); + + String::from_utf8(buffer.clone()).unwrap() + }); + println!("Metrics server starting on port 9091"); + warp::serve(metrics_route).run(([0, 0, 0, 0], 9091)).await; +} + pub async fn runner( mango_client: Arc, debugging_handle: impl Future, @@ -83,7 +141,8 @@ pub async fn runner( mango_client.clone(), interval_check_new_listings_and_abort ), - debugging_handle + serve_metrics(), + debugging_handle, ); Ok(()) @@ -165,19 +224,24 @@ pub async fn loop_update_index_and_rate( .send_and_confirm_permissionless_tx(instructions) .await; + let confirmation_time = pre.elapsed().as_millis(); + METRIC_CONFIRMATION_TIMES.observe(confirmation_time as f64); + if let Err(e) = sig_result { + METRIC_UPDATE_TOKENS_FAILURE.inc(); log::info!( "metricName=UpdateTokensV4Failure tokens={} durationMs={} error={}", token_names, - pre.elapsed().as_millis(), + confirmation_time, e ); log::error!("{:?}", e) } else { + METRIC_UPDATE_TOKENS_SUCCESS.inc(); log::info!( "metricName=UpdateTokensV4Success tokens={} durationMs={}", token_names, - pre.elapsed().as_millis(), + confirmation_time, ); log::info!("{:?}", sig_result); } @@ -278,20 +342,25 @@ pub async fn loop_consume_events( let sig_result = client.send_and_confirm_permissionless_tx(vec![ix]).await; + let confirmation_time = pre.elapsed().as_millis(); + METRIC_CONFIRMATION_TIMES.observe(confirmation_time as f64); + if let Err(e) = sig_result { + METRIC_CONSUME_EVENTS_FAILURE.inc(); log::info!( "metricName=ConsumeEventsV4Failure market={} durationMs={} consumed={} error={}", perp_market.name(), - pre.elapsed().as_millis(), + confirmation_time, num_of_events, e.to_string() ); log::error!("{:?}", e) } else { + METRIC_CONSUME_EVENTS_SUCCESS.inc(); log::info!( "metricName=ConsumeEventsV4Success market={} durationMs={} consumed={}", perp_market.name(), - pre.elapsed().as_millis(), + confirmation_time, num_of_events, ); log::info!("{:?}", sig_result); @@ -328,19 +397,24 @@ pub async fn loop_update_funding( }; let sig_result = client.send_and_confirm_permissionless_tx(vec![ix]).await; + let confirmation_time = pre.elapsed().as_millis(); + METRIC_CONFIRMATION_TIMES.observe(confirmation_time as f64); + if let Err(e) = sig_result { + METRIC_UPDATE_FUNDING_FAILURE.inc(); log::error!( "metricName=UpdateFundingV4Error market={} durationMs={} error={}", perp_market.name(), - pre.elapsed().as_millis(), + confirmation_time, e.to_string() ); log::error!("{:?}", e) } else { + METRIC_UPDATE_FUNDING_SUCCESS.inc(); log::info!( "metricName=UpdateFundingV4Success market={} durationMs={}", perp_market.name(), - pre.elapsed().as_millis(), + confirmation_time, ); log::info!("{:?}", sig_result); } diff --git a/cd/keeper.toml b/cd/keeper.toml index 8d126e8b9..2ea15a03b 100644 --- a/cd/keeper.toml +++ b/cd/keeper.toml @@ -4,3 +4,7 @@ kill_timeout = 30 [build] dockerfile = "../bin/keeper/Dockerfile.keeper" + +[metrics] + port = 9091 + path = "/metrics" \ No newline at end of file