From 21f614b286f81a3df3c089d0c2b3c24cecc30b7d Mon Sep 17 00:00:00 2001 From: Riordan Panayides Date: Mon, 21 Nov 2022 03:47:20 +0000 Subject: [PATCH] Update fills feed, disable connector-mango --- Cargo.lock | 136 +++++++++++++++++++++++--------- Cargo.toml | 12 ++- lib/src/fill_event_filter.rs | 2 - service-mango-fills/src/main.rs | 2 +- service-mango-pnl/Cargo.toml | 3 + service-mango-pnl/src/main.rs | 128 +++++++++++++++++++++--------- 6 files changed, 197 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5e77621..a3a0c62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -104,7 +104,7 @@ checksum = "6b2d54853319fd101b8dd81de382bcbf3e03410a64d8928bbee85a3e7dcde483" [[package]] name = "anchor-attribute-access-control" version = "0.25.0" -source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12" +source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9" dependencies = [ "anchor-syn", "anyhow", @@ -117,7 +117,7 @@ dependencies = [ [[package]] name = "anchor-attribute-account" version = "0.25.0" -source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12" +source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9" dependencies = [ "anchor-syn", "anyhow", @@ -131,7 +131,7 @@ dependencies = [ [[package]] name = "anchor-attribute-constant" version = "0.25.0" -source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12" +source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9" dependencies = [ "anchor-syn", "proc-macro2 1.0.47", @@ -141,7 +141,7 @@ dependencies = [ [[package]] name = "anchor-attribute-error" version = "0.25.0" -source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12" +source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9" dependencies = [ "anchor-syn", "proc-macro2 1.0.47", @@ -152,7 +152,7 @@ dependencies = [ [[package]] name = "anchor-attribute-event" version = "0.25.0" -source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12" +source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9" dependencies = [ "anchor-syn", "anyhow", @@ -164,7 +164,7 @@ dependencies = [ [[package]] name = "anchor-attribute-interface" version = "0.25.0" -source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12" +source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9" dependencies = [ "anchor-syn", "anyhow", @@ -177,7 +177,7 @@ dependencies = [ [[package]] name = "anchor-attribute-program" version = "0.25.0" -source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12" +source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9" dependencies = [ "anchor-syn", "anyhow", @@ -189,7 +189,7 @@ dependencies = [ [[package]] name = "anchor-attribute-state" version = "0.25.0" -source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12" +source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9" dependencies = [ "anchor-syn", "anyhow", @@ -198,10 +198,26 @@ dependencies = [ "syn 1.0.103", ] +[[package]] +name = "anchor-client" +version = "0.25.0" +source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9" +dependencies = [ + "anchor-lang", + "anyhow", + "regex", + "serde", + "solana-account-decoder", + "solana-client", + "solana-sdk", + "thiserror", + "url 2.3.1", +] + [[package]] name = "anchor-derive-accounts" version = "0.25.0" -source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12" +source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9" dependencies = [ "anchor-syn", "anyhow", @@ -213,7 +229,7 @@ dependencies = [ [[package]] name = "anchor-lang" version = "0.25.0" -source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12" +source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#5d780a86dca69ef23c527217ae18358c3c9c2724" dependencies = [ "anchor-attribute-access-control", "anchor-attribute-account", @@ -236,7 +252,7 @@ dependencies = [ [[package]] name = "anchor-spl" version = "0.25.0" -source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12" +source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9" dependencies = [ "anchor-lang", "solana-program", @@ -247,7 +263,7 @@ dependencies = [ [[package]] name = "anchor-syn" version = "0.25.0" -source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12" +source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9" dependencies = [ "anyhow", "bs58 0.3.1", @@ -925,7 +941,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "checked_math" version = "0.1.0" -source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12" +source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9" dependencies = [ "proc-macro-error", "proc-macro2 1.0.47", @@ -1003,6 +1019,36 @@ dependencies = [ "vec_map", ] +[[package]] +name = "client" +version = "0.1.0" +source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9" +dependencies = [ + "anchor-client", + "anchor-lang", + "anchor-spl", + "anyhow", + "base64 0.13.1", + "bincode", + "fixed", + "fixed-macro", + "itertools 0.10.5", + "log 0.4.17", + "mango-v4", + "pyth-sdk-solana", + "reqwest", + "serde", + "serde_json", + "serum_dex", + "shellexpand", + "solana-account-decoder", + "solana-client", + "solana-sdk", + "spl-associated-token-account", + "thiserror", + "tokio", +] + [[package]] name = "cloudabi" version = "0.0.3" @@ -1423,6 +1469,15 @@ dependencies = [ "walkdir", ] +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -1433,6 +1488,17 @@ dependencies = [ "dirs-sys-next", ] +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi 0.3.9", +] + [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -2885,7 +2951,7 @@ dependencies = [ [[package]] name = "mango-macro" version = "0.0.1" -source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12" +source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9" dependencies = [ "bytemuck", "quote 1.0.21", @@ -2895,7 +2961,7 @@ dependencies = [ [[package]] name = "mango-v4" version = "0.1.0" -source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12" +source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9" dependencies = [ "anchor-lang", "anchor-spl", @@ -2914,6 +2980,7 @@ dependencies = [ "serum_dex", "solana-address-lookup-table-program", "solana-program", + "solana-sdk", "static_assertions", "switchboard-program", "switchboard-utils", @@ -4880,8 +4947,8 @@ dependencies = [ [[package]] name = "serum_dex" -version = "0.4.0" -source = "git+ssh://git@github.com/blockworks-foundation/serum-dex.git?branch=v4-compat#7033229b816af91cdeef3f14aff23a5e20be11f1" +version = "0.5.6" +source = "git+https://github.com/blockworks-foundation/serum-dex.git?branch=ckamm/find_by_key#c0b9f559daa74ec5960cc371a8bed610029a656b" dependencies = [ "arrayref", "bincode", @@ -4911,7 +4978,7 @@ dependencies = [ "bs58 0.4.0", "futures-channel", "futures-util", - "log 0.4.17", + "log 0.3.9", "serde", "serde_derive", "serde_json", @@ -4927,10 +4994,13 @@ dependencies = [ name = "service-mango-pnl" version = "0.1.0" dependencies = [ + "anchor-client", + "anchor-lang", "anyhow", "async-trait", "bs58 0.3.1", "bytemuck", + "client", "fixed", "jsonrpsee", "log 0.4.17", @@ -5052,6 +5122,15 @@ dependencies = [ "keccak", ] +[[package]] +name = "shellexpand" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ccc8076840c4da029af4f87e4e8daeb0fca6b87bbb02e10cb60b791450e11e4" +dependencies = [ + "dirs", +] + [[package]] name = "shlex" version = "1.1.0" @@ -5461,7 +5540,7 @@ dependencies = [ "async-channel", "async-stream 0.2.1", "async-trait", - "base64 0.13.1", + "base64 0.9.3", "bs58 0.4.0", "bytemuck", "bytes 1.2.1", @@ -5495,25 +5574,6 @@ dependencies = [ "warp", ] -[[package]] -name = "solana-geyser-connector-mango" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "bs58 0.3.1", - "fixed", - "log 0.4.17", - "mango-v4", - "postgres-types", - "postgres_query", - "solana-geyser-connector-lib", - "solana-logger", - "tokio", - "tokio-postgres", - "toml", -] - [[package]] name = "solana-geyser-connector-plugin-grpc" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index bbe1001..dbe30be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = [ "geyser-plugin-grpc", "lib", "connector-raw", - "connector-mango", +# "connector-mango", "service-mango-fills", "service-mango-pnl", ] @@ -12,9 +12,7 @@ members = [ [patch.crates-io] # for gzip encoded responses jsonrpc-core-client = { git = "https://github.com/ckamm/jsonrpc.git", branch = "ckamm/http-with-gzip" } -anchor-spl = { git = "ssh://git@github.com/blockworks-foundation/mango-v4", branch = "solana-1.10.35" } -anchor-lang = { git = "ssh://git@github.com/blockworks-foundation/mango-v4", branch = "solana-1.10.35" } - -[patch.'https://github.com/blockworks-foundation/serum-dex.git'] -serum_dex = { git = 'ssh://git@github.com/blockworks-foundation/serum-dex.git', branch = "v4-compat" } - +# force usage of mango-v4 anchor submodule with 1.10.35 support +anchor-spl = { git = "ssh://git@github.com/blockworks-foundation/mango-v4", branch = "dev" } +anchor-lang = { git = "ssh://git@github.com/blockworks-foundation/mango-v4", branch = "dev" } +anchor-client = { git = "ssh://git@github.com/blockworks-foundation/mango-v4", branch = "dev" } diff --git a/lib/src/fill_event_filter.rs b/lib/src/fill_event_filter.rs index 6bdcd9c..3c5562c 100644 --- a/lib/src/fill_event_filter.rs +++ b/lib/src/fill_event_filter.rs @@ -14,13 +14,11 @@ use std::{ borrow::BorrowMut, cmp::max, collections::{HashMap, HashSet}, - mem::size_of, str::FromStr, }; use crate::metrics::MetricU64; use anchor_lang::AccountDeserialize; -use arrayref::array_ref; use mango_v4::state::{ AnyEvent, EventQueue, EventQueueHeader, EventType, FillEvent, MAX_NUM_EVENTS, }; diff --git a/service-mango-fills/src/main.rs b/service-mango-fills/src/main.rs index 2b9cb52..91492c3 100644 --- a/service-mango-fills/src/main.rs +++ b/service-mango-fills/src/main.rs @@ -177,7 +177,7 @@ async fn main() -> anyhow::Result<()> { .map(|c| c.connection_string.clone()) .collect::() ); - let use_geyser = false; + let use_geyser = true; if use_geyser { grpc_plugin_source::process_events( &config.source, diff --git a/service-mango-pnl/Cargo.toml b/service-mango-pnl/Cargo.toml index 340ad35..ad26ff9 100644 --- a/service-mango-pnl/Cargo.toml +++ b/service-mango-pnl/Cargo.toml @@ -22,3 +22,6 @@ tokio = { version = "1", features = ["full"] } serde = "1.0.130" serde_derive = "1.0.130" mango-v4 = { git = "ssh://git@github.com/blockworks-foundation/mango-v4", branch = "dev" } +client = { git = "ssh://git@github.com/blockworks-foundation/mango-v4", branch = "dev" } +anchor-lang = "0.25.0" +anchor-client = "0.25.0" diff --git a/service-mango-pnl/src/main.rs b/service-mango-pnl/src/main.rs index 5394d8a..d4badef 100644 --- a/service-mango-pnl/src/main.rs +++ b/service-mango-pnl/src/main.rs @@ -8,22 +8,25 @@ use { std::{ fs::File, io::Read, + mem::size_of, sync::{Arc, RwLock}, + time::Duration, }, }; +use anchor_client::Cluster; +use anchor_lang::Discriminator; +use client::{chain_data, health_cache, AccountFetcher, Client, MangoGroupContext}; use fixed::types::I80F48; -use mango::state::{DataType, MangoAccount, MangoCache, MangoGroup, MAX_PAIRS}; +use mango_v4::state::{MangoAccount, MangoAccountValue, PerpMarketIndex}; use solana_geyser_connector_lib::metrics::*; -use solana_sdk::account::ReadableAccount; -use std::mem::size_of; - +use solana_sdk::commitment_config::CommitmentConfig; +use solana_sdk::{account::ReadableAccount, signature::Keypair}; #[derive(Clone, Debug, Deserialize)] pub struct PnlConfig { pub update_interval_millis: u64, pub mango_program: String, pub mango_group: String, - pub mango_cache: String, } #[derive(Clone, Debug, Deserialize)] @@ -34,39 +37,54 @@ pub struct JsonRpcConfig { #[derive(Clone, Debug, Deserialize)] pub struct Config { pub source: SourceConfig, + pub snapshot_source: SnapshotSourceConfig, pub metrics: MetricsConfig, pub pnl: PnlConfig, pub jsonrpc_server: JsonRpcConfig, } -type PnlData = Vec<(Pubkey, [I80F48; MAX_PAIRS])>; +type PnlData = Vec<(Pubkey, Vec<(PerpMarketIndex, I80F48)>)>; fn compute_pnl( - account: &MangoAccount, - market_index: usize, - group: &MangoGroup, - cache: &MangoCache, -) -> I80F48 { - let perp_account = &account.perp_accounts[market_index]; - let perp_market_cache = &cache.perp_market_cache[market_index]; - let price = cache.price_cache[market_index].price; - let contract_size = group.perp_markets[market_index].base_lot_size; + context: Arc, + account_fetcher: Arc, + account: &MangoAccountValue, +) -> anyhow::Result> { + let health_cache = health_cache::new(&context, account_fetcher.as_ref(), account)?; + let perp_settle_health = health_cache.perp_settle_health(); - let base_pos = I80F48::from_num(perp_account.base_position * contract_size) * price; - let quote_pos = perp_account.get_quote_position(&perp_market_cache); - base_pos + quote_pos + let pnls = account + .active_perp_positions() + .filter_map(|pp| { + if pp.base_position_lots() != 0 { + return None; + } + let pnl = pp.quote_position_native(); + let settleable_pnl = if pnl > 0 { + pnl + } else if pnl < 0 && perp_settle_health > 0 { + pnl.max(-perp_settle_health) + } else { + return None; + }; + Some((pp.market_index, settleable_pnl)) + }) + .collect::>(); + + Ok(pnls) } // regularly updates pnl_data from chain_data fn start_pnl_updater( config: PnlConfig, + context: Arc, + account_fetcher: Arc, chain_data: Arc>, pnl_data: Arc>, metrics_pnls_tracked: MetricU64, ) { let program_pk = Pubkey::from_str(&config.mango_program).unwrap(); let group_pk = Pubkey::from_str(&config.mango_group).unwrap(); - let cache_pk = Pubkey::from_str(&config.mango_cache).unwrap(); tokio::spawn(async move { loop { @@ -79,33 +97,29 @@ fn start_pnl_updater( // get the group and cache now let group = snapshot.get(&group_pk); - let cache = snapshot.get(&cache_pk); - if group.is_none() || cache.is_none() { + if group.is_none() { continue; } - let group: &MangoGroup = bytemuck::from_bytes(group.unwrap().account.data()); - let cache: &MangoCache = bytemuck::from_bytes(cache.unwrap().account.data()); let mut pnls = Vec::with_capacity(snapshot.len()); for (pubkey, account) in snapshot.iter() { let owner = account.account.owner(); let data = account.account.data(); + if data.len() != size_of::() - || data[0] != DataType::MangoAccount as u8 + || data[0..8] != MangoAccount::discriminator() || owner != &program_pk { continue; } - let mango_account: &MangoAccount = bytemuck::from_bytes(data); - if mango_account.mango_group != group_pk { + let mango_account = MangoAccountValue::from_bytes(&data[8..]).unwrap(); + if mango_account.fixed.group != group_pk { continue; } - let mut pnl_vals = [I80F48::ZERO; MAX_PAIRS]; - for market_index in 0..MAX_PAIRS { - pnl_vals[market_index] = compute_pnl(mango_account, market_index, group, cache); - } + let pnl_vals = + compute_pnl(context.clone(), account_fetcher.clone(), &mango_account).unwrap(); // Alternatively, we could prepare the sorted and limited lists for each // market here. That would be faster and cause less contention on the pnl_data @@ -158,11 +172,11 @@ fn start_jsonrpc_server( metrics_invalid_reqs.clone().increment(); return invalid("'limit' must be <= 20"); } - let market_index = req.market_index as usize; - if market_index >= MAX_PAIRS { - metrics_invalid_reqs.clone().increment(); - return invalid("'market_index' must be < MAX_PAIRS"); - } + let market_index = req.market_index as u16; + // if market_index >= MAX_PAIRS { + // metrics_invalid_reqs.clone().increment(); + // return invalid("'market_index' must be < MAX_PAIRS"); + // } if req.order != "ASC" && req.order != "DESC" { metrics_invalid_reqs.clone().increment(); return invalid("'order' must be ASC or DESC"); @@ -171,15 +185,29 @@ fn start_jsonrpc_server( // write lock, because we sort in-place... let mut pnls = pnl_data.write().unwrap(); if req.order == "ASC" { - pnls.sort_unstable_by(|a, b| a.1[market_index].cmp(&b.1[market_index])); + pnls.sort_unstable_by(|a, b| { + a.1.iter() + .find(|x| x.0 == market_index) + .cmp(&b.1.iter().find(|x| x.0 == market_index)) + }); } else { - pnls.sort_unstable_by(|a, b| b.1[market_index].cmp(&a.1[market_index])); + pnls.sort_unstable_by(|a, b| { + b.1.iter() + .find(|x| x.0 == market_index) + .cmp(&a.1.iter().find(|x| x.0 == market_index)) + }); } let response = pnls .iter() .take(limit) .map(|p| PnlResponseItem { - pnl: p.1[market_index].to_num::(), + pnl: p + .1 + .iter() + .find(|x| x.0 == market_index) + .unwrap() + .1 + .to_num::(), pubkey: p.0.to_string(), }) .collect::>(); @@ -208,6 +236,28 @@ async fn main() -> anyhow::Result<()> { solana_logger::setup_with_default("info"); info!("startup"); + let rpc_url = config.snapshot_source.rpc_http_url; + let ws_url = rpc_url.replace("https", "wss"); + let rpc_timeout = Duration::from_secs(10); + let cluster = Cluster::Custom(rpc_url.clone(), ws_url.clone()); + let commitment = CommitmentConfig::processed(); + let client = Client::new( + cluster.clone(), + commitment, + &Keypair::new(), + Some(rpc_timeout), + ); + let group_context = Arc::new(MangoGroupContext::new_from_rpc( + Pubkey::from_str(&config.pnl.mango_group).unwrap(), + client.cluster.clone(), + client.commitment, + )?); + let chain_data = Arc::new(RwLock::new(chain_data::ChainData::new())); + let account_fetcher = Arc::new(chain_data::AccountFetcher { + chain_data: chain_data.clone(), + rpc: client.rpc(), + }); + let metrics_tx = metrics::start(config.metrics, "pnl".into()); let metrics_reqs = @@ -221,6 +271,8 @@ async fn main() -> anyhow::Result<()> { start_pnl_updater( config.pnl.clone(), + group_context.clone(), + account_fetcher.clone(), chain_data.clone(), pnl_data.clone(), metrics_pnls_tracked,