Update fills feed, disable connector-mango

This commit is contained in:
Riordan Panayides 2022-11-21 03:47:20 +00:00
parent cf26a885f2
commit 21f614b286
6 changed files with 197 additions and 86 deletions

136
Cargo.lock generated
View File

@ -104,7 +104,7 @@ checksum = "6b2d54853319fd101b8dd81de382bcbf3e03410a64d8928bbee85a3e7dcde483"
[[package]]
name = "anchor-attribute-access-control"
version = "0.25.0"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9"
dependencies = [
"anchor-syn",
"anyhow",
@ -117,7 +117,7 @@ dependencies = [
[[package]]
name = "anchor-attribute-account"
version = "0.25.0"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9"
dependencies = [
"anchor-syn",
"anyhow",
@ -131,7 +131,7 @@ dependencies = [
[[package]]
name = "anchor-attribute-constant"
version = "0.25.0"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9"
dependencies = [
"anchor-syn",
"proc-macro2 1.0.47",
@ -141,7 +141,7 @@ dependencies = [
[[package]]
name = "anchor-attribute-error"
version = "0.25.0"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9"
dependencies = [
"anchor-syn",
"proc-macro2 1.0.47",
@ -152,7 +152,7 @@ dependencies = [
[[package]]
name = "anchor-attribute-event"
version = "0.25.0"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9"
dependencies = [
"anchor-syn",
"anyhow",
@ -164,7 +164,7 @@ dependencies = [
[[package]]
name = "anchor-attribute-interface"
version = "0.25.0"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9"
dependencies = [
"anchor-syn",
"anyhow",
@ -177,7 +177,7 @@ dependencies = [
[[package]]
name = "anchor-attribute-program"
version = "0.25.0"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9"
dependencies = [
"anchor-syn",
"anyhow",
@ -189,7 +189,7 @@ dependencies = [
[[package]]
name = "anchor-attribute-state"
version = "0.25.0"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9"
dependencies = [
"anchor-syn",
"anyhow",
@ -198,10 +198,26 @@ dependencies = [
"syn 1.0.103",
]
[[package]]
name = "anchor-client"
version = "0.25.0"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9"
dependencies = [
"anchor-lang",
"anyhow",
"regex",
"serde",
"solana-account-decoder",
"solana-client",
"solana-sdk",
"thiserror",
"url 2.3.1",
]
[[package]]
name = "anchor-derive-accounts"
version = "0.25.0"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9"
dependencies = [
"anchor-syn",
"anyhow",
@ -213,7 +229,7 @@ dependencies = [
[[package]]
name = "anchor-lang"
version = "0.25.0"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#5d780a86dca69ef23c527217ae18358c3c9c2724"
dependencies = [
"anchor-attribute-access-control",
"anchor-attribute-account",
@ -236,7 +252,7 @@ dependencies = [
[[package]]
name = "anchor-spl"
version = "0.25.0"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9"
dependencies = [
"anchor-lang",
"solana-program",
@ -247,7 +263,7 @@ dependencies = [
[[package]]
name = "anchor-syn"
version = "0.25.0"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9"
dependencies = [
"anyhow",
"bs58 0.3.1",
@ -925,7 +941,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "checked_math"
version = "0.1.0"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9"
dependencies = [
"proc-macro-error",
"proc-macro2 1.0.47",
@ -1003,6 +1019,36 @@ dependencies = [
"vec_map",
]
[[package]]
name = "client"
version = "0.1.0"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9"
dependencies = [
"anchor-client",
"anchor-lang",
"anchor-spl",
"anyhow",
"base64 0.13.1",
"bincode",
"fixed",
"fixed-macro",
"itertools 0.10.5",
"log 0.4.17",
"mango-v4",
"pyth-sdk-solana",
"reqwest",
"serde",
"serde_json",
"serum_dex",
"shellexpand",
"solana-account-decoder",
"solana-client",
"solana-sdk",
"spl-associated-token-account",
"thiserror",
"tokio",
]
[[package]]
name = "cloudabi"
version = "0.0.3"
@ -1423,6 +1469,15 @@ dependencies = [
"walkdir",
]
[[package]]
name = "dirs"
version = "4.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059"
dependencies = [
"dirs-sys",
]
[[package]]
name = "dirs-next"
version = "2.0.0"
@ -1433,6 +1488,17 @@ dependencies = [
"dirs-sys-next",
]
[[package]]
name = "dirs-sys"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
dependencies = [
"libc",
"redox_users",
"winapi 0.3.9",
]
[[package]]
name = "dirs-sys-next"
version = "0.1.2"
@ -2885,7 +2951,7 @@ dependencies = [
[[package]]
name = "mango-macro"
version = "0.0.1"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9"
dependencies = [
"bytemuck",
"quote 1.0.21",
@ -2895,7 +2961,7 @@ dependencies = [
[[package]]
name = "mango-v4"
version = "0.1.0"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=solana-1.10.35#ca11d73be8bde6730dcdf9a6e467d6dd8341ab12"
source = "git+ssh://git@github.com/blockworks-foundation/mango-v4?branch=dev#19e2b8639407c8599c1f62d3b7fd282420e3c9d9"
dependencies = [
"anchor-lang",
"anchor-spl",
@ -2914,6 +2980,7 @@ dependencies = [
"serum_dex",
"solana-address-lookup-table-program",
"solana-program",
"solana-sdk",
"static_assertions",
"switchboard-program",
"switchboard-utils",
@ -4880,8 +4947,8 @@ dependencies = [
[[package]]
name = "serum_dex"
version = "0.4.0"
source = "git+ssh://git@github.com/blockworks-foundation/serum-dex.git?branch=v4-compat#7033229b816af91cdeef3f14aff23a5e20be11f1"
version = "0.5.6"
source = "git+https://github.com/blockworks-foundation/serum-dex.git?branch=ckamm/find_by_key#c0b9f559daa74ec5960cc371a8bed610029a656b"
dependencies = [
"arrayref",
"bincode",
@ -4911,7 +4978,7 @@ dependencies = [
"bs58 0.4.0",
"futures-channel",
"futures-util",
"log 0.4.17",
"log 0.3.9",
"serde",
"serde_derive",
"serde_json",
@ -4927,10 +4994,13 @@ dependencies = [
name = "service-mango-pnl"
version = "0.1.0"
dependencies = [
"anchor-client",
"anchor-lang",
"anyhow",
"async-trait",
"bs58 0.3.1",
"bytemuck",
"client",
"fixed",
"jsonrpsee",
"log 0.4.17",
@ -5052,6 +5122,15 @@ dependencies = [
"keccak",
]
[[package]]
name = "shellexpand"
version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ccc8076840c4da029af4f87e4e8daeb0fca6b87bbb02e10cb60b791450e11e4"
dependencies = [
"dirs",
]
[[package]]
name = "shlex"
version = "1.1.0"
@ -5461,7 +5540,7 @@ dependencies = [
"async-channel",
"async-stream 0.2.1",
"async-trait",
"base64 0.13.1",
"base64 0.9.3",
"bs58 0.4.0",
"bytemuck",
"bytes 1.2.1",
@ -5495,25 +5574,6 @@ dependencies = [
"warp",
]
[[package]]
name = "solana-geyser-connector-mango"
version = "0.1.0"
dependencies = [
"anyhow",
"async-trait",
"bs58 0.3.1",
"fixed",
"log 0.4.17",
"mango-v4",
"postgres-types",
"postgres_query",
"solana-geyser-connector-lib",
"solana-logger",
"tokio",
"tokio-postgres",
"toml",
]
[[package]]
name = "solana-geyser-connector-plugin-grpc"
version = "0.1.0"

View File

@ -3,7 +3,7 @@ members = [
"geyser-plugin-grpc",
"lib",
"connector-raw",
"connector-mango",
# "connector-mango",
"service-mango-fills",
"service-mango-pnl",
]
@ -12,9 +12,7 @@ members = [
[patch.crates-io]
# for gzip encoded responses
jsonrpc-core-client = { git = "https://github.com/ckamm/jsonrpc.git", branch = "ckamm/http-with-gzip" }
anchor-spl = { git = "ssh://git@github.com/blockworks-foundation/mango-v4", branch = "solana-1.10.35" }
anchor-lang = { git = "ssh://git@github.com/blockworks-foundation/mango-v4", branch = "solana-1.10.35" }
[patch.'https://github.com/blockworks-foundation/serum-dex.git']
serum_dex = { git = 'ssh://git@github.com/blockworks-foundation/serum-dex.git', branch = "v4-compat" }
# force usage of mango-v4 anchor submodule with 1.10.35 support
anchor-spl = { git = "ssh://git@github.com/blockworks-foundation/mango-v4", branch = "dev" }
anchor-lang = { git = "ssh://git@github.com/blockworks-foundation/mango-v4", branch = "dev" }
anchor-client = { git = "ssh://git@github.com/blockworks-foundation/mango-v4", branch = "dev" }

View File

@ -14,13 +14,11 @@ use std::{
borrow::BorrowMut,
cmp::max,
collections::{HashMap, HashSet},
mem::size_of,
str::FromStr,
};
use crate::metrics::MetricU64;
use anchor_lang::AccountDeserialize;
use arrayref::array_ref;
use mango_v4::state::{
AnyEvent, EventQueue, EventQueueHeader, EventType, FillEvent, MAX_NUM_EVENTS,
};

View File

@ -177,7 +177,7 @@ async fn main() -> anyhow::Result<()> {
.map(|c| c.connection_string.clone())
.collect::<String>()
);
let use_geyser = false;
let use_geyser = true;
if use_geyser {
grpc_plugin_source::process_events(
&config.source,

View File

@ -22,3 +22,6 @@ tokio = { version = "1", features = ["full"] }
serde = "1.0.130"
serde_derive = "1.0.130"
mango-v4 = { git = "ssh://git@github.com/blockworks-foundation/mango-v4", branch = "dev" }
client = { git = "ssh://git@github.com/blockworks-foundation/mango-v4", branch = "dev" }
anchor-lang = "0.25.0"
anchor-client = "0.25.0"

View File

@ -8,22 +8,25 @@ use {
std::{
fs::File,
io::Read,
mem::size_of,
sync::{Arc, RwLock},
time::Duration,
},
};
use anchor_client::Cluster;
use anchor_lang::Discriminator;
use client::{chain_data, health_cache, AccountFetcher, Client, MangoGroupContext};
use fixed::types::I80F48;
use mango::state::{DataType, MangoAccount, MangoCache, MangoGroup, MAX_PAIRS};
use mango_v4::state::{MangoAccount, MangoAccountValue, PerpMarketIndex};
use solana_geyser_connector_lib::metrics::*;
use solana_sdk::account::ReadableAccount;
use std::mem::size_of;
use solana_sdk::commitment_config::CommitmentConfig;
use solana_sdk::{account::ReadableAccount, signature::Keypair};
#[derive(Clone, Debug, Deserialize)]
pub struct PnlConfig {
pub update_interval_millis: u64,
pub mango_program: String,
pub mango_group: String,
pub mango_cache: String,
}
#[derive(Clone, Debug, Deserialize)]
@ -34,39 +37,54 @@ pub struct JsonRpcConfig {
#[derive(Clone, Debug, Deserialize)]
pub struct Config {
pub source: SourceConfig,
pub snapshot_source: SnapshotSourceConfig,
pub metrics: MetricsConfig,
pub pnl: PnlConfig,
pub jsonrpc_server: JsonRpcConfig,
}
type PnlData = Vec<(Pubkey, [I80F48; MAX_PAIRS])>;
type PnlData = Vec<(Pubkey, Vec<(PerpMarketIndex, I80F48)>)>;
fn compute_pnl(
account: &MangoAccount,
market_index: usize,
group: &MangoGroup,
cache: &MangoCache,
) -> I80F48 {
let perp_account = &account.perp_accounts[market_index];
let perp_market_cache = &cache.perp_market_cache[market_index];
let price = cache.price_cache[market_index].price;
let contract_size = group.perp_markets[market_index].base_lot_size;
context: Arc<MangoGroupContext>,
account_fetcher: Arc<impl AccountFetcher>,
account: &MangoAccountValue,
) -> anyhow::Result<Vec<(PerpMarketIndex, I80F48)>> {
let health_cache = health_cache::new(&context, account_fetcher.as_ref(), account)?;
let perp_settle_health = health_cache.perp_settle_health();
let base_pos = I80F48::from_num(perp_account.base_position * contract_size) * price;
let quote_pos = perp_account.get_quote_position(&perp_market_cache);
base_pos + quote_pos
let pnls = account
.active_perp_positions()
.filter_map(|pp| {
if pp.base_position_lots() != 0 {
return None;
}
let pnl = pp.quote_position_native();
let settleable_pnl = if pnl > 0 {
pnl
} else if pnl < 0 && perp_settle_health > 0 {
pnl.max(-perp_settle_health)
} else {
return None;
};
Some((pp.market_index, settleable_pnl))
})
.collect::<Vec<(PerpMarketIndex, I80F48)>>();
Ok(pnls)
}
// regularly updates pnl_data from chain_data
fn start_pnl_updater(
config: PnlConfig,
context: Arc<MangoGroupContext>,
account_fetcher: Arc<impl AccountFetcher + 'static>,
chain_data: Arc<RwLock<ChainData>>,
pnl_data: Arc<RwLock<PnlData>>,
metrics_pnls_tracked: MetricU64,
) {
let program_pk = Pubkey::from_str(&config.mango_program).unwrap();
let group_pk = Pubkey::from_str(&config.mango_group).unwrap();
let cache_pk = Pubkey::from_str(&config.mango_cache).unwrap();
tokio::spawn(async move {
loop {
@ -79,33 +97,29 @@ fn start_pnl_updater(
// get the group and cache now
let group = snapshot.get(&group_pk);
let cache = snapshot.get(&cache_pk);
if group.is_none() || cache.is_none() {
if group.is_none() {
continue;
}
let group: &MangoGroup = bytemuck::from_bytes(group.unwrap().account.data());
let cache: &MangoCache = bytemuck::from_bytes(cache.unwrap().account.data());
let mut pnls = Vec::with_capacity(snapshot.len());
for (pubkey, account) in snapshot.iter() {
let owner = account.account.owner();
let data = account.account.data();
if data.len() != size_of::<MangoAccount>()
|| data[0] != DataType::MangoAccount as u8
|| data[0..8] != MangoAccount::discriminator()
|| owner != &program_pk
{
continue;
}
let mango_account: &MangoAccount = bytemuck::from_bytes(data);
if mango_account.mango_group != group_pk {
let mango_account = MangoAccountValue::from_bytes(&data[8..]).unwrap();
if mango_account.fixed.group != group_pk {
continue;
}
let mut pnl_vals = [I80F48::ZERO; MAX_PAIRS];
for market_index in 0..MAX_PAIRS {
pnl_vals[market_index] = compute_pnl(mango_account, market_index, group, cache);
}
let pnl_vals =
compute_pnl(context.clone(), account_fetcher.clone(), &mango_account).unwrap();
// Alternatively, we could prepare the sorted and limited lists for each
// market here. That would be faster and cause less contention on the pnl_data
@ -158,11 +172,11 @@ fn start_jsonrpc_server(
metrics_invalid_reqs.clone().increment();
return invalid("'limit' must be <= 20");
}
let market_index = req.market_index as usize;
if market_index >= MAX_PAIRS {
metrics_invalid_reqs.clone().increment();
return invalid("'market_index' must be < MAX_PAIRS");
}
let market_index = req.market_index as u16;
// if market_index >= MAX_PAIRS {
// metrics_invalid_reqs.clone().increment();
// return invalid("'market_index' must be < MAX_PAIRS");
// }
if req.order != "ASC" && req.order != "DESC" {
metrics_invalid_reqs.clone().increment();
return invalid("'order' must be ASC or DESC");
@ -171,15 +185,29 @@ fn start_jsonrpc_server(
// write lock, because we sort in-place...
let mut pnls = pnl_data.write().unwrap();
if req.order == "ASC" {
pnls.sort_unstable_by(|a, b| a.1[market_index].cmp(&b.1[market_index]));
pnls.sort_unstable_by(|a, b| {
a.1.iter()
.find(|x| x.0 == market_index)
.cmp(&b.1.iter().find(|x| x.0 == market_index))
});
} else {
pnls.sort_unstable_by(|a, b| b.1[market_index].cmp(&a.1[market_index]));
pnls.sort_unstable_by(|a, b| {
b.1.iter()
.find(|x| x.0 == market_index)
.cmp(&a.1.iter().find(|x| x.0 == market_index))
});
}
let response = pnls
.iter()
.take(limit)
.map(|p| PnlResponseItem {
pnl: p.1[market_index].to_num::<f64>(),
pnl: p
.1
.iter()
.find(|x| x.0 == market_index)
.unwrap()
.1
.to_num::<f64>(),
pubkey: p.0.to_string(),
})
.collect::<Vec<_>>();
@ -208,6 +236,28 @@ async fn main() -> anyhow::Result<()> {
solana_logger::setup_with_default("info");
info!("startup");
let rpc_url = config.snapshot_source.rpc_http_url;
let ws_url = rpc_url.replace("https", "wss");
let rpc_timeout = Duration::from_secs(10);
let cluster = Cluster::Custom(rpc_url.clone(), ws_url.clone());
let commitment = CommitmentConfig::processed();
let client = Client::new(
cluster.clone(),
commitment,
&Keypair::new(),
Some(rpc_timeout),
);
let group_context = Arc::new(MangoGroupContext::new_from_rpc(
Pubkey::from_str(&config.pnl.mango_group).unwrap(),
client.cluster.clone(),
client.commitment,
)?);
let chain_data = Arc::new(RwLock::new(chain_data::ChainData::new()));
let account_fetcher = Arc::new(chain_data::AccountFetcher {
chain_data: chain_data.clone(),
rpc: client.rpc(),
});
let metrics_tx = metrics::start(config.metrics, "pnl".into());
let metrics_reqs =
@ -221,6 +271,8 @@ async fn main() -> anyhow::Result<()> {
start_pnl_updater(
config.pnl.clone(),
group_context.clone(),
account_fetcher.clone(),
chain_data.clone(),
pnl_data.clone(),
metrics_pnls_tracked,