From 348d8cfcd85a702f7f0ea5cc74ad6c74c414a54b Mon Sep 17 00:00:00 2001 From: Christian Kamm Date: Sat, 16 Jul 2022 14:37:15 +0200 Subject: [PATCH] Liq/Client: Various improvements - Abstract away account fetching, so it can be done via RPC or from a websocket stream (or a geyser plugin) that populates a ChainData instance. - Separate out information about tokens, markets into MangoGroupContext. - Separate all gPA calls into functions in a new file - The liquidator re-fetches critical accounts via RPC before liquidation. Unfortunately the websocket stream seems slower :/ - Don't re-implement health account derivation in the liquidator. Instead reuse the existing code from the client. - More smaller stuff. --- client/src/account_fetcher.rs | 72 +++ client/src/client.rs | 627 ++++++------------- client/src/context.rs | 263 ++++++++ client/src/gpa.rs | 66 ++ client/src/lib.rs | 7 +- keeper/src/crank.rs | 40 +- keeper/src/main.rs | 2 +- keeper/src/taker.rs | 40 +- liquidator/src/chain_data.rs | 16 + liquidator/src/chain_data_fetcher.rs | 81 +++ liquidator/src/liquidate.rs | 190 ++---- liquidator/src/main.rs | 91 +-- programs/mango-v4/src/state/mango_account.rs | 7 +- 13 files changed, 840 insertions(+), 662 deletions(-) create mode 100644 client/src/account_fetcher.rs create mode 100644 client/src/context.rs create mode 100644 client/src/gpa.rs create mode 100644 liquidator/src/chain_data_fetcher.rs diff --git a/client/src/account_fetcher.rs b/client/src/account_fetcher.rs new file mode 100644 index 000000000..11be7b441 --- /dev/null +++ b/client/src/account_fetcher.rs @@ -0,0 +1,72 @@ +use std::collections::HashMap; +use std::sync::Mutex; + +use anchor_client::ClientError; + +use anchor_lang::AccountDeserialize; + +use solana_client::rpc_client::RpcClient; + +use anyhow::Context; +use solana_sdk::account::Account; +use solana_sdk::pubkey::Pubkey; + +pub trait AccountFetcher: Sync + Send { + fn fetch_raw_account(&self, address: Pubkey) -> anyhow::Result; +} + +// Can't be in the trait, since then it would no longer be object-safe... +pub fn account_fetcher_fetch_anchor_account( + fetcher: &dyn AccountFetcher, + address: Pubkey, +) -> anyhow::Result { + let account = fetcher.fetch_raw_account(address)?; + let mut data: &[u8] = &account.data; + T::try_deserialize(&mut data).with_context(|| format!("deserializing account {}", address)) +} + +pub struct RpcAccountFetcher { + pub rpc: RpcClient, +} + +impl AccountFetcher for RpcAccountFetcher { + fn fetch_raw_account(&self, address: Pubkey) -> anyhow::Result { + self.rpc + .get_account_with_commitment(&address, self.rpc.commitment()) + .with_context(|| format!("fetch account {}", address))? + .value + .ok_or(ClientError::AccountNotFound) + .with_context(|| format!("fetch account {}", address)) + } +} + +pub struct CachedAccountFetcher { + fetcher: T, + cache: Mutex>, +} + +impl CachedAccountFetcher { + pub fn new(fetcher: T) -> Self { + Self { + fetcher, + cache: Mutex::new(HashMap::new()), + } + } + + pub fn clear_cache(&self) { + let mut cache = self.cache.lock().unwrap(); + cache.clear(); + } +} + +impl AccountFetcher for CachedAccountFetcher { + fn fetch_raw_account(&self, address: Pubkey) -> anyhow::Result { + let mut cache = self.cache.lock().unwrap(); + if let Some(account) = cache.get(&address) { + return Ok(account.clone()); + } + let account = self.fetcher.fetch_raw_account(address)?; + cache.insert(address, account.clone()); + Ok(account) + } +} diff --git a/client/src/client.rs b/client/src/client.rs index 1fa24d20f..63842e849 100644 --- a/client/src/client.rs +++ b/client/src/client.rs @@ -1,25 +1,26 @@ -use std::collections::HashMap; +use std::sync::Arc; -use anchor_client::{Client, Cluster, Program}; +use anchor_client::{Client, ClientError, Cluster, Program}; use anchor_lang::__private::bytemuck; use anchor_lang::prelude::System; -use anchor_lang::{AccountDeserialize, Id}; +use anchor_lang::Id; use anchor_spl::associated_token::get_associated_token_address; -use anchor_spl::token::{Mint, Token}; +use anchor_spl::token::Token; use fixed::types::I80F48; use itertools::Itertools; use mango_v4::instructions::{Serum3OrderType, Serum3SelfTradeBehavior, Serum3Side}; -use mango_v4::state::{ - Bank, Group, MangoAccount, MintInfo, PerpMarket, PerpMarketIndex, Serum3Market, TokenIndex, -}; +use mango_v4::state::{Bank, Group, MangoAccount, Serum3MarketIndex, TokenIndex}; use solana_client::rpc_client::RpcClient; +use crate::account_fetcher::*; +use crate::context::{MangoGroupContext, Serum3MarketContext, TokenContext}; +use crate::gpa::fetch_mango_accounts; use crate::util::MyClone; + use anyhow::Context; -use solana_client::rpc_filter::{Memcmp, MemcmpEncodedBytes, RpcFilterType}; use solana_sdk::instruction::{AccountMeta, Instruction}; use solana_sdk::signature::{Keypair, Signature}; use solana_sdk::sysvar; @@ -30,20 +31,15 @@ pub struct MangoClient { pub rpc: RpcClient, pub cluster: Cluster, pub commitment: CommitmentConfig, + + // todo: possibly this object should have cache-functions, so there can be one getMultipleAccounts + // call to refresh banks etc -- if it's backed by websockets, these could just do nothing + pub account_fetcher: Arc, + pub payer: Keypair, - pub mango_account_cache: (Pubkey, MangoAccount), - pub group: Pubkey, - pub group_cache: Group, - // TODO: future: this may not scale if there's thousands of mints, probably some function - // wrapping getMultipleAccounts is needed (or bettew: we provide this data as a service) - pub banks_cache: HashMap>, - pub banks_cache_by_token_index: HashMap>, - pub mint_infos_cache: HashMap, - pub mint_infos_cache_by_token_index: HashMap, - pub serum3_markets_cache: HashMap, - pub serum3_external_markets_cache: HashMap)>, - pub perp_markets_cache: HashMap, - pub perp_markets_cache_by_perp_market_index: HashMap, + pub mango_account_address: Pubkey, + + pub context: MangoGroupContext, } // TODO: add retry framework for sending tx and rpc calls @@ -60,34 +56,48 @@ impl MangoClient { .0 } + /// Conveniently creates a RPC based client pub fn new( cluster: Cluster, commitment: CommitmentConfig, - payer: Keypair, group: Pubkey, + payer: Keypair, mango_account_name: &str, + ) -> anyhow::Result { + let group_context = MangoGroupContext::new_from_rpc(group, cluster.clone(), commitment)?; + + let rpc = RpcClient::new_with_commitment(cluster.url().to_string(), commitment); + let account_fetcher = Arc::new(CachedAccountFetcher::new(RpcAccountFetcher { rpc })); + + Self::new_detail( + cluster, + commitment, + payer, + mango_account_name, + group_context, + account_fetcher, + ) + } + + /// Allows control of AccountFetcher and externally created MangoGroupContext + pub fn new_detail( + cluster: Cluster, + commitment: CommitmentConfig, + payer: Keypair, + mango_account_name: &str, + // future: maybe pass Arc, so it can be extenally updated? + group_context: MangoGroupContext, + account_fetcher: Arc, ) -> anyhow::Result { let program = Client::new_with_options(cluster.clone(), std::rc::Rc::new(payer.clone()), commitment) .program(mango_v4::ID); let rpc = program.rpc(); - - let group_data = program.account::(group)?; + let group = group_context.group; // Mango Account - let mut mango_account_tuples = program.accounts::(vec![ - RpcFilterType::Memcmp(Memcmp { - offset: 8, - bytes: MemcmpEncodedBytes::Base58(group.to_string()), - encoding: None, - }), - RpcFilterType::Memcmp(Memcmp { - offset: 40, - bytes: MemcmpEncodedBytes::Base58(payer.pubkey().to_string()), - encoding: None, - }), - ])?; + let mut mango_account_tuples = fetch_mango_accounts(&program, group, payer.pubkey())?; let mango_account_opt = mango_account_tuples .iter() .find(|tuple| tuple.1.name() == mango_account_name); @@ -133,129 +143,24 @@ impl MangoClient { .send() .context("Failed to create account...")?; } - let mango_account_tuples = program.accounts::(vec![ - RpcFilterType::Memcmp(Memcmp { - offset: 8, - bytes: MemcmpEncodedBytes::Base58(group.to_string()), - encoding: None, - }), - RpcFilterType::Memcmp(Memcmp { - offset: 40, - bytes: MemcmpEncodedBytes::Base58(payer.pubkey().to_string()), - encoding: None, - }), - ])?; + let mango_account_tuples = fetch_mango_accounts(&program, group, payer.pubkey())?; let index = mango_account_tuples .iter() .position(|tuple| tuple.1.name() == mango_account_name) .unwrap(); let mango_account_cache = mango_account_tuples[index]; - // banks cache - let mut banks_cache = HashMap::new(); - let mut banks_cache_by_token_index = HashMap::new(); - let bank_tuples = program.accounts::(vec![RpcFilterType::Memcmp(Memcmp { - offset: 8, - bytes: MemcmpEncodedBytes::Base58(group.to_string()), - encoding: None, - })])?; - for (k, v) in bank_tuples { - banks_cache - .entry(v.name().to_owned()) - .or_insert_with(|| Vec::new()) - .push((k, v)); - banks_cache_by_token_index - .entry(v.token_index) - .or_insert_with(|| Vec::new()) - .push((k, v)); - } - - // mintinfo cache - let mut mint_infos_cache = HashMap::new(); - let mut mint_infos_cache_by_token_index = HashMap::new(); - let mint_info_tuples = - program.accounts::(vec![RpcFilterType::Memcmp(Memcmp { - offset: 8, - bytes: MemcmpEncodedBytes::Base58(group.to_string()), - encoding: None, - })])?; - for (k, v) in mint_info_tuples { - let data = program - .rpc() - .get_account_with_commitment(&v.mint, commitment)? - .value - .unwrap() - .data; - let mint = Mint::try_deserialize(&mut &data[..])?; - - mint_infos_cache.insert(v.mint, (k, v, mint.clone())); - mint_infos_cache_by_token_index.insert(v.token_index, (k, v, mint)); - } - - // serum3 markets cache - let mut serum3_markets_cache = HashMap::new(); - let mut serum3_external_markets_cache = HashMap::new(); - let serum3_market_tuples = - program.accounts::(vec![RpcFilterType::Memcmp(Memcmp { - offset: 8, - bytes: MemcmpEncodedBytes::Base58(group.to_string()), - encoding: None, - })])?; - for (k, v) in serum3_market_tuples { - serum3_markets_cache.insert(v.name().to_owned(), (k, v)); - - let market_external_bytes = program - .rpc() - .get_account_with_commitment(&v.serum_market_external, commitment)? - .value - .unwrap() - .data; - serum3_external_markets_cache.insert( - v.name().to_owned(), - (v.serum_market_external, market_external_bytes), - ); - } - - // perp markets cache - let mut perp_markets_cache = HashMap::new(); - let mut perp_markets_cache_by_perp_market_index = HashMap::new(); - let perp_market_tuples = - program.accounts::(vec![RpcFilterType::Memcmp(Memcmp { - offset: 8, - bytes: MemcmpEncodedBytes::Base58(group.to_string()), - encoding: None, - })])?; - for (k, v) in perp_market_tuples { - perp_markets_cache.insert(v.name().to_owned(), (k, v)); - perp_markets_cache_by_perp_market_index.insert(v.perp_market_index, (k, v)); - } - Ok(Self { rpc, - cluster, + cluster: cluster.clone(), commitment, + account_fetcher, payer, - mango_account_cache, - group, - group_cache: group_data, - banks_cache, - banks_cache_by_token_index, - mint_infos_cache, - mint_infos_cache_by_token_index, - serum3_markets_cache, - serum3_external_markets_cache, - perp_markets_cache, - perp_markets_cache_by_perp_market_index, + mango_account_address: mango_account_cache.0, + context: group_context, }) } - pub fn get_mint_info(&self, token_index: &TokenIndex) -> Pubkey { - self.mint_infos_cache_by_token_index - .get(token_index) - .unwrap() - .0 - } - pub fn client(&self) -> Client { Client::new_with_options( self.cluster.clone(), @@ -273,98 +178,29 @@ impl MangoClient { } pub fn group(&self) -> Pubkey { - self.group + self.context.group } - pub fn get_account(&self) -> Result<(Pubkey, MangoAccount), anchor_client::ClientError> { - let mango_accounts = self.program().accounts::(vec![ - RpcFilterType::Memcmp(Memcmp { - offset: 8, - bytes: MemcmpEncodedBytes::Base58(self.group().to_string()), - encoding: None, - }), - RpcFilterType::Memcmp(Memcmp { - offset: 40, - bytes: MemcmpEncodedBytes::Base58(self.payer().to_string()), - encoding: None, - }), - ])?; - Ok(mango_accounts[0]) + pub fn mango_account(&self) -> anyhow::Result { + account_fetcher_fetch_anchor_account(&*self.account_fetcher, self.mango_account_address) + } + + pub fn first_bank(&self, token_index: TokenIndex) -> anyhow::Result { + let bank_address = self.context.mint_info(token_index).first_bank(); + account_fetcher_fetch_anchor_account(&*self.account_fetcher, bank_address) } pub fn derive_health_check_remaining_account_metas( &self, - affected_bank: Option<(Pubkey, Bank)>, + affected_token: Option, writable_banks: bool, - ) -> Result, anchor_client::ClientError> { - // figure out all the banks/oracles that need to be passed for the health check - let mut banks = vec![]; - let mut oracles = vec![]; - let account = self.get_account()?; - for position in account.1.tokens.iter_active() { - let mint_info = self - .mint_infos_cache_by_token_index - .get(&position.token_index) - .unwrap() - .1; - // TODO: ALTs are unavailable - // let lookup_table = account_loader - // .load_bytes(&mint_info.address_lookup_table) - // .await - // .unwrap(); - // let addresses = mango_v4::address_lookup_table::addresses(&lookup_table); - // banks.push(addresses[mint_info.address_lookup_table_bank_index as usize]); - // oracles.push(addresses[mint_info.address_lookup_table_oracle_index as usize]); - banks.push(mint_info.first_bank()); - oracles.push(mint_info.oracle); - } - if let Some(affected_bank) = affected_bank { - if !banks.iter().any(|&v| v == affected_bank.0) { - // If there is not yet an active position for the token, we need to pass - // the bank/oracle for health check anyway. - let new_position = account - .1 - .tokens - .values - .iter() - .position(|p| !p.is_active()) - .unwrap(); - banks.insert(new_position, affected_bank.0); - oracles.insert(new_position, affected_bank.1.oracle); - } - } - - let serum_oos = account.1.serum3.iter_active().map(|&s| s.open_orders); - let perp_markets = account.1.perps.iter_active_accounts().map(|&pa| { - self.perp_markets_cache_by_perp_market_index - .get(&pa.market_index) - .unwrap() - .0 - }); - - Ok(banks - .iter() - .map(|&pubkey| AccountMeta { - pubkey, - is_writable: writable_banks, - is_signer: false, - }) - .chain(oracles.iter().map(|&pubkey| AccountMeta { - pubkey, - is_writable: false, - is_signer: false, - })) - .chain(serum_oos.map(|pubkey| AccountMeta { - pubkey, - is_writable: false, - is_signer: false, - })) - .chain(perp_markets.map(|pubkey| AccountMeta { - pubkey, - is_writable: false, - is_signer: false, - })) - .collect()) + ) -> anyhow::Result> { + let account = self.mango_account()?; + self.context.derive_health_check_remaining_account_metas( + &account, + affected_token, + writable_banks, + ) } pub fn derive_liquidation_health_check_remaining_account_metas( @@ -372,34 +208,22 @@ impl MangoClient { liqee: &MangoAccount, asset_token_index: TokenIndex, liab_token_index: TokenIndex, - ) -> Result, anchor_client::ClientError> { + ) -> anyhow::Result> { // figure out all the banks/oracles that need to be passed for the health check let mut banks = vec![]; let mut oracles = vec![]; - let account = self.get_account()?; + let account = self.mango_account()?; let token_indexes = liqee .tokens .iter_active() - .chain(account.1.tokens.iter_active()) + .chain(account.tokens.iter_active()) .map(|ta| ta.token_index) .unique(); for token_index in token_indexes { - let mint_info = self - .mint_infos_cache_by_token_index - .get(&token_index) - .unwrap() - .1; + let mint_info = self.context.mint_info(token_index); let writable_bank = token_index == asset_token_index || token_index == liab_token_index; - // TODO: ALTs are unavailable - // let lookup_table = account_loader - // .load_bytes(&mint_info.address_lookup_table) - // .await - // .unwrap(); - // let addresses = mango_v4::address_lookup_table::addresses(&lookup_table); - // banks.push(addresses[mint_info.address_lookup_table_bank_index as usize]); - // oracles.push(addresses[mint_info.address_lookup_table_oracle_index as usize]); banks.push((mint_info.first_bank(), writable_bank)); oracles.push(mint_info.oracle); } @@ -407,18 +231,13 @@ impl MangoClient { let serum_oos = liqee .serum3 .iter_active() - .chain(account.1.serum3.iter_active()) + .chain(account.serum3.iter_active()) .map(|&s| s.open_orders); let perp_markets = liqee .perps .iter_active_accounts() - .chain(account.1.perps.iter_active_accounts()) - .map(|&pa| { - self.perp_markets_cache_by_perp_market_index - .get(&pa.market_index) - .unwrap() - .0 - }); + .chain(account.perps.iter_active_accounts()) + .map(|&pa| self.context.perp_market_address(pa.market_index)); let to_account_meta = |pubkey| AccountMeta { pubkey, @@ -434,17 +253,17 @@ impl MangoClient { is_signer: false, }) .chain(oracles.into_iter().map(to_account_meta)) - .chain(serum_oos.map(to_account_meta)) .chain(perp_markets.map(to_account_meta)) + .chain(serum_oos.map(to_account_meta)) .collect()) } pub fn token_deposit(&self, token_name: &str, amount: u64) -> anyhow::Result { - let bank = self.banks_cache.get(token_name).unwrap().get(0).unwrap(); - let mint_info: MintInfo = self.mint_infos_cache.get(&bank.1.mint).unwrap().1; + let token_index = *self.context.token_indexes_by_name.get(token_name).unwrap(); + let mint_info = self.context.mint_info(token_index); let health_check_metas = - self.derive_health_check_remaining_account_metas(Some(*bank), false)?; + self.derive_health_check_remaining_account_metas(Some(token_index), false)?; self.program() .request() @@ -454,9 +273,9 @@ impl MangoClient { let mut ams = anchor_lang::ToAccountMetas::to_account_metas( &mango_v4::accounts::TokenDeposit { group: self.group(), - account: self.mango_account_cache.0, - bank: bank.0, - vault: bank.1.vault, + account: self.mango_account_address, + bank: mint_info.first_bank(), + vault: mint_info.first_vault(), token_account: get_associated_token_address( &self.payer(), &mint_info.mint, @@ -481,17 +300,10 @@ impl MangoClient { &self, token_name: &str, ) -> Result { - let bank = self.banks_cache.get(token_name).unwrap().get(0).unwrap().1; - - let data = self - .program() - .rpc() - .get_account_with_commitment(&bank.oracle, self.commitment)? - .value - .unwrap() - .data; - - Ok(pyth_sdk_solana::load_price(&data).unwrap()) + let token_index = *self.context.token_indexes_by_name.get(token_name).unwrap(); + let mint_info = self.context.mint_info(token_index); + let oracle_account = self.account_fetcher.fetch_raw_account(mint_info.oracle)?; + Ok(pyth_sdk_solana::load_price(&oracle_account.data).unwrap()) } // @@ -499,15 +311,20 @@ impl MangoClient { // pub fn serum3_create_open_orders(&self, name: &str) -> anyhow::Result { - let (account_pubkey, _) = self.mango_account_cache; + let account_pubkey = self.mango_account_address; - let serum3_market = self.serum3_markets_cache.get(name).unwrap(); + let market_index = *self + .context + .serum3_market_indexes_by_name + .get(name) + .unwrap(); + let serum3_info = self.context.serum3_markets.get(&market_index).unwrap(); let open_orders = Pubkey::find_program_address( &[ account_pubkey.as_ref(), b"Serum3OO".as_ref(), - serum3_market.0.as_ref(), + serum3_info.address.as_ref(), ], &self.program().id(), ) @@ -522,9 +339,9 @@ impl MangoClient { group: self.group(), account: account_pubkey, - serum_market: serum3_market.0, - serum_program: serum3_market.1.serum_program, - serum_market_external: serum3_market.1.serum_market_external, + serum_market: serum3_info.address, + serum_program: serum3_info.market.serum_program, + serum_market_external: serum3_info.market.serum_market_external, open_orders, owner: self.payer(), payer: self.payer(), @@ -541,6 +358,25 @@ impl MangoClient { .map_err(prettify_client_error) } + fn serum3_data<'a>(&'a self, name: &str) -> Result, ClientError> { + let market_index = *self + .context + .serum3_market_indexes_by_name + .get(name) + .unwrap(); + let serum3_info = self.context.serum3_markets.get(&market_index).unwrap(); + + let quote_info = self.context.token(serum3_info.market.quote_token_index); + let base_info = self.context.token(serum3_info.market.base_token_index); + + Ok(Serum3Data { + market_index, + market: serum3_info, + quote: quote_info, + base: base_info, + }) + } + #[allow(clippy::too_many_arguments)] pub fn serum3_place_order( &self, @@ -553,54 +389,22 @@ impl MangoClient { client_order_id: u64, limit: u16, ) -> anyhow::Result { - let (_, account) = self.get_account()?; + let s3 = self.serum3_data(name)?; - let serum3_market = self.serum3_markets_cache.get(name).unwrap(); - let open_orders = account - .serum3 - .find(serum3_market.1.market_index) - .unwrap() - .open_orders; - let (_, quote_info, quote_mint) = self - .mint_infos_cache_by_token_index - .get(&serum3_market.1.quote_token_index) - .unwrap(); - let (_, base_info, base_mint) = self - .mint_infos_cache_by_token_index - .get(&serum3_market.1.base_token_index) - .unwrap(); - - let market_external: &serum_dex::state::MarketState = bytemuck::from_bytes( - &(self.serum3_external_markets_cache.get(name).unwrap().1) - [5..5 + std::mem::size_of::()], - ); - let bids = market_external.bids; - let asks = market_external.asks; - let event_q = market_external.event_q; - let req_q = market_external.req_q; - let coin_vault = market_external.coin_vault; - let pc_vault = market_external.pc_vault; - let vault_signer = serum_dex::state::gen_vault_signer_key( - market_external.vault_signer_nonce, - &serum3_market.1.serum_market_external, - &serum3_market.1.serum_program, - ) - .unwrap(); + let account = self.mango_account()?; + let open_orders = account.serum3.find(s3.market_index).unwrap().open_orders; let health_check_metas = self.derive_health_check_remaining_account_metas(None, false)?; // https://github.com/project-serum/serum-ts/blob/master/packages/serum/src/market.ts#L1306 let limit_price = { - (price - * ((10u64.pow(quote_mint.decimals as u32) * market_external.coin_lot_size) as f64)) + (price * ((10u64.pow(s3.quote.decimals as u32) * s3.market.coin_lot_size) as f64)) as u64 - / (10u64.pow(base_mint.decimals as u32) * market_external.pc_lot_size) + / (10u64.pow(s3.base.decimals as u32) * s3.market.pc_lot_size) }; // https://github.com/project-serum/serum-ts/blob/master/packages/serum/src/market.ts#L1333 - let max_base_qty = { - (size * 10u64.pow(base_mint.decimals as u32) as f64) as u64 - / market_external.coin_lot_size - }; + let max_base_qty = + { (size * 10u64.pow(s3.base.decimals as u32) as f64) as u64 / s3.market.coin_lot_size }; let max_native_quote_qty_including_fees = { fn get_fee_tier(msrm_balance: u64, srm_balance: u64) -> u64 { if msrm_balance >= 1 { @@ -646,8 +450,7 @@ impl MangoClient { let fee_tier = get_fee_tier(0, 0); let rates = get_fee_rates(fee_tier); - (market_external.pc_lot_size as f64 * (1f64 + rates.0)) as u64 - * (limit_price * max_base_qty) + (s3.market.pc_lot_size as f64 * (1f64 + rates.0)) as u64 * (limit_price * max_base_qty) }; self.program() @@ -658,22 +461,22 @@ impl MangoClient { let mut ams = anchor_lang::ToAccountMetas::to_account_metas( &mango_v4::accounts::Serum3PlaceOrder { group: self.group(), - account: self.mango_account_cache.0, + account: self.mango_account_address, open_orders, - quote_bank: quote_info.first_bank(), - quote_vault: quote_info.first_vault(), - base_bank: base_info.first_bank(), - base_vault: base_info.first_vault(), - serum_market: serum3_market.0, - serum_program: serum3_market.1.serum_program, - serum_market_external: serum3_market.1.serum_market_external, - market_bids: from_serum_style_pubkey(&bids), - market_asks: from_serum_style_pubkey(&asks), - market_event_queue: from_serum_style_pubkey(&event_q), - market_request_queue: from_serum_style_pubkey(&req_q), - market_base_vault: from_serum_style_pubkey(&coin_vault), - market_quote_vault: from_serum_style_pubkey(&pc_vault), - market_vault_signer: vault_signer, + quote_bank: s3.quote.mint_info.first_bank(), + quote_vault: s3.quote.mint_info.first_vault(), + base_bank: s3.base.mint_info.first_bank(), + base_vault: s3.base.mint_info.first_vault(), + serum_market: s3.market.address, + serum_program: s3.market.market.serum_program, + serum_market_external: s3.market.market.serum_market_external, + market_bids: s3.market.bids, + market_asks: s3.market.asks, + market_event_queue: s3.market.event_q, + market_request_queue: s3.market.req_q, + market_base_vault: s3.market.coin_vault, + market_quote_vault: s3.market.pc_vault, + market_vault_signer: s3.market.vault_signer, owner: self.payer(), token_program: Token::id(), }, @@ -700,35 +503,10 @@ impl MangoClient { } pub fn serum3_settle_funds(&self, name: &str) -> anyhow::Result { - let (_, account) = self.get_account()?; + let s3 = self.serum3_data(name)?; - let serum3_market = self.serum3_markets_cache.get(name).unwrap(); - let open_orders = account - .serum3 - .find(serum3_market.1.market_index) - .unwrap() - .open_orders; - let (_, quote_info, _) = self - .mint_infos_cache_by_token_index - .get(&serum3_market.1.quote_token_index) - .unwrap(); - let (_, base_info, _) = self - .mint_infos_cache_by_token_index - .get(&serum3_market.1.base_token_index) - .unwrap(); - - let market_external: &serum_dex::state::MarketState = bytemuck::from_bytes( - &(self.serum3_external_markets_cache.get(name).unwrap().1) - [5..5 + std::mem::size_of::()], - ); - let coin_vault = market_external.coin_vault; - let pc_vault = market_external.pc_vault; - let vault_signer = serum_dex::state::gen_vault_signer_key( - market_external.vault_signer_nonce, - &serum3_market.1.serum_market_external, - &serum3_market.1.serum_program, - ) - .unwrap(); + let account = self.mango_account()?; + let open_orders = account.serum3.find(s3.market_index).unwrap().open_orders; self.program() .request() @@ -737,18 +515,18 @@ impl MangoClient { accounts: anchor_lang::ToAccountMetas::to_account_metas( &mango_v4::accounts::Serum3SettleFunds { group: self.group(), - account: self.mango_account_cache.0, + account: self.mango_account_address, open_orders, - quote_bank: quote_info.first_bank(), - quote_vault: quote_info.first_vault(), - base_bank: base_info.first_bank(), - base_vault: base_info.first_vault(), - serum_market: serum3_market.0, - serum_program: serum3_market.1.serum_program, - serum_market_external: serum3_market.1.serum_market_external, - market_base_vault: from_serum_style_pubkey(&coin_vault), - market_quote_vault: from_serum_style_pubkey(&pc_vault), - market_vault_signer: vault_signer, + quote_bank: s3.quote.mint_info.first_bank(), + quote_vault: s3.quote.mint_info.first_vault(), + base_bank: s3.base.mint_info.first_bank(), + base_vault: s3.base.mint_info.first_vault(), + serum_market: s3.market.address, + serum_program: s3.market.market.serum_program, + serum_market_external: s3.market.market.serum_market_external, + market_base_vault: s3.market.coin_vault, + market_quote_vault: s3.market.pc_vault, + market_vault_signer: s3.market.vault_signer, owner: self.payer(), token_program: Token::id(), }, @@ -763,25 +541,15 @@ impl MangoClient { } pub fn serum3_cancel_all_orders(&self, market_name: &str) -> Result, anyhow::Error> { - let serum3_market = self.serum3_markets_cache.get(market_name).unwrap(); + let market_index = *self + .context + .serum3_market_indexes_by_name + .get(market_name) + .unwrap(); + let account = self.mango_account()?; + let open_orders = account.serum3.find(market_index).unwrap().open_orders; - let open_orders = Pubkey::find_program_address( - &[ - self.mango_account_cache.0.as_ref(), - b"Serum3OO".as_ref(), - serum3_market.0.as_ref(), - ], - &self.program().id(), - ) - .0; - - let open_orders_bytes = self - .program() - .rpc() - .get_account_with_commitment(&open_orders, self.commitment)? - .value - .unwrap() - .data; + let open_orders_bytes = self.account_fetcher.fetch_raw_account(open_orders)?.data; let open_orders_data: &serum_dex::state::OpenOrders = bytemuck::from_bytes( &open_orders_bytes[5..5 + std::mem::size_of::()], ); @@ -808,30 +576,10 @@ impl MangoClient { side: Serum3Side, order_id: u128, ) -> anyhow::Result<()> { - let (account_pubkey, _account) = self.get_account()?; + let s3 = self.serum3_data(market_name)?; - let serum3_market = self.serum3_markets_cache.get(market_name).unwrap(); - - let open_orders = Pubkey::find_program_address( - &[ - account_pubkey.as_ref(), - b"Serum3OO".as_ref(), - serum3_market.0.as_ref(), - ], - &self.program().id(), - ) - .0; - - let market_external: &serum_dex::state::MarketState = bytemuck::from_bytes( - &(self - .serum3_external_markets_cache - .get(market_name) - .unwrap() - .1)[5..5 + std::mem::size_of::()], - ); - let bids = market_external.bids; - let asks = market_external.asks; - let event_q = market_external.event_q; + let account = self.mango_account()?; + let open_orders = account.serum3.find(s3.market_index).unwrap().open_orders; self.program() .request() @@ -841,14 +589,14 @@ impl MangoClient { anchor_lang::ToAccountMetas::to_account_metas( &mango_v4::accounts::Serum3CancelOrder { group: self.group(), - account: account_pubkey, - serum_market: serum3_market.0, - serum_program: serum3_market.1.serum_program, - serum_market_external: serum3_market.1.serum_market_external, + account: self.mango_account_address, + serum_market: s3.market.address, + serum_program: s3.market.market.serum_program, + serum_market_external: s3.market.market.serum_market_external, open_orders, - market_bids: from_serum_style_pubkey(&bids), - market_asks: from_serum_style_pubkey(&asks), - market_event_queue: from_serum_style_pubkey(&event_q), + market_bids: s3.market.bids, + market_asks: s3.market.asks, + market_event_queue: s3.market.event_q, owner: self.payer(), }, None, @@ -896,7 +644,7 @@ impl MangoClient { &mango_v4::accounts::LiqTokenWithToken { group: self.group(), liqee: *liqee.0, - liqor: self.mango_account_cache.0, + liqor: self.mango_account_address, liqor_owner: self.payer.pubkey(), }, None, @@ -924,16 +672,11 @@ impl MangoClient { ) -> anyhow::Result { let quote_token_index = 0; - let (_, quote_mint_info, _) = self - .mint_infos_cache_by_token_index - .get("e_token_index) - .unwrap(); - let (liab_mint_info_key, liab_mint_info, _) = self - .mint_infos_cache_by_token_index - .get(&liab_token_index) - .unwrap(); + let quote_info = self.context.token(quote_token_index); + let liab_info = self.context.token(liab_token_index); - let bank_remaining_ams = liab_mint_info + let bank_remaining_ams = liab_info + .mint_info .banks() .iter() .map(|bank_pubkey| AccountMeta { @@ -951,6 +694,11 @@ impl MangoClient { ) .unwrap(); + let group = account_fetcher_fetch_anchor_account::( + &*self.account_fetcher, + self.context.group, + )?; + self.program() .request() .instruction(Instruction { @@ -960,11 +708,11 @@ impl MangoClient { &mango_v4::accounts::LiqTokenBankruptcy { group: self.group(), liqee: *liqee.0, - liqor: self.mango_account_cache.0, + liqor: self.mango_account_address, liqor_owner: self.payer.pubkey(), - liab_mint_info: *liab_mint_info_key, - quote_vault: quote_mint_info.first_vault(), - insurance_vault: self.group_cache.insurance_vault, + liab_mint_info: liab_info.mint_info_address, + quote_vault: quote_info.mint_info.first_vault(), + insurance_vault: group.insurance_vault, token_program: Token::id(), }, None, @@ -985,8 +733,11 @@ impl MangoClient { } } -fn from_serum_style_pubkey(d: &[u64; 4]) -> Pubkey { - Pubkey::new(bytemuck::cast_slice(d as &[_])) +struct Serum3Data<'a> { + market_index: Serum3MarketIndex, + market: &'a Serum3MarketContext, + quote: &'a TokenContext, + base: &'a TokenContext, } /// Do some manual unpacking on some ClientErrors diff --git a/client/src/context.rs b/client/src/context.rs new file mode 100644 index 000000000..3fc417cbb --- /dev/null +++ b/client/src/context.rs @@ -0,0 +1,263 @@ +use std::collections::HashMap; + +use anchor_client::{Client, ClientError, Cluster, Program}; + +use anchor_lang::__private::bytemuck; + +use mango_v4::state::{ + MangoAccount, MintInfo, PerpMarket, PerpMarketIndex, Serum3Market, Serum3MarketIndex, + TokenIndex, +}; + +use crate::gpa::*; + +use solana_sdk::account::Account; +use solana_sdk::instruction::AccountMeta; +use solana_sdk::signature::Keypair; +use solana_sdk::{commitment_config::CommitmentConfig, pubkey::Pubkey}; + +pub struct TokenContext { + pub name: String, + pub mint_info: MintInfo, + pub mint_info_address: Pubkey, + pub decimals: u8, +} + +pub struct Serum3MarketContext { + pub address: Pubkey, + pub market: Serum3Market, + pub bids: Pubkey, + pub asks: Pubkey, + pub event_q: Pubkey, + pub req_q: Pubkey, + pub coin_vault: Pubkey, + pub pc_vault: Pubkey, + pub vault_signer: Pubkey, + pub coin_lot_size: u64, + pub pc_lot_size: u64, +} + +pub struct PerpMarketContext { + pub address: Pubkey, + pub market: PerpMarket, +} + +pub struct MangoGroupContext { + pub group: Pubkey, + + pub tokens: HashMap, + pub token_indexes_by_name: HashMap, + + pub serum3_markets: HashMap, + pub serum3_market_indexes_by_name: HashMap, + + pub perp_markets: HashMap, + pub perp_market_indexes_by_name: HashMap, +} + +impl MangoGroupContext { + pub fn mint_info_address(&self, token_index: TokenIndex) -> Pubkey { + self.token(token_index).mint_info_address + } + + pub fn mint_info(&self, token_index: TokenIndex) -> MintInfo { + self.token(token_index).mint_info + } + + pub fn token(&self, token_index: TokenIndex) -> &TokenContext { + self.tokens.get(&token_index).unwrap() + } + + pub fn perp_market_address(&self, perp_market_index: PerpMarketIndex) -> Pubkey { + self.perp_markets.get(&perp_market_index).unwrap().address + } + + pub fn new_from_rpc( + group: Pubkey, + cluster: Cluster, + commitment: CommitmentConfig, + ) -> Result { + let program = + Client::new_with_options(cluster, std::rc::Rc::new(Keypair::new()), commitment) + .program(mango_v4::ID); + + // tokens + let mint_info_tuples = fetch_mint_infos(&program, group)?; + let mut tokens = mint_info_tuples + .iter() + .map(|(pk, mi)| { + ( + mi.token_index, + TokenContext { + name: String::new(), + mint_info: *mi, + mint_info_address: *pk, + decimals: u8::MAX, + }, + ) + }) + .collect::>(); + + // reading the banks is only needed for the token names and decimals + // FUTURE: either store the names on MintInfo as well, or maybe don't store them at all + // because they are in metaplex? + let bank_tuples = fetch_banks(&program, group)?; + for (_, bank) in bank_tuples { + let token = tokens.get_mut(&bank.token_index).unwrap(); + token.name = bank.name().into(); + token.decimals = bank.mint_decimals; + } + assert!(tokens.values().all(|t| t.decimals != u8::MAX)); + + // serum3 markets + let serum3_market_tuples = fetch_serum3_markets(&program, group)?; + let serum3_markets = serum3_market_tuples + .iter() + .map(|(pk, s)| { + let market_external_account = fetch_raw_account(&program, s.serum_market_external)?; + let market_external: &serum_dex::state::MarketState = bytemuck::from_bytes( + &market_external_account.data + [5..5 + std::mem::size_of::()], + ); + let vault_signer = serum_dex::state::gen_vault_signer_key( + market_external.vault_signer_nonce, + &s.serum_market_external, + &s.serum_program, + ) + .unwrap(); + Ok(( + s.market_index, + Serum3MarketContext { + address: *pk, + market: *s, + bids: from_serum_style_pubkey(market_external.bids), + asks: from_serum_style_pubkey(market_external.asks), + event_q: from_serum_style_pubkey(market_external.event_q), + req_q: from_serum_style_pubkey(market_external.req_q), + coin_vault: from_serum_style_pubkey(market_external.coin_vault), + pc_vault: from_serum_style_pubkey(market_external.pc_vault), + vault_signer, + coin_lot_size: market_external.coin_lot_size, + pc_lot_size: market_external.pc_lot_size, + }, + )) + }) + .collect::, ClientError>>()?; + + // perp markets + let perp_market_tuples = fetch_perp_markets(&program, group)?; + let perp_markets = perp_market_tuples + .iter() + .map(|(pk, pm)| { + ( + pm.perp_market_index, + PerpMarketContext { + address: *pk, + market: *pm, + }, + ) + }) + .collect::>(); + + // Name lookup tables + let token_indexes_by_name = tokens + .iter() + .map(|(i, t)| (t.name.clone(), *i)) + .collect::>(); + let serum3_market_indexes_by_name = serum3_markets + .iter() + .map(|(i, s)| (s.market.name().to_string(), *i)) + .collect::>(); + let perp_market_indexes_by_name = perp_markets + .iter() + .map(|(i, p)| (p.market.name().to_string(), *i)) + .collect::>(); + + Ok(MangoGroupContext { + group, + tokens, + token_indexes_by_name, + serum3_markets, + serum3_market_indexes_by_name, + perp_markets, + perp_market_indexes_by_name, + }) + } + + pub fn derive_health_check_remaining_account_metas( + &self, + account: &MangoAccount, + affected_token: Option, + writable_banks: bool, + ) -> anyhow::Result> { + // figure out all the banks/oracles that need to be passed for the health check + let mut banks = vec![]; + let mut oracles = vec![]; + for position in account.tokens.iter_active() { + let mint_info = self.mint_info(position.token_index); + banks.push(mint_info.first_bank()); + oracles.push(mint_info.oracle); + } + if let Some(affected_token_index) = affected_token { + if account + .tokens + .iter_active() + .find(|p| p.token_index == affected_token_index) + .is_none() + { + // If there is not yet an active position for the token, we need to pass + // the bank/oracle for health check anyway. + let new_position = account + .tokens + .values + .iter() + .position(|p| !p.is_active()) + .unwrap(); + let mint_info = self.mint_info(affected_token_index); + banks.insert(new_position, mint_info.first_bank()); + oracles.insert(new_position, mint_info.oracle); + } + } + + let serum_oos = account.serum3.iter_active().map(|&s| s.open_orders); + let perp_markets = account + .perps + .iter_active_accounts() + .map(|&pa| self.perp_market_address(pa.market_index)); + + Ok(banks + .iter() + .map(|&pubkey| AccountMeta { + pubkey, + is_writable: writable_banks, + is_signer: false, + }) + .chain(oracles.iter().map(|&pubkey| AccountMeta { + pubkey, + is_writable: false, + is_signer: false, + })) + .chain(perp_markets.map(|pubkey| AccountMeta { + pubkey, + is_writable: false, + is_signer: false, + })) + .chain(serum_oos.map(|pubkey| AccountMeta { + pubkey, + is_writable: false, + is_signer: false, + })) + .collect()) + } +} + +fn from_serum_style_pubkey(d: [u64; 4]) -> Pubkey { + Pubkey::new(bytemuck::cast_slice(&d as &[_])) +} + +fn fetch_raw_account(program: &Program, address: Pubkey) -> Result { + let rpc = program.rpc(); + rpc.get_account_with_commitment(&address, rpc.commitment())? + .value + .ok_or(ClientError::AccountNotFound) +} diff --git a/client/src/gpa.rs b/client/src/gpa.rs new file mode 100644 index 000000000..1f0b7d2bd --- /dev/null +++ b/client/src/gpa.rs @@ -0,0 +1,66 @@ +use anchor_client::{ClientError, Program}; + +use mango_v4::state::{Bank, MangoAccount, MintInfo, PerpMarket, Serum3Market}; + +use solana_client::rpc_filter::{Memcmp, MemcmpEncodedBytes, RpcFilterType}; +use solana_sdk::pubkey::Pubkey; + +pub fn fetch_mango_accounts( + program: &Program, + group: Pubkey, + owner: Pubkey, +) -> Result, ClientError> { + program.accounts::(vec![ + RpcFilterType::Memcmp(Memcmp { + offset: 8, + bytes: MemcmpEncodedBytes::Base58(group.to_string()), + encoding: None, + }), + RpcFilterType::Memcmp(Memcmp { + offset: 40, + bytes: MemcmpEncodedBytes::Base58(owner.to_string()), + encoding: None, + }), + ]) +} + +pub fn fetch_banks(program: &Program, group: Pubkey) -> Result, ClientError> { + program.accounts::(vec![RpcFilterType::Memcmp(Memcmp { + offset: 8, + bytes: MemcmpEncodedBytes::Base58(group.to_string()), + encoding: None, + })]) +} + +pub fn fetch_mint_infos( + program: &Program, + group: Pubkey, +) -> Result, ClientError> { + program.accounts::(vec![RpcFilterType::Memcmp(Memcmp { + offset: 8, + bytes: MemcmpEncodedBytes::Base58(group.to_string()), + encoding: None, + })]) +} + +pub fn fetch_serum3_markets( + program: &Program, + group: Pubkey, +) -> Result, ClientError> { + program.accounts::(vec![RpcFilterType::Memcmp(Memcmp { + offset: 8, + bytes: MemcmpEncodedBytes::Base58(group.to_string()), + encoding: None, + })]) +} + +pub fn fetch_perp_markets( + program: &Program, + group: Pubkey, +) -> Result, ClientError> { + program.accounts::(vec![RpcFilterType::Memcmp(Memcmp { + offset: 8, + bytes: MemcmpEncodedBytes::Base58(group.to_string()), + encoding: None, + })]) +} diff --git a/client/src/lib.rs b/client/src/lib.rs index 13d450e92..b6ccaaedc 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -1,5 +1,10 @@ +pub use account_fetcher::*; pub use client::*; +pub use context::*; pub use util::*; -pub mod client; +mod account_fetcher; +mod client; +mod context; +mod gpa; mod util; diff --git a/keeper/src/crank.rs b/keeper/src/crank.rs index c65c24e22..ae2398053 100644 --- a/keeper/src/crank.rs +++ b/keeper/src/crank.rs @@ -16,26 +16,24 @@ pub async fn runner( debugging_handle: impl Future, ) -> Result<(), anyhow::Error> { let handles1 = mango_client - .banks_cache - .values() - .map(|banks_for_a_token| { - loop_update_index_and_rate( - mango_client.clone(), - banks_for_a_token.get(0).unwrap().1.token_index, - ) - }) + .context + .tokens + .keys() + .map(|&token_index| loop_update_index_and_rate(mango_client.clone(), token_index)) .collect::>(); let handles2 = mango_client - .perp_markets_cache + .context + .perp_markets .values() - .map(|(pk, perp_market)| loop_consume_events(mango_client.clone(), *pk, *perp_market)) + .map(|perp| loop_consume_events(mango_client.clone(), perp.address, perp.market)) .collect::>(); let handles3 = mango_client - .perp_markets_cache + .context + .perp_markets .values() - .map(|(pk, perp_market)| loop_update_funding(mango_client.clone(), *pk, *perp_market)) + .map(|perp| loop_update_funding(mango_client.clone(), perp.address, perp.market)) .collect::>(); futures::join!( @@ -56,16 +54,10 @@ pub async fn loop_update_index_and_rate(mango_client: Arc, token_in let client = mango_client.clone(); let res = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - let mint_info = client.get_mint_info(&token_index); - let banks_for_a_token = client.banks_cache_by_token_index.get(&token_index).unwrap(); - let some_bank = banks_for_a_token.get(0).unwrap().1; - let token_name = some_bank.name(); - let oracle = some_bank.oracle; - - let bank_pubkeys_for_a_token = banks_for_a_token - .into_iter() - .map(|bank| bank.0) - .collect::>(); + let token = client.context.token(token_index); + let banks_for_a_token = token.mint_info.banks(); + let token_name = &token.name; + let oracle = token.mint_info.oracle; let sig_result = client .program() @@ -75,7 +67,7 @@ pub async fn loop_update_index_and_rate(mango_client: Arc, token_in program_id: mango_v4::id(), accounts: anchor_lang::ToAccountMetas::to_account_metas( &mango_v4::accounts::TokenUpdateIndexAndRate { - mint_info, + mint_info: token.mint_info_address, oracle, instructions: solana_program::sysvar::instructions::id(), }, @@ -85,7 +77,7 @@ pub async fn loop_update_index_and_rate(mango_client: Arc, token_in &mango_v4::instruction::TokenUpdateIndexAndRate {}, ), }; - let mut banks = bank_pubkeys_for_a_token + let mut banks = banks_for_a_token .iter() .map(|bank_pubkey| AccountMeta { pubkey: *bank_pubkey, diff --git a/keeper/src/main.rs b/keeper/src/main.rs index 117767be5..1a9c294e6 100644 --- a/keeper/src/main.rs +++ b/keeper/src/main.rs @@ -101,8 +101,8 @@ fn main() -> Result<(), anyhow::Error> { let mango_client = Arc::new(MangoClient::new( cluster, commitment, - payer, group, + payer, &cli.mango_account_name, )?); diff --git a/keeper/src/taker.rs b/keeper/src/taker.rs index 0c8faf35f..1d9aa369d 100644 --- a/keeper/src/taker.rs +++ b/keeper/src/taker.rs @@ -6,10 +6,7 @@ use std::{ use fixed::types::I80F48; use futures::Future; -use mango_v4::{ - instructions::{Serum3OrderType, Serum3SelfTradeBehavior, Serum3Side}, - state::Bank, -}; +use mango_v4::instructions::{Serum3OrderType, Serum3SelfTradeBehavior, Serum3Side}; use tokio::time; @@ -24,7 +21,7 @@ pub async fn runner( ensure_oo(&mango_client)?; let mut price_arcs = HashMap::new(); - for market_name in mango_client.serum3_markets_cache.keys() { + for market_name in mango_client.context.serum3_market_indexes_by_name.keys() { let price = mango_client .get_oracle_price( market_name @@ -43,7 +40,8 @@ pub async fn runner( } let handles1 = mango_client - .serum3_markets_cache + .context + .serum3_market_indexes_by_name .keys() .map(|market_name| { loop_blocking_price_update( @@ -55,7 +53,8 @@ pub async fn runner( .collect::>(); let handles2 = mango_client - .serum3_markets_cache + .context + .serum3_market_indexes_by_name .keys() .map(|market_name| { loop_blocking_orders( @@ -75,11 +74,11 @@ pub async fn runner( } fn ensure_oo(mango_client: &Arc) -> Result<(), anyhow::Error> { - let account = mango_client.get_account()?.1; + let account = mango_client.mango_account()?; - for (_, serum3_market) in mango_client.serum3_markets_cache.values() { - if account.serum3.find(serum3_market.market_index).is_none() { - mango_client.serum3_create_open_orders(serum3_market.name())?; + for (market_index, serum3_market) in mango_client.context.serum3_markets.iter() { + if account.serum3.find(*market_index).is_none() { + mango_client.serum3_create_open_orders(serum3_market.market.name())?; } } @@ -87,25 +86,18 @@ fn ensure_oo(mango_client: &Arc) -> Result<(), anyhow::Error> { } fn ensure_deposit(mango_client: &Arc) -> Result<(), anyhow::Error> { - let mango_account = mango_client.get_account()?.1; + let mango_account = mango_client.mango_account()?; - let banks: Vec = mango_client - .banks_cache - .values() - .map(|vec| vec.get(0).unwrap().1) - .collect::>(); + for &token_index in mango_client.context.tokens.keys() { + let bank = mango_client.first_bank(token_index)?; + let desired_balance = I80F48::from_num(10_000 * 10u64.pow(bank.mint_decimals as u32)); - for bank in banks { - let mint = &mango_client.mint_infos_cache.get(&bank.mint).unwrap().2; - let desired_balance = I80F48::from_num(10_000 * 10u64.pow(mint.decimals as u32)); - - let token_account_opt = mango_account.tokens.find(bank.token_index); + let token_account_opt = mango_account.tokens.find(token_index); let deposit_native = match token_account_opt { Some(token_account) => { let native = token_account.native(&bank); - - let ui = token_account.ui(&bank, mint); + let ui = token_account.ui(&bank); log::info!("Current balance {} {}", ui, bank.name()); if native < I80F48::ZERO { diff --git a/liquidator/src/chain_data.rs b/liquidator/src/chain_data.rs index 811f1e7b5..a22c8b731 100644 --- a/liquidator/src/chain_data.rs +++ b/liquidator/src/chain_data.rs @@ -244,6 +244,22 @@ impl ChainData { } } + pub fn update_from_rpc(&mut self, pubkey: &Pubkey, account: AccountData) { + // Add a stub slot if the rpc has information about the future. + // If it's in the past, either the slot already exists (and maybe we have + // the data already), or it was a skipped slot and adding it now makes no difference. + if account.slot > self.newest_processed_slot { + self.update_slot(SlotData { + slot: account.slot, + parent: Some(self.newest_processed_slot), + status: SlotStatus::Processed, + chain: 0, + }); + } + + self.update_account(*pubkey, account) + } + fn is_account_write_live(&self, write: &AccountData) -> bool { self.slots .get(&write.slot) diff --git a/liquidator/src/chain_data_fetcher.rs b/liquidator/src/chain_data_fetcher.rs new file mode 100644 index 000000000..2a19612fb --- /dev/null +++ b/liquidator/src/chain_data_fetcher.rs @@ -0,0 +1,81 @@ +use std::sync::{Arc, RwLock}; + +use crate::chain_data::*; + +use client::AccountFetcher; +use mango_v4::accounts_zerocopy::LoadZeroCopy; + +use anyhow::Context; + +use solana_client::rpc_client::RpcClient; +use solana_sdk::account::AccountSharedData; +use solana_sdk::pubkey::Pubkey; + +pub struct ChainDataAccountFetcher { + pub chain_data: Arc>, + pub rpc: RpcClient, +} + +impl ChainDataAccountFetcher { + // loads from ChainData + pub fn fetch( + &self, + address: &Pubkey, + ) -> anyhow::Result { + Ok(self + .fetch_raw(address)? + .load::() + .with_context(|| format!("loading account {}", address))? + .clone()) + } + + // fetches via RPC, stores in ChainData, returns new version + pub fn fetch_fresh( + &self, + address: &Pubkey, + ) -> anyhow::Result { + self.refresh_account_via_rpc(address)?; + self.fetch(address) + } + + pub fn fetch_raw(&self, address: &Pubkey) -> anyhow::Result { + let chain_data = self.chain_data.read().unwrap(); + Ok(chain_data + .account(address) + .with_context(|| format!("fetch account {} via chain_data", address))? + .clone()) + } + + pub fn refresh_account_via_rpc(&self, address: &Pubkey) -> anyhow::Result<()> { + let response = self + .rpc + .get_account_with_commitment(&address, self.rpc.commitment()) + .with_context(|| format!("refresh account {} via rpc", address))?; + let account = response + .value + .ok_or(anchor_client::ClientError::AccountNotFound) + .with_context(|| format!("refresh account {} via rpc", address))?; + + let mut chain_data = self.chain_data.write().unwrap(); + chain_data.update_from_rpc( + address, + AccountData { + slot: response.context.slot, + account: account.into(), + }, + ); + log::trace!( + "refreshed data of account {} via rpc, got context slot {}", + address, + response.context.slot + ); + + Ok(()) + } +} + +impl AccountFetcher for ChainDataAccountFetcher { + fn fetch_raw_account(&self, address: Pubkey) -> anyhow::Result { + self.fetch_raw(&address).map(|a| a.clone().into()) + } +} diff --git a/liquidator/src/liquidate.rs b/liquidator/src/liquidate.rs index 363c00603..f74540b2a 100644 --- a/liquidator/src/liquidate.rs +++ b/liquidator/src/liquidate.rs @@ -1,132 +1,72 @@ -use std::collections::HashMap; - use crate::account_shared_data::KeyedAccountSharedData; +use crate::ChainDataAccountFetcher; -use client::MangoClient; -use mango_v4::accounts_zerocopy::LoadZeroCopy; +use client::{AccountFetcher, MangoClient, MangoGroupContext}; use mango_v4::state::{ new_health_cache, oracle_price, Bank, FixedOrderAccountRetriever, HealthCache, HealthType, - MangoAccount, MintInfo, PerpMarketIndex, TokenIndex, + MangoAccount, TokenIndex, }; -use { - crate::chain_data::ChainData, anyhow::Context, fixed::types::I80F48, - solana_sdk::account::AccountSharedData, solana_sdk::pubkey::Pubkey, -}; - -pub fn load_mango_account( - account: &AccountSharedData, -) -> anyhow::Result<&T> { - account.load::().map_err(|e| e.into()) -} - -fn load_mango_account_from_chain<'a, T: anchor_lang::ZeroCopy + anchor_lang::Owner>( - chain_data: &'a ChainData, - pubkey: &Pubkey, -) -> anyhow::Result<&'a T> { - load_mango_account::( - chain_data - .account(pubkey) - .context("retrieving account from chain")?, - ) -} +use {anyhow::Context, fixed::types::I80F48, solana_sdk::pubkey::Pubkey}; pub fn new_health_cache_( - chain_data: &ChainData, - mint_infos: &HashMap, - perp_markets: &HashMap, + context: &MangoGroupContext, + account_fetcher: &ChainDataAccountFetcher, account: &MangoAccount, -) -> anchor_lang::Result { - let mut health_accounts = vec![]; - let mut banks = vec![]; - let mut oracles = vec![]; +) -> anyhow::Result { + let active_token_len = account.tokens.iter_active().count(); + let active_perp_len = account.perps.iter_active_accounts().count(); - // collect banks and oracles for active token positions - for position in account.tokens.iter_active() { - let mint_info = load_mango_account_from_chain::( - chain_data, - mint_infos - .get(&position.token_index) - .expect("mint_infos cache missing entry"), - ) - .unwrap(); - - banks.push(( - mint_info.first_bank(), - chain_data - .account(&mint_info.first_bank()) - .expect("chain data is missing bank"), - )); - oracles.push(( - mint_info.oracle, - chain_data - .account(&mint_info.oracle) - .expect("chain data is missing oracle"), - )); - } - - // collect active perp markets - let mut perp_markets = account - .perps - .iter_active_accounts() - .map(|&s| { - ( - *perp_markets - .get(&s.market_index) - .expect("perp markets cache is missing entry"), - chain_data - .account( - perp_markets - .get(&s.market_index) - .expect("perp markets cache is missing entry"), - ) - .expect("chain data is missing perp market"), - ) + let metas = context.derive_health_check_remaining_account_metas(account, None, false)?; + let accounts = metas + .iter() + .map(|meta| { + Ok(KeyedAccountSharedData::new( + meta.pubkey, + account_fetcher.fetch_raw(&meta.pubkey)?, + )) }) - .collect::>(); - let active_perp_len = perp_markets.len(); - - // collect OO for active serum markets - let mut serum_oos = account - .serum3 - .iter_active() - .map(|&s| (s.open_orders, chain_data.account(&s.open_orders).unwrap())) - .collect::>(); - - let active_token_len = banks.len(); - health_accounts.append(&mut banks); - health_accounts.append(&mut oracles); - health_accounts.append(&mut perp_markets); - health_accounts.append(&mut serum_oos); + .collect::>>()?; let retriever = FixedOrderAccountRetriever { - ais: health_accounts - .into_iter() - .map(|asd| KeyedAccountSharedData::new(asd.0, asd.1.clone())) - .collect::>(), + ais: accounts, n_banks: active_token_len, begin_perp: active_token_len * 2, begin_serum3: active_token_len * 2 + active_perp_len, }; - new_health_cache(account, &retriever) + new_health_cache(account, &retriever).context("make health cache") } #[allow(clippy::too_many_arguments)] pub fn process_account( mango_client: &MangoClient, - chain_data: &ChainData, - mint_infos: &HashMap, - perp_markets: &HashMap, + account_fetcher: &ChainDataAccountFetcher, pubkey: &Pubkey, ) -> anyhow::Result<()> { // TODO: configurable let min_health_ratio = I80F48::from_num(50.0f64); let quote_token_index = 0; - let account = load_mango_account_from_chain::(chain_data, pubkey)?; + let account = account_fetcher.fetch::(pubkey)?; + let maint_health = new_health_cache_(&mango_client.context, account_fetcher, &account) + .expect("always ok") + .health(HealthType::Maint); - // compute maint health for account - let maint_health = new_health_cache_(chain_data, mint_infos, perp_markets, account) + if maint_health >= 0 && !account.is_bankrupt() { + return Ok(()); + } + + log::trace!( + "possible candidate: {}, with owner: {}, maint health: {}, bankrupt: {}", + pubkey, + account.owner, + maint_health, + account.is_bankrupt(), + ); + + // Fetch a fresh account and re-compute + let account = account_fetcher.fetch_fresh::(pubkey)?; + let maint_health = new_health_cache_(&mango_client.context, account_fetcher, &account) .expect("always ok") .health(HealthType::Maint); @@ -134,28 +74,29 @@ pub fn process_account( let mut tokens = account .tokens .iter_active() - .map(|token| { - let mint_info_pk = mint_infos.get(&token.token_index).expect("always Ok"); - let mint_info = load_mango_account_from_chain::(chain_data, mint_info_pk)?; - let bank = load_mango_account_from_chain::(chain_data, &mint_info.first_bank())?; - let oracle = chain_data.account(&mint_info.oracle)?; + .map(|token_position| { + let token = mango_client.context.token(token_position.token_index); + let bank = account_fetcher.fetch::(&token.mint_info.first_bank())?; + let oracle = account_fetcher.fetch_raw_account(token.mint_info.oracle)?; let price = oracle_price( - &KeyedAccountSharedData::new(mint_info.oracle, oracle.clone()), + &KeyedAccountSharedData::new(token.mint_info.oracle, oracle.into()), bank.oracle_config.conf_filter, bank.mint_decimals, )?; - Ok((token.token_index, bank, token.native(bank) * price)) + Ok(( + token_position.token_index, + bank, + token_position.native(&bank) * price, + )) }) - .collect::>>()?; + .collect::>>()?; tokens.sort_by(|a, b| a.2.cmp(&b.2)); let get_max_liab_transfer = |source, target| -> anyhow::Result { - let mut liqor = load_mango_account_from_chain::( - chain_data, - &mango_client.mango_account_cache.0, - ) - .context("getting liquidator account")? - .clone(); + let mut liqor = account_fetcher + .fetch_fresh::(&mango_client.mango_account_address) + .context("getting liquidator account")? + .clone(); // Ensure the tokens are activated, so they appear in the health cache and // max_swap_source() will work. @@ -163,20 +104,13 @@ pub fn process_account( liqor.tokens.get_mut_or_create(target)?; let health_cache = - new_health_cache_(chain_data, mint_infos, perp_markets, &liqor).expect("always ok"); + new_health_cache_(&mango_client.context, account_fetcher, &liqor).expect("always ok"); let amount = health_cache .max_swap_source_for_health_ratio(source, target, min_health_ratio) .context("getting max_swap_source")?; Ok(amount) }; - log::trace!( - "checking account {} with owner {}: maint health: {}", - pubkey, - account.owner, - maint_health - ); - // try liquidating if account.is_bankrupt() { if tokens.is_empty() { @@ -187,7 +121,7 @@ pub fn process_account( let max_liab_transfer = get_max_liab_transfer(*liab_token_index, quote_token_index)?; let sig = mango_client - .liq_token_bankruptcy((pubkey, account), *liab_token_index, max_liab_transfer) + .liq_token_bankruptcy((pubkey, &account), *liab_token_index, max_liab_transfer) .context("sending liq_token_bankruptcy")?; log::info!( "Liquidated bankruptcy for {}..., maint_health was {}, tx sig {:?}", @@ -209,12 +143,10 @@ pub fn process_account( // TODO: log liqor's assets in UI form // TODO: log liquee's liab_needed, need to refactor program code to be able to be accessed from client side // TODO: swap inherited liabs to desired asset for liqor - // TODO: hook ChainData into MangoClient - // TODO: liq_token_with_token() re-gets the liqor account via rpc unnecessarily // let sig = mango_client .liq_token_with_token( - (pubkey, account), + (pubkey, &account), *asset_token_index, *liab_token_index, max_liab_transfer, @@ -233,13 +165,11 @@ pub fn process_account( #[allow(clippy::too_many_arguments)] pub fn process_accounts<'a>( mango_client: &MangoClient, - chain_data: &ChainData, + account_fetcher: &ChainDataAccountFetcher, accounts: impl Iterator, - mint_infos: &HashMap, - perp_markets: &HashMap, ) -> anyhow::Result<()> { for pubkey in accounts { - match process_account(mango_client, chain_data, mint_infos, perp_markets, pubkey) { + match process_account(mango_client, account_fetcher, pubkey) { Err(err) => log::error!("error liquidating account {}: {:?}", pubkey, err), _ => {} }; diff --git a/liquidator/src/main.rs b/liquidator/src/main.rs index 7992b9de7..3df596574 100644 --- a/liquidator/src/main.rs +++ b/liquidator/src/main.rs @@ -1,17 +1,16 @@ use std::collections::HashMap; -use std::sync::Arc; - -use crate::chain_data::*; -use crate::util::{is_mango_account, is_mango_bank, is_mint_info, is_perp_market}; +use std::sync::{Arc, RwLock}; +use std::time::Duration; use anchor_client::Cluster; -use client::MangoClient; +use client::{MangoClient, MangoGroupContext}; use log::*; use mango_v4::state::{PerpMarketIndex, TokenIndex}; use once_cell::sync::OnceCell; use serde_derive::Deserialize; +use solana_client::rpc_client::RpcClient; use solana_sdk::commitment_config::CommitmentConfig; use solana_sdk::pubkey::Pubkey; use solana_sdk::signer::keypair; @@ -22,12 +21,17 @@ use std::str::FromStr; pub mod account_shared_data; pub mod chain_data; +pub mod chain_data_fetcher; pub mod liquidate; pub mod metrics; pub mod snapshot_source; pub mod util; pub mod websocket_source; +use crate::chain_data::*; +use crate::chain_data_fetcher::*; +use crate::util::{is_mango_account, is_mango_bank, is_mint_info, is_perp_market}; + // jemalloc seems to be better at keeping the memory footprint reasonable over // longer periods of time #[global_allocator] @@ -90,31 +94,19 @@ async fn main() -> anyhow::Result<()> { let mango_group_id = Pubkey::from_str(&config.mango_group_id)?; - // - // mango client setup - // - let mango_client = { - let payer = keypair::read_keypair_file(&config.payer).unwrap(); + let rpc_url = config.rpc_http_url.to_owned(); + let ws_url = rpc_url.replace("https", "wss"); + let rpc_timeout = Duration::from_secs(1); + let cluster = Cluster::Custom(rpc_url, ws_url); + let commitment = CommitmentConfig::processed(); + let group_context = + MangoGroupContext::new_from_rpc(mango_group_id, cluster.clone(), commitment)?; - let rpc_url = config.rpc_http_url.to_owned(); - let ws_url = rpc_url.replace("https", "wss"); - - let cluster = Cluster::Custom(rpc_url, ws_url); - let commitment = CommitmentConfig::confirmed(); - - Arc::new(MangoClient::new( - cluster, - commitment, - payer, - mango_group_id, - &config.mango_account_name, - )?) - }; - - let mango_pyth_oracles = mango_client - .mint_infos_cache + // TODO: this is all oracles, not just pyth! + let mango_pyth_oracles = group_context + .tokens .values() - .map(|value| value.1.oracle) + .map(|value| value.mint_info.oracle) .collect::>(); // @@ -142,7 +134,16 @@ async fn main() -> anyhow::Result<()> { snapshot_source::start(config.clone(), mango_pyth_oracles, snapshot_sender); // The representation of current on-chain account data - let mut chain_data = ChainData::new(&metrics); + let chain_data = Arc::new(RwLock::new(ChainData::new(&metrics))); + // Reading accounts from chain_data + let account_fetcher = Arc::new(ChainDataAccountFetcher { + chain_data: chain_data.clone(), + rpc: RpcClient::new_with_timeout_and_commitment( + cluster.url().to_string(), + rpc_timeout, + commitment, + ), + }); // Addresses of the MangoAccounts belonging to the mango program. // Needed to check health of them all when the cache updates. @@ -167,6 +168,22 @@ async fn main() -> anyhow::Result<()> { let mut metric_snapshot_queue_len = metrics.register_u64("snapshot_queue_length".into()); let mut metric_mango_accounts = metrics.register_u64("mango_accouns".into()); + // + // mango client setup + // + let mango_client = { + let payer = keypair::read_keypair_file(&config.payer).unwrap(); + + Arc::new(MangoClient::new_detail( + cluster, + commitment, + payer, + &config.mango_account_name, + group_context, + account_fetcher.clone(), + )?) + }; + info!("main loop"); loop { tokio::select! { @@ -177,7 +194,7 @@ async fn main() -> anyhow::Result<()> { // build a model of slots and accounts in `chain_data` // this code should be generic so it can be reused in future projects - chain_data.update_from_websocket(message.clone()); + chain_data.write().unwrap().update_from_websocket(message.clone()); // specific program logic using the mirrored data if let websocket_source::Message::Account(account_write) = message { @@ -197,10 +214,8 @@ async fn main() -> anyhow::Result<()> { if let Err(err) = liquidate::process_accounts( &mango_client, - &chain_data, + &account_fetcher, std::iter::once(&account_write.pubkey), - &mint_infos, - &perp_markets, ) { warn!("could not process account {}: {:?}", account_write.pubkey, err); @@ -230,10 +245,8 @@ async fn main() -> anyhow::Result<()> { // so optimizing much seems unnecessary. if let Err(err) = liquidate::process_accounts( &mango_client, - &chain_data, + &account_fetcher, mango_accounts.iter(), - &mint_infos, - &perp_markets, ) { warn!("could not process accounts: {:?}", err); } @@ -260,16 +273,14 @@ async fn main() -> anyhow::Result<()> { } metric_mango_accounts.set(mango_accounts.len() as u64); - chain_data.update_from_snapshot(message); + chain_data.write().unwrap().update_from_snapshot(message); one_snapshot_done = true; // trigger a full health check if let Err(err) = liquidate::process_accounts( &mango_client, - &chain_data, + &account_fetcher, mango_accounts.iter(), - &mint_infos, - &perp_markets, ) { warn!("could not process accounts: {:?}", err); } diff --git a/programs/mango-v4/src/state/mango_account.rs b/programs/mango-v4/src/state/mango_account.rs index 4eb28dab0..34b181e30 100644 --- a/programs/mango-v4/src/state/mango_account.rs +++ b/programs/mango-v4/src/state/mango_account.rs @@ -1,5 +1,4 @@ use anchor_lang::prelude::*; -use anchor_spl::token::Mint; use checked_math as cm; use fixed::types::I80F48; use static_assertions::const_assert_eq; @@ -63,13 +62,13 @@ impl TokenPosition { } } - pub fn ui(&self, bank: &Bank, mint: &Mint) -> I80F48 { + pub fn ui(&self, bank: &Bank) -> I80F48 { if self.indexed_position.is_positive() { (self.indexed_position * bank.deposit_index) - / I80F48::from_num(10u64.pow(mint.decimals as u32)) + / I80F48::from_num(10u64.pow(bank.mint_decimals as u32)) } else { (self.indexed_position * bank.borrow_index) - / I80F48::from_num(10u64.pow(mint.decimals as u32)) + / I80F48::from_num(10u64.pow(bank.mint_decimals as u32)) } }