From 0380b3f09749d5f80955a9fa64d17f507f4a43f3 Mon Sep 17 00:00:00 2001 From: Christian Kamm Date: Wed, 7 Feb 2024 12:50:35 +0100 Subject: [PATCH] Settler: Keep going on error (#873) in particular: don't panic when a health cache can't be constructed (cherry picked from commit 712a2e3bd6d28212fab0e572e8499a13024d7bc8) --- bin/settler/src/main.rs | 8 ++++-- bin/settler/src/settle.rs | 53 ++++++++++++++++++++++++------------ bin/settler/src/tcs_start.rs | 25 +++++++++++++---- 3 files changed, 61 insertions(+), 25 deletions(-) diff --git a/bin/settler/src/main.rs b/bin/settler/src/main.rs index ae357518b..3dde16acf 100644 --- a/bin/settler/src/main.rs +++ b/bin/settler/src/main.rs @@ -311,7 +311,9 @@ async fn main() -> anyhow::Result<()> { account_addresses = state.mango_accounts.iter().cloned().collect(); } - settlement.settle(account_addresses).await.unwrap(); + if let Err(err) = settlement.settle(account_addresses).await { + warn!("settle error: {err:?}"); + } } } }); @@ -332,7 +334,9 @@ async fn main() -> anyhow::Result<()> { account_addresses = state.mango_accounts.iter().cloned().collect(); } - tcs_start.run_pass(account_addresses).await.unwrap(); + if let Err(err) = tcs_start.run_pass(account_addresses).await { + warn!("tcs-start error: {err:?}"); + } } } }); diff --git a/bin/settler/src/settle.rs b/bin/settler/src/settle.rs index af938006a..bac70a040 100644 --- a/bin/settler/src/settle.rs +++ b/bin/settler/src/settle.rs @@ -16,7 +16,7 @@ use solana_sdk::signature::Signature; use solana_sdk::signer::Signer; use solana_sdk::transaction::VersionedTransaction; use tracing::*; -use {anyhow::Context, fixed::types::I80F48, solana_sdk::pubkey::Pubkey}; +use {fixed::types::I80F48, solana_sdk::pubkey::Pubkey}; pub struct Config { /// Amount of time to wait before reusing a positive-pnl account @@ -100,7 +100,13 @@ impl SettlementState { let mut all_negative_settleable = HashMap::>::new(); for account_key in accounts.iter() { - let mut account = account_fetcher.fetch_mango_account(account_key)?; + let mut account = match account_fetcher.fetch_mango_account(account_key) { + Ok(acc) => acc, + Err(e) => { + info!("could not fetch account, skipping {account_key}: {e:?}"); + continue; + } + }; if account.fixed.group != mango_client.group() { continue; } @@ -115,9 +121,11 @@ impl SettlementState { continue; } - let health_cache = health_cache::new(&mango_client.context, account_fetcher, &account) - .await - .context("creating health cache")?; + let health_cache = + match health_cache::new(&mango_client.context, account_fetcher, &account).await { + Ok(hc) => hc, + Err(_) => continue, // Skip for stale/unconfident oracles + }; let liq_end_health = health_cache.health(HealthType::LiquidationEnd); for perp_market_index in perp_indexes { @@ -126,14 +134,19 @@ impl SettlementState { Some(v) => v, None => continue, // skip accounts with perp positions where we couldn't get the price and market }; - let perp_max_settle = - health_cache.perp_max_settle(perp_market.settle_token_index)?; + let perp_max_settle = health_cache + .perp_max_settle(perp_market.settle_token_index) + .expect("perp_max_settle always succeeds when the token index is valid"); - let perp_position = account.perp_position_mut(perp_market_index).unwrap(); + let perp_position = account + .perp_position_mut(perp_market_index) + .expect("index comes from active_perp_positions()"); perp_position.settle_funding(perp_market); perp_position.update_settle_limit(perp_market, now_ts); - let unsettled = perp_position.unsettled_pnl(perp_market, *perp_price)?; + let unsettled = perp_position + .unsettled_pnl(perp_market, *perp_price) + .expect("unsettled_pnl always succeeds with the right perp market"); let limited = perp_position.apply_pnl_settle_limit(perp_market, unsettled); let settleable = if limited >= 0 { limited @@ -160,7 +173,7 @@ impl SettlementState { liq_end_health, maint_health, ) - .unwrap(); + .expect("always ok"); // Assume that settle_fee_flat is near the tx fee, and if we can't possibly // make up for the tx fee even with multiple settle ix in one tx, skip. @@ -184,7 +197,9 @@ impl SettlementState { let address_lookup_tables = mango_client.mango_address_lookup_tables().await?; for (perp_market_index, mut positive_settleable) in all_positive_settleable { - let (perp_market, _, _) = perp_market_info.get(&perp_market_index).unwrap(); + let (perp_market, _, _) = perp_market_info + .get(&perp_market_index) + .expect("perp market must exist"); let negative_settleable = match all_negative_settleable.get_mut(&perp_market_index) { None => continue, Some(v) => v, @@ -295,14 +310,16 @@ impl<'a> SettleBatchProcessor<'a> { .await .map_err(prettify_solana_client_error); - if let Err(err) = send_result { - info!("error while sending settle batch: {}", err); - return Ok(None); + match send_result { + Ok(txsig) => { + info!("sent settle tx: {txsig}"); + Ok(Some(txsig)) + } + Err(err) => { + info!("error while sending settle batch: {}", err); + Ok(None) + } } - - let txsig = send_result.unwrap(); - info!("sent settle tx: {txsig}"); - Ok(Some(txsig)) } async fn add_and_maybe_send( diff --git a/bin/settler/src/tcs_start.rs b/bin/settler/src/tcs_start.rs index 6c98eb012..d2b96461e 100644 --- a/bin/settler/src/tcs_start.rs +++ b/bin/settler/src/tcs_start.rs @@ -59,7 +59,13 @@ impl State { let mut startable = vec![]; for account_key in accounts.iter() { - let account = account_fetcher.fetch_mango_account(account_key).unwrap(); + let account = match account_fetcher.fetch_mango_account(account_key) { + Ok(acc) => acc, + Err(e) => { + info!("could not fetch account, skipping {account_key}: {e:?}"); + continue; + } + }; if account.fixed.group != mango_client.group() { continue; } @@ -94,6 +100,11 @@ impl State { let mut ix_targets = vec![]; let mut liqor_account = mango_client.mango_account().await?; for (pubkey, tcs_id, incentive_token_index) in startable_chunk { + // can only batch until all token positions are full + if let Err(_) = liqor_account.ensure_token_position(*incentive_token_index) { + break; + } + let ixs = match self.make_start_ix(pubkey, *tcs_id).await { Ok(v) => v, Err(e) => { @@ -107,18 +118,22 @@ impl State { }; instructions.append(ixs); ix_targets.push((*pubkey, *tcs_id)); - liqor_account.ensure_token_position(*incentive_token_index)?; } // Clear newly created token positions, so the liqor account is mostly empty for token_index in startable_chunk.iter().map(|(_, _, ti)| *ti).unique() { let mint = mango_client.context.token(token_index).mint; - instructions.append(mango_client.token_withdraw_instructions( + let ix = match mango_client.token_withdraw_instructions( &liqor_account, mint, u64::MAX, false, - )?); + ) { + Ok(ix) => ix, + Err(_) => continue, + }; + + instructions.append(ix) } let txsig = match mango_client @@ -159,7 +174,7 @@ impl State { pubkey: &Pubkey, tcs_id: u64, ) -> anyhow::Result { - let account = self.account_fetcher.fetch_mango_account(pubkey).unwrap(); + let account = self.account_fetcher.fetch_mango_account(pubkey)?; self.mango_client .token_conditional_swap_start_instruction((pubkey, &account), tcs_id) .await