Settler: Keep going on error (#873)

in particular: don't panic when a health cache can't be constructed

(cherry picked from commit 712a2e3bd6)

bin/settler/src/main.rs

@@ -311,7 +311,9 @@ async fn main() -> anyhow::Result<()> {
                     account_addresses = state.mango_accounts.iter().cloned().collect();
                 }
 
-                settlement.settle(account_addresses).await.unwrap();
+                if let Err(err) = settlement.settle(account_addresses).await {
+                    warn!("settle error: {err:?}");
+                }
             }
         }
     });
@@ -332,7 +334,9 @@ async fn main() -> anyhow::Result<()> {
                     account_addresses = state.mango_accounts.iter().cloned().collect();
                 }
 
-                tcs_start.run_pass(account_addresses).await.unwrap();
+                if let Err(err) = tcs_start.run_pass(account_addresses).await {
+                    warn!("tcs-start error: {err:?}");
+                }
             }
         }
     });

bin/settler/src/settle.rs

@@ -16,7 +16,7 @@ use solana_sdk::signature::Signature;
 use solana_sdk::signer::Signer;
 use solana_sdk::transaction::VersionedTransaction;
 use tracing::*;
-use {anyhow::Context, fixed::types::I80F48, solana_sdk::pubkey::Pubkey};
+use {fixed::types::I80F48, solana_sdk::pubkey::Pubkey};
 
 pub struct Config {
     /// Amount of time to wait before reusing a positive-pnl account
@@ -100,7 +100,13 @@ impl SettlementState {
         let mut all_negative_settleable =
             HashMap::<PerpMarketIndex, priority_queue::PriorityQueue<Pubkey, I80F48>>::new();
         for account_key in accounts.iter() {
-            let mut account = account_fetcher.fetch_mango_account(account_key)?;
+            let mut account = match account_fetcher.fetch_mango_account(account_key) {
+                Ok(acc) => acc,
+                Err(e) => {
+                    info!("could not fetch account, skipping {account_key}: {e:?}");
+                    continue;
+                }
+            };
             if account.fixed.group != mango_client.group() {
                 continue;
             }
@@ -115,9 +121,11 @@ impl SettlementState {
                 continue;
             }
 
-            let health_cache = health_cache::new(&mango_client.context, account_fetcher, &account)
+            let health_cache =
-                .await
+                match health_cache::new(&mango_client.context, account_fetcher, &account).await {
-                .context("creating health cache")?;
+                    Ok(hc) => hc,
+                    Err(_) => continue, // Skip for stale/unconfident oracles
+                };
             let liq_end_health = health_cache.health(HealthType::LiquidationEnd);
 
             for perp_market_index in perp_indexes {
@@ -126,14 +134,19 @@ impl SettlementState {
                         Some(v) => v,
                         None => continue, // skip accounts with perp positions where we couldn't get the price and market
                     };
-                let perp_max_settle =
+                let perp_max_settle = health_cache
-                    health_cache.perp_max_settle(perp_market.settle_token_index)?;
+                    .perp_max_settle(perp_market.settle_token_index)
+                    .expect("perp_max_settle always succeeds when the token index is valid");
 
-                let perp_position = account.perp_position_mut(perp_market_index).unwrap();
+                let perp_position = account
+                    .perp_position_mut(perp_market_index)
+                    .expect("index comes from active_perp_positions()");
                 perp_position.settle_funding(perp_market);
                 perp_position.update_settle_limit(perp_market, now_ts);
 
-                let unsettled = perp_position.unsettled_pnl(perp_market, *perp_price)?;
+                let unsettled = perp_position
+                    .unsettled_pnl(perp_market, *perp_price)
+                    .expect("unsettled_pnl always succeeds with the right perp market");
                 let limited = perp_position.apply_pnl_settle_limit(perp_market, unsettled);
                 let settleable = if limited >= 0 {
                     limited
@@ -160,7 +173,7 @@ impl SettlementState {
                             liq_end_health,
                             maint_health,
                         )
-                        .unwrap();
+                        .expect("always ok");
 
                     // Assume that settle_fee_flat is near the tx fee, and if we can't possibly
                     // make up for the tx fee even with multiple settle ix in one tx, skip.
@@ -184,7 +197,9 @@ impl SettlementState {
         let address_lookup_tables = mango_client.mango_address_lookup_tables().await?;
 
         for (perp_market_index, mut positive_settleable) in all_positive_settleable {
-            let (perp_market, _, _) = perp_market_info.get(&perp_market_index).unwrap();
+            let (perp_market, _, _) = perp_market_info
+                .get(&perp_market_index)
+                .expect("perp market must exist");
             let negative_settleable = match all_negative_settleable.get_mut(&perp_market_index) {
                 None => continue,
                 Some(v) => v,
@@ -295,15 +310,17 @@ impl<'a> SettleBatchProcessor<'a> {
             .await
             .map_err(prettify_solana_client_error);
 
-        if let Err(err) = send_result {
+        match send_result {
-            info!("error while sending settle batch: {}", err);
+            Ok(txsig) => {
-            return Ok(None);
-        }
-
-        let txsig = send_result.unwrap();
                 info!("sent settle tx: {txsig}");
                 Ok(Some(txsig))
             }
+            Err(err) => {
+                info!("error while sending settle batch: {}", err);
+                Ok(None)
+            }
+        }
+    }
 
     async fn add_and_maybe_send(
         &mut self,

bin/settler/src/tcs_start.rs

@@ -59,7 +59,13 @@ impl State {
 
         let mut startable = vec![];
         for account_key in accounts.iter() {
-            let account = account_fetcher.fetch_mango_account(account_key).unwrap();
+            let account = match account_fetcher.fetch_mango_account(account_key) {
+                Ok(acc) => acc,
+                Err(e) => {
+                    info!("could not fetch account, skipping {account_key}: {e:?}");
+                    continue;
+                }
+            };
             if account.fixed.group != mango_client.group() {
                 continue;
             }
@@ -94,6 +100,11 @@ impl State {
             let mut ix_targets = vec![];
             let mut liqor_account = mango_client.mango_account().await?;
             for (pubkey, tcs_id, incentive_token_index) in startable_chunk {
+                // can only batch until all token positions are full
+                if let Err(_) = liqor_account.ensure_token_position(*incentive_token_index) {
+                    break;
+                }
+
                 let ixs = match self.make_start_ix(pubkey, *tcs_id).await {
                     Ok(v) => v,
                     Err(e) => {
@@ -107,18 +118,22 @@ impl State {
                 };
                 instructions.append(ixs);
                 ix_targets.push((*pubkey, *tcs_id));
-                liqor_account.ensure_token_position(*incentive_token_index)?;
             }
 
             // Clear newly created token positions, so the liqor account is mostly empty
             for token_index in startable_chunk.iter().map(|(_, _, ti)| *ti).unique() {
                 let mint = mango_client.context.token(token_index).mint;
-                instructions.append(mango_client.token_withdraw_instructions(
+                let ix = match mango_client.token_withdraw_instructions(
                     &liqor_account,
                     mint,
                     u64::MAX,
                     false,
-                )?);
+                ) {
+                    Ok(ix) => ix,
+                    Err(_) => continue,
+                };
+
+                instructions.append(ix)
             }
 
             let txsig = match mango_client
@@ -159,7 +174,7 @@ impl State {
         pubkey: &Pubkey,
         tcs_id: u64,
     ) -> anyhow::Result<PreparedInstructions> {
-        let account = self.account_fetcher.fetch_mango_account(pubkey).unwrap();
+        let account = self.account_fetcher.fetch_mango_account(pubkey)?;
         self.mango_client
             .token_conditional_swap_start_instruction((pubkey, &account), tcs_id)
             .await