Liq: Don't log common+expected simulation errors

Cargo.lock

@@ -918,6 +918,7 @@ dependencies = [
  "serum_dex",
  "solana-client",
  "solana-sdk",
+ "thiserror",
 ]
 
 [[package]]

client/Cargo.toml

@@ -14,8 +14,9 @@ anyhow = "1.0"
 fixed = { version = "=1.11.0", features = ["serde", "borsh"] }
 fixed-macro = "^1.1.1"
 itertools = "0.10.3"
-mango-v4 = { path = "../programs/mango-v4" } 
+mango-v4 = { path = "../programs/mango-v4" }
 pyth-sdk-solana = "0.1.0"
 serum_dex = { version = "0.4.0", git = "https://github.com/blockworks-foundation/serum-dex.git", default-features=false, features = ["no-entrypoint", "program"] }
 solana-client = "~1.9.13"
 solana-sdk = "~1.9.13"
+thiserror = "1.0.31"

client/src/client.rs

@@ -740,6 +740,12 @@ struct Serum3Data<'a> {
     base: &'a TokenContext,
 }
 
+#[derive(Debug, thiserror::Error)]
+pub enum MangoClientError {
+    #[error("Transaction simulation error. Logs: {logs}")]
+    SendTransactionPreflightFailure { logs: String },
+}
+
 /// Do some manual unpacking on some ClientErrors
 ///
 /// Unfortunately solana's RpcResponseError will very unhelpfully print [N log messages]
@@ -754,10 +760,10 @@ fn prettify_client_error(err: anchor_client::ClientError) -> anyhow::Error {
                 ClientErrorKind::RpcError(RpcError::RpcResponseError { data, .. }) => match data {
                     RpcResponseErrorData::SendTransactionPreflightFailure(s) => {
                         if let Some(logs) = s.logs.as_ref() {
-                            return anyhow::anyhow!(
-                                "transaction simulation error. logs:\n{}",
-                                logs.iter().map(|l| format!("    {}", l)).join("\n")
-                            );
+                            return MangoClientError::SendTransactionPreflightFailure {
+                                logs: logs.iter().join("; "),
+                            }
+                            .into();
                         }
                     }
                     _ => {}

liquidator/src/liquidate.rs

@@ -1,7 +1,7 @@
 use crate::account_shared_data::KeyedAccountSharedData;
 use crate::ChainDataAccountFetcher;
 
-use client::{AccountFetcher, MangoClient, MangoGroupContext};
+use client::{AccountFetcher, MangoClient, MangoClientError, MangoGroupContext};
 use mango_v4::state::{
     new_health_cache, oracle_price, Bank, FixedOrderAccountRetriever, HealthCache, HealthType,
     MangoAccount, TokenIndex,
@@ -172,7 +172,22 @@ pub fn process_accounts<'a>(
 ) -> anyhow::Result<()> {
     for pubkey in accounts {
         match process_account(mango_client, account_fetcher, pubkey) {
-            Err(err) => log::error!("error liquidating account {}: {:?}", pubkey, err),
+            Err(err) => {
+                // Not all errors need to be raised to the user's attention.
+                let mut log_level = log::Level::Error;
+
+                // Simulation errors due to liqee precondition failures on the liquidation instructions
+                // will commonly happen if our liquidator is late or if there are chain forks.
+                match err.downcast_ref::<MangoClientError>() {
+                    Some(MangoClientError::SendTransactionPreflightFailure { logs }) => {
+                        if logs.contains("HealthMustBeNegative") || logs.contains("IsNotBankrupt") {
+                            log_level = log::Level::Trace;
+                        }
+                    }
+                    _ => {}
+                };
+                log::log!(log_level, "liquidating account {}: {:?}", pubkey, err);
+            }
             _ => {}
         };
     }

programs/mango-v4/src/error.rs

@@ -17,6 +17,8 @@ pub enum MangoError {
     InvalidFlashLoanTargetCpiProgram,
     #[msg("health must be positive")]
     HealthMustBePositive,
+    #[msg("health must be negative")]
+    HealthMustBeNegative,
     #[msg("the account is bankrupt")]
     IsBankrupt,
     #[msg("the account is not bankrupt")]

programs/mango-v4/src/instructions/liq_token_with_token.rs

@@ -49,6 +49,7 @@ pub fn liq_token_with_token(
     let mut liqee_health_cache =
         new_health_cache(&liqee, &account_retriever).context("create liqee health cache")?;
     let init_health = liqee_health_cache.health(HealthType::Init);
+    msg!("init health: {}", init_health);
     if liqee.being_liquidated() {
         if init_health > I80F48::ZERO {
             liqee.set_being_liquidated(false);
@@ -57,7 +58,11 @@ pub fn liq_token_with_token(
         }
     } else {
         let maint_health = liqee_health_cache.health(HealthType::Maint);
-        require!(maint_health < I80F48::ZERO, MangoError::SomeError);
+        msg!("maint health: {}", maint_health);
+        require!(
+            maint_health < I80F48::ZERO,
+            MangoError::HealthMustBeNegative
+        );
         liqee.set_being_liquidated(true);
     }
 

ts/client/src/mango_v4.ts

@@ -4747,33 +4747,43 @@ export type MangoV4 = {
     },
     {
       "code": 6006,
+      "name": "HealthMustBeNegative",
+      "msg": "health must be negative"
+    },
+    {
+      "code": 6007,
       "name": "IsBankrupt",
       "msg": "the account is bankrupt"
     },
     {
-      "code": 6007,
+      "code": 6008,
       "name": "IsNotBankrupt",
       "msg": "the account is not bankrupt"
     },
     {
-      "code": 6008,
+      "code": 6009,
       "name": "NoFreeTokenPositionIndex",
       "msg": "no free token position index"
     },
     {
-      "code": 6009,
+      "code": 6010,
       "name": "NoFreeSerum3OpenOrdersIndex",
       "msg": "no free serum3 open orders index"
     },
     {
-      "code": 6010,
+      "code": 6011,
       "name": "NoFreePerpPositionIndex",
       "msg": "no free perp position index"
     },
     {
-      "code": 6011,
+      "code": 6012,
       "name": "Serum3OpenOrdersExistAlready",
       "msg": "serum3 open orders exist already"
+    },
+    {
+      "code": 6013,
+      "name": "InsufficentBankVaultFunds",
+      "msg": "bank vault has insufficent funds"
     }
   ]
 };
@@ -9527,33 +9537,43 @@ export const IDL: MangoV4 = {
     },
     {
       "code": 6006,
+      "name": "HealthMustBeNegative",
+      "msg": "health must be negative"
+    },
+    {
+      "code": 6007,
       "name": "IsBankrupt",
       "msg": "the account is bankrupt"
     },
     {
-      "code": 6007,
+      "code": 6008,
       "name": "IsNotBankrupt",
       "msg": "the account is not bankrupt"
     },
     {
-      "code": 6008,
+      "code": 6009,
       "name": "NoFreeTokenPositionIndex",
       "msg": "no free token position index"
     },
     {
-      "code": 6009,
+      "code": 6010,
       "name": "NoFreeSerum3OpenOrdersIndex",
       "msg": "no free serum3 open orders index"
     },
     {
-      "code": 6010,
+      "code": 6011,
       "name": "NoFreePerpPositionIndex",
       "msg": "no free perp position index"
     },
     {
-      "code": 6011,
+      "code": 6012,
       "name": "Serum3OpenOrdersExistAlready",
       "msg": "serum3 open orders exist already"
+    },
+    {
+      "code": 6013,
+      "name": "InsufficentBankVaultFunds",
+      "msg": "bank vault has insufficent funds"
     }
   ]
 };