liquidator: add a sequence check in rebalancing (#926)

liquidator: add a sequence check in rebalancing

bin/liquidator/src/main.rs

@@ -543,6 +543,7 @@ fn spawn_rebalance_job(
                 if let Err(err) = rebalancer.zero_all_non_quote().await {
                     error!("failed to rebalance liqor: {:?}", err);
 
+                    // TODO FAS Are there other scenario where this sleep is useful ?
                     // Workaround: We really need a sequence enforcer in the liquidator since we don't want to
                     // accidentally send a similar tx again when we incorrectly believe an earlier one got forked
                     // off. For now, hard sleep on error to avoid the most frequent error cases.

bin/liquidator/src/rebalance.rs

@@ -133,6 +133,7 @@ impl Rebalancer {
     /// Use 1. if it fits into a tx. Otherwise use the better of 2./3.
     async fn token_swap_buy(
         &self,
+        account: &MangoAccountValue,
         output_mint: Pubkey,
         in_amount_quote: u64,
     ) -> anyhow::Result<(Signature, jupiter::Quote)> {
@@ -174,13 +175,20 @@ impl Rebalancer {
         let full_route = results.remove(0)?;
         let alternatives = results.into_iter().filter_map(|v| v.ok()).collect_vec();
 
-        let (tx_builder, route) = self
+        let (mut tx_builder, route) = self
             .determine_best_jupiter_tx(
                 // If the best_route couldn't be fetched, something is wrong
                 &full_route,
                 &alternatives,
             )
             .await?;
+
+        let seq_check_ix = self
+            .mango_client
+            .sequence_check_instruction(&self.mango_account_address, account)
+            .await?;
+        tx_builder.append(seq_check_ix);
+
         let sig = tx_builder
             .send_and_confirm(&self.mango_client.client)
             .await?;
@@ -194,6 +202,7 @@ impl Rebalancer {
     /// Use 1. if it fits into a tx. Otherwise use the better of 2./3.
     async fn token_swap_sell(
         &self,
+        account: &MangoAccountValue,
         input_mint: Pubkey,
         in_amount: u64,
     ) -> anyhow::Result<(Signature, jupiter::Quote)> {
@@ -218,7 +227,7 @@ impl Rebalancer {
         let full_route = results.remove(0)?;
         let alternatives = results.into_iter().filter_map(|v| v.ok()).collect_vec();
 
-        let (tx_builder, route) = self
+        let (mut tx_builder, route) = self
             .determine_best_jupiter_tx(
                 // If the best_route couldn't be fetched, something is wrong
                 &full_route,
@@ -226,6 +235,12 @@ impl Rebalancer {
             )
             .await?;
 
+        let seq_check_ix = self
+            .mango_client
+            .sequence_check_instruction(&self.mango_account_address, account)
+            .await?;
+        tx_builder.append(seq_check_ix);
+
         let sig = tx_builder
             .send_and_confirm(&self.mango_client.client)
             .await?;
@@ -331,7 +346,7 @@ impl Rebalancer {
                 let input_amount =
                     buy_amount * token_price * I80F48::from_num(self.config.borrow_settle_excess);
                 let (txsig, route) = self
-                    .token_swap_buy(token_mint, input_amount.to_num())
+                    .token_swap_buy(&account, token_mint, input_amount.to_num())
                     .await?;
                 let in_token = self
                     .mango_client
@@ -355,7 +370,7 @@ impl Rebalancer {
             if amount > dust_threshold {
                 // Sell
                 let (txsig, route) = self
-                    .token_swap_sell(token_mint, amount.to_num::<u64>())
+                    .token_swap_sell(&account, token_mint, amount.to_num::<u64>())
                     .await?;
                 let out_token = self
                     .mango_client
@@ -477,9 +492,10 @@ impl Rebalancer {
                 return Ok(true);
             }
 
-            let txsig = self
+            let mut ixs = self
                 .mango_client
-                .perp_place_order(
+                .perp_place_order_instruction(
+                    account,
                     perp_position.market_index,
                     side,
                     price_lots,
@@ -493,6 +509,23 @@ impl Rebalancer {
                     mango_v4::state::SelfTradeBehavior::DecrementTake,
                 )
                 .await?;
+
+            let seq_check_ix = self
+                .mango_client
+                .sequence_check_instruction(&self.mango_account_address, account)
+                .await?;
+            ixs.append(seq_check_ix);
+
+            let tx_builder = TransactionBuilder {
+                instructions: ixs.to_instructions(),
+                signers: vec![self.mango_client.owner.clone()],
+                ..self.mango_client.transaction_builder().await?
+            };
+
+            let txsig = tx_builder
+                .send_and_confirm(&self.mango_client.client)
+                .await?;
+
             info!(
                 %txsig,
                 %order_price,

lib/client/src/client.rs

@@ -610,6 +610,34 @@ impl MangoClient {
         Ok(ixs)
     }
 
+    /// Avoid executing same instruction multiple time
+    pub async fn sequence_check_instruction(
+        &self,
+        mango_account_address: &Pubkey,
+        mango_account: &MangoAccountValue,
+    ) -> anyhow::Result<PreparedInstructions> {
+        let ixs = PreparedInstructions::from_vec(
+            vec![Instruction {
+                program_id: mango_v4::id(),
+                accounts: {
+                    anchor_lang::ToAccountMetas::to_account_metas(
+                        &mango_v4::accounts::SequenceCheck {
+                            group: self.group(),
+                            account: *mango_account_address,
+                            owner: mango_account.fixed.owner,
+                        },
+                        None,
+                    )
+                },
+                data: anchor_lang::InstructionData::data(&mango_v4::instruction::SequenceCheck {
+                    expected_sequence_number: mango_account.fixed.sequence_number,
+                }),
+            }],
+            self.context.compute_estimates.cu_for_sequence_check,
+        );
+        Ok(ixs)
+    }
+
     /// Creates token withdraw instructions for the MangoClient's account/owner.
     /// The `account` state is passed in separately so changes during the tx can be
     /// accounted for when deriving health accounts.
@@ -2476,6 +2504,11 @@ impl TransactionBuilder {
             length: bytes.len(),
         })
     }
+
+    pub fn append(&mut self, prepared_instructions: PreparedInstructions) {
+        self.instructions
+            .extend(prepared_instructions.to_instructions());
+    }
 }
 
 /// Do some manual unpacking on some ClientErrors

lib/client/src/context.rs

@@ -122,6 +122,7 @@ pub struct ComputeEstimates {
     pub cu_per_oracle_fallback: u32,
     pub cu_per_charge_collateral_fees: u32,
     pub cu_per_charge_collateral_fees_token: u32,
+    pub cu_for_sequence_check: u32,
 }
 
 impl Default for ComputeEstimates {
@@ -145,6 +146,8 @@ impl Default for ComputeEstimates {
             cu_per_charge_collateral_fees: 20_000,
             // per-chargable-token cost
             cu_per_charge_collateral_fees_token: 15_000,
+            // measured around 8k, see test_basics
+            cu_for_sequence_check: 10_000,
         }
     }
 }