liquidator: add a sequence check in rebalancing (#926)

liquidator: add a sequence check in rebalancing

bin/liquidator/src/main.rs

@@ -577,6 +577,7 @@ fn spawn_rebalance_job(
                 if let Err(err) = rebalancer.zero_all_non_quote().await {
                     error!("failed to rebalance liqor: {:?}", err);
 
+                    // TODO FAS Are there other scenario where this sleep is useful ?
                     // Workaround: We really need a sequence enforcer in the liquidator since we don't want to
                     // accidentally send a similar tx again when we incorrectly believe an earlier one got forked
                     // off. For now, hard sleep on error to avoid the most frequent error cases.

bin/liquidator/src/rebalance.rs

@@ -141,6 +141,7 @@ impl Rebalancer {
     /// Otherwise use the best of 4.
     async fn token_swap_buy(
         &self,
+        account: &MangoAccountValue,
         output_mint: Pubkey,
         in_amount_quote: u64,
     ) -> anyhow::Result<(Signature, swap::Quote)> {
@@ -186,7 +187,7 @@ impl Rebalancer {
             .determine_best_swap_tx(routes, quote_mint, output_mint)
             .await;
 
-        let (tx_builder, route) = match best_route_res {
+        let (mut tx_builder, route) = match best_route_res {
             Ok(x) => x,
             Err(e) => {
                 warn!("could not use simple routes because of {}, trying with an alternative one (if configured)", e);
@@ -211,6 +212,12 @@ impl Rebalancer {
             }
         };
 
+        let seq_check_ix = self
+            .mango_client
+            .sequence_check_instruction(&self.mango_account_address, account)
+            .await?;
+        tx_builder.append(seq_check_ix);
+
         let sig = tx_builder
             .send_and_confirm(&self.mango_client.client)
             .await?;
@@ -226,6 +233,7 @@ impl Rebalancer {
     /// Otherwise use the best of 4.
     async fn token_swap_sell(
         &self,
+        account: &MangoAccountValue,
         input_mint: Pubkey,
         in_amount: u64,
     ) -> anyhow::Result<(Signature, swap::Quote)> {
@@ -260,7 +268,7 @@ impl Rebalancer {
             .determine_best_swap_tx(routes, input_mint, quote_mint)
             .await;
 
-        let (tx_builder, route) = match best_route_res {
+        let (mut tx_builder, route) = match best_route_res {
             Ok(x) => x,
             Err(e) => {
                 warn!("could not use simple routes because of {}, trying with an alternative one (if configured)", e);
@@ -283,6 +291,12 @@ impl Rebalancer {
             }
         };
 
+        let seq_check_ix = self
+            .mango_client
+            .sequence_check_instruction(&self.mango_account_address, account)
+            .await?;
+        tx_builder.append(seq_check_ix);
+
         let sig = tx_builder
             .send_and_confirm(&self.mango_client.client)
             .await?;
@@ -475,7 +489,7 @@ impl Rebalancer {
                 let input_amount =
                     buy_amount * token_price * I80F48::from_num(self.config.borrow_settle_excess);
                 let (txsig, route) = self
-                    .token_swap_buy(token_mint, input_amount.to_num())
+                    .token_swap_buy(&account, token_mint, input_amount.to_num())
                     .await?;
                 let in_token = self
                     .mango_client
@@ -502,7 +516,7 @@ impl Rebalancer {
                 // To avoid creating a borrow when paying flash loan fees, sell only a fraction
                 let input_amount = amount * I80F48::from_num(0.99);
                 let (txsig, route) = self
-                    .token_swap_sell(token_mint, input_amount.to_num::<u64>())
+                    .token_swap_sell(&account, token_mint, input_amount.to_num::<u64>())
                     .await?;
                 let out_token = self
                     .mango_client
@@ -556,13 +570,13 @@ impl Rebalancer {
     }
 
     #[instrument(
-    skip_all,
-    fields(
-    perp_market_name = perp.name,
-    base_lots = perp_position.base_position_lots(),
-    effective_lots = perp_position.effective_base_position_lots(),
-    quote_native = %perp_position.quote_position_native()
-    )
+        skip_all,
+        fields(
+            perp_market_name = perp.name,
+            base_lots = perp_position.base_position_lots(),
+            effective_lots = perp_position.effective_base_position_lots(),
+            quote_native = %perp_position.quote_position_native()
+        )
     )]
     async fn rebalance_perp(
         &self,
@@ -624,7 +638,7 @@ impl Rebalancer {
                 return Ok(true);
             }
 
-            let ixs = self
+            let mut ixs = self
                 .mango_client
                 .perp_place_order_instruction(
                     account,
@@ -642,6 +656,12 @@ impl Rebalancer {
                 )
                 .await?;
 
+            let seq_check_ix = self
+                .mango_client
+                .sequence_check_instruction(&self.mango_account_address, account)
+                .await?;
+            ixs.append(seq_check_ix);
+
             let tx_builder = TransactionBuilder {
                 instructions: ixs.to_instructions(),
                 signers: vec![self.mango_client.authority.clone()],

lib/client/src/client.rs

@@ -623,6 +623,34 @@ impl MangoClient {
         Ok(ixs)
     }
 
+    /// Avoid executing same instruction multiple time
+    pub async fn sequence_check_instruction(
+        &self,
+        mango_account_address: &Pubkey,
+        mango_account: &MangoAccountValue,
+    ) -> anyhow::Result<PreparedInstructions> {
+        let ixs = PreparedInstructions::from_vec(
+            vec![Instruction {
+                program_id: mango_v4::id(),
+                accounts: {
+                    anchor_lang::ToAccountMetas::to_account_metas(
+                        &mango_v4::accounts::SequenceCheck {
+                            group: self.group(),
+                            account: *mango_account_address,
+                            owner: mango_account.fixed.owner,
+                        },
+                        None,
+                    )
+                },
+                data: anchor_lang::InstructionData::data(&mango_v4::instruction::SequenceCheck {
+                    expected_sequence_number: mango_account.fixed.sequence_number,
+                }),
+            }],
+            self.context.compute_estimates.cu_for_sequence_check,
+        );
+        Ok(ixs)
+    }
+
     /// Creates token withdraw instructions for the MangoClient's account/owner.
     /// The `account` state is passed in separately so changes during the tx can be
     /// accounted for when deriving health accounts.

lib/client/src/context.rs

@@ -128,6 +128,7 @@ pub struct ComputeEstimates {
     pub cu_perp_consume_events_base: u32,
     pub cu_perp_consume_events_per_event: u32,
     pub cu_token_update_index_and_rates: u32,
+    pub cu_for_sequence_check: u32,
 }
 
 impl Default for ComputeEstimates {
@@ -156,6 +157,8 @@ impl Default for ComputeEstimates {
             cu_perp_consume_events_base: 10_000,
             cu_perp_consume_events_per_event: 18_000,
             cu_token_update_index_and_rates: 90_000,
+            // measured around 8k, see test_basics
+            cu_for_sequence_check: 10_000,
         }
     }
 }