Print errors that we encounter while quoting swap, and list missing accounts (#46)

lib/dex-infinity/src/infinity.rs

@@ -47,8 +47,8 @@ impl DexInterface for InfinityDex {
             lst_state_list,
             pool_state,
         } = SPoolJup::init_keys(program_id);
-        let lst_state_list_account = rpc.get_account(&lst_state_list).await.unwrap();
-        let pool_state_account = rpc.get_account(&pool_state).await.unwrap();
+        let lst_state_list_account = rpc.get_account(&lst_state_list).await.unwrap().unwrap();
+        let pool_state_account = rpc.get_account(&pool_state).await.unwrap().unwrap();
 
         let amm: s_jup_interface::SPool<Account, Account> = SPoolJup::from_init_accounts(
             program_id,

lib/dex-infinity/tests/test_infinity.rs

@@ -1,6 +1,7 @@
 use std::collections::HashMap;
 use std::env;
 
+use router_feed_lib::utils::tracing_subscriber_init;
 use solana_program_test::tokio;
 
 use router_lib::dex::DexInterface;
@@ -8,6 +9,7 @@ use router_lib::test_tools::{generate_dex_rpc_dump, rpc};
 
 #[tokio::test]
 async fn test_dump_input_data_infinity() -> anyhow::Result<()> {
+    tracing_subscriber_init();
     if router_test_lib::config_should_dump_mainnet_data() {
         step_1_infinity().await?;
     }

lib/dex-invariant/tests/test_invariant.rs

@@ -1,3 +1,4 @@
+use router_feed_lib::utils::tracing_subscriber_init;
 use solana_program_test::tokio;
 use std::collections::HashMap;
 use std::env;
@@ -7,6 +8,7 @@ use router_lib::test_tools::{generate_dex_rpc_dump, rpc};
 
 #[tokio::test]
 async fn test_dump_input_data_invariant() -> anyhow::Result<()> {
+    tracing_subscriber_init();
     let options = HashMap::from([]);
 
     if router_test_lib::config_should_dump_mainnet_data() {

lib/dex-openbook-v2/tests/test_openbook_v2_cp.rs

@@ -2,12 +2,14 @@ use std::collections::HashMap;
 use std::env;
 
 use dex_openbook_v2::OpenbookV2Edge;
+use router_feed_lib::utils::tracing_subscriber_init;
 use router_lib::dex::DexInterface;
 use router_lib::test_tools::{generate_dex_rpc_dump, rpc};
 use solana_program_test::tokio;
 
 #[tokio::test]
 async fn test_dump_input_data_openbook_v2() -> anyhow::Result<()> {
+    tracing_subscriber_init();
     let options = HashMap::from([]);
     if router_test_lib::config_should_dump_mainnet_data() {
         openbook_v2_step_1(&options).await?;

lib/dex-orca/tests/test_cropper.rs

@@ -1,6 +1,7 @@
 use std::collections::HashMap;
 use std::env;
 
+use router_feed_lib::utils::tracing_subscriber_init;
 use solana_program_test::tokio;
 
 use router_lib::dex::DexInterface;
@@ -8,6 +9,7 @@ use router_lib::test_tools::{generate_dex_rpc_dump, rpc};
 
 #[tokio::test]
 async fn test_dump_input_data_cropper() -> anyhow::Result<()> {
+    tracing_subscriber_init();
     let is_eclipse = std::env::var("ECLIPSE")
         .map(|x| {
             let value: bool = x.parse().unwrap();

lib/dex-orca/tests/test_orca.rs

@@ -1,6 +1,7 @@
 use std::collections::HashMap;
 use std::env;
 
+use router_feed_lib::utils::tracing_subscriber_init;
 use solana_program_test::tokio;
 
 use router_lib::dex::DexInterface;
@@ -8,6 +9,7 @@ use router_lib::test_tools::{generate_dex_rpc_dump, rpc};
 
 #[tokio::test]
 async fn test_dump_input_data_orca() -> anyhow::Result<()> {
+    tracing_subscriber_init();
     let options = HashMap::from([
         ("program_id".to_string(), whirlpools_client::ID.to_string()),
         ("program_name".to_string(), "Orca".to_string()),

lib/dex-raydium-cp/tests/test_raydium_cp.rs

@@ -1,6 +1,7 @@
 use std::collections::HashMap;
 use std::env;
 
+use router_feed_lib::utils::tracing_subscriber_init;
 use solana_program_test::tokio;
 
 use router_lib::dex::DexInterface;
@@ -8,6 +9,7 @@ use router_lib::test_tools::{generate_dex_rpc_dump, rpc};
 
 #[tokio::test]
 async fn test_dump_input_data_raydium_cp() -> anyhow::Result<()> {
+    tracing_subscriber_init();
     let options = HashMap::from([]);
 
     if router_test_lib::config_should_dump_mainnet_data() {

lib/dex-raydium/tests/test_raydium.rs

@@ -1,3 +1,4 @@
+use router_feed_lib::utils::tracing_subscriber_init;
 use solana_program_test::tokio;
 use std::collections::HashMap;
 use std::env;
@@ -7,6 +8,7 @@ use router_lib::test_tools::{generate_dex_rpc_dump, rpc};
 
 #[tokio::test]
 async fn test_dump_input_data_raydium() -> anyhow::Result<()> {
+    tracing_subscriber_init();
     let options = HashMap::from([]);
     if router_test_lib::config_should_dump_mainnet_data() {
         raydium_step_1(&options).await?;

lib/dex-saber/tests/test_saber.rs

@@ -1,6 +1,7 @@
 use std::collections::HashMap;
 use std::env;
 
+use router_feed_lib::utils::tracing_subscriber_init;
 use solana_program_test::tokio;
 
 use router_lib::dex::DexInterface;
@@ -8,6 +9,7 @@ use router_lib::test_tools::{generate_dex_rpc_dump, rpc};
 
 #[tokio::test]
 async fn test_dump_input_data_saber() -> anyhow::Result<()> {
+    tracing_subscriber_init();
     let options = HashMap::from([]);
 
     if router_test_lib::config_should_dump_mainnet_data() {

lib/router-feed-lib/src/router_rpc_client.rs

@@ -7,7 +7,7 @@ use solana_sdk::pubkey::Pubkey;
 
 #[async_trait::async_trait]
 pub trait RouterRpcClientTrait: Sync + Send {
-    async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result<Account>;
+    async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result<Option<Account>>;
 
     async fn get_multiple_accounts(
         &mut self,
@@ -30,7 +30,7 @@ pub struct RouterRpcClient {
 
 #[async_trait::async_trait]
 impl RouterRpcClientTrait for RouterRpcClient {
-    async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result<Account> {
+    async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result<Option<Account>> {
         self.rpc.get_account(pubkey).await
     }
 

lib/router-feed-lib/src/router_rpc_wrapper.rs

@@ -22,7 +22,7 @@ pub struct RouterRpcWrapper {
 
 #[async_trait]
 impl RouterRpcClientTrait for RouterRpcWrapper {
-    async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result<Account> {
+    async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result<Option<Account>> {
         let response = self
             .rpc
             .get_account_with_config(
@@ -36,10 +36,7 @@ impl RouterRpcClientTrait for RouterRpcWrapper {
             )
             .await?;
 
-        match response.value {
-            None => Err(anyhow::format_err!("missing account")),
-            Some(x) => Ok(x),
-        }
+        Ok(response.value)
     }
 
     async fn get_multiple_accounts(

lib/router-lib/src/test_tools/generate_dex_rpc_dump.rs

@@ -17,7 +17,7 @@ use solana_sdk::signature::Keypair;
 use solana_sdk::signer::Signer;
 use solana_sdk::sysvar::SysvarId;
 use std::sync::Arc;
-use tracing::{debug, error};
+use tracing::{debug, error, warn};
 
 pub async fn run_dump_mainnet_data(
     dex: Arc<dyn DexInterface>,
@@ -44,7 +44,7 @@ pub async fn run_dump_mainnet_data_with_custom_amount(
     );
 
     // always insert clock into chain data so it's available in dump during test run
-    let clock_account = rpc_client.get_account(&Clock::id()).await?;
+    let clock_account = rpc_client.get_account(&Clock::id()).await?.unwrap();
     let clock = clock_account.deserialize_data::<Clock>()?;
     let clock_data = AccountData {
         slot: clock.slot,
@@ -78,9 +78,17 @@ pub async fn run_dump_mainnet_data_with_custom_amount(
             continue;
         };
 
-        let Ok(quote) = dex.quote(&id, &edge, &chain_data, q(edge.clone())) else {
-            errors += 1;
-            continue;
+        let quote = match dex.quote(&id, &edge, &chain_data, q(edge.clone())) {
+            Ok(quote) => quote,
+            Err(e) => {
+                tracing::error!(
+                    "Error : quote failed {:?} -> {:?} : {e:?}",
+                    id.input_mint(),
+                    id.output_mint()
+                );
+                errors += 1;
+                continue;
+            }
         };
 
         if quote.in_amount == 0 {
@@ -93,16 +101,20 @@ pub async fn run_dump_mainnet_data_with_custom_amount(
             continue;
         }
 
-        let Ok(swap_ix) = dex.build_swap_ix(
+        let swap_ix = match dex.build_swap_ix(
             &id,
             &chain_data,
             &wallet.pubkey(),
             quote.in_amount,
             quote.out_amount,
             1000,
-        ) else {
-            errors += 1;
-            continue;
+        ) {
+            Ok(swap_ix) => swap_ix,
+            Err(e) => {
+                tracing::error!("Error : creating swap instruction {e:?}");
+                errors += 1;
+                continue;
+            }
         };
 
         accounts_needed.extend(
@@ -182,6 +194,7 @@ pub async fn run_dump_swap_ix_with_custom_amount(
         programs: dex.program_ids().into_iter().collect(),
         cache: vec![],
         accounts: Default::default(),
+        missing_accounts: Default::default(),
     };
 
     // always include clock sysvar in dump to ensure we can set the sysvar during test run
@@ -200,9 +213,17 @@ pub async fn run_dump_swap_ix_with_custom_amount(
             continue;
         };
 
-        let Ok(quote) = dex.quote(&id, &edge, &account_provider, q(edge.clone())) else {
-            errors += 1;
-            continue;
+        let quote = match dex.quote(&id, &edge, &account_provider, q(edge.clone())) {
+            Ok(quote) => quote,
+            Err(e) => {
+                tracing::error!(
+                    "Error : quote failed {:?} -> {:?} : {e:?}",
+                    id.input_mint(),
+                    id.output_mint()
+                );
+                errors += 1;
+                continue;
+            }
         };
 
         if quote.in_amount == 0 {
@@ -215,16 +236,20 @@ pub async fn run_dump_swap_ix_with_custom_amount(
             continue;
         }
 
-        let Ok(swap_ix) = dex.build_swap_ix(
+        let swap_ix = match dex.build_swap_ix(
             &id,
             &account_provider,
             &wallet.pubkey(),
             quote.in_amount,
             quote.out_amount,
             1000,
-        ) else {
-            errors += 1;
-            continue;
+        ) {
+            Ok(swap_ix) => swap_ix,
+            Err(e) => {
+                tracing::error!("Error : creating swap instruction {e:?}");
+                errors += 1;
+                continue;
+            }
         };
 
         println!(
@@ -251,7 +276,11 @@ pub async fn run_dump_swap_ix_with_custom_amount(
             if let Ok(acc) = chain_data_reader.account(&account.pubkey) {
                 dump.accounts.insert(account.pubkey, acc.account.clone());
             } else {
-                error!("Missing account (needed for swap) {}", account.pubkey);
+                if !is_ata(&account.pubkey, &wallet.pubkey(), &id.input_mint())
+                    && !is_ata(&account.pubkey, &wallet.pubkey(), &id.output_mint())
+                {
+                    dump.missing_accounts.insert(account.pubkey);
+                }
             }
         }
         let account = chain_data_reader
@@ -311,7 +340,12 @@ pub async fn run_dump_swap_ix_with_custom_amount(
                     if let Ok(acc) = chain_data_reader.account(&account.pubkey) {
                         dump.accounts.insert(account.pubkey, acc.account.clone());
                     } else {
-                        error!("Missing account (needed for swap) {}", account.pubkey);
+                        if !is_ata(&account.pubkey, &wallet.pubkey(), &id.input_mint())
+                            && !is_ata(&account.pubkey, &wallet.pubkey(), &id.output_mint())
+                            && !dump.missing_accounts.contains(&account.pubkey)
+                        {
+                            error!("Missing account in dump {}", account.pubkey);
+                        }
                     }
                 }
             }
@@ -355,6 +389,14 @@ pub async fn run_dump_swap_ix_with_custom_amount(
         let base64 = base64::encode(result);
         debug!("account : {pk:?} dump : {base64:?}");
     }
+
+    for pk in &dump.missing_accounts {
+        warn!(
+            "following account is in the transaction but does not exists : {:?}",
+            pk
+        );
+    }
+
     serialize::serialize_to_file(
         &dump,
         &format!("../../programs/simulator/tests/fixtures/{}", dump_name).to_string(),

lib/router-lib/src/test_tools/rpc.rs

@@ -22,6 +22,7 @@ use std::time::Duration;
 struct RpcDump {
     pub cache: HashMap<Pubkey, Option<Account>>,
     pub cache_gpa: HashMap<(Pubkey, String), Option<Vec<AccountWrite>>>,
+    pub missing_accounts: HashSet<Pubkey>,
 }
 
 pub struct DumpRpcClient {
@@ -37,9 +38,12 @@ pub struct ReplayerRpcClient {
 
 #[async_trait::async_trait]
 impl RouterRpcClientTrait for ReplayerRpcClient {
-    async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result<Account> {
+    async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result<Option<Account>> {
+        if self.dump.missing_accounts.contains(pubkey) {
+            return Ok(None);
+        }
         match self.dump.cache.get(pubkey).unwrap() {
-            Some(x) => Ok(x.clone()),
+            Some(x) => Ok(Some(x.clone())),
             None => anyhow::bail!("Invalid account"),
         }
     }
@@ -87,11 +91,31 @@ impl RouterRpcClientTrait for ReplayerRpcClient {
 
 #[async_trait::async_trait]
 impl RouterRpcClientTrait for DumpRpcClient {
-    async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result<Account> {
+    async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result<Option<Account>> {
         match self.rpc.get_account(pubkey).await {
             Ok(r) => {
-                insert_into_arc_chain_data(&self.chain_data, *pubkey, r.clone());
-                self.dump.cache.insert(*pubkey, Some(r.clone()));
+                match &r {
+                    Some(acc) => {
+                        insert_into_arc_chain_data(&self.chain_data, *pubkey, acc.clone());
+                        self.dump.cache.insert(*pubkey, Some(acc.clone()));
+                    }
+                    None => {
+                        self.dump.cache.insert(*pubkey, None);
+                        self.dump.missing_accounts.insert(*pubkey);
+                        // add empty account in chain data
+                        insert_into_arc_chain_data(
+                            &self.chain_data,
+                            *pubkey,
+                            Account {
+                                lamports: 0,
+                                data: vec![],
+                                owner: Pubkey::default(),
+                                executable: false,
+                                rent_epoch: 0,
+                            },
+                        );
+                    }
+                }
                 Ok(r)
             }
             Err(e) => {
@@ -177,6 +201,7 @@ pub fn rpc_dumper_client(url: String, out_path: &str) -> (RouterRpcClient, Chain
             dump: RpcDump {
                 cache: Default::default(),
                 cache_gpa: Default::default(),
+                missing_accounts: Default::default(),
             },
             chain_data: chain_data.clone(),
             rpc: RouterRpcClient {

lib/router-test-lib/src/execution_dump.rs

@@ -20,4 +20,5 @@ pub struct ExecutionDump {
     pub programs: HashSet<Pubkey>,
     pub cache: Vec<ExecutionItem>,
     pub accounts: HashMap<Pubkey, AccountSharedData>,
+    pub missing_accounts: HashSet<Pubkey>,
 }