diff --git a/lib/dex-infinity/src/infinity.rs b/lib/dex-infinity/src/infinity.rs index 9f4012c..8cb9a2f 100644 --- a/lib/dex-infinity/src/infinity.rs +++ b/lib/dex-infinity/src/infinity.rs @@ -47,8 +47,8 @@ impl DexInterface for InfinityDex { lst_state_list, pool_state, } = SPoolJup::init_keys(program_id); - let lst_state_list_account = rpc.get_account(&lst_state_list).await.unwrap(); - let pool_state_account = rpc.get_account(&pool_state).await.unwrap(); + let lst_state_list_account = rpc.get_account(&lst_state_list).await.unwrap().unwrap(); + let pool_state_account = rpc.get_account(&pool_state).await.unwrap().unwrap(); let amm: s_jup_interface::SPool = SPoolJup::from_init_accounts( program_id, diff --git a/lib/dex-infinity/tests/test_infinity.rs b/lib/dex-infinity/tests/test_infinity.rs index 48d6317..1f1bc22 100644 --- a/lib/dex-infinity/tests/test_infinity.rs +++ b/lib/dex-infinity/tests/test_infinity.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::env; +use router_feed_lib::utils::tracing_subscriber_init; use solana_program_test::tokio; use router_lib::dex::DexInterface; @@ -8,6 +9,7 @@ use router_lib::test_tools::{generate_dex_rpc_dump, rpc}; #[tokio::test] async fn test_dump_input_data_infinity() -> anyhow::Result<()> { + tracing_subscriber_init(); if router_test_lib::config_should_dump_mainnet_data() { step_1_infinity().await?; } diff --git a/lib/dex-invariant/tests/test_invariant.rs b/lib/dex-invariant/tests/test_invariant.rs index 62e0522..07c67dd 100644 --- a/lib/dex-invariant/tests/test_invariant.rs +++ b/lib/dex-invariant/tests/test_invariant.rs @@ -1,3 +1,4 @@ +use router_feed_lib::utils::tracing_subscriber_init; use solana_program_test::tokio; use std::collections::HashMap; use std::env; @@ -7,6 +8,7 @@ use router_lib::test_tools::{generate_dex_rpc_dump, rpc}; #[tokio::test] async fn test_dump_input_data_invariant() -> anyhow::Result<()> { + tracing_subscriber_init(); let options = HashMap::from([]); if router_test_lib::config_should_dump_mainnet_data() { diff --git a/lib/dex-openbook-v2/tests/test_openbook_v2_cp.rs b/lib/dex-openbook-v2/tests/test_openbook_v2_cp.rs index 8752fab..ed8249b 100644 --- a/lib/dex-openbook-v2/tests/test_openbook_v2_cp.rs +++ b/lib/dex-openbook-v2/tests/test_openbook_v2_cp.rs @@ -2,12 +2,14 @@ use std::collections::HashMap; use std::env; use dex_openbook_v2::OpenbookV2Edge; +use router_feed_lib::utils::tracing_subscriber_init; use router_lib::dex::DexInterface; use router_lib::test_tools::{generate_dex_rpc_dump, rpc}; use solana_program_test::tokio; #[tokio::test] async fn test_dump_input_data_openbook_v2() -> anyhow::Result<()> { + tracing_subscriber_init(); let options = HashMap::from([]); if router_test_lib::config_should_dump_mainnet_data() { openbook_v2_step_1(&options).await?; diff --git a/lib/dex-orca/tests/test_cropper.rs b/lib/dex-orca/tests/test_cropper.rs index 51939cc..57a41ab 100644 --- a/lib/dex-orca/tests/test_cropper.rs +++ b/lib/dex-orca/tests/test_cropper.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::env; +use router_feed_lib::utils::tracing_subscriber_init; use solana_program_test::tokio; use router_lib::dex::DexInterface; @@ -8,6 +9,7 @@ use router_lib::test_tools::{generate_dex_rpc_dump, rpc}; #[tokio::test] async fn test_dump_input_data_cropper() -> anyhow::Result<()> { + tracing_subscriber_init(); let is_eclipse = std::env::var("ECLIPSE") .map(|x| { let value: bool = x.parse().unwrap(); diff --git a/lib/dex-orca/tests/test_orca.rs b/lib/dex-orca/tests/test_orca.rs index 61f8b96..d528fd6 100644 --- a/lib/dex-orca/tests/test_orca.rs +++ b/lib/dex-orca/tests/test_orca.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::env; +use router_feed_lib::utils::tracing_subscriber_init; use solana_program_test::tokio; use router_lib::dex::DexInterface; @@ -8,6 +9,7 @@ use router_lib::test_tools::{generate_dex_rpc_dump, rpc}; #[tokio::test] async fn test_dump_input_data_orca() -> anyhow::Result<()> { + tracing_subscriber_init(); let options = HashMap::from([ ("program_id".to_string(), whirlpools_client::ID.to_string()), ("program_name".to_string(), "Orca".to_string()), diff --git a/lib/dex-raydium-cp/tests/test_raydium_cp.rs b/lib/dex-raydium-cp/tests/test_raydium_cp.rs index acd0008..e3f5b3d 100644 --- a/lib/dex-raydium-cp/tests/test_raydium_cp.rs +++ b/lib/dex-raydium-cp/tests/test_raydium_cp.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::env; +use router_feed_lib::utils::tracing_subscriber_init; use solana_program_test::tokio; use router_lib::dex::DexInterface; @@ -8,6 +9,7 @@ use router_lib::test_tools::{generate_dex_rpc_dump, rpc}; #[tokio::test] async fn test_dump_input_data_raydium_cp() -> anyhow::Result<()> { + tracing_subscriber_init(); let options = HashMap::from([]); if router_test_lib::config_should_dump_mainnet_data() { diff --git a/lib/dex-raydium/tests/test_raydium.rs b/lib/dex-raydium/tests/test_raydium.rs index ac84b3d..7f67e6d 100644 --- a/lib/dex-raydium/tests/test_raydium.rs +++ b/lib/dex-raydium/tests/test_raydium.rs @@ -1,3 +1,4 @@ +use router_feed_lib::utils::tracing_subscriber_init; use solana_program_test::tokio; use std::collections::HashMap; use std::env; @@ -7,6 +8,7 @@ use router_lib::test_tools::{generate_dex_rpc_dump, rpc}; #[tokio::test] async fn test_dump_input_data_raydium() -> anyhow::Result<()> { + tracing_subscriber_init(); let options = HashMap::from([]); if router_test_lib::config_should_dump_mainnet_data() { raydium_step_1(&options).await?; diff --git a/lib/dex-saber/tests/test_saber.rs b/lib/dex-saber/tests/test_saber.rs index 00950c6..cc768a4 100644 --- a/lib/dex-saber/tests/test_saber.rs +++ b/lib/dex-saber/tests/test_saber.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::env; +use router_feed_lib::utils::tracing_subscriber_init; use solana_program_test::tokio; use router_lib::dex::DexInterface; @@ -8,6 +9,7 @@ use router_lib::test_tools::{generate_dex_rpc_dump, rpc}; #[tokio::test] async fn test_dump_input_data_saber() -> anyhow::Result<()> { + tracing_subscriber_init(); let options = HashMap::from([]); if router_test_lib::config_should_dump_mainnet_data() { diff --git a/lib/router-feed-lib/src/router_rpc_client.rs b/lib/router-feed-lib/src/router_rpc_client.rs index 93e87e3..636f8a8 100644 --- a/lib/router-feed-lib/src/router_rpc_client.rs +++ b/lib/router-feed-lib/src/router_rpc_client.rs @@ -7,7 +7,7 @@ use solana_sdk::pubkey::Pubkey; #[async_trait::async_trait] pub trait RouterRpcClientTrait: Sync + Send { - async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result; + async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result>; async fn get_multiple_accounts( &mut self, @@ -30,7 +30,7 @@ pub struct RouterRpcClient { #[async_trait::async_trait] impl RouterRpcClientTrait for RouterRpcClient { - async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result { + async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result> { self.rpc.get_account(pubkey).await } diff --git a/lib/router-feed-lib/src/router_rpc_wrapper.rs b/lib/router-feed-lib/src/router_rpc_wrapper.rs index 7d95160..a5882bf 100644 --- a/lib/router-feed-lib/src/router_rpc_wrapper.rs +++ b/lib/router-feed-lib/src/router_rpc_wrapper.rs @@ -22,7 +22,7 @@ pub struct RouterRpcWrapper { #[async_trait] impl RouterRpcClientTrait for RouterRpcWrapper { - async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result { + async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result> { let response = self .rpc .get_account_with_config( @@ -36,10 +36,7 @@ impl RouterRpcClientTrait for RouterRpcWrapper { ) .await?; - match response.value { - None => Err(anyhow::format_err!("missing account")), - Some(x) => Ok(x), - } + Ok(response.value) } async fn get_multiple_accounts( diff --git a/lib/router-lib/src/test_tools/generate_dex_rpc_dump.rs b/lib/router-lib/src/test_tools/generate_dex_rpc_dump.rs index 8bd587d..900d76e 100644 --- a/lib/router-lib/src/test_tools/generate_dex_rpc_dump.rs +++ b/lib/router-lib/src/test_tools/generate_dex_rpc_dump.rs @@ -17,7 +17,7 @@ use solana_sdk::signature::Keypair; use solana_sdk::signer::Signer; use solana_sdk::sysvar::SysvarId; use std::sync::Arc; -use tracing::{debug, error}; +use tracing::{debug, error, warn}; pub async fn run_dump_mainnet_data( dex: Arc, @@ -44,7 +44,7 @@ pub async fn run_dump_mainnet_data_with_custom_amount( ); // always insert clock into chain data so it's available in dump during test run - let clock_account = rpc_client.get_account(&Clock::id()).await?; + let clock_account = rpc_client.get_account(&Clock::id()).await?.unwrap(); let clock = clock_account.deserialize_data::()?; let clock_data = AccountData { slot: clock.slot, @@ -78,9 +78,17 @@ pub async fn run_dump_mainnet_data_with_custom_amount( continue; }; - let Ok(quote) = dex.quote(&id, &edge, &chain_data, q(edge.clone())) else { - errors += 1; - continue; + let quote = match dex.quote(&id, &edge, &chain_data, q(edge.clone())) { + Ok(quote) => quote, + Err(e) => { + tracing::error!( + "Error : quote failed {:?} -> {:?} : {e:?}", + id.input_mint(), + id.output_mint() + ); + errors += 1; + continue; + } }; if quote.in_amount == 0 { @@ -93,16 +101,20 @@ pub async fn run_dump_mainnet_data_with_custom_amount( continue; } - let Ok(swap_ix) = dex.build_swap_ix( + let swap_ix = match dex.build_swap_ix( &id, &chain_data, &wallet.pubkey(), quote.in_amount, quote.out_amount, 1000, - ) else { - errors += 1; - continue; + ) { + Ok(swap_ix) => swap_ix, + Err(e) => { + tracing::error!("Error : creating swap instruction {e:?}"); + errors += 1; + continue; + } }; accounts_needed.extend( @@ -182,6 +194,7 @@ pub async fn run_dump_swap_ix_with_custom_amount( programs: dex.program_ids().into_iter().collect(), cache: vec![], accounts: Default::default(), + missing_accounts: Default::default(), }; // always include clock sysvar in dump to ensure we can set the sysvar during test run @@ -200,9 +213,17 @@ pub async fn run_dump_swap_ix_with_custom_amount( continue; }; - let Ok(quote) = dex.quote(&id, &edge, &account_provider, q(edge.clone())) else { - errors += 1; - continue; + let quote = match dex.quote(&id, &edge, &account_provider, q(edge.clone())) { + Ok(quote) => quote, + Err(e) => { + tracing::error!( + "Error : quote failed {:?} -> {:?} : {e:?}", + id.input_mint(), + id.output_mint() + ); + errors += 1; + continue; + } }; if quote.in_amount == 0 { @@ -215,16 +236,20 @@ pub async fn run_dump_swap_ix_with_custom_amount( continue; } - let Ok(swap_ix) = dex.build_swap_ix( + let swap_ix = match dex.build_swap_ix( &id, &account_provider, &wallet.pubkey(), quote.in_amount, quote.out_amount, 1000, - ) else { - errors += 1; - continue; + ) { + Ok(swap_ix) => swap_ix, + Err(e) => { + tracing::error!("Error : creating swap instruction {e:?}"); + errors += 1; + continue; + } }; println!( @@ -251,7 +276,11 @@ pub async fn run_dump_swap_ix_with_custom_amount( if let Ok(acc) = chain_data_reader.account(&account.pubkey) { dump.accounts.insert(account.pubkey, acc.account.clone()); } else { - error!("Missing account (needed for swap) {}", account.pubkey); + if !is_ata(&account.pubkey, &wallet.pubkey(), &id.input_mint()) + && !is_ata(&account.pubkey, &wallet.pubkey(), &id.output_mint()) + { + dump.missing_accounts.insert(account.pubkey); + } } } let account = chain_data_reader @@ -311,7 +340,12 @@ pub async fn run_dump_swap_ix_with_custom_amount( if let Ok(acc) = chain_data_reader.account(&account.pubkey) { dump.accounts.insert(account.pubkey, acc.account.clone()); } else { - error!("Missing account (needed for swap) {}", account.pubkey); + if !is_ata(&account.pubkey, &wallet.pubkey(), &id.input_mint()) + && !is_ata(&account.pubkey, &wallet.pubkey(), &id.output_mint()) + && !dump.missing_accounts.contains(&account.pubkey) + { + error!("Missing account in dump {}", account.pubkey); + } } } } @@ -355,6 +389,14 @@ pub async fn run_dump_swap_ix_with_custom_amount( let base64 = base64::encode(result); debug!("account : {pk:?} dump : {base64:?}"); } + + for pk in &dump.missing_accounts { + warn!( + "following account is in the transaction but does not exists : {:?}", + pk + ); + } + serialize::serialize_to_file( &dump, &format!("../../programs/simulator/tests/fixtures/{}", dump_name).to_string(), diff --git a/lib/router-lib/src/test_tools/rpc.rs b/lib/router-lib/src/test_tools/rpc.rs index 5a16e3b..a1fdb3d 100644 --- a/lib/router-lib/src/test_tools/rpc.rs +++ b/lib/router-lib/src/test_tools/rpc.rs @@ -22,6 +22,7 @@ use std::time::Duration; struct RpcDump { pub cache: HashMap>, pub cache_gpa: HashMap<(Pubkey, String), Option>>, + pub missing_accounts: HashSet, } pub struct DumpRpcClient { @@ -37,9 +38,12 @@ pub struct ReplayerRpcClient { #[async_trait::async_trait] impl RouterRpcClientTrait for ReplayerRpcClient { - async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result { + async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result> { + if self.dump.missing_accounts.contains(pubkey) { + return Ok(None); + } match self.dump.cache.get(pubkey).unwrap() { - Some(x) => Ok(x.clone()), + Some(x) => Ok(Some(x.clone())), None => anyhow::bail!("Invalid account"), } } @@ -87,11 +91,31 @@ impl RouterRpcClientTrait for ReplayerRpcClient { #[async_trait::async_trait] impl RouterRpcClientTrait for DumpRpcClient { - async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result { + async fn get_account(&mut self, pubkey: &Pubkey) -> anyhow::Result> { match self.rpc.get_account(pubkey).await { Ok(r) => { - insert_into_arc_chain_data(&self.chain_data, *pubkey, r.clone()); - self.dump.cache.insert(*pubkey, Some(r.clone())); + match &r { + Some(acc) => { + insert_into_arc_chain_data(&self.chain_data, *pubkey, acc.clone()); + self.dump.cache.insert(*pubkey, Some(acc.clone())); + } + None => { + self.dump.cache.insert(*pubkey, None); + self.dump.missing_accounts.insert(*pubkey); + // add empty account in chain data + insert_into_arc_chain_data( + &self.chain_data, + *pubkey, + Account { + lamports: 0, + data: vec![], + owner: Pubkey::default(), + executable: false, + rent_epoch: 0, + }, + ); + } + } Ok(r) } Err(e) => { @@ -177,6 +201,7 @@ pub fn rpc_dumper_client(url: String, out_path: &str) -> (RouterRpcClient, Chain dump: RpcDump { cache: Default::default(), cache_gpa: Default::default(), + missing_accounts: Default::default(), }, chain_data: chain_data.clone(), rpc: RouterRpcClient { diff --git a/lib/router-test-lib/src/execution_dump.rs b/lib/router-test-lib/src/execution_dump.rs index 1632dc8..1a8900f 100644 --- a/lib/router-test-lib/src/execution_dump.rs +++ b/lib/router-test-lib/src/execution_dump.rs @@ -20,4 +20,5 @@ pub struct ExecutionDump { pub programs: HashSet, pub cache: Vec, pub accounts: HashMap, + pub missing_accounts: HashSet, }