Migrating testing to lib svm (#28)

* Using libSVM for simulation

* Some more fixes for invalid mint authority

* avoiding disabled lst mints

* performance improvements, reduced log-levels

* adding a todo for token extention program

Cargo.lock

@@ -3865,6 +3865,26 @@ version = "0.4.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c"
 
+[[package]]
+name = "litesvm"
+version = "0.1.0"
+source = "git+https://github.com/blockworks-foundation/litesvm.git?branch=v0.1.0+solana_1.7#4c884606289e484090be42a08b79dc3bc5f449f7"
+dependencies = [
+ "bincode",
+ "indexmap 2.2.6",
+ "itertools 0.12.1",
+ "log 0.4.21",
+ "solana-address-lookup-table-program",
+ "solana-bpf-loader-program",
+ "solana-compute-budget-program",
+ "solana-loader-v4-program",
+ "solana-program",
+ "solana-program-runtime",
+ "solana-sdk",
+ "solana-system-program",
+ "thiserror",
+]
+
 [[package]]
 name = "lock_api"
 version = "0.3.4"
@@ -6763,11 +6783,13 @@ dependencies = [
  "bincode",
  "bonfida-test-utils",
  "env_logger",
+ "litesvm",
  "log 0.4.21",
  "router-test-lib",
  "sha2 0.10.8",
  "solana-address-lookup-table-program",
  "solana-program",
+ "solana-program-runtime",
  "solana-program-test",
  "solana-sdk",
  "spl-associated-token-account 1.1.3",
@@ -6775,6 +6797,8 @@ dependencies = [
  "spl-token-2022 1.0.0",
  "test-case",
  "tokio",
+ "tracing",
+ "tracing-subscriber",
 ]
 
 [[package]]

lib/dex-infinity/src/infinity.rs

@@ -68,6 +68,14 @@ impl DexInterface for InfinityDex {
             let lst_mint = lst_data.sol_val_calc.lst_mint();
             let account_metas = lst_data.sol_val_calc.ix_accounts();
             let num_accounts_for_tx = account_metas.len();
+            let Ok((lst_state, lst_data)) = amm.find_ready_lst(lst_mint) else {
+                continue;
+            };
+
+            if lst_state.is_input_disabled != 0 {
+                continue;
+            }
+
             for pk in lst_data.sol_val_calc.get_accounts_to_update() {
                 let edges = vec![
                     Arc::new(InfinityEdgeIdentifier {

lib/router-lib/src/test_tools/generate_dex_rpc_dump.rs

@@ -126,7 +126,7 @@ pub async fn run_dump_mainnet_data_with_custom_amount(
     }
     let accounts = rpc_client.get_multiple_accounts(&accounts_needed).await?;
 
-    for (_, account) in accounts {
+    for (_pk, account) in accounts {
         // get buffer for upgradable programs
         if account.owner == solana_sdk::bpf_loader_upgradeable::ID {
             let state = bincode::deserialize::<UpgradeableLoaderState>(&account.data).unwrap();
@@ -260,7 +260,7 @@ pub async fn run_dump_swap_ix_with_custom_amount(
         dump.accounts
             .insert(id.input_mint(), account.account.clone());
         let account = chain_data_reader
-            .account(&id.input_mint())
+            .account(&id.output_mint())
             .expect("missing mint");
         dump.accounts
             .insert(id.output_mint(), account.account.clone());
@@ -307,9 +307,6 @@ pub async fn run_dump_swap_ix_with_custom_amount(
                     instruction: bincode::serialize(&swap_exact_out_ix.instruction).unwrap(),
                     is_exact_out: true,
                 });
-
-                // add exact out accounts
-                let chain_data_reader = chain_data.read().unwrap();
                 for account in swap_exact_out_ix.instruction.accounts {
                     if let Ok(acc) = chain_data_reader.account(&account.pubkey) {
                         dump.accounts.insert(account.pubkey, acc.account.clone());
@@ -351,9 +348,9 @@ pub async fn run_dump_swap_ix_with_custom_amount(
         debug!("program : {program:?}");
     }
 
-    for (pk, program) in &dump.accounts {
+    for (pk, account_data) in &dump.accounts {
         let mut hasher = Sha256::new();
-        hasher.update(program.data());
+        hasher.update(account_data.data());
         let result = hasher.finalize();
         let base64 = base64::encode(result);
         debug!("account : {pk:?} dump : {base64:?}");

programs/simulator/Cargo.toml

@@ -23,6 +23,7 @@ test-case = "*"
 tokio = "1.37.0"
 solana-address-lookup-table-program = "1.17"
 solana-program-test = "1.17"
+solana-program-runtime = "1.17"
 solana-sdk = "1.17"
 spl-token = { version = "^3.0.0", features = ["no-entrypoint"] }
 spl-token-2022 = { version = "1.0.0", features = ["no-entrypoint"] }
@@ -33,6 +34,9 @@ env_logger = "0.9.0"
 bincode = "1.3.3"
 sha2 = "0.10.8"
 base64 = "0.12.3"
+litesvm = { git = "https://github.com/blockworks-foundation/litesvm.git", branch = "v0.1.0+solana_1.7" }
+tracing = "0.1.37"
+tracing-subscriber = "0.3.16"
 
 [profile.test]
 inherits = "release"

programs/simulator/tests/cases/test_swap_from_dump.rs

@@ -1,30 +1,30 @@
-use anyhow::{Context, Error};
+use anyhow::Error;
-use bonfida_test_utils::error::TestError;
+use litesvm::LiteSVM;
-use bonfida_test_utils::ProgramTestContextExt;
+use log::{error, info, warn};
-use log::{debug, error, info, warn};
 use router_test_lib::execution_dump::{ExecutionDump, ExecutionItem};
 use router_test_lib::{execution_dump, serialize};
-use solana_program::clock::{Clock, Epoch};
+use sha2::Digest;
+use sha2::Sha256;
+use solana_program::clock::Clock;
 use solana_program::instruction::Instruction;
 use solana_program::program_pack::Pack;
 use solana_program::program_stubs::{set_syscall_stubs, SyscallStubs};
 use solana_program::pubkey::Pubkey;
 use solana_program::sysvar::SysvarId;
-use solana_program_test::BanksClientError;
-use solana_program_test::{ProgramTest, ProgramTestContext};
 use solana_sdk::account::{Account, AccountSharedData, ReadableAccount};
-use solana_sdk::epoch_info::EpochInfo;
+use solana_sdk::bpf_loader_upgradeable::UpgradeableLoaderState;
+use solana_sdk::message::{Message, VersionedMessage};
 use solana_sdk::signature::Keypair;
 use solana_sdk::signer::Signer;
-use solana_sdk::transaction::Transaction;
+use solana_sdk::transaction::VersionedTransaction;
-use spl_associated_token_account::get_associated_token_address;
+use spl_associated_token_account::{
+    get_associated_token_address, get_associated_token_address_with_program_id,
+};
 use spl_token::state::AccountState;
 use spl_token_2022::state::AccountState as AccountState2022;
 use std::collections::HashMap;
-use std::process::exit;
+use std::path::PathBuf;
 use std::str::FromStr;
-// use sha2::Sha256;
-// use sha2::Digest;
 
 struct TestLogSyscallStubs;
 impl SyscallStubs for TestLogSyscallStubs {
@@ -73,13 +73,15 @@ async fn test_quote_match_swap_for_infinity() -> anyhow::Result<()> {
 }
 
 async fn run_all_swap_from_dump(dump_name: &str) -> Result<Result<(), Error>, Error> {
+    tracing_subscriber::fmt::init();
+
     let mut skip_count = option_env!("SKIP_COUNT")
         .map(|x| u32::from_str(x).unwrap_or(0))
         .unwrap_or(0);
     let mut stop_at = u32::MAX;
     let skip_ixs_index = vec![];
 
-    let mut run_lot_size = option_env!("RUN_LOT_SIZE")
+    let run_lot_size = option_env!("RUN_LOT_SIZE")
         .map(|x| u32::from_str(x).unwrap_or(500))
         .unwrap_or(500);
 
@@ -126,7 +128,7 @@ async fn run_all_swap_from_dump(dump_name: &str) -> Result<Result<(), Error>, Er
         let instruction = deserialize_instruction(&quote.instruction)?;
 
         let programs = data.programs.iter().copied().collect();
-        let mut ctx = setup_test_chain(&programs, &clock, &data).await;
+        let mut ctx = setup_test_chain(&programs, &clock, &data, &instruction)?;
 
         create_wallet(&mut ctx, wallet.pubkey());
 
@@ -145,28 +147,31 @@ async fn run_all_swap_from_dump(dump_name: &str) -> Result<Result<(), Error>, Er
             quote.input_mint,
             initial_in_balance,
             input_mint_is_2022,
-        )
+        )?;
-        .await?;
         set_balance(
             &mut ctx,
             wallet.pubkey(),
             quote.output_mint,
             initial_out_balance,
             output_mint_is_2022,
-        )
+        )?;
-        .await?;
 
         for meta in &instruction.accounts {
-            let Ok(Some(account)) = ctx.banks_client.get_account(meta.pubkey).await else {
+            let Some(account) = ctx.get_account(&meta.pubkey) else {
                 log::warn!("missing account : {:?}", meta.pubkey);
                 continue;
             };
+
             // keep code to test hashses
-            // let mut hasher = Sha256::new();
+            let mut hasher = Sha256::new();
-            // hasher.update(account.data());
+            hasher.update(account.data());
-            // let result = hasher.finalize();
+            let result = hasher.finalize();
-            // let base64 = base64::encode(result);
+            let base64 = base64::encode(result);
-            // log::debug!("account : {:?} dump : {base64:?} executable : {}", meta.pubkey, account.executable());
+            log::debug!(
+                "account : {:?} dump : {base64:?} executable : {}",
+                meta.pubkey,
+                account.executable()
+            );
         }
 
         if let Some(cus) = simulate_cu_usage(&mut ctx, &wallet, &instruction).await {
@@ -278,7 +283,7 @@ async fn debug_print_ix(
     success: &mut i32,
     index: &mut u32,
     quote: &ExecutionItem,
-    ctx: &mut ProgramTestContext,
+    ctx: &mut LiteSVM,
     instruction: &Instruction,
     input_mint_is_2022: bool,
     output_mint_is_2022: bool,
@@ -310,13 +315,8 @@ async fn debug_print_ix(
 
     for acc in &instruction.accounts {
         let account = ctx
-            .banks_client
+            .get_account(&acc.pubkey)
-            .get_account(acc.pubkey)
+            .map(|x| (x.executable, x.owner.to_string()))
-            .await
-            .map(|x| {
-                x.map(|y| (y.executable, y.owner.to_string()))
-                    .unwrap_or((false, "???".to_string()))
-            })
             .unwrap_or((false, "???".to_string()));
 
         warn!(
@@ -341,14 +341,53 @@ fn deserialize_instruction(swap_ix: &Vec<u8>) -> anyhow::Result<Instruction> {
     Ok(instruction)
 }
 
-async fn initialize_accounts(
+fn initialize_accounts(
-    program_test: &mut ProgramTest,
+    program_test: &mut LiteSVM,
     dump: &ExecutionDump,
+    accounts_list: &Vec<Pubkey>,
 ) -> anyhow::Result<()> {
-    println!("initializing accounts : {:?}", dump.accounts.len());
+    log::debug!("initializing accounts : {:?}", dump.accounts.len());
-    for (pk, account) in &dump.accounts {
+    for pk in accounts_list {
-        println!("Setting data for {}", pk);
+        let Some(account) = dump.accounts.get(pk) else {
-        program_test.add_account(
+            continue;
+        };
+        if *account.owner() == solana_sdk::bpf_loader_upgradeable::ID {
+            log::debug!("{pk:?} has upgradable loader");
+            let state = bincode::deserialize::<UpgradeableLoaderState>(&account.data()).unwrap();
+            if let UpgradeableLoaderState::Program {
+                programdata_address,
+            } = state
+            {
+                // load buffer accounts first
+                match dump.accounts.get(&programdata_address) {
+                    Some(program_buffer) => {
+                        log::debug!("loading buffer:  {programdata_address:?}");
+                        program_test.set_account(
+                            programdata_address,
+                            solana_sdk::account::Account {
+                                lamports: program_buffer.lamports(),
+                                owner: *program_buffer.owner(),
+                                data: program_buffer.data().to_vec(),
+                                rent_epoch: program_buffer.rent_epoch(),
+                                executable: program_buffer.executable(),
+                            },
+                        )?;
+                    }
+                    None => {
+                        error!("{programdata_address:?} is not there");
+                    }
+                }
+            }
+        }
+        log::debug!(
+            "Setting data for {} with owner {} and is executable {}",
+            pk,
+            account.owner(),
+            account.executable()
+        );
+
+        log::debug!("Setting data for {}", pk);
+        program_test.set_account(
             *pk,
             solana_sdk::account::Account {
                 lamports: account.lamports(),
@@ -357,110 +396,109 @@ async fn initialize_accounts(
                 rent_epoch: account.rent_epoch(),
                 executable: account.executable(),
             },
-        );
+        )?;
     }
 
     Ok(())
 }
 
 async fn simulate_cu_usage(
-    ctx: &mut ProgramTestContext,
+    ctx: &mut LiteSVM,
     owner: &Keypair,
     instruction: &Instruction,
 ) -> Option<u64> {
-    let mut transaction =
+    let tx = VersionedTransaction::try_new(
-        Transaction::new_with_payer(&[instruction.clone()], Some(&ctx.payer.pubkey()));
+        VersionedMessage::Legacy(Message::new(&[instruction.clone()], Some(&owner.pubkey()))),
+        &[owner],
+    )
+    .unwrap();
 
-    transaction.sign(&[&ctx.payer, owner], ctx.last_blockhash);
+    let sim = ctx.simulate_transaction(tx);
-    let sim = ctx
-        .banks_client
-        .simulate_transaction(transaction.clone())
-        .await;
     match sim {
         Ok(sim) => {
-            log::debug!("{:?}", sim.result);
+            let cus = sim.compute_units_consumed;
-            let simulation_details = sim.simulation_details.unwrap();
+            log::debug!("----logs");
-            let cus = simulation_details.units_consumed;
+            for log in sim.logs {
-            if sim.result.is_some() && sim.result.clone().unwrap().is_ok() {
+                log::debug!("{log:?}");
-                log::debug!("units consumed : {}", cus);
+            }
+            if cus > 0 {
                 Some(cus)
-            } else if sim.result.is_some() && sim.result.clone().unwrap().is_err() {
-                log::debug!("simluation failed : {:?}", sim.result.unwrap());
-                log::debug!("----logs");
-                for log in simulation_details.logs {
-                    log::debug!("{log:?}");
-                }
-                None
             } else {
                 None
             }
         }
         Err(e) => {
-            log::warn!("Error simulating : {}", e);
+            log::warn!("Error simulating : {:?}", e);
             None
         }
     }
 }
 
-async fn swap(
+async fn swap(ctx: &mut LiteSVM, owner: &Keypair, instruction: &Instruction) -> anyhow::Result<()> {
-    ctx: &mut ProgramTestContext,
+    let tx = VersionedTransaction::try_new(
-    owner: &Keypair,
+        VersionedMessage::Legacy(Message::new(&[instruction.clone()], Some(&owner.pubkey()))),
-    instruction: &Instruction,
+        &[owner],
-) -> anyhow::Result<()> {
+    )
-    ctx.get_new_latest_blockhash().await?;
+    .unwrap();
-
-    log::info!("swapping");
-    let result = ctx
-        .sign_send_instructions(&[instruction.clone()], &[&owner])
-        .await;
 
+    let result = ctx.send_transaction(tx);
     match result {
-        Ok(()) => Ok(()),
+        Ok(_) => Ok(()),
-        Err(e) => Err(anyhow::format_err!("Failed to swap {:?}", e)),
+        Err(e) => {
+            log::error!("------------- LOGS ------------------");
+            for log in &e.meta.logs {
+                log::error!("{log:?}");
+            }
+            Err(anyhow::format_err!("Failed to swap {:?}", e.err))
+        }
     }
 }
 
 async fn get_balance(
-    ctx: &mut ProgramTestContext,
+    ctx: &mut LiteSVM,
     owner: Pubkey,
     mint: Pubkey,
     is_2022: bool,
 ) -> anyhow::Result<u64> {
     let ata_address = get_associated_token_address(&owner, &mint);
 
+    let Some(ata) = ctx.get_account(&ata_address) else {
+        return Ok(0);
+    };
+
     if is_2022 {
-        let Ok(ata) = ctx.banks_client.get_account(ata_address).await else {
-            return Ok(0);
-        };
-
-        let Some(ata) = ata else {
-            return Ok(0);
-        };
-
         let ata = spl_token_2022::state::Account::unpack(&ata.data);
         if let Ok(ata) = ata {
             return Ok(ata.amount);
         }
     };
 
-    if let Ok(ata) = ctx.get_token_account(ata_address).await {
+    if let Ok(ata) = spl_token::state::Account::unpack(&ata.data) {
         Ok(ata.amount)
     } else {
         Ok(0u64)
     }
 }
 
-async fn set_balance(
+fn set_balance(
-    ctx: &mut ProgramTestContext,
+    ctx: &mut LiteSVM,
     owner: Pubkey,
     mint: Pubkey,
     amount: u64,
     is_2022: bool,
 ) -> anyhow::Result<()> {
-    let ata_address = get_associated_token_address(&owner, &mint);
+    let token_program_id = if is_2022 {
+        spl_token_2022::ID
+    } else {
+        spl_token::ID
+    };
+
+    let ata_address =
+        get_associated_token_address_with_program_id(&owner, &mint, &token_program_id);
+    let mut data = vec![0u8; 165];
 
     if is_2022 {
-        let mut data = vec![0u8; 165];
+        // TODO: to properly setup extensions, this is not sufficient
         let account = spl_token_2022::state::Account {
             mint,
             owner,
@@ -472,101 +510,83 @@ async fn set_balance(
             close_authority: Default::default(),
         };
         account.pack_into_slice(data.as_mut_slice());
-
+    } else {
-        ctx.set_account(
+        let account = spl_token::state::Account {
-            &ata_address,
+            mint,
-            &AccountSharedData::from(Account {
+            owner,
-                lamports: 1_000_000_000,
+            amount,
-                data: data,
+            delegate: Default::default(),
-                owner: spl_token_2022::ID,
+            state: AccountState::Initialized,
-                executable: false,
+            is_native: Default::default(),
-                rent_epoch: 0,
+            delegated_amount: 0,
-            }),
+            close_authority: Default::default(),
-        );
+        };
-
+        account.pack_into_slice(data.as_mut_slice());
-        return Ok(());
-    }
-
-    let mut data = vec![0u8; 165];
-    let account = spl_token::state::Account {
-        mint,
-        owner,
-        amount,
-        delegate: Default::default(),
-        state: AccountState::Initialized,
-        is_native: Default::default(),
-        delegated_amount: 0,
-        close_authority: Default::default(),
     };
-    account.pack_into_slice(data.as_mut_slice());
 
     ctx.set_account(
-        &ata_address,
+        ata_address,
-        &AccountSharedData::from(Account {
+        Account {
             lamports: 1_000_000_000,
             data: data,
-            owner: spl_token::ID,
+            owner: token_program_id,
             executable: false,
-            rent_epoch: 0,
+            rent_epoch: u64::MAX,
-        }),
+        },
-    );
+    )?;
 
     Ok(())
 }
 
-fn create_wallet(ctx: &mut ProgramTestContext, address: Pubkey) {
+fn create_wallet(ctx: &mut LiteSVM, address: Pubkey) {
-    ctx.set_account(
+    let _ = ctx.airdrop(&address, 1_000_000_000);
-        &address,
-        &AccountSharedData::from(Account {
-            lamports: 1_000_000_000,
-            data: vec![],
-            owner: address,
-            executable: false,
-            rent_epoch: 0,
-        }),
-    );
 }
 
-async fn setup_test_chain(
+pub fn find_file(filename: &str) -> Option<PathBuf> {
+    for dir in default_shared_object_dirs() {
+        let candidate = dir.join(filename);
+        if candidate.exists() {
+            return Some(candidate);
+        }
+    }
+    None
+}
+
+fn default_shared_object_dirs() -> Vec<PathBuf> {
+    let mut search_path = vec![];
+    if let Ok(bpf_out_dir) = std::env::var("BPF_OUT_DIR") {
+        search_path.push(PathBuf::from(bpf_out_dir));
+    } else if let Ok(bpf_out_dir) = std::env::var("SBF_OUT_DIR") {
+        search_path.push(PathBuf::from(bpf_out_dir));
+    }
+    search_path.push(PathBuf::from("tests/fixtures"));
+    if let Ok(dir) = std::env::current_dir() {
+        search_path.push(dir);
+    }
+    log::trace!("SBF .so search path: {:?}", search_path);
+    search_path
+}
+
+fn setup_test_chain(
     programs: &Vec<Pubkey>,
     clock: &Clock,
     dump: &ExecutionDump,
-) -> ProgramTestContext {
+    instruction: &Instruction,
-    // We need to intercept logs to capture program log output
+) -> anyhow::Result<LiteSVM> {
-    let log_filter = "solana_rbpf=trace,\
+    let mut program_test = LiteSVM::new();
-                    solana_runtime::message_processor=debug,\
+    program_test.set_sysvar(clock);
-                    solana_runtime::system_instruction_processor=trace,\
+    let mut accounts_list = programs.clone();
-                    solana_program_test=info,\
+    accounts_list.extend(instruction.accounts.iter().map(|x| x.pubkey));
-                    solana_metrics::metrics=warn,\
-                    tarpc=error,\
-                    info";
-    let env_logger =
-        env_logger::Builder::from_env(env_logger::Env::new().default_filter_or(log_filter))
-            .format_timestamp_nanos()
-            .build();
-    let _ = log::set_boxed_logger(Box::new(env_logger));
 
-    let mut program_test = ProgramTest::default();
+    initialize_accounts(&mut program_test, dump, &accounts_list)?;
-
+    let path = find_file(format!("autobahn_executor.so").as_str()).unwrap();
-    initialize_accounts(&mut program_test, dump).await.unwrap();
+    log::debug!("Adding program: {:?} at {path:?}", autobahn_executor::ID);
-
+    program_test.add_program_from_file(autobahn_executor::ID, path)?;
-    program_test.prefer_bpf(true);
-    for &key in programs {
-        program_test.add_program(key.to_string().as_str(), key, None);
-    }
-    program_test.add_program("autobahn_executor", autobahn_executor::ID, None);
 
     // TODO: make this dynamic based on routes
-    program_test.set_compute_max_units(1_400_000);
+    let mut cb = solana_program_runtime::compute_budget::ComputeBudget::default();
+    cb.compute_unit_limit = 1_400_000;
+    program_test.set_compute_budget(cb);
 
-    let mut program_test_context = program_test.start_with_context().await;
+    Ok(program_test)
-
-    // Set clock
-    program_test_context.set_sysvar(clock);
-
-    info!("Setting clock to: {}", clock.unix_timestamp);
-
-    program_test_context.warp_to_slot(40).unwrap();
-
-    program_test_context
 }