diff --git a/bin/autobahn-router/Dockerfile b/bin/autobahn-router/Dockerfile index b671b3b..2469708 100644 --- a/bin/autobahn-router/Dockerfile +++ b/bin/autobahn-router/Dockerfile @@ -16,10 +16,11 @@ COPY . . RUN cargo build --release --bin autobahn-router FROM debian:bookworm-slim as run +ARG CONFIG_PATH=/app/bin/autobahn-router/template-config.toml RUN apt-get update && apt-get -y install ca-certificates libc6 libssl3 libssl-dev openssl COPY --from=build /app/target/release/autobahn-router /usr/local/bin/ -COPY --from=build /app/bin/autobahn-router/template-config.toml /usr/local/bin/template-config.toml +COPY --from=build $CONFIG_PATH /usr/local/bin/template-config.toml RUN adduser --system --group --no-create-home mangouser USER mangouser diff --git a/bin/autobahn-router/template-config-eclipse.toml b/bin/autobahn-router/template-config-eclipse.toml new file mode 100644 index 0000000..25bd6e5 --- /dev/null +++ b/bin/autobahn-router/template-config-eclipse.toml @@ -0,0 +1,99 @@ +snapshot_timeout_in_seconds = 900 + +[infinity] +enabled = false + +[orca] +enabled = true +mints = [] +take_all_mints = true +add_mango_tokens = false + +[cropper] +enabled = false +mints = [] +take_all_mints = true +add_mango_tokens = false + +[openbook_v2] +enabled = false +mints = [] +take_all_mints = true +add_mango_tokens = false + +[raydium] +enabled = false +mints = [] +take_all_mints = true +add_mango_tokens = false + +[raydium_cp] +enabled = false +mints = [] +take_all_mints = true +add_mango_tokens = false + +[saber] +enabled = false +mints = [] +take_all_mints = true +add_mango_tokens = false + + +[routing] +path_cache_validity_ms = 30000 +path_warming_mode = "ConfiguredMints" +#path_warming_mode = "HotMints" +path_warming_amounts = [100, 1000, 10_000] +path_warming_for_mints = [ + "So11111111111111111111111111111111111111112", # SOL +] +path_warming_interval_secs = 5 +path_warming_max_accounts = [20, 30, 40, 64] +lookup_tables = [] +cooldown_duration_multihop_secs = 30 +cooldown_duration_singlehop_secs = 60 +max_path_length = 3 +retain_path_count = 5 +max_edge_per_pair = 5 +max_edge_per_cold_pair = 2 +slot_excessive_lag_max_duration_secs = 1600 + +[server] +address = "[::]:8888" + +[metrics] +output_http = true +prometheus_address = "[::]:9091" +output_stdout = false + +[[sources]] +dedup_queue_size = 50000 +rpc_http_url = "$RPC_HTTP_URL" +rpc_support_compression = false +re_snapshot_interval_secs = 1200 +request_timeout_in_seconds = 300 + +[[sources.grpc_sources]] +name = "router-eclipse" +connection_string = "$RPC_HTTP_URL_WITHOUT_TOKEN" +token = "$RPC_TOKEN" +retry_connection_sleep_secs = 30 + +[price_feed] +birdeye_token = "$BIRDEYE_TOKEN" +refresh_interval_secs = 1200 # every 20 min + +[safety_checks] +check_quote_out_amount_deviation = true +min_quote_out_to_in_amount_ratio = 0.65 + +[hot_mints] +always_hot_mints = [ + "So11111111111111111111111111111111111111112", # SOL +] +keep_latest_count = 50 + +[debug_config] +reprice_using_live_rpc = true +reprice_probability = 0.05 diff --git a/bin/autobahn-router/template-config.toml b/bin/autobahn-router/template-config.toml index 1a521c4..a51ccc7 100644 --- a/bin/autobahn-router/template-config.toml +++ b/bin/autobahn-router/template-config.toml @@ -52,7 +52,7 @@ path_warming_for_mints = [ # "27G8MtK7VtTcCHkpASjSDdkWWYfoqT6ggEuKidVJidD4" # JLP ] path_warming_interval_secs = 5 -path_warming_max_accounts = [20, 25, 30, 35, 40, 64] +path_warming_max_accounts = [20, 30, 40, 64] lookup_tables = ["87TgskchTNEv1uXkGQk1U4zt65tjqbfGAZWNMGAcRRPx", "AgCBUZ6UMWqPLftTxeAqpQxtrfiCyL2HgRfmmM6QTfCj", "A1v3qxN7HbUvtyPnnaoCrKonXjkFLaDHXk3S6R2QfEaw", diff --git a/fly-eclipse.toml b/fly-eclipse.toml new file mode 100644 index 0000000..765febc --- /dev/null +++ b/fly-eclipse.toml @@ -0,0 +1,25 @@ +app = "router-eclipse" +primary_region = "fra" +kill_signal = "SIGTERM" +kill_timeout = "30s" + +[build] + dockerfile = 'bin/autobahn-router/Dockerfile' + +[build.args] + CONFIG_PATH="/app/bin/autobahn-router/template-config-eclipse.toml" + +[experimental] + cmd = ["autobahn-router", "/usr/local/bin/template-config.toml"] + +[[vm]] + size = "shared-cpu-4x" + memory = "8gb" + +[[restart]] + policy = "always" + retries = 10 + +[metrics] + port = 9091 + path = "/metrics" \ No newline at end of file diff --git a/fly.toml b/fly.toml index 0927c77..77b8498 100644 --- a/fly.toml +++ b/fly.toml @@ -1,5 +1,5 @@ app = "router-2" -primary_region = "dfw" +primary_region = "ams" kill_signal = "SIGTERM" kill_timeout = "30s" diff --git a/lib/dex-orca/src/orca.rs b/lib/dex-orca/src/orca.rs index d489bdc..c415644 100644 --- a/lib/dex-orca/src/orca.rs +++ b/lib/dex-orca/src/orca.rs @@ -310,10 +310,17 @@ pub async fn fetch_all_whirlpools( .await?; let result = whirlpools .iter() - .map(|account| { - let whirlpool: Whirlpool = - AnchorDeserialize::deserialize(&mut &account.data[8..]).unwrap(); - (account.pubkey, whirlpool) + .filter_map(|account| { + let pubkey = account.pubkey; + let whirlpool: Result = + AnchorDeserialize::deserialize(&mut &account.data[8..]); + match whirlpool { + Ok(whirlpool) => Some((account.pubkey, whirlpool)), + Err(e) => { + error!("Error deserializing whirlpool account : {pubkey:?} error: {e:?}"); + None + } + } }) .collect_vec(); Ok(result) diff --git a/lib/dex-orca/src/orca_dex.rs b/lib/dex-orca/src/orca_dex.rs index 7412879..85023e4 100644 --- a/lib/dex-orca/src/orca_dex.rs +++ b/lib/dex-orca/src/orca_dex.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use anchor_lang::Id; use anchor_spl::token::spl_token; -use anchor_spl::token::spl_token::state::AccountState; +use anchor_spl::token::spl_token::state::{Account, AccountState}; use anchor_spl::token_2022::Token2022; use anyhow::Context; use itertools::Itertools; @@ -229,7 +229,12 @@ impl OrcaDex { .iter() .filter(|x| { x.1.owner == Token2022::id() - || spl_token::state::Account::unpack(x.1.data()).unwrap().state + || spl_token::state::Account::unpack(x.1.data()) + .unwrap_or(Account { + state: AccountState::Frozen, + ..Default::default() + }) + .state == AccountState::Frozen }) .map(|x| x.0) @@ -246,7 +251,9 @@ impl OrcaDex { // TODO: actually need to dynamically adjust subscriptions based on the tick? let tick_arrays = filtered_pools .iter() - .map(|(pk, wp)| whirlpool_tick_array_pks(wp, pk, program_id)) + .map(|(pk, wp)| { + whirlpool_tick_array_pks(wp, pk, program_id) + }) .collect_vec(); let edge_pairs = filtered_pools diff --git a/lib/dex-orca/tests/test_cropper.rs b/lib/dex-orca/tests/test_cropper.rs index b46354d..51939cc 100644 --- a/lib/dex-orca/tests/test_cropper.rs +++ b/lib/dex-orca/tests/test_cropper.rs @@ -8,6 +8,16 @@ use router_lib::test_tools::{generate_dex_rpc_dump, rpc}; #[tokio::test] async fn test_dump_input_data_cropper() -> anyhow::Result<()> { + let is_eclipse = std::env::var("ECLIPSE") + .map(|x| { + let value: bool = x.parse().unwrap(); + value + }) + .unwrap_or_default(); + if is_eclipse { + // crooper is not yet on eclipse + return Ok(()); + } let options = HashMap::from([ ( "program_id".to_string(), diff --git a/lib/router-feed-lib/src/get_program_account.rs b/lib/router-feed-lib/src/get_program_account.rs index 2c87b37..5b9ae2b 100644 --- a/lib/router-feed-lib/src/get_program_account.rs +++ b/lib/router-feed-lib/src/get_program_account.rs @@ -205,6 +205,47 @@ pub async fn get_compressed_program_account_rpc( Ok((min_slot, snap_result)) } +// called on startup to get the required accounts, few calls with some 100 thousand accounts +#[tracing::instrument(skip_all, level = "trace")] +pub async fn get_uncompressed_program_account_rpc( + rpc_client: &RpcClient, + filters: &HashSet, + config: RpcProgramAccountsConfig, +) -> anyhow::Result<(u64, Vec)> { + let slot = rpc_client.get_slot().await?; + let config = RpcProgramAccountsConfig { + with_context: Some(true), + account_config: RpcAccountInfoConfig { + encoding: Some(UiAccountEncoding::Base64), + min_context_slot: None, + commitment: config.account_config.commitment, + data_slice: config.account_config.data_slice, + }, + filters: config.filters, + }; + + let mut snap_result = vec![]; + let mut min_slot = u64::MAX; + + // use getGPA compressed if available + for program_id in filters.iter() { + info!("gPA for {}", program_id); + min_slot = slot.min(min_slot); + let account_snapshot = rpc_client + .get_program_accounts_with_config(&program_id, config.clone()) + .await + .map_err_anyhow()?; + tracing::log::debug!("gpa snapshot received {}", program_id); + + let iter = account_snapshot.iter().map(|(pk, account)| { + account_write_from(*pk, slot, SNAP_ACCOUNT_WRITE_VERSION, account.clone()) + }); + snap_result.extend(iter); + } + + Ok((min_slot, snap_result)) +} + // called on startup to get the required accounts, few calls with some 100 thousand accounts #[tracing::instrument(skip_all, level = "trace")] pub async fn get_uncompressed_program_account( diff --git a/lib/router-feed-lib/src/router_rpc_wrapper.rs b/lib/router-feed-lib/src/router_rpc_wrapper.rs index a08369c..9177417 100644 --- a/lib/router-feed-lib/src/router_rpc_wrapper.rs +++ b/lib/router-feed-lib/src/router_rpc_wrapper.rs @@ -9,7 +9,10 @@ use solana_sdk::account::Account; use solana_sdk::pubkey::Pubkey; use crate::account_write::AccountWrite; -use crate::get_program_account::{fetch_multiple_accounts, get_compressed_program_account_rpc}; +use crate::get_program_account::{ + fetch_multiple_accounts, get_compressed_program_account_rpc, + get_uncompressed_program_account_rpc, +}; use crate::router_rpc_client::RouterRpcClientTrait; pub struct RouterRpcWrapper { @@ -52,10 +55,21 @@ impl RouterRpcClientTrait for RouterRpcWrapper { pubkey: &Pubkey, config: RpcProgramAccountsConfig, ) -> anyhow::Result> { - Ok( - get_compressed_program_account_rpc(&self.rpc, &HashSet::from([*pubkey]), config) - .await? - .1, - ) + let disable_compressed = std::env::var::("DISABLE_COMRPESSED_GPA".to_string()) + .unwrap_or("false".to_string()); + let disable_compressed: bool = disable_compressed.trim().parse().unwrap(); + if disable_compressed { + Ok( + get_uncompressed_program_account_rpc(&self.rpc, &HashSet::from([*pubkey]), config) + .await? + .1, + ) + } else { + Ok( + get_compressed_program_account_rpc(&self.rpc, &HashSet::from([*pubkey]), config) + .await? + .1, + ) + } } } diff --git a/lib/router-lib/src/dex/interface.rs b/lib/router-lib/src/dex/interface.rs index 09cd8bd..d79a560 100644 --- a/lib/router-lib/src/dex/interface.rs +++ b/lib/router-lib/src/dex/interface.rs @@ -194,7 +194,6 @@ pub trait DexInterface: Sync + Send { /// simulation tests. fn program_ids(&self) -> HashSet; - /// Initializes an Edge from ChainData (production) or BanksClient (test). /// The Edge will be dropped once a new Edge for the same EdgeIndentifier /// has been initialized. After calling initialize the DexInterface needs diff --git a/lib/router-lib/src/test_tools/generate_dex_rpc_dump.rs b/lib/router-lib/src/test_tools/generate_dex_rpc_dump.rs index 1133986..8bd587d 100644 --- a/lib/router-lib/src/test_tools/generate_dex_rpc_dump.rs +++ b/lib/router-lib/src/test_tools/generate_dex_rpc_dump.rs @@ -227,7 +227,7 @@ pub async fn run_dump_swap_ix_with_custom_amount( continue; }; - debug!( + println!( "#{} || quote: {} => {} : {} => {}", success, id.input_mint(), diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index d63da2b..71a1c7e 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -8,7 +8,9 @@ export DUMP_MAINNET_DATA=1 RUST_LOG=info # define in addition # RPC_HTTP_URL="http://fcs-ams1._peer.internal:18899" - +# for eclipse +# export ECLIPSE=true +# export DISABLE_COMRPESSED_GPA=true # saber DUMP_SABER_START=$(date)