2022-06-18 07:31:28 -07:00
|
|
|
use std::collections::HashMap;
|
2022-07-16 05:37:15 -07:00
|
|
|
use std::sync::{Arc, RwLock};
|
|
|
|
use std::time::Duration;
|
2022-06-18 07:31:28 -07:00
|
|
|
|
|
|
|
use anchor_client::Cluster;
|
2022-07-19 00:59:30 -07:00
|
|
|
use clap::Parser;
|
2022-08-01 05:19:52 -07:00
|
|
|
use client::{chain_data, keypair_from_cli, Client, MangoClient, MangoGroupContext};
|
2022-06-18 07:31:28 -07:00
|
|
|
use log::*;
|
|
|
|
use mango_v4::state::{PerpMarketIndex, TokenIndex};
|
|
|
|
|
|
|
|
use solana_sdk::commitment_config::CommitmentConfig;
|
|
|
|
use solana_sdk::pubkey::Pubkey;
|
|
|
|
use std::collections::HashSet;
|
|
|
|
|
|
|
|
pub mod account_shared_data;
|
|
|
|
pub mod liquidate;
|
|
|
|
pub mod metrics;
|
2022-08-05 11:28:14 -07:00
|
|
|
pub mod rebalance;
|
2022-06-18 07:31:28 -07:00
|
|
|
pub mod snapshot_source;
|
|
|
|
pub mod util;
|
|
|
|
pub mod websocket_source;
|
|
|
|
|
2022-07-16 05:37:15 -07:00
|
|
|
use crate::util::{is_mango_account, is_mango_bank, is_mint_info, is_perp_market};
|
|
|
|
|
2022-06-18 07:31:28 -07:00
|
|
|
// jemalloc seems to be better at keeping the memory footprint reasonable over
|
|
|
|
// longer periods of time
|
|
|
|
#[global_allocator]
|
|
|
|
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
|
|
|
|
|
|
|
|
trait AnyhowWrap {
|
|
|
|
type Value;
|
|
|
|
fn map_err_anyhow(self) -> anyhow::Result<Self::Value>;
|
|
|
|
}
|
|
|
|
|
|
|
|
impl<T, E: std::fmt::Debug> AnyhowWrap for Result<T, E> {
|
|
|
|
type Value = T;
|
|
|
|
fn map_err_anyhow(self) -> anyhow::Result<Self::Value> {
|
|
|
|
self.map_err(|err| anyhow::anyhow!("{:?}", err))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-07-19 04:43:14 -07:00
|
|
|
#[derive(Parser, Debug)]
|
|
|
|
#[clap()]
|
|
|
|
struct CliDotenv {
|
|
|
|
// When --dotenv <file> is passed, read the specified dotenv file before parsing args
|
|
|
|
#[clap(long)]
|
|
|
|
dotenv: std::path::PathBuf,
|
|
|
|
|
|
|
|
remaining_args: Vec<std::ffi::OsString>,
|
|
|
|
}
|
|
|
|
|
2022-07-19 00:59:30 -07:00
|
|
|
#[derive(Parser)]
|
|
|
|
#[clap()]
|
|
|
|
struct Cli {
|
|
|
|
#[clap(short, long, env)]
|
|
|
|
rpc_url: String,
|
|
|
|
|
|
|
|
// TODO: different serum markets could use different serum programs, should come from registered markets
|
|
|
|
#[clap(long, env)]
|
|
|
|
serum_program: Pubkey,
|
|
|
|
|
|
|
|
#[clap(long, env)]
|
2022-08-01 05:19:52 -07:00
|
|
|
liqor_mango_account: Pubkey,
|
2022-07-19 00:59:30 -07:00
|
|
|
|
|
|
|
#[clap(long, env)]
|
2022-08-01 05:19:52 -07:00
|
|
|
liqor_owner: String,
|
2022-07-19 00:59:30 -07:00
|
|
|
|
|
|
|
#[clap(long, env, default_value = "300")]
|
|
|
|
snapshot_interval_secs: u64,
|
|
|
|
|
2022-08-05 04:50:44 -07:00
|
|
|
/// how many getMultipleAccounts requests to send in parallel
|
2022-07-19 00:59:30 -07:00
|
|
|
#[clap(long, env, default_value = "10")]
|
|
|
|
parallel_rpc_requests: usize,
|
|
|
|
|
2022-08-05 04:50:44 -07:00
|
|
|
/// typically 100 is the max number of accounts getMultipleAccounts will retrieve at once
|
2022-07-19 00:59:30 -07:00
|
|
|
#[clap(long, env, default_value = "100")]
|
|
|
|
get_multiple_accounts_count: usize,
|
2022-08-05 04:50:44 -07:00
|
|
|
|
|
|
|
/// liquidator health ratio should not fall below this value
|
|
|
|
#[clap(long, env, default_value = "50")]
|
|
|
|
min_health_ratio: f64,
|
2022-08-05 11:28:14 -07:00
|
|
|
|
|
|
|
#[clap(long, env, default_value = "1")]
|
|
|
|
rebalance_slippage: f64,
|
2022-07-19 00:59:30 -07:00
|
|
|
}
|
2022-06-18 07:31:28 -07:00
|
|
|
|
|
|
|
pub fn encode_address(addr: &Pubkey) -> String {
|
|
|
|
bs58::encode(&addr.to_bytes()).into_string()
|
|
|
|
}
|
|
|
|
|
|
|
|
#[tokio::main]
|
|
|
|
async fn main() -> anyhow::Result<()> {
|
2022-07-19 04:43:14 -07:00
|
|
|
let args = if let Ok(cli_dotenv) = CliDotenv::try_parse() {
|
|
|
|
dotenv::from_path(cli_dotenv.dotenv)?;
|
|
|
|
cli_dotenv.remaining_args
|
|
|
|
} else {
|
|
|
|
dotenv::dotenv().ok();
|
|
|
|
std::env::args_os().collect()
|
|
|
|
};
|
|
|
|
let cli = Cli::parse_from(args);
|
2022-07-19 00:59:30 -07:00
|
|
|
|
2022-07-31 00:25:11 -07:00
|
|
|
let liqor_owner = keypair_from_cli(&cli.liqor_owner);
|
2022-07-19 00:59:30 -07:00
|
|
|
|
|
|
|
let rpc_url = cli.rpc_url;
|
2022-07-16 05:37:15 -07:00
|
|
|
let ws_url = rpc_url.replace("https", "wss");
|
2022-07-19 00:59:30 -07:00
|
|
|
|
2022-08-08 04:40:33 -07:00
|
|
|
let rpc_timeout = Duration::from_secs(10);
|
2022-07-19 00:59:30 -07:00
|
|
|
let cluster = Cluster::Custom(rpc_url.clone(), ws_url.clone());
|
2022-07-16 05:37:15 -07:00
|
|
|
let commitment = CommitmentConfig::processed();
|
2022-08-04 08:01:00 -07:00
|
|
|
let client = Client::new(cluster.clone(), commitment, &liqor_owner, Some(rpc_timeout));
|
2022-08-01 05:19:52 -07:00
|
|
|
|
|
|
|
// The representation of current on-chain account data
|
|
|
|
let chain_data = Arc::new(RwLock::new(chain_data::ChainData::new()));
|
|
|
|
// Reading accounts from chain_data
|
|
|
|
let account_fetcher = Arc::new(chain_data::AccountFetcher {
|
|
|
|
chain_data: chain_data.clone(),
|
2022-08-04 08:01:00 -07:00
|
|
|
rpc: client.rpc(),
|
2022-08-01 05:19:52 -07:00
|
|
|
});
|
|
|
|
|
|
|
|
let mango_account = account_fetcher.fetch_fresh_mango_account(&cli.liqor_mango_account)?;
|
|
|
|
let mango_group = mango_account.fixed.group;
|
|
|
|
|
2022-07-19 00:59:30 -07:00
|
|
|
let group_context = MangoGroupContext::new_from_rpc(mango_group, cluster.clone(), commitment)?;
|
2022-07-16 05:37:15 -07:00
|
|
|
|
2022-08-05 04:50:44 -07:00
|
|
|
let mango_oracles = group_context
|
2022-07-16 05:37:15 -07:00
|
|
|
.tokens
|
2022-06-18 07:31:28 -07:00
|
|
|
.values()
|
2022-07-16 05:37:15 -07:00
|
|
|
.map(|value| value.mint_info.oracle)
|
2022-06-18 07:31:28 -07:00
|
|
|
.collect::<Vec<Pubkey>>();
|
|
|
|
|
|
|
|
//
|
|
|
|
// feed setup
|
|
|
|
//
|
|
|
|
// FUTURE: decouple feed setup and liquidator business logic
|
|
|
|
// feed should send updates to a channel which liquidator can consume
|
2022-07-19 00:59:30 -07:00
|
|
|
|
|
|
|
let mango_program = mango_v4::ID;
|
2022-06-18 07:31:28 -07:00
|
|
|
|
|
|
|
solana_logger::setup_with_default("info");
|
|
|
|
info!("startup");
|
|
|
|
|
|
|
|
let metrics = metrics::start();
|
|
|
|
|
|
|
|
// Sourcing account and slot data from solana via websockets
|
|
|
|
// FUTURE: websocket feed should take which accounts to listen to as an input
|
|
|
|
let (websocket_sender, websocket_receiver) =
|
|
|
|
async_channel::unbounded::<websocket_source::Message>();
|
2022-07-19 00:59:30 -07:00
|
|
|
websocket_source::start(
|
|
|
|
websocket_source::Config {
|
|
|
|
rpc_ws_url: ws_url.clone(),
|
|
|
|
mango_program,
|
|
|
|
serum_program: cli.serum_program,
|
|
|
|
open_orders_authority: mango_group,
|
|
|
|
},
|
2022-08-05 04:50:44 -07:00
|
|
|
mango_oracles.clone(),
|
2022-07-19 00:59:30 -07:00
|
|
|
websocket_sender,
|
|
|
|
);
|
2022-06-18 07:31:28 -07:00
|
|
|
|
2022-07-21 04:03:28 -07:00
|
|
|
let first_websocket_slot = websocket_source::get_next_create_bank_slot(
|
|
|
|
websocket_receiver.clone(),
|
|
|
|
Duration::from_secs(10),
|
|
|
|
)
|
|
|
|
.await?;
|
|
|
|
|
2022-06-18 07:31:28 -07:00
|
|
|
// Getting solana account snapshots via jsonrpc
|
|
|
|
let (snapshot_sender, snapshot_receiver) =
|
|
|
|
async_channel::unbounded::<snapshot_source::AccountSnapshot>();
|
|
|
|
// FUTURE: of what to fetch a snapshot - should probably take as an input
|
2022-07-19 00:59:30 -07:00
|
|
|
snapshot_source::start(
|
|
|
|
snapshot_source::Config {
|
|
|
|
rpc_http_url: rpc_url.clone(),
|
|
|
|
mango_program,
|
|
|
|
mango_group,
|
|
|
|
get_multiple_accounts_count: cli.get_multiple_accounts_count,
|
|
|
|
parallel_rpc_requests: cli.parallel_rpc_requests,
|
|
|
|
snapshot_interval: std::time::Duration::from_secs(cli.snapshot_interval_secs),
|
2022-07-21 04:03:28 -07:00
|
|
|
min_slot: first_websocket_slot + 10,
|
2022-07-19 00:59:30 -07:00
|
|
|
},
|
2022-08-05 04:50:44 -07:00
|
|
|
mango_oracles,
|
2022-07-19 00:59:30 -07:00
|
|
|
snapshot_sender,
|
|
|
|
);
|
2022-06-18 07:31:28 -07:00
|
|
|
|
2022-07-21 04:03:28 -07:00
|
|
|
start_chain_data_metrics(chain_data.clone(), &metrics);
|
|
|
|
|
2022-06-18 07:31:28 -07:00
|
|
|
// Addresses of the MangoAccounts belonging to the mango program.
|
|
|
|
// Needed to check health of them all when the cache updates.
|
|
|
|
let mut mango_accounts = HashSet::<Pubkey>::new();
|
|
|
|
|
|
|
|
let mut mint_infos = HashMap::<TokenIndex, Pubkey>::new();
|
|
|
|
let mut oracles = HashSet::<Pubkey>::new();
|
|
|
|
let mut perp_markets = HashMap::<PerpMarketIndex, Pubkey>::new();
|
|
|
|
|
|
|
|
// Is the first snapshot done? Only start checking account health when it is.
|
|
|
|
let mut one_snapshot_done = false;
|
|
|
|
|
|
|
|
let mut metric_websocket_queue_len = metrics.register_u64("websocket_queue_length".into());
|
|
|
|
let mut metric_snapshot_queue_len = metrics.register_u64("snapshot_queue_length".into());
|
|
|
|
let mut metric_mango_accounts = metrics.register_u64("mango_accouns".into());
|
|
|
|
|
2022-07-16 05:37:15 -07:00
|
|
|
//
|
|
|
|
// mango client setup
|
|
|
|
//
|
|
|
|
let mango_client = {
|
|
|
|
Arc::new(MangoClient::new_detail(
|
2022-08-01 05:19:52 -07:00
|
|
|
client,
|
|
|
|
cli.liqor_mango_account,
|
2022-07-19 00:59:30 -07:00
|
|
|
liqor_owner,
|
2022-07-16 05:37:15 -07:00
|
|
|
group_context,
|
|
|
|
account_fetcher.clone(),
|
|
|
|
)?)
|
|
|
|
};
|
|
|
|
|
2022-08-05 04:50:44 -07:00
|
|
|
let liq_config = liquidate::Config {
|
|
|
|
min_health_ratio: cli.min_health_ratio,
|
2022-08-07 11:04:19 -07:00
|
|
|
// TODO: config
|
|
|
|
refresh_timeout: Duration::from_secs(30),
|
2022-08-05 04:50:44 -07:00
|
|
|
};
|
|
|
|
|
2022-08-05 11:28:14 -07:00
|
|
|
let mut rebalance_interval = tokio::time::interval(Duration::from_secs(5));
|
|
|
|
let rebalance_config = rebalance::Config {
|
|
|
|
slippage: cli.rebalance_slippage,
|
2022-08-07 11:04:19 -07:00
|
|
|
// TODO: config
|
|
|
|
refresh_timeout: Duration::from_secs(30),
|
2022-08-05 11:28:14 -07:00
|
|
|
};
|
|
|
|
|
2022-06-18 07:31:28 -07:00
|
|
|
info!("main loop");
|
|
|
|
loop {
|
|
|
|
tokio::select! {
|
|
|
|
message = websocket_receiver.recv() => {
|
|
|
|
|
|
|
|
metric_websocket_queue_len.set(websocket_receiver.len() as u64);
|
|
|
|
let message = message.expect("channel not closed");
|
|
|
|
|
|
|
|
// build a model of slots and accounts in `chain_data`
|
2022-07-21 04:03:28 -07:00
|
|
|
websocket_source::update_chain_data(&mut chain_data.write().unwrap(), message.clone());
|
2022-06-18 07:31:28 -07:00
|
|
|
|
|
|
|
// specific program logic using the mirrored data
|
|
|
|
if let websocket_source::Message::Account(account_write) = message {
|
|
|
|
|
2022-07-19 00:59:30 -07:00
|
|
|
if is_mango_account(&account_write.account, &mango_program, &mango_group).is_some() {
|
2022-06-18 07:31:28 -07:00
|
|
|
|
|
|
|
// e.g. to render debug logs RUST_LOG="liquidator=debug"
|
|
|
|
log::debug!("change to mango account {}...", &account_write.pubkey.to_string()[0..3]);
|
|
|
|
|
|
|
|
// Track all MangoAccounts: we need to iterate over them later
|
|
|
|
mango_accounts.insert(account_write.pubkey);
|
|
|
|
metric_mango_accounts.set(mango_accounts.len() as u64);
|
|
|
|
|
|
|
|
if !one_snapshot_done {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2022-08-05 11:28:14 -07:00
|
|
|
liquidate(
|
2022-08-05 04:50:44 -07:00
|
|
|
&mango_client,
|
|
|
|
&account_fetcher,
|
|
|
|
std::iter::once(&account_write.pubkey),
|
|
|
|
&liq_config,
|
2022-08-05 11:28:14 -07:00
|
|
|
&rebalance_config,
|
|
|
|
)?;
|
2022-06-18 07:31:28 -07:00
|
|
|
}
|
|
|
|
|
2022-07-19 00:59:30 -07:00
|
|
|
if is_mango_bank(&account_write.account, &mango_program, &mango_group).is_some() || oracles.contains(&account_write.pubkey) {
|
2022-06-18 07:31:28 -07:00
|
|
|
if !one_snapshot_done {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2022-07-19 00:59:30 -07:00
|
|
|
if is_mango_bank(&account_write.account, &mango_program, &mango_group).is_some() {
|
2022-06-18 07:31:28 -07:00
|
|
|
log::debug!("change to bank {}", &account_write.pubkey);
|
|
|
|
}
|
|
|
|
|
|
|
|
if oracles.contains(&account_write.pubkey) {
|
|
|
|
log::debug!("change to oracle {}", &account_write.pubkey);
|
|
|
|
}
|
|
|
|
|
2022-08-05 11:28:14 -07:00
|
|
|
liquidate(
|
2022-08-05 04:50:44 -07:00
|
|
|
&mango_client,
|
|
|
|
&account_fetcher,
|
|
|
|
mango_accounts.iter(),
|
|
|
|
&liq_config,
|
2022-08-05 11:28:14 -07:00
|
|
|
&rebalance_config,
|
|
|
|
)?;
|
2022-06-18 07:31:28 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
},
|
|
|
|
|
|
|
|
message = snapshot_receiver.recv() => {
|
|
|
|
metric_snapshot_queue_len.set(snapshot_receiver.len() as u64);
|
|
|
|
let message = message.expect("channel not closed");
|
|
|
|
|
|
|
|
// Track all mango account pubkeys
|
|
|
|
for update in message.accounts.iter() {
|
2022-07-19 00:59:30 -07:00
|
|
|
if is_mango_account(&update.account, &mango_program, &mango_group).is_some() {
|
2022-06-18 07:31:28 -07:00
|
|
|
mango_accounts.insert(update.pubkey);
|
|
|
|
}
|
2022-07-19 00:59:30 -07:00
|
|
|
if let Some(mint_info) = is_mint_info(&update.account, &mango_program, &mango_group) {
|
2022-06-18 07:31:28 -07:00
|
|
|
mint_infos.insert(mint_info.token_index, update.pubkey);
|
|
|
|
oracles.insert(mint_info.oracle);
|
|
|
|
}
|
2022-07-19 00:59:30 -07:00
|
|
|
if let Some(perp_market) = is_perp_market(&update.account, &mango_program, &mango_group) {
|
2022-06-18 07:31:28 -07:00
|
|
|
perp_markets.insert(perp_market.perp_market_index, update.pubkey);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
metric_mango_accounts.set(mango_accounts.len() as u64);
|
|
|
|
|
2022-07-21 04:03:28 -07:00
|
|
|
snapshot_source::update_chain_data(&mut chain_data.write().unwrap(), message);
|
2022-06-18 07:31:28 -07:00
|
|
|
one_snapshot_done = true;
|
|
|
|
|
2022-08-05 11:28:14 -07:00
|
|
|
liquidate(
|
2022-08-05 04:50:44 -07:00
|
|
|
&mango_client,
|
|
|
|
&account_fetcher,
|
|
|
|
mango_accounts.iter(),
|
|
|
|
&liq_config,
|
2022-08-05 11:28:14 -07:00
|
|
|
&rebalance_config,
|
|
|
|
)?;
|
2022-06-18 07:31:28 -07:00
|
|
|
},
|
2022-08-05 11:28:14 -07:00
|
|
|
|
|
|
|
_ = rebalance_interval.tick() => {
|
2022-08-07 11:04:19 -07:00
|
|
|
if one_snapshot_done {
|
|
|
|
if let Err(err) = rebalance::zero_all_non_quote(&mango_client, &account_fetcher, &cli.liqor_mango_account, &rebalance_config) {
|
|
|
|
log::error!("failed to rebalance liqor: {:?}", err);
|
2022-09-02 01:20:02 -07:00
|
|
|
|
|
|
|
// Workaround: We really need a sequence enforcer in the liquidator since we don't want to
|
|
|
|
// accidentally send a similar tx again when we incorrectly believe an earlier one got forked
|
|
|
|
// off. For now, hard sleep on error to avoid the most frequent error cases.
|
|
|
|
std::thread::sleep(Duration::from_secs(10));
|
2022-08-07 11:04:19 -07:00
|
|
|
}
|
|
|
|
}
|
2022-08-05 11:28:14 -07:00
|
|
|
}
|
2022-06-18 07:31:28 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2022-07-21 04:03:28 -07:00
|
|
|
|
2022-08-05 11:28:14 -07:00
|
|
|
fn liquidate<'a>(
|
|
|
|
mango_client: &MangoClient,
|
|
|
|
account_fetcher: &chain_data::AccountFetcher,
|
|
|
|
accounts: impl Iterator<Item = &'a Pubkey>,
|
|
|
|
config: &liquidate::Config,
|
|
|
|
rebalance_config: &rebalance::Config,
|
|
|
|
) -> anyhow::Result<()> {
|
2022-08-30 04:46:39 -07:00
|
|
|
if !liquidate::maybe_liquidate_one(mango_client, account_fetcher, accounts, config) {
|
2022-08-05 11:28:14 -07:00
|
|
|
return Ok(());
|
|
|
|
}
|
|
|
|
|
|
|
|
let liqor = &mango_client.mango_account_address;
|
2022-08-07 11:04:19 -07:00
|
|
|
if let Err(err) =
|
2022-08-30 04:46:39 -07:00
|
|
|
rebalance::zero_all_non_quote(mango_client, account_fetcher, liqor, rebalance_config)
|
2022-08-07 11:04:19 -07:00
|
|
|
{
|
|
|
|
log::error!("failed to rebalance liqor: {:?}", err);
|
|
|
|
}
|
2022-08-05 11:28:14 -07:00
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2022-07-21 04:03:28 -07:00
|
|
|
fn start_chain_data_metrics(chain: Arc<RwLock<chain_data::ChainData>>, metrics: &metrics::Metrics) {
|
|
|
|
let mut interval = tokio::time::interval(std::time::Duration::from_secs(5));
|
|
|
|
|
|
|
|
let mut metric_slots_count = metrics.register_u64("chain_data_slots_count".into());
|
|
|
|
let mut metric_accounts_count = metrics.register_u64("chain_data_accounts_count".into());
|
|
|
|
let mut metric_account_write_count =
|
|
|
|
metrics.register_u64("chain_data_account_write_count".into());
|
|
|
|
|
|
|
|
tokio::spawn(async move {
|
|
|
|
loop {
|
|
|
|
interval.tick().await;
|
|
|
|
let chain_lock = chain.read().unwrap();
|
|
|
|
metric_slots_count.set(chain_lock.slots_count() as u64);
|
|
|
|
metric_accounts_count.set(chain_lock.accounts_count() as u64);
|
|
|
|
metric_account_write_count.set(chain_lock.account_writes_count() as u64);
|
|
|
|
}
|
|
|
|
});
|
|
|
|
}
|