2023-05-07 20:16:29 -07:00
|
|
|
use anchor_lang::prelude::Pubkey;
|
2023-05-17 09:11:17 -07:00
|
|
|
use chrono::{DateTime, Duration, NaiveDateTime, Utc};
|
2023-06-07 21:03:54 -07:00
|
|
|
use deadpool_postgres::Pool;
|
2023-05-07 20:16:29 -07:00
|
|
|
use futures::future::join_all;
|
2023-06-07 21:03:54 -07:00
|
|
|
use log::debug;
|
2023-05-17 09:11:17 -07:00
|
|
|
use openbook_candles::{
|
2023-06-07 21:03:54 -07:00
|
|
|
database::{
|
|
|
|
fetch::fetch_worker_transactions,
|
|
|
|
initialize::{connect_to_database, setup_database},
|
|
|
|
insert::{add_fills_atomically, build_transactions_insert_statement},
|
|
|
|
},
|
2023-05-17 09:11:17 -07:00
|
|
|
structs::{
|
|
|
|
markets::{fetch_market_infos, load_markets},
|
2023-06-07 21:03:54 -07:00
|
|
|
transaction::{PgTransaction, NUM_TRANSACTION_PARTITIONS},
|
2023-05-17 09:11:17 -07:00
|
|
|
},
|
2023-06-07 21:03:54 -07:00
|
|
|
utils::{AnyhowWrap, Config, OPENBOOK_KEY},
|
2023-05-17 09:11:17 -07:00
|
|
|
worker::trade_fetching::parsing::parse_trades_from_openbook_txns,
|
|
|
|
};
|
|
|
|
use solana_client::{
|
|
|
|
nonblocking::rpc_client::RpcClient, rpc_client::GetConfirmedSignaturesForAddress2Config,
|
2023-06-07 21:03:54 -07:00
|
|
|
rpc_config::RpcTransactionConfig
|
2023-05-17 09:11:17 -07:00
|
|
|
};
|
2023-05-07 20:16:29 -07:00
|
|
|
use solana_sdk::{commitment_config::CommitmentConfig, signature::Signature};
|
|
|
|
use solana_transaction_status::UiTransactionEncoding;
|
2023-06-07 21:03:54 -07:00
|
|
|
use std::{collections::HashMap, env, str::FromStr,time::Duration as WaitDuration };
|
2023-05-06 18:06:42 -07:00
|
|
|
|
2023-05-19 17:15:13 -07:00
|
|
|
#[tokio::main(flavor = "multi_thread", worker_threads = 10)]
|
2023-05-17 09:11:17 -07:00
|
|
|
async fn main() -> anyhow::Result<()> {
|
2023-05-07 20:16:29 -07:00
|
|
|
dotenv::dotenv().ok();
|
|
|
|
let args: Vec<String> = env::args().collect();
|
|
|
|
assert!(args.len() == 2);
|
2023-05-17 09:11:17 -07:00
|
|
|
|
2023-05-07 20:16:29 -07:00
|
|
|
let path_to_markets_json = &args[1];
|
2023-06-07 21:03:54 -07:00
|
|
|
// let num_days = args[2].parse::<i64>().unwrap(); // TODO: implement
|
|
|
|
let num_days = 1;
|
2023-05-07 20:16:29 -07:00
|
|
|
let rpc_url: String = dotenv::var("RPC_URL").unwrap();
|
|
|
|
|
|
|
|
let config = Config {
|
|
|
|
rpc_url: rpc_url.clone(),
|
|
|
|
};
|
2023-05-19 17:16:37 -07:00
|
|
|
let markets = load_markets(path_to_markets_json);
|
2023-05-07 20:16:29 -07:00
|
|
|
let market_infos = fetch_market_infos(&config, markets.clone()).await?;
|
|
|
|
let mut target_markets = HashMap::new();
|
|
|
|
for m in market_infos.clone() {
|
2023-05-30 10:01:47 -07:00
|
|
|
target_markets.insert(Pubkey::from_str(&m.address)?, m.name);
|
2023-05-07 20:16:29 -07:00
|
|
|
}
|
|
|
|
println!("{:?}", target_markets);
|
|
|
|
|
2023-05-17 09:11:17 -07:00
|
|
|
let pool = connect_to_database().await?;
|
2023-06-07 21:03:54 -07:00
|
|
|
setup_database(&pool).await?;
|
|
|
|
|
|
|
|
let mut handles = vec![];
|
2023-05-07 20:16:29 -07:00
|
|
|
|
2023-06-07 21:03:54 -07:00
|
|
|
let rpc_clone = rpc_url.clone();
|
|
|
|
let pool_clone = pool.clone();
|
|
|
|
handles.push(tokio::spawn(async move {
|
|
|
|
fetch_signatures(rpc_clone, &pool_clone, num_days).await.unwrap();
|
|
|
|
}));
|
|
|
|
|
|
|
|
for id in 0..NUM_TRANSACTION_PARTITIONS {
|
|
|
|
let rpc_clone = rpc_url.clone();
|
|
|
|
let pool_clone = pool.clone();
|
|
|
|
let markets_clone = target_markets.clone();
|
|
|
|
handles.push(tokio::spawn(async move {
|
|
|
|
backfill(id as i32, rpc_clone, &pool_clone, &markets_clone)
|
2023-05-07 20:16:29 -07:00
|
|
|
.await
|
|
|
|
.unwrap();
|
2023-06-07 21:03:54 -07:00
|
|
|
}));
|
|
|
|
}
|
2023-05-07 20:16:29 -07:00
|
|
|
|
2023-06-07 21:03:54 -07:00
|
|
|
// TODO: spawn status thread
|
|
|
|
|
|
|
|
futures::future::join_all(handles).await;
|
2023-05-07 20:16:29 -07:00
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2023-06-07 21:03:54 -07:00
|
|
|
pub async fn fetch_signatures(rpc_url: String, pool: &Pool, num_days: i64) -> anyhow::Result<()> {
|
2023-05-07 20:16:29 -07:00
|
|
|
let mut before_sig: Option<Signature> = None;
|
|
|
|
let mut now_time = Utc::now().timestamp();
|
2023-06-07 21:03:54 -07:00
|
|
|
let end_time = (Utc::now() - Duration::days(num_days)).timestamp();
|
|
|
|
let rpc_client = RpcClient::new_with_commitment(rpc_url.clone(), CommitmentConfig::confirmed());
|
2023-05-07 20:16:29 -07:00
|
|
|
|
|
|
|
while now_time > end_time {
|
2023-06-07 21:03:54 -07:00
|
|
|
let rpc_config = GetConfirmedSignaturesForAddress2Config {
|
|
|
|
before: before_sig,
|
|
|
|
until: None,
|
|
|
|
limit: None,
|
|
|
|
commitment: Some(CommitmentConfig::confirmed()),
|
|
|
|
};
|
|
|
|
|
|
|
|
let sigs = match rpc_client
|
|
|
|
.get_signatures_for_address_with_config(
|
|
|
|
&OPENBOOK_KEY,
|
|
|
|
rpc_config,
|
|
|
|
)
|
|
|
|
.await
|
|
|
|
{
|
|
|
|
Ok(sigs) => sigs,
|
|
|
|
Err(e) => {
|
|
|
|
println!("Error fetching signatures: {}", e);
|
|
|
|
continue;
|
2023-05-17 09:11:17 -07:00
|
|
|
}
|
2023-06-07 21:03:54 -07:00
|
|
|
};
|
|
|
|
if sigs.is_empty() {
|
|
|
|
println!("No signatures found, trying again");
|
|
|
|
continue;
|
2023-05-07 20:16:29 -07:00
|
|
|
}
|
2023-06-07 21:03:54 -07:00
|
|
|
let last = sigs.last().unwrap();
|
|
|
|
let last_time = last.block_time.unwrap().clone();
|
|
|
|
let last_signature = last.signature.clone();
|
|
|
|
let transactions = sigs
|
|
|
|
.into_iter()
|
|
|
|
.map(|s| PgTransaction::from_rpc_confirmed_transaction(s))
|
|
|
|
.collect::<Vec<PgTransaction>>();
|
|
|
|
|
|
|
|
if transactions.is_empty() {
|
|
|
|
println!("No transactions found, trying again");
|
2023-05-07 20:16:29 -07:00
|
|
|
}
|
2023-06-07 21:03:54 -07:00
|
|
|
debug!("writing: {:?} txns to DB\n", transactions.len());
|
|
|
|
let upsert_statement = build_transactions_insert_statement(transactions);
|
|
|
|
let client = pool.get().await?;
|
|
|
|
client
|
|
|
|
.execute(&upsert_statement, &[])
|
|
|
|
.await
|
|
|
|
.map_err_anyhow()?;
|
|
|
|
|
|
|
|
now_time = last_time;
|
|
|
|
before_sig = Some(Signature::from_str(&last_signature)?);
|
|
|
|
let time_left = backfill_time_left(now_time, end_time);
|
|
|
|
println!(
|
|
|
|
"{} minutes ~ {} days remaining in the backfill\n",
|
|
|
|
time_left.num_minutes(),
|
|
|
|
time_left.num_days()
|
|
|
|
);
|
2023-05-17 09:11:17 -07:00
|
|
|
}
|
2023-06-07 21:03:54 -07:00
|
|
|
Ok(())
|
2023-05-17 09:11:17 -07:00
|
|
|
}
|
2023-05-07 20:16:29 -07:00
|
|
|
|
2023-06-07 21:03:54 -07:00
|
|
|
pub async fn backfill(
|
|
|
|
worker_id: i32,
|
|
|
|
rpc_url: String,
|
|
|
|
pool: &Pool,
|
2023-05-30 10:01:47 -07:00
|
|
|
target_markets: &HashMap<Pubkey, String>,
|
2023-06-07 21:03:54 -07:00
|
|
|
) -> anyhow::Result<()> {
|
|
|
|
println!("Worker {} up \n", worker_id);
|
|
|
|
let rpc_client = RpcClient::new_with_commitment(rpc_url, CommitmentConfig::confirmed());
|
|
|
|
|
|
|
|
loop {
|
|
|
|
let transactions = fetch_worker_transactions(worker_id, pool).await?;
|
|
|
|
if transactions.len() == 0 {
|
|
|
|
println!("No signatures found by worker {}", worker_id);
|
|
|
|
tokio::time::sleep(WaitDuration::from_secs(1)).await;
|
|
|
|
continue;
|
|
|
|
};
|
|
|
|
|
|
|
|
// for each signature, fetch the transaction
|
|
|
|
let txn_config = RpcTransactionConfig {
|
|
|
|
encoding: Some(UiTransactionEncoding::Json),
|
|
|
|
commitment: Some(CommitmentConfig::confirmed()),
|
|
|
|
max_supported_transaction_version: Some(0),
|
|
|
|
};
|
|
|
|
|
|
|
|
let sig_strings = transactions
|
|
|
|
.iter()
|
|
|
|
.map(|t| t.signature.clone())
|
|
|
|
.collect::<Vec<String>>();
|
|
|
|
|
|
|
|
let signatures: Vec<_> = transactions
|
|
|
|
.into_iter()
|
|
|
|
.map(|t| t.signature.parse::<Signature>().unwrap())
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
let txn_futs: Vec<_> = signatures
|
|
|
|
.iter()
|
|
|
|
.map(|s| rpc_client.get_transaction_with_config(s, txn_config))
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
let mut txns = join_all(txn_futs).await;
|
|
|
|
|
|
|
|
let fills = parse_trades_from_openbook_txns(&mut txns, target_markets);
|
|
|
|
|
|
|
|
// Write any fills to the database, and mark the transactions as processed
|
|
|
|
add_fills_atomically(pool, worker_id, fills, sig_strings).await?;
|
2023-05-07 20:16:29 -07:00
|
|
|
}
|
|
|
|
|
2023-06-07 21:03:54 -07:00
|
|
|
// TODO: graceful shutdown
|
|
|
|
// println!("Worker {} down \n", worker_id);
|
|
|
|
Ok(())
|
2023-05-07 20:16:29 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
fn backfill_time_left(current_time: i64, backfill_end: i64) -> Duration {
|
|
|
|
let naive_cur = NaiveDateTime::from_timestamp_millis(current_time * 1000).unwrap();
|
|
|
|
let naive_bf = NaiveDateTime::from_timestamp_millis(backfill_end * 1000).unwrap();
|
|
|
|
let cur_date = DateTime::<Utc>::from_utc(naive_cur, Utc);
|
|
|
|
let bf_date = DateTime::<Utc>::from_utc(naive_bf, Utc);
|
|
|
|
cur_date - bf_date
|
2023-05-17 09:11:17 -07:00
|
|
|
}
|