2023-05-14 00:15:10 -07:00
|
|
|
use deadpool_postgres::Pool;
|
2023-03-07 22:45:37 -08:00
|
|
|
use std::{
|
2023-03-12 16:32:12 -07:00
|
|
|
collections::{hash_map::DefaultHasher, HashMap},
|
2023-03-07 22:45:37 -08:00
|
|
|
hash::{Hash, Hasher},
|
|
|
|
};
|
2023-03-27 12:01:48 -07:00
|
|
|
use tokio::sync::mpsc::{error::TryRecvError, Receiver};
|
2023-03-05 22:52:42 -08:00
|
|
|
|
2023-03-13 09:51:30 -07:00
|
|
|
use crate::{
|
2023-05-19 12:50:56 -07:00
|
|
|
structs::{candle::Candle, openbook::OpenBookFillEvent},
|
|
|
|
utils::{to_timestampz, AnyhowWrap},
|
2023-03-13 09:51:30 -07:00
|
|
|
};
|
2023-03-11 12:15:36 -08:00
|
|
|
|
2023-03-12 00:13:57 -08:00
|
|
|
pub async fn persist_fill_events(
|
2023-05-14 00:15:10 -07:00
|
|
|
pool: &Pool,
|
2023-05-19 12:50:56 -07:00
|
|
|
fill_receiver: &mut Receiver<OpenBookFillEvent>,
|
2023-03-27 10:11:58 -07:00
|
|
|
) -> anyhow::Result<()> {
|
2023-05-14 00:15:10 -07:00
|
|
|
let client = pool.get().await?;
|
2023-03-05 22:52:42 -08:00
|
|
|
loop {
|
2023-03-12 16:32:12 -07:00
|
|
|
let mut write_batch = HashMap::new();
|
2023-03-13 23:21:15 -07:00
|
|
|
while write_batch.len() < 10 {
|
2023-03-07 22:45:37 -08:00
|
|
|
match fill_receiver.try_recv() {
|
2023-03-05 22:52:42 -08:00
|
|
|
Ok(event) => {
|
2023-03-12 16:32:12 -07:00
|
|
|
if !write_batch.contains_key(&event) {
|
|
|
|
write_batch.insert(event, 0);
|
2023-03-05 22:52:42 -08:00
|
|
|
}
|
|
|
|
}
|
2023-03-13 23:21:15 -07:00
|
|
|
Err(TryRecvError::Empty) => {
|
|
|
|
if write_batch.len() > 0 {
|
|
|
|
break;
|
|
|
|
} else {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
2023-03-05 22:52:42 -08:00
|
|
|
Err(TryRecvError::Disconnected) => {
|
2023-03-12 00:13:57 -08:00
|
|
|
panic!("Fills sender must stay alive")
|
2023-03-05 22:52:42 -08:00
|
|
|
}
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
if write_batch.len() > 0 {
|
2023-03-27 10:11:58 -07:00
|
|
|
// print!("writing: {:?} events to DB\n", write_batch.len());
|
|
|
|
|
2023-05-14 00:15:10 -07:00
|
|
|
// match conn.ping().await {
|
|
|
|
// Ok(_) => {
|
|
|
|
let upsert_statement = build_fills_upsert_statement(write_batch);
|
|
|
|
client
|
|
|
|
.execute(&upsert_statement, &[])
|
|
|
|
.await
|
|
|
|
.map_err_anyhow()
|
|
|
|
.unwrap();
|
|
|
|
// }
|
|
|
|
// Err(_) => {
|
|
|
|
// println!("Fills ping failed");
|
|
|
|
// break;
|
|
|
|
// }
|
|
|
|
// }
|
2023-03-05 22:52:42 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-03-27 10:11:58 -07:00
|
|
|
pub async fn persist_candles(
|
2023-05-14 00:15:10 -07:00
|
|
|
pool: Pool,
|
2023-03-27 10:11:58 -07:00
|
|
|
candles_receiver: &mut Receiver<Vec<Candle>>,
|
|
|
|
) -> anyhow::Result<()> {
|
2023-05-14 00:15:10 -07:00
|
|
|
let client = pool.get().await.unwrap();
|
2023-03-13 22:31:00 -07:00
|
|
|
loop {
|
2023-05-14 00:15:10 -07:00
|
|
|
// match client.ping().await {
|
|
|
|
// Ok(_) => {
|
|
|
|
match candles_receiver.try_recv() {
|
|
|
|
Ok(candles) => {
|
|
|
|
if candles.len() == 0 {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
// print!("writing: {:?} candles to DB\n", candles.len());
|
2023-05-19 17:15:13 -07:00
|
|
|
let upsert_statement = build_candles_upsert_statement(candles);
|
2023-05-14 00:15:10 -07:00
|
|
|
client
|
|
|
|
.execute(&upsert_statement, &[])
|
|
|
|
.await
|
|
|
|
.map_err_anyhow()
|
|
|
|
.unwrap();
|
2023-03-13 22:31:00 -07:00
|
|
|
}
|
2023-05-14 00:15:10 -07:00
|
|
|
Err(TryRecvError::Empty) => continue,
|
|
|
|
Err(TryRecvError::Disconnected) => {
|
|
|
|
panic!("Candles sender must stay alive")
|
2023-03-13 22:31:00 -07:00
|
|
|
}
|
|
|
|
};
|
2023-05-14 00:15:10 -07:00
|
|
|
// }
|
|
|
|
// Err(_) => {
|
|
|
|
// println!("Candle ping failed");
|
|
|
|
// break;
|
|
|
|
// }
|
|
|
|
// };
|
2023-03-13 22:31:00 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-05-19 12:50:56 -07:00
|
|
|
#[allow(deprecated)]
|
|
|
|
fn build_fills_upsert_statement(events: HashMap<OpenBookFillEvent, u8>) -> String {
|
2023-03-11 14:50:22 -08:00
|
|
|
let mut stmt = String::from("INSERT INTO fills (id, time, market, open_orders, open_orders_owner, bid, maker, native_qty_paid, native_qty_received, native_fee_or_rebate, fee_tier, order_id) VALUES");
|
2023-03-12 16:32:12 -07:00
|
|
|
for (idx, event) in events.keys().enumerate() {
|
2023-03-05 22:52:42 -08:00
|
|
|
let mut hasher = DefaultHasher::new();
|
|
|
|
event.hash(&mut hasher);
|
|
|
|
let val_str = format!(
|
2023-03-11 14:50:22 -08:00
|
|
|
"({}, \'{}\', \'{}\', \'{}\', \'{}\', {}, {}, {}, {}, {}, {}, {})",
|
2023-03-05 22:52:42 -08:00
|
|
|
hasher.finish(),
|
2023-05-19 12:50:56 -07:00
|
|
|
to_timestampz(event.block_time as u64).to_rfc3339(),
|
2023-03-05 22:52:42 -08:00
|
|
|
event.market,
|
|
|
|
event.open_orders,
|
|
|
|
event.open_orders_owner,
|
|
|
|
event.bid,
|
|
|
|
event.maker,
|
|
|
|
event.native_qty_paid,
|
|
|
|
event.native_qty_received,
|
|
|
|
event.native_fee_or_rebate,
|
|
|
|
event.fee_tier,
|
|
|
|
event.order_id,
|
|
|
|
);
|
|
|
|
|
|
|
|
if idx == 0 {
|
|
|
|
stmt = format!("{} {}", &stmt, val_str);
|
|
|
|
} else {
|
|
|
|
stmt = format!("{}, {}", &stmt, val_str);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
let handle_conflict = "ON CONFLICT (id) DO UPDATE SET market=excluded.market";
|
|
|
|
|
|
|
|
stmt = format!("{} {}", stmt, handle_conflict);
|
|
|
|
stmt
|
|
|
|
}
|
|
|
|
|
2023-05-19 17:15:13 -07:00
|
|
|
pub fn build_candles_upsert_statement(candles: Vec<Candle>) -> String {
|
2023-03-12 18:03:39 -07:00
|
|
|
let mut stmt = String::from("INSERT INTO candles (market_name, start_time, end_time, resolution, open, close, high, low, volume, complete) VALUES");
|
2023-03-12 00:13:57 -08:00
|
|
|
for (idx, candle) in candles.iter().enumerate() {
|
|
|
|
let val_str = format!(
|
|
|
|
"(\'{}\', \'{}\', \'{}\', \'{}\', {}, {}, {}, {}, {}, {})",
|
2023-03-12 18:03:39 -07:00
|
|
|
candle.market_name,
|
2023-03-12 00:13:57 -08:00
|
|
|
candle.start_time.to_rfc3339(),
|
|
|
|
candle.end_time.to_rfc3339(),
|
|
|
|
candle.resolution,
|
|
|
|
candle.open,
|
|
|
|
candle.close,
|
|
|
|
candle.high,
|
|
|
|
candle.low,
|
|
|
|
candle.volume,
|
|
|
|
candle.complete,
|
|
|
|
);
|
|
|
|
|
|
|
|
if idx == 0 {
|
|
|
|
stmt = format!("{} {}", &stmt, val_str);
|
|
|
|
} else {
|
|
|
|
stmt = format!("{}, {}", &stmt, val_str);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-03-12 18:03:39 -07:00
|
|
|
let handle_conflict = "ON CONFLICT (market_name, start_time, resolution)
|
2023-03-12 00:13:57 -08:00
|
|
|
DO UPDATE SET
|
|
|
|
open=excluded.open,
|
|
|
|
close=excluded.close,
|
|
|
|
high=excluded.high,
|
|
|
|
low=excluded.low,
|
|
|
|
volume=excluded.volume,
|
|
|
|
complete=excluded.complete
|
|
|
|
";
|
|
|
|
|
|
|
|
stmt = format!("{} {}", stmt, handle_conflict);
|
|
|
|
stmt
|
|
|
|
}
|
|
|
|
|
2023-03-05 22:52:42 -08:00
|
|
|
#[cfg(test)]
|
|
|
|
mod tests {
|
|
|
|
use super::*;
|
2023-03-07 22:45:37 -08:00
|
|
|
use solana_sdk::pubkey::Pubkey;
|
|
|
|
use std::str::FromStr;
|
2023-03-05 22:52:42 -08:00
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_event_hashing() {
|
2023-05-19 12:50:56 -07:00
|
|
|
let event_1 = OpenBookFillEvent {
|
2023-03-05 22:52:42 -08:00
|
|
|
market: Pubkey::from_str("8BnEgHoWFysVcuFFX7QztDmzuH8r5ZFvyP3sYwn1XTh6").unwrap(),
|
|
|
|
open_orders: Pubkey::from_str("CKo9nGfgekYYfjHw4K22qMAtVeqBXET3pSGm8k5DSJi7").unwrap(),
|
|
|
|
open_orders_owner: Pubkey::from_str("JCNCMFXo5M5qwUPg2Utu1u6YWp3MbygxqBsBeXXJfrw")
|
|
|
|
.unwrap(),
|
|
|
|
bid: false,
|
|
|
|
maker: false,
|
|
|
|
native_qty_paid: 200000000,
|
|
|
|
native_qty_received: 4204317,
|
|
|
|
native_fee_or_rebate: 1683,
|
|
|
|
order_id: 387898134381964481824213,
|
|
|
|
owner_slot: 0,
|
|
|
|
fee_tier: 0,
|
|
|
|
client_order_id: None,
|
|
|
|
referrer_rebate: Some(841),
|
2023-05-19 12:50:56 -07:00
|
|
|
block_time: 0,
|
2023-03-05 22:52:42 -08:00
|
|
|
};
|
|
|
|
|
2023-05-19 12:50:56 -07:00
|
|
|
let event_2 = OpenBookFillEvent {
|
2023-03-05 22:52:42 -08:00
|
|
|
market: Pubkey::from_str("8BnEgHoWFysVcuFFX7QztDmzuH8r5ZFvyP3sYwn1XTh6").unwrap(),
|
|
|
|
open_orders: Pubkey::from_str("CKo9nGfgekYYfjHw4K22qMAtVeqBXET3pSGm8k5DSJi7").unwrap(),
|
|
|
|
open_orders_owner: Pubkey::from_str("JCNCMFXo5M5qwUPg2Utu1u6YWp3MbygxqBsBeXXJfrw")
|
|
|
|
.unwrap(),
|
|
|
|
bid: false,
|
|
|
|
maker: false,
|
|
|
|
native_qty_paid: 200000001,
|
|
|
|
native_qty_received: 4204317,
|
|
|
|
native_fee_or_rebate: 1683,
|
|
|
|
order_id: 387898134381964481824213,
|
|
|
|
owner_slot: 0,
|
|
|
|
fee_tier: 0,
|
|
|
|
client_order_id: None,
|
|
|
|
referrer_rebate: Some(841),
|
2023-05-19 12:50:56 -07:00
|
|
|
block_time: 0,
|
2023-03-05 22:52:42 -08:00
|
|
|
};
|
|
|
|
|
|
|
|
let mut h1 = DefaultHasher::new();
|
|
|
|
event_1.hash(&mut h1);
|
|
|
|
|
|
|
|
let mut h2 = DefaultHasher::new();
|
|
|
|
event_2.hash(&mut h2);
|
|
|
|
|
|
|
|
assert_ne!(h1.finish(), h2.finish());
|
|
|
|
}
|
|
|
|
}
|