feat: can batch 1m candles
This commit is contained in:
dboures
parent
16789aa0b0
commit
9216cc17d2
|
|
@ -254,7 +254,7 @@ dependencies = [
|
|||
"arrayref",
|
||||
"base64 0.13.1",
|
||||
"bincode",
|
||||
"borsh",
|
||||
"borsh 0.9.3",
|
||||
"bytemuck",
|
||||
"solana-program",
|
||||
"thiserror",
|
||||
|
|
@ -662,7 +662,17 @@ version = "0.9.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "15bf3650200d8bffa99015595e10f1fbd17de07abbc25bb067da79e769939bfa"
|
||||
dependencies = [
|
||||
"borsh-derive",
|
||||
"borsh-derive 0.9.3",
|
||||
"hashbrown 0.11.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "borsh"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40f9ca3698b2e4cb7c15571db0abc5551dca417a21ae8140460b50309bb2cc62"
|
||||
dependencies = [
|
||||
"borsh-derive 0.10.2",
|
||||
"hashbrown 0.11.2",
|
||||
]
|
||||
|
||||
|
|
@ -672,8 +682,21 @@ version = "0.9.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6441c552f230375d18e3cc377677914d2ca2b0d36e52129fe15450a2dce46775"
|
||||
dependencies = [
|
||||
"borsh-derive-internal",
|
||||
"borsh-schema-derive-internal",
|
||||
"borsh-derive-internal 0.9.3",
|
||||
"borsh-schema-derive-internal 0.9.3",
|
||||
"proc-macro-crate 0.1.5",
|
||||
"proc-macro2 1.0.50",
|
||||
"syn 1.0.107",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "borsh-derive"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "598b3eacc6db9c3ee57b22707ad8f6a8d2f6d442bfe24ffeb8cbb70ca59e6a35"
|
||||
dependencies = [
|
||||
"borsh-derive-internal 0.10.2",
|
||||
"borsh-schema-derive-internal 0.10.2",
|
||||
"proc-macro-crate 0.1.5",
|
||||
"proc-macro2 1.0.50",
|
||||
"syn 1.0.107",
|
||||
|
|
@ -690,6 +713,17 @@ dependencies = [
|
|||
"syn 1.0.107",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "borsh-derive-internal"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "186b734fa1c9f6743e90c95d7233c9faab6360d1a96d4ffa19d9cfd1e9350f8a"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.50",
|
||||
"quote 1.0.23",
|
||||
"syn 1.0.107",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "borsh-schema-derive-internal"
|
||||
version = "0.9.3"
|
||||
|
|
@ -701,6 +735,17 @@ dependencies = [
|
|||
"syn 1.0.107",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "borsh-schema-derive-internal"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "99b7ff1008316626f485991b960ade129253d4034014616b94f309a15366cc49"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.50",
|
||||
"quote 1.0.23",
|
||||
"syn 1.0.107",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "brotli"
|
||||
version = "3.3.4"
|
||||
|
|
@ -760,6 +805,28 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bytecheck"
|
||||
version = "0.6.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13fe11640a23eb24562225322cd3e452b93a3d4091d62fab69c70542fcd17d1f"
|
||||
dependencies = [
|
||||
"bytecheck_derive",
|
||||
"ptr_meta",
|
||||
"simdutf8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bytecheck_derive"
|
||||
version = "0.6.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e31225543cb46f81a7e224762764f4a6a0f097b1db0b175f69e8065efaa42de5"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.50",
|
||||
"quote 1.0.23",
|
||||
"syn 1.0.107",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bytemuck"
|
||||
version = "1.12.3"
|
||||
|
|
@ -3122,11 +3189,12 @@ dependencies = [
|
|||
"anchor-lang",
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"borsh",
|
||||
"borsh 0.9.3",
|
||||
"chrono",
|
||||
"dotenv",
|
||||
"jsonrpc-core-client",
|
||||
"log 0.4.17",
|
||||
"num-traits",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
|
|
@ -3138,6 +3206,7 @@ dependencies = [
|
|||
"solana-transaction-status",
|
||||
"spl-token",
|
||||
"sqlx",
|
||||
"strum",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
]
|
||||
|
|
@ -3602,6 +3671,26 @@ dependencies = [
|
|||
"autotools",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ptr_meta"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1"
|
||||
dependencies = [
|
||||
"ptr_meta_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ptr_meta_derive"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.50",
|
||||
"quote 1.0.23",
|
||||
"syn 1.0.107",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "qstring"
|
||||
version = "0.7.2"
|
||||
|
|
@ -3978,6 +4067,15 @@ dependencies = [
|
|||
"winapi 0.3.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rend"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "581008d2099240d37fb08d77ad713bcaec2c4d89d50b5b21a8bb1996bbab68ab"
|
||||
dependencies = [
|
||||
"bytecheck",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.11.14"
|
||||
|
|
@ -4037,6 +4135,31 @@ dependencies = [
|
|||
"winapi 0.3.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rkyv"
|
||||
version = "0.7.40"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c30f1d45d9aa61cbc8cd1eb87705470892289bb2d01943e7803b873a57404dc3"
|
||||
dependencies = [
|
||||
"bytecheck",
|
||||
"hashbrown 0.12.3",
|
||||
"ptr_meta",
|
||||
"rend",
|
||||
"rkyv_derive",
|
||||
"seahash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rkyv_derive"
|
||||
version = "0.7.40"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff26ed6c7c4dfc2aa9480b86a60e3c7233543a270a680e10758a507c5a4ce476"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.50",
|
||||
"quote 1.0.23",
|
||||
"syn 1.0.107",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rocksdb"
|
||||
version = "0.19.0"
|
||||
|
|
@ -4059,6 +4182,24 @@ dependencies = [
|
|||
"winapi 0.3.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust_decimal"
|
||||
version = "1.28.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e13cf35f7140155d02ba4ec3294373d513a3c7baa8364c162b030e33c61520a8"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"borsh 0.10.2",
|
||||
"bytecheck",
|
||||
"byteorder",
|
||||
"bytes 1.3.0",
|
||||
"num-traits",
|
||||
"rand 0.8.5",
|
||||
"rkyv",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.21"
|
||||
|
|
@ -4233,6 +4374,12 @@ dependencies = [
|
|||
"untrusted",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "seahash"
|
||||
version = "4.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
|
||||
|
||||
[[package]]
|
||||
name = "security-framework"
|
||||
version = "2.8.0"
|
||||
|
|
@ -4508,6 +4655,12 @@ version = "1.6.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c"
|
||||
|
||||
[[package]]
|
||||
name = "simdutf8"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a"
|
||||
|
||||
[[package]]
|
||||
name = "simpl"
|
||||
version = "0.1.0"
|
||||
|
|
@ -5122,8 +5275,8 @@ dependencies = [
|
|||
"bincode",
|
||||
"bitflags",
|
||||
"blake3",
|
||||
"borsh",
|
||||
"borsh-derive",
|
||||
"borsh 0.9.3",
|
||||
"borsh-derive 0.9.3",
|
||||
"bs58 0.4.0",
|
||||
"bv",
|
||||
"bytemuck",
|
||||
|
|
@ -5341,7 +5494,7 @@ dependencies = [
|
|||
"base64 0.13.1",
|
||||
"bincode",
|
||||
"bitflags",
|
||||
"borsh",
|
||||
"borsh 0.9.3",
|
||||
"bs58 0.4.0",
|
||||
"bytemuck",
|
||||
"byteorder",
|
||||
|
|
@ -5545,7 +5698,7 @@ dependencies = [
|
|||
"Inflector",
|
||||
"base64 0.13.1",
|
||||
"bincode",
|
||||
"borsh",
|
||||
"borsh 0.9.3",
|
||||
"bs58 0.4.0",
|
||||
"lazy_static",
|
||||
"log 0.4.17",
|
||||
|
|
@ -5695,7 +5848,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "fbc000f0fdf1f12f99d77d398137c1751345b18c88258ce0f99b7872cf6c9bd6"
|
||||
dependencies = [
|
||||
"assert_matches",
|
||||
"borsh",
|
||||
"borsh 0.9.3",
|
||||
"num-derive",
|
||||
"num-traits",
|
||||
"solana-program",
|
||||
|
|
@ -5779,6 +5932,7 @@ dependencies = [
|
|||
"bitflags",
|
||||
"byteorder",
|
||||
"bytes 1.3.0",
|
||||
"chrono",
|
||||
"crc",
|
||||
"crossbeam-queue",
|
||||
"dirs",
|
||||
|
|
@ -5799,10 +5953,12 @@ dependencies = [
|
|||
"log 0.4.17",
|
||||
"md-5",
|
||||
"memchr",
|
||||
"num-bigint 0.4.3",
|
||||
"once_cell",
|
||||
"paste",
|
||||
"percent-encoding 2.2.0",
|
||||
"rand 0.8.5",
|
||||
"rust_decimal",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha1 0.10.5",
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ tokio-stream = "0.1"
|
|||
|
||||
jsonrpc-core-client = { version = "18.0.0", features = ["ws", "http"] }
|
||||
|
||||
sqlx = { version = "0.6", features = [ "runtime-tokio-native-tls" , "postgres" ] }
|
||||
sqlx = { version = "0.6", features = [ "runtime-tokio-native-tls" , "postgres", "chrono", "decimal" ] }
|
||||
chrono = "0.4.23"
|
||||
|
||||
solana-client = "=1.14.13"
|
||||
|
|
@ -31,6 +31,8 @@ dotenv = "0.15.0"
|
|||
serde = "1.0"
|
||||
serde_json = "1.0"
|
||||
serde_derive = "1.0"
|
||||
strum = { version = "0.24", features = ["derive"] }
|
||||
num-traits = "0.2"
|
||||
|
||||
serum_dex = { version = "0.5.10", git = "https://github.com/openbook-dex/program.git", default-features=false, features = ["no-entrypoint", "program"] }
|
||||
anchor-lang = ">=0.25.0"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,178 @@
|
|||
use std::cmp::{max, min};
|
||||
|
||||
use chrono::{DateTime, Duration, DurationRound, SubsecRound, Utc};
|
||||
use num_traits::{FromPrimitive, Zero};
|
||||
use sqlx::{types::Decimal, Pool, Postgres};
|
||||
|
||||
use crate::database::{
|
||||
fetchers::{fetch_earliest_fill, fetch_fills_from, fetch_latest_finished_candle},
|
||||
Candle, MarketInfo, PgOpenBookFill, Resolution,
|
||||
};
|
||||
|
||||
pub async fn batch_candles(pool: &Pool<Postgres>, markets: Vec<MarketInfo>) {
|
||||
let m = MarketInfo {
|
||||
name: "BTC/USDC".to_owned(),
|
||||
address: "A8YFbxQYFVqKZaoYJLLUVcQiWP7G2MeEgW5wsAQgMvFw".to_owned(),
|
||||
base_decimals: 6,
|
||||
quote_decimals: 6,
|
||||
base_mint_key: "GVXRSBjFk6e6J3NbVPXohDJetcTjaeeuykUpbQF8UoMU".to_owned(),
|
||||
quote_mint_key: "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v".to_owned(),
|
||||
base_lot_size: 10_000,
|
||||
quote_lot_size: 1,
|
||||
};
|
||||
batch_for_market(&pool.clone(), m).await.unwrap();
|
||||
|
||||
// for market in markets.iter() {
|
||||
// tokio::spawn(async move {
|
||||
// loop {
|
||||
// batch_for_market(&pool.clone(), &market.address.clone()).await;
|
||||
// }
|
||||
// });
|
||||
// }
|
||||
}
|
||||
|
||||
async fn batch_for_market(pool: &Pool<Postgres>, market: MarketInfo) -> anyhow::Result<()> {
|
||||
let candles = batch_1m_candles(pool, market).await?;
|
||||
// println!("candles {:?}", candles[10]);
|
||||
// database::insert_candles(pool, candles)
|
||||
|
||||
// for resolution in Resolution.iter
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn batch_1m_candles(
|
||||
pool: &Pool<Postgres>,
|
||||
market: MarketInfo,
|
||||
) -> anyhow::Result<Vec<Candle>> {
|
||||
let market_address_string = &market.address;
|
||||
let latest_candle =
|
||||
fetch_latest_finished_candle(pool, market_address_string, Resolution::R1m).await?;
|
||||
|
||||
match latest_candle {
|
||||
Some(candle) => {
|
||||
let start_time = candle.end_time;
|
||||
let end_time = min(
|
||||
start_time + Duration::hours(6),
|
||||
Utc::now().duration_trunc(Duration::minutes(1))?,
|
||||
);
|
||||
let mut fills =
|
||||
fetch_fills_from(pool, market_address_string, start_time, end_time).await?;
|
||||
let candles = combine_fills_into_1m_candles(
|
||||
&mut fills,
|
||||
market,
|
||||
start_time,
|
||||
end_time,
|
||||
Some(candle.close),
|
||||
);
|
||||
Ok(candles)
|
||||
}
|
||||
None => {
|
||||
let earliest_fill = fetch_earliest_fill(pool, market_address_string).await?;
|
||||
|
||||
if earliest_fill.is_none() {
|
||||
println!("No fills found for: {:?}", market_address_string);
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let start_time = earliest_fill
|
||||
.unwrap()
|
||||
.time
|
||||
.duration_trunc(Duration::minutes(1))?;
|
||||
let end_time = min(
|
||||
start_time + Duration::hours(6),
|
||||
Utc::now().duration_trunc(Duration::minutes(1))?,
|
||||
);
|
||||
let mut fills =
|
||||
fetch_fills_from(pool, market_address_string, start_time, end_time).await?;
|
||||
let candles =
|
||||
combine_fills_into_1m_candles(&mut fills, market, start_time, end_time, None);
|
||||
Ok(candles)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn combine_fills_into_1m_candles(
|
||||
fills: &mut Vec<PgOpenBookFill>,
|
||||
market: MarketInfo,
|
||||
st: DateTime<Utc>,
|
||||
et: DateTime<Utc>,
|
||||
maybe_last_price: Option<Decimal>,
|
||||
) -> Vec<Candle> {
|
||||
let empty_candle = Candle::create_empty_candle(market.address, Resolution::R1m);
|
||||
|
||||
let minutes = (et - st).num_minutes();
|
||||
let mut candles = vec![empty_candle; minutes as usize];
|
||||
|
||||
let mut fills_iter = fills.iter_mut().peekable();
|
||||
let mut start_time = st.clone();
|
||||
let mut end_time = start_time + Duration::minutes(1);
|
||||
|
||||
let mut last_price = maybe_last_price.unwrap_or(Decimal::zero());
|
||||
|
||||
for i in 0..candles.len() {
|
||||
candles[i].open = last_price;
|
||||
candles[i].close = last_price;
|
||||
candles[i].low = last_price;
|
||||
candles[i].high = last_price;
|
||||
|
||||
while matches!(fills_iter.peek(), Some(f) if f.time < end_time) {
|
||||
let fill = fills_iter.next().unwrap();
|
||||
|
||||
let (price, volume) =
|
||||
calculate_fill_price_and_size(*fill, market.base_decimals, market.quote_decimals);
|
||||
|
||||
println!("{:?}", price);
|
||||
|
||||
candles[i].close = price;
|
||||
candles[i].low = min(price, candles[i].low);
|
||||
candles[i].high = max(price, candles[i].high);
|
||||
candles[i].volume += volume;
|
||||
|
||||
last_price = price;
|
||||
}
|
||||
|
||||
candles[i].start_time = start_time;
|
||||
candles[i].end_time = end_time;
|
||||
candles[i].complete = matches!(fills_iter.peek(), Some(f) if f.time > end_time);
|
||||
|
||||
println!("{:?}", candles[i]);
|
||||
|
||||
start_time = end_time;
|
||||
end_time = end_time + Duration::minutes(1);
|
||||
}
|
||||
|
||||
candles
|
||||
}
|
||||
|
||||
fn calculate_fill_price_and_size(
|
||||
fill: PgOpenBookFill,
|
||||
base_decimals: u8,
|
||||
quote_decimals: u8,
|
||||
) -> (Decimal, Decimal) {
|
||||
if fill.bid {
|
||||
let price_before_fees = if fill.maker {
|
||||
fill.native_qty_paid + fill.native_fee_or_rebate
|
||||
} else {
|
||||
fill.native_qty_paid - fill.native_fee_or_rebate
|
||||
};
|
||||
let price = (price_before_fees * token_factor(base_decimals))
|
||||
/ (token_factor(quote_decimals) * fill.native_qty_received);
|
||||
let size = fill.native_qty_received / token_factor(base_decimals);
|
||||
(price, size)
|
||||
} else {
|
||||
let price_before_fees = if fill.maker {
|
||||
fill.native_qty_received - fill.native_fee_or_rebate
|
||||
} else {
|
||||
fill.native_qty_received + fill.native_fee_or_rebate
|
||||
};
|
||||
let price = (price_before_fees * token_factor(base_decimals))
|
||||
/ (token_factor(quote_decimals) * fill.native_qty_paid);
|
||||
let size = fill.native_qty_paid / token_factor(base_decimals);
|
||||
(price, size)
|
||||
}
|
||||
}
|
||||
|
||||
fn token_factor(decimals: u8) -> Decimal {
|
||||
Decimal::from_u64(10u64.pow(decimals as u32)).unwrap()
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
pub mod batcher;
|
||||
|
|
@ -89,9 +89,7 @@ pub async fn create_fills_table(pool: &Pool<Postgres>) -> anyhow::Result<()> {
|
|||
native_qty_received numeric not null,
|
||||
native_fee_or_rebate numeric not null,
|
||||
fee_tier text not null,
|
||||
order_id text not null,
|
||||
client_order_id numeric not null,
|
||||
referrer_rebate numeric not null
|
||||
order_id text not null
|
||||
)",
|
||||
)
|
||||
.execute(&mut tx)
|
||||
|
|
@ -147,12 +145,12 @@ pub async fn handle_fill_events(
|
|||
}
|
||||
|
||||
fn build_fills_upsert_statement(events: Vec<OpenBookFillEventLog>) -> String {
|
||||
let mut stmt = String::from("INSERT INTO fills (id, time, market, open_orders, open_orders_owner, bid, maker, native_qty_paid, native_qty_received, native_fee_or_rebate, fee_tier, order_id, client_order_id, referrer_rebate) VALUES");
|
||||
let mut stmt = String::from("INSERT INTO fills (id, time, market, open_orders, open_orders_owner, bid, maker, native_qty_paid, native_qty_received, native_fee_or_rebate, fee_tier, order_id) VALUES");
|
||||
for (idx, event) in events.iter().enumerate() {
|
||||
let mut hasher = DefaultHasher::new();
|
||||
event.hash(&mut hasher);
|
||||
let val_str = format!(
|
||||
"({}, \'{}\', \'{}\', \'{}\', \'{}\', {}, {}, {}, {}, {}, {}, {}, {}, {})",
|
||||
"({}, \'{}\', \'{}\', \'{}\', \'{}\', {}, {}, {}, {}, {}, {}, {})",
|
||||
hasher.finish(),
|
||||
Utc::now().to_rfc3339(),
|
||||
event.market,
|
||||
|
|
@ -165,8 +163,6 @@ fn build_fills_upsert_statement(events: Vec<OpenBookFillEventLog>) -> String {
|
|||
event.native_fee_or_rebate,
|
||||
event.fee_tier,
|
||||
event.order_id,
|
||||
event.client_order_id.unwrap_or_else(|| 0),
|
||||
event.referrer_rebate.unwrap_or_else(|| 0),
|
||||
);
|
||||
|
||||
if idx == 0 {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,93 @@
|
|||
use chrono::{DateTime, Utc};
|
||||
use sqlx::{Pool, Postgres};
|
||||
|
||||
use crate::{database::PgOpenBookFill, utils::AnyhowWrap};
|
||||
|
||||
use super::{Candle, Resolution};
|
||||
|
||||
// use super::PgMarketInfo;
|
||||
|
||||
pub async fn fetch_earliest_fill(
|
||||
pool: &Pool<Postgres>,
|
||||
market_address_string: &str,
|
||||
) -> anyhow::Result<Option<PgOpenBookFill>> {
|
||||
sqlx::query_as!(
|
||||
PgOpenBookFill,
|
||||
r#"SELECT
|
||||
time as "time!",
|
||||
bid as "bid!",
|
||||
maker as "maker!",
|
||||
native_qty_paid as "native_qty_paid!",
|
||||
native_qty_received as "native_qty_received!",
|
||||
native_fee_or_rebate as "native_fee_or_rebate!"
|
||||
from fills
|
||||
where market = $1
|
||||
ORDER BY time asc LIMIT 1"#,
|
||||
market_address_string
|
||||
)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.map_err_anyhow()
|
||||
}
|
||||
|
||||
pub async fn fetch_fills_from(
|
||||
pool: &Pool<Postgres>,
|
||||
market_address_string: &str,
|
||||
start_time: DateTime<Utc>,
|
||||
end_time: DateTime<Utc>,
|
||||
) -> anyhow::Result<Vec<PgOpenBookFill>> {
|
||||
sqlx::query_as!(
|
||||
PgOpenBookFill,
|
||||
r#"SELECT
|
||||
time as "time!",
|
||||
bid as "bid!",
|
||||
maker as "maker!",
|
||||
native_qty_paid as "native_qty_paid!",
|
||||
native_qty_received as "native_qty_received!",
|
||||
native_fee_or_rebate as "native_fee_or_rebate!"
|
||||
from fills
|
||||
where market = $1
|
||||
and time >= $2
|
||||
and time < $3
|
||||
ORDER BY time asc"#,
|
||||
market_address_string,
|
||||
start_time,
|
||||
end_time
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err_anyhow()
|
||||
}
|
||||
|
||||
pub async fn fetch_latest_finished_candle(
|
||||
pool: &Pool<Postgres>,
|
||||
market_address_string: &str,
|
||||
resolution: Resolution,
|
||||
) -> anyhow::Result<Option<Candle>> {
|
||||
sqlx::query_as!(
|
||||
Candle,
|
||||
r#"SELECT
|
||||
start_time as "start_time!",
|
||||
end_time as "end_time!",
|
||||
resolution as "resolution!",
|
||||
market as "market!",
|
||||
open as "open!",
|
||||
close as "close!",
|
||||
high as "high!",
|
||||
low as "low!",
|
||||
volume as "volume!",
|
||||
complete as "complete!"
|
||||
from candles
|
||||
where market = $1
|
||||
and resolution = $2
|
||||
and complete = true
|
||||
ORDER BY start_time desc LIMIT 1"#,
|
||||
market_address_string,
|
||||
resolution.to_string()
|
||||
)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.map_err_anyhow()
|
||||
}
|
||||
|
||||
// fetch_candles
|
||||
|
|
@ -1,15 +1,129 @@
|
|||
use chrono::{DateTime, Utc};
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
use std::fmt;
|
||||
|
||||
use chrono::{DateTime, NaiveDateTime, Utc};
|
||||
use num_traits::Zero;
|
||||
use sqlx::types::Decimal;
|
||||
use strum::EnumIter;
|
||||
|
||||
pub mod database;
|
||||
pub mod fetchers;
|
||||
|
||||
pub struct Candle {}
|
||||
pub trait Summary {
|
||||
fn summarize(&self) -> String;
|
||||
}
|
||||
|
||||
#[derive(EnumIter)]
|
||||
pub enum Resolution {
|
||||
R1m,
|
||||
R3m,
|
||||
R5m,
|
||||
R15m,
|
||||
R30m,
|
||||
R1h,
|
||||
R2h,
|
||||
R4h,
|
||||
R1d,
|
||||
R1w,
|
||||
}
|
||||
|
||||
impl fmt::Display for Resolution {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
match self {
|
||||
Resolution::R1m => write!(f, "1M"),
|
||||
Resolution::R3m => write!(f, "3M"),
|
||||
Resolution::R5m => write!(f, "5M"),
|
||||
Resolution::R15m => write!(f, "15M"),
|
||||
Resolution::R30m => write!(f, "30M"),
|
||||
Resolution::R1h => write!(f, "1H"),
|
||||
Resolution::R2h => write!(f, "2H"),
|
||||
Resolution::R4h => write!(f, "4H"),
|
||||
Resolution::R1d => write!(f, "1D"),
|
||||
Resolution::R1w => write!(f, "1W"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Resolution {
|
||||
pub fn get_constituent_resolution(self) -> Resolution {
|
||||
match self {
|
||||
Resolution::R1m => panic!("have to use fills to make 1M candles"),
|
||||
Resolution::R3m => Resolution::R1m,
|
||||
Resolution::R5m => Resolution::R1m,
|
||||
Resolution::R15m => Resolution::R5m,
|
||||
Resolution::R30m => Resolution::R15m,
|
||||
Resolution::R1h => Resolution::R30m,
|
||||
Resolution::R2h => Resolution::R1h,
|
||||
Resolution::R4h => Resolution::R2h,
|
||||
Resolution::R1d => Resolution::R4h,
|
||||
Resolution::R1w => Resolution::R1d,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_constituent_resolution_factor(self) -> u8 {
|
||||
match self {
|
||||
Resolution::R1m => panic!("have to use fills to make 1M candles"),
|
||||
Resolution::R3m => 3,
|
||||
Resolution::R5m => 5,
|
||||
Resolution::R15m => 3,
|
||||
Resolution::R30m => 2,
|
||||
Resolution::R1h => 2,
|
||||
Resolution::R2h => 2,
|
||||
Resolution::R4h => 2,
|
||||
Resolution::R1d => 6,
|
||||
Resolution::R1w => 7,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Candle {
|
||||
pub market: String,
|
||||
pub start_time: DateTime<Utc>,
|
||||
pub end_time: DateTime<Utc>,
|
||||
pub resolution: String,
|
||||
pub open: Decimal,
|
||||
pub close: Decimal,
|
||||
pub high: Decimal,
|
||||
pub low: Decimal,
|
||||
pub volume: Decimal,
|
||||
pub complete: bool,
|
||||
}
|
||||
|
||||
impl Candle {
|
||||
pub fn create_empty_candle(market: String, resolution: Resolution) -> Candle {
|
||||
Candle {
|
||||
market,
|
||||
start_time: DateTime::from_utc(NaiveDateTime::MIN, Utc),
|
||||
end_time: DateTime::from_utc(NaiveDateTime::MIN, Utc),
|
||||
resolution: resolution.to_string(),
|
||||
open: Decimal::zero(),
|
||||
close: Decimal::zero(),
|
||||
high: Decimal::zero(),
|
||||
low: Decimal::zero(),
|
||||
volume: Decimal::zero(),
|
||||
complete: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, PartialEq)]
|
||||
pub struct PgOpenBookFill {
|
||||
pub time: DateTime<Utc>,
|
||||
pub bid: bool,
|
||||
pub maker: bool,
|
||||
pub native_qty_paid: Decimal,
|
||||
pub native_qty_received: Decimal,
|
||||
pub native_fee_or_rebate: Decimal,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct MarketInfo {
|
||||
pub market_key: Pubkey,
|
||||
pub market_name: String,
|
||||
pub base_symbol: String,
|
||||
pub quote_symbol: String,
|
||||
pub name: String,
|
||||
pub address: String,
|
||||
pub base_decimals: u8,
|
||||
pub quote_decimals: u8,
|
||||
pub base_mint_key: String,
|
||||
pub quote_mint_key: String,
|
||||
pub base_lot_size: u64,
|
||||
pub quote_lot_size: u64,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,17 @@
|
|||
use crate::{trade_fetching::{parsing::OpenBookFillEventLog, scrape::fetch_market_infos}, utils::Config};
|
||||
use database::database::{connect_to_database, setup_database};
|
||||
use crate::{
|
||||
database::{fetchers::fetch_latest_finished_candle, Resolution},
|
||||
trade_fetching::{parsing::OpenBookFillEventLog, scrape::fetch_market_infos},
|
||||
utils::Config,
|
||||
};
|
||||
use database::{
|
||||
database::{connect_to_database, setup_database},
|
||||
fetchers::fetch_earliest_fill,
|
||||
};
|
||||
use dotenv;
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
mod candle_batching;
|
||||
mod database;
|
||||
mod trade_fetching;
|
||||
mod utils;
|
||||
|
|
@ -14,7 +23,6 @@ async fn main() -> anyhow::Result<()> {
|
|||
let rpc_url: String = dotenv::var("RPC_URL").unwrap();
|
||||
let database_url: String = dotenv::var("DATABASE_URL").unwrap();
|
||||
|
||||
|
||||
let config = Config {
|
||||
rpc_url,
|
||||
database_url,
|
||||
|
|
@ -23,26 +31,23 @@ async fn main() -> anyhow::Result<()> {
|
|||
|
||||
let markets = utils::load_markets("/Users/dboures/dev/openbook-candles/markets.json");
|
||||
let market_infos = fetch_market_infos(&config, markets).await?;
|
||||
println!("{:?}", market_infos);
|
||||
|
||||
let pool = connect_to_database(&config).await?;
|
||||
setup_database(&pool, market_infos).await?;
|
||||
// setup_database(&pool, market_infos).await?;
|
||||
|
||||
let (fill_sender, fill_receiver) = mpsc::channel::<OpenBookFillEventLog>(1000);
|
||||
|
||||
// spawn a thread for each market?
|
||||
// what are the memory implications?
|
||||
// let (fill_sender, fill_receiver) = mpsc::channel::<OpenBookFillEventLog>(1000);
|
||||
|
||||
// tokio::spawn(async move {
|
||||
// trade_fetching::scrape::scrape(&config, fill_event_sender.clone()).await;
|
||||
// trade_fetching::scrape::scrape(&config, fill_sender.clone()).await;
|
||||
// });
|
||||
|
||||
tokio::spawn(async move {
|
||||
trade_fetching::scrape::scrape(&config, fill_sender.clone()).await;
|
||||
});
|
||||
|
||||
database::database::handle_fill_events(&pool, fill_receiver).await;
|
||||
// database::database::handle_fill_events(&pool, fill_receiver).await;
|
||||
|
||||
// trade_fetching::websocket::listen_logs().await?;
|
||||
|
||||
candle_batching::batcher::batch_candles(&pool, market_infos).await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,15 +3,21 @@ use solana_account_decoder::UiAccountEncoding;
|
|||
use solana_client::{
|
||||
client_error::Result as ClientResult,
|
||||
rpc_client::{GetConfirmedSignaturesForAddress2Config, RpcClient},
|
||||
rpc_config::{RpcTransactionConfig, RpcAccountInfoConfig},
|
||||
rpc_config::{RpcAccountInfoConfig, RpcTransactionConfig},
|
||||
};
|
||||
use solana_sdk::{
|
||||
commitment_config::CommitmentConfig, program_pack::Pack, pubkey::Pubkey, signature::Signature,
|
||||
};
|
||||
use solana_sdk::{commitment_config::CommitmentConfig, pubkey::Pubkey, signature::Signature, program_pack::Pack};
|
||||
use solana_transaction_status::{EncodedConfirmedTransactionWithStatusMeta, UiTransactionEncoding};
|
||||
use spl_token::state::Mint;
|
||||
use std::{str::FromStr, time::Duration};
|
||||
use std::{collections::HashMap, str::FromStr, time::Duration};
|
||||
use tokio::sync::mpsc::Sender;
|
||||
|
||||
use crate::{utils::{Config, MarketInfo, MarketConfig}, trade_fetching::parsing::{MarketState}};
|
||||
use crate::{
|
||||
database::MarketInfo,
|
||||
trade_fetching::parsing::MarketState,
|
||||
utils::{Config, MarketConfig},
|
||||
};
|
||||
|
||||
use super::parsing::{parse_trades_from_openbook_txns, OpenBookFillEventLog};
|
||||
|
||||
|
|
@ -28,37 +34,41 @@ pub async fn scrape(config: &Config, fill_sender: Sender<OpenBookFillEventLog>)
|
|||
}
|
||||
}
|
||||
|
||||
pub async fn backfill(config: &Config, fill_sender: Sender<OpenBookFillEventLog>) {
|
||||
let url = &config.rpc_url;
|
||||
let rpc_client = RpcClient::new_with_commitment(url, CommitmentConfig::processed());
|
||||
// pub async fn backfill(config: &Config, fill_sender: Sender<OpenBookFillEventLog>) {
|
||||
// let url = &config.rpc_url;
|
||||
// let rpc_client = RpcClient::new_with_commitment(url, CommitmentConfig::processed());
|
||||
|
||||
let mut before_slot: Option<Signature> = None;
|
||||
// let mut before_slot: Option<Signature> = None;
|
||||
|
||||
loop {
|
||||
let last_sig = scrape_transactions(&rpc_client, before_slot, &fill_sender).await;
|
||||
// loop {
|
||||
// let last_sig_option = scrape_transactions(&rpc_client, before_slot, &fill_sender).await;
|
||||
|
||||
match rpc_client.get_transaction(&last_sig, UiTransactionEncoding::Json) {
|
||||
Ok(txn) => {
|
||||
let unix_sig_time = rpc_client.get_block_time(txn.slot).unwrap();
|
||||
if unix_sig_time > 0 {
|
||||
// TODO: is 3 months in past
|
||||
break;
|
||||
}
|
||||
println!("backfill at {}", unix_sig_time);
|
||||
}
|
||||
Err(_) => continue,
|
||||
}
|
||||
before_slot = Some(last_sig);
|
||||
}
|
||||
// if last_sig_option.is_none() {
|
||||
// continue;
|
||||
// }
|
||||
|
||||
print!("Backfill complete \n");
|
||||
}
|
||||
// match rpc_client.get_transaction(&last_sig_option.unwrap(), UiTransactionEncoding::Json) {
|
||||
// Ok(txn) => {
|
||||
// let unix_sig_time = rpc_client.get_block_time(txn.slot).unwrap();
|
||||
// if unix_sig_time > 0 {
|
||||
// // TODO: is 3 months in past
|
||||
// break;
|
||||
// }
|
||||
// println!("backfill at {}", unix_sig_time);
|
||||
// }
|
||||
// Err(_) => continue,
|
||||
// }
|
||||
// before_slot = last_sig_option;
|
||||
// }
|
||||
|
||||
// print!("Backfill complete \n");
|
||||
// }
|
||||
|
||||
pub async fn scrape_transactions(
|
||||
rpc_client: &RpcClient,
|
||||
before_slot: Option<Signature>,
|
||||
fill_sender: &Sender<OpenBookFillEventLog>,
|
||||
) -> Signature {
|
||||
) -> Option<Signature> {
|
||||
let rpc_config = GetConfirmedSignaturesForAddress2Config {
|
||||
before: before_slot,
|
||||
until: None,
|
||||
|
|
@ -71,11 +81,11 @@ pub async fn scrape_transactions(
|
|||
rpc_config,
|
||||
) {
|
||||
Ok(s) => s,
|
||||
Err(_) => return before_slot.unwrap(),
|
||||
Err(_) => return before_slot, // TODO: add error log
|
||||
};
|
||||
|
||||
sigs.retain(|sig| sig.err.is_none());
|
||||
let last_sig = sigs.last().unwrap().clone();
|
||||
let last_sig = sigs.last().unwrap().clone(); // Failed here
|
||||
|
||||
let txn_config = RpcTransactionConfig {
|
||||
encoding: Some(UiTransactionEncoding::Json),
|
||||
|
|
@ -102,10 +112,13 @@ pub async fn scrape_transactions(
|
|||
}
|
||||
}
|
||||
|
||||
Signature::from_str(&last_sig.signature).unwrap()
|
||||
Some(Signature::from_str(&last_sig.signature).unwrap())
|
||||
}
|
||||
|
||||
pub async fn fetch_market_infos(config: &Config, markets: Vec<MarketConfig>) -> anyhow::Result<Vec<MarketInfo>> {
|
||||
pub async fn fetch_market_infos(
|
||||
config: &Config,
|
||||
markets: Vec<MarketConfig>,
|
||||
) -> anyhow::Result<Vec<MarketInfo>> {
|
||||
let url = &config.rpc_url;
|
||||
let rpc_client = RpcClient::new_with_commitment(url, CommitmentConfig::processed());
|
||||
|
||||
|
|
@ -116,46 +129,74 @@ pub async fn fetch_market_infos(config: &Config, markets: Vec<MarketConfig>) ->
|
|||
min_context_slot: None,
|
||||
};
|
||||
|
||||
let market_keys = markets.iter().map(|x| Pubkey::from_str(&x.address).unwrap()).collect::<Vec<Pubkey>>();
|
||||
let mut market_results = rpc_client.get_multiple_accounts_with_config(&market_keys, rpc_config.clone()).unwrap().value;
|
||||
let market_keys = markets
|
||||
.iter()
|
||||
.map(|x| Pubkey::from_str(&x.address).unwrap())
|
||||
.collect::<Vec<Pubkey>>();
|
||||
let mut market_results = rpc_client
|
||||
.get_multiple_accounts_with_config(&market_keys, rpc_config.clone())
|
||||
.unwrap()
|
||||
.value;
|
||||
|
||||
let mut mint_keys = Vec::new();
|
||||
let mut mint_key_map = HashMap::new();
|
||||
|
||||
let mut market_infos = market_results.iter_mut().map(|mut r| {
|
||||
let get_account_result = r.as_mut().unwrap();
|
||||
let mut market_infos = market_results
|
||||
.iter_mut()
|
||||
.map(|r| {
|
||||
let get_account_result = r.as_mut().unwrap();
|
||||
|
||||
let mut market_bytes: &[u8] = &mut get_account_result.data[5..];
|
||||
let raw_market: MarketState = AnchorDeserialize::deserialize(&mut market_bytes).unwrap();
|
||||
let mut market_bytes: &[u8] = &mut get_account_result.data[5..];
|
||||
let raw_market: MarketState =
|
||||
AnchorDeserialize::deserialize(&mut market_bytes).unwrap();
|
||||
|
||||
let base_mint = serum_bytes_to_pubkey(raw_market.coin_mint);
|
||||
let quote_mint = serum_bytes_to_pubkey(raw_market.pc_mint);
|
||||
mint_keys.push(base_mint);
|
||||
mint_keys.push(quote_mint);
|
||||
let market_address_string = serum_bytes_to_pubkey(raw_market.own_address).to_string();
|
||||
let base_mint_key = serum_bytes_to_pubkey(raw_market.coin_mint);
|
||||
let quote_mint_key = serum_bytes_to_pubkey(raw_market.pc_mint);
|
||||
mint_key_map.insert(base_mint_key, 0);
|
||||
mint_key_map.insert(quote_mint_key, 0);
|
||||
|
||||
MarketInfo {
|
||||
name: "".to_string(),
|
||||
address: serum_bytes_to_pubkey(raw_market.own_address).to_string(),
|
||||
base_decimals: 0,
|
||||
quote_decimals: 0,
|
||||
base_lot_size: raw_market.coin_lot_size,
|
||||
quote_lot_size: raw_market.pc_lot_size,
|
||||
}
|
||||
}).collect::<Vec<MarketInfo>>();
|
||||
let market_name = markets
|
||||
.iter()
|
||||
.find(|x| x.address == market_address_string)
|
||||
.unwrap()
|
||||
.name
|
||||
.clone();
|
||||
|
||||
let mint_results = rpc_client.get_multiple_accounts_with_config(&mint_keys, rpc_config).unwrap().value;
|
||||
for i in (0..mint_results.len()).step_by(2) {
|
||||
let mut base_mint_account = mint_results[i].as_ref().unwrap().clone();
|
||||
let mut quote_mint_account = mint_results[i+1].as_ref().unwrap().clone();
|
||||
MarketInfo {
|
||||
name: market_name,
|
||||
address: market_address_string,
|
||||
base_decimals: 0,
|
||||
quote_decimals: 0,
|
||||
base_mint_key: base_mint_key.to_string(),
|
||||
quote_mint_key: quote_mint_key.to_string(),
|
||||
base_lot_size: raw_market.coin_lot_size,
|
||||
quote_lot_size: raw_market.pc_lot_size,
|
||||
}
|
||||
})
|
||||
.collect::<Vec<MarketInfo>>();
|
||||
|
||||
let mut base_mint_bytes: &[u8] = &mut base_mint_account.data[..];
|
||||
let mut quote_mint_bytes: &[u8] = &mut quote_mint_account.data[..];
|
||||
let mint_keys = mint_key_map.keys().cloned().collect::<Vec<Pubkey>>();
|
||||
|
||||
let base_mint = Mint::unpack_from_slice(&mut base_mint_bytes).unwrap();
|
||||
let quote_mint = Mint::unpack_from_slice(&mut quote_mint_bytes).unwrap();
|
||||
let mint_results = rpc_client
|
||||
.get_multiple_accounts_with_config(&mint_keys, rpc_config)
|
||||
.unwrap()
|
||||
.value;
|
||||
// println!("{:?}", mint_results);
|
||||
// println!("{:?}", mint_keys);
|
||||
// println!("{:?}", mint_results.len());
|
||||
for i in 0..mint_results.len() {
|
||||
let mut mint_account = mint_results[i].as_ref().unwrap().clone();
|
||||
let mut mint_bytes: &[u8] = &mut mint_account.data[..];
|
||||
let mint = Mint::unpack_from_slice(&mut mint_bytes).unwrap();
|
||||
|
||||
market_infos[i / 2].name = markets[i / 2].name.clone();
|
||||
market_infos[i / 2].base_decimals = base_mint.decimals;
|
||||
market_infos[i / 2].quote_decimals = quote_mint.decimals;
|
||||
mint_key_map.insert(mint_keys[i], mint.decimals);
|
||||
}
|
||||
|
||||
for i in 0..market_infos.len() {
|
||||
let base_key = Pubkey::from_str(&market_infos[i].base_mint_key).unwrap();
|
||||
let quote_key = Pubkey::from_str(&market_infos[i].quote_mint_key).unwrap();
|
||||
market_infos[i].base_decimals = *mint_key_map.get(&base_key).unwrap();
|
||||
market_infos[i].quote_decimals = *mint_key_map.get("e_key).unwrap();
|
||||
}
|
||||
|
||||
Ok(market_infos)
|
||||
|
|
@ -164,7 +205,7 @@ pub async fn fetch_market_infos(config: &Config, markets: Vec<MarketConfig>) ->
|
|||
fn serum_bytes_to_pubkey(data: [u64; 4]) -> Pubkey {
|
||||
let mut res = [0; 32];
|
||||
for i in 0..4 {
|
||||
res[8*i..][..8].copy_from_slice(&data[i].to_le_bytes());
|
||||
res[8 * i..][..8].copy_from_slice(&data[i].to_le_bytes());
|
||||
}
|
||||
Pubkey::new_from_array(res)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue