use std::{cmp::min, collections::HashMap}; use chrono::{DateTime, Duration, DurationRound, Utc}; use deadpool_postgres::Pool; use itertools::Itertools; use log::debug; use crate::database::backfill::{ fetch_earliest_fill_multiple_markets, fetch_fills_multiple_markets_from, fetch_last_minute_candles, }; use crate::{ database::{ fetch::{fetch_earliest_fill, fetch_fills_from, fetch_latest_finished_candle}, insert::build_candles_upsert_statement, }, structs::{ candle::{Candle}, markets::MarketInfo, openbook::PgOpenBookFill, resolution::{day, Resolution}, }, utils::{f64_max, f64_min, AnyhowWrap}, }; pub async fn batch_1m_candles(pool: &Pool, market: &MarketInfo) -> anyhow::Result> { let market_name = &market.name; let market_address = &market.address; let latest_candle = fetch_latest_finished_candle(pool, market_name, Resolution::R1m).await?; match latest_candle { Some(candle) => { let start_time = candle.end_time; let end_time = min( start_time + day(), (Utc::now() + Duration::minutes(1)).duration_trunc(Duration::minutes(1))?, ); let mut fills = fetch_fills_from(pool, market_address, start_time, end_time).await?; let candles = combine_fills_into_1m_candles( &mut fills, market, start_time, end_time, Some(candle.close), ); Ok(candles) } None => { let earliest_fill = fetch_earliest_fill(pool, market_address).await?; if earliest_fill.is_none() { debug!("No fills found for: {:?}", market_name); return Ok(Vec::new()); } let start_time = earliest_fill .unwrap() .time .duration_trunc(Duration::minutes(1))?; let end_time = min( start_time + day(), Utc::now().duration_trunc(Duration::minutes(1))?, ); let mut fills = fetch_fills_from(pool, market_address, start_time, end_time).await?; if !fills.is_empty() { let candles = combine_fills_into_1m_candles(&mut fills, market, start_time, end_time, None); Ok(candles) } else { Ok(Vec::new()) } } } } fn combine_fills_into_1m_candles( fills: &mut Vec, market: &MarketInfo, st: DateTime, et: DateTime, maybe_last_price: Option, ) -> Vec { let empty_candle = Candle::create_empty_candle(market.name.clone(), Resolution::R1m); let minutes = (et - st).num_minutes(); let mut candles = vec![empty_candle; minutes as usize]; let mut fills_iter = fills.iter_mut().peekable(); let mut start_time = st; let mut end_time = start_time + Duration::minutes(1); let mut last_price = match maybe_last_price { Some(p) => p, None => { let first = fills_iter.peek().unwrap(); first.price } }; for i in 0..candles.len() { candles[i].open = last_price; candles[i].close = last_price; candles[i].low = last_price; candles[i].high = last_price; while matches!(fills_iter.peek(), Some(f) if f.time < end_time) { let fill = fills_iter.next().unwrap(); candles[i].close = fill.price; candles[i].low = f64_min(fill.price, candles[i].low); candles[i].high = f64_max(fill.price, candles[i].high); candles[i].volume += fill.size; last_price = fill.price; } candles[i].start_time = start_time; candles[i].end_time = end_time; candles[i].complete = matches!(fills_iter.peek(), Some(f) if f.time > end_time) || end_time < Utc::now() - Duration::minutes(10); start_time = end_time; end_time += Duration::minutes(1); } candles } /// Goes from the earliest fill to the most recent. Will mark candles as complete if there are missing gaps of fills between the start and end. pub async fn backfill_batch_1m_candles( pool: &Pool, markets: Vec, ) -> anyhow::Result<()> { let market_address_strings: Vec = markets.iter().map(|m| m.address.clone()).collect(); let mut candle_container = HashMap::new(); let client = pool.get().await?; let earliest_fill = fetch_earliest_fill_multiple_markets(&client, &market_address_strings).await?; if earliest_fill.is_none() { println!("No fills found for backfill"); return Ok(()); } println!("Found earliset fill for backfill"); let mut start_time = earliest_fill .unwrap() .time .duration_trunc(Duration::minutes(1))?; while start_time < Utc::now() { let end_time = min( start_time + day(), Utc::now().duration_trunc(Duration::minutes(1))?, ); let last_candles = fetch_last_minute_candles(&client).await?; let all_fills = fetch_fills_multiple_markets_from( &client, &market_address_strings, start_time, end_time, ) .await?; // println!("{:?} {:?}", start_time, end_time); // println!("all fills len : {:?}", all_fills.len()); println!("{:?}", all_fills[0]); // println!("{:?}", all_fills[1]); // println!("Fetched multiple fills for backfill"); let fills_groups = all_fills .into_iter() .sorted_by(|a, b| Ord::cmp(&a.market_key, &b.market_key)) .group_by(|f| f.market_key.clone()); let fills_by_market: Vec<(String, Vec)> = fills_groups .into_iter() .map(|(m, group)| (m, group.collect())) .collect(); println!("fbm len : {:?}", fills_by_market.len()); // sort fills by market, make candles for (_, mut fills) in fills_by_market { let market = markets .iter() .find(|m| m.address == fills[0].market_key) .unwrap(); let minute_candles = combine_fills_into_1m_candles(&mut fills, market, start_time, end_time, None); candle_container.insert(&market.address, minute_candles); } // where no candles, make empty ones for (k, v) in candle_container.iter_mut() { if v.is_empty() { let market = markets.iter().find(|m| &m.address == *k).unwrap(); let last_candle = last_candles .iter() .find(|c| c.market_name == market.name) .unwrap(); let empty_candles = combine_fills_into_1m_candles( &mut vec![], market, start_time, end_time, Some(last_candle.close), ); *v = empty_candles; } } // insert candles in batches for candles in candle_container.values() { let candle_chunks: Vec> = candles.chunks(1500).map(|chunk| chunk.to_vec()).collect(); // 1440 minutes in a day for c in candle_chunks { let upsert_statement = build_candles_upsert_statement(&c); client .execute(&upsert_statement, &[]) .await .map_err_anyhow()?; } } // reset entries but keep markets we've seen for blank candles for (_, v) in candle_container.iter_mut() { *v = vec![]; } println!("day done"); start_time += day(); } Ok(()) }