2023-05-14 00:15:10 -07:00
use std ::cmp ::min ;
2023-03-11 14:50:22 -08:00
2023-03-12 00:13:57 -08:00
use chrono ::{ DateTime , Duration , DurationRound , Utc } ;
2023-05-14 00:15:10 -07:00
use deadpool_postgres ::Pool ;
2023-05-31 06:32:34 -07:00
use log ::debug ;
2023-03-11 14:50:22 -08:00
2023-03-13 09:51:30 -07:00
use crate ::{
2023-05-31 22:58:57 -07:00
database ::fetch ::{ fetch_earliest_fill , fetch_fills_from , fetch_latest_finished_candle , fetch_candles_from } ,
2023-03-13 09:51:30 -07:00
structs ::{
candle ::Candle ,
markets ::MarketInfo ,
openbook ::{ calculate_fill_price_and_size , PgOpenBookFill } ,
resolution ::{ day , Resolution } ,
} ,
2023-05-14 00:15:10 -07:00
utils ::{ f64_max , f64_min } ,
2023-03-13 09:51:30 -07:00
} ;
2023-03-11 14:50:22 -08:00
2023-05-14 00:15:10 -07:00
pub async fn batch_1m_candles ( pool : & Pool , market : & MarketInfo ) -> anyhow ::Result < Vec < Candle > > {
2023-03-12 18:03:39 -07:00
let market_name = & market . name ;
let market_address = & market . address ;
2023-05-14 00:15:10 -07:00
let latest_candle = fetch_latest_finished_candle ( pool , market_name , Resolution ::R1m ) . await ? ;
2023-03-11 14:50:22 -08:00
match latest_candle {
Some ( candle ) = > {
2023-06-01 07:30:54 -07:00
println! ( " {} : latest finished candle time {} " , market_name , candle . end_time ) ;
2023-03-11 14:50:22 -08:00
let start_time = candle . end_time ;
let end_time = min (
2023-03-12 16:32:12 -07:00
start_time + day ( ) ,
2023-03-11 14:50:22 -08:00
Utc ::now ( ) . duration_trunc ( Duration ::minutes ( 1 ) ) ? ,
) ;
2023-05-14 00:15:10 -07:00
let mut fills = fetch_fills_from ( pool , market_address , start_time , end_time ) . await ? ;
2023-05-31 22:58:57 -07:00
let existing_candles = fetch_candles_from ( pool , market_name , Resolution ::R1m , candle . start_time , end_time ) . await ? ;
2023-06-01 07:30:54 -07:00
println! ( " {} : combining {} fills from {} to {} " , market_name , fills . clone ( ) . len ( ) , start_time , end_time ) ;
2023-05-31 22:58:57 -07:00
2023-03-11 14:50:22 -08:00
let candles = combine_fills_into_1m_candles (
& mut fills ,
market ,
start_time ,
end_time ,
Some ( candle . close ) ,
) ;
2023-06-01 07:30:54 -07:00
println! ( " {} : filtering {} new candles on {} existing candles from {} to {} " , market_name , candles . clone ( ) . len ( ) , existing_candles . clone ( ) . len ( ) , start_time , end_time ) ;
2023-05-31 22:58:57 -07:00
Ok ( filter_redundant_candles ( existing_candles , candles . clone ( ) ) )
2023-03-11 14:50:22 -08:00
}
None = > {
2023-06-01 07:30:54 -07:00
println! ( " {} : no finished candle " , market_name ) ;
2023-05-14 00:15:10 -07:00
let earliest_fill = fetch_earliest_fill ( pool , market_address ) . await ? ;
2023-03-11 14:50:22 -08:00
if earliest_fill . is_none ( ) {
2023-05-31 06:32:34 -07:00
debug! ( " No fills found for: {:?} " , market_name ) ;
2023-03-11 14:50:22 -08:00
return Ok ( Vec ::new ( ) ) ;
}
let start_time = earliest_fill
. unwrap ( )
. time
. duration_trunc ( Duration ::minutes ( 1 ) ) ? ;
let end_time = min (
2023-03-12 16:32:12 -07:00
start_time + day ( ) ,
2023-03-11 14:50:22 -08:00
Utc ::now ( ) . duration_trunc ( Duration ::minutes ( 1 ) ) ? ,
) ;
2023-05-14 00:15:10 -07:00
let mut fills = fetch_fills_from ( pool , market_address , start_time , end_time ) . await ? ;
2023-06-01 07:30:54 -07:00
println! ( " {} : combining {} fills from {} to {} " , market_name , fills . clone ( ) . len ( ) , start_time , end_time ) ;
2023-05-19 17:16:37 -07:00
if ! fills . is_empty ( ) {
2023-03-27 12:01:48 -07:00
let candles =
combine_fills_into_1m_candles ( & mut fills , market , start_time , end_time , None ) ;
Ok ( candles )
} else {
Ok ( Vec ::new ( ) )
}
2023-03-11 14:50:22 -08:00
}
}
}
fn combine_fills_into_1m_candles (
fills : & mut Vec < PgOpenBookFill > ,
2023-03-13 22:31:00 -07:00
market : & MarketInfo ,
2023-03-11 14:50:22 -08:00
st : DateTime < Utc > ,
et : DateTime < Utc > ,
2023-05-14 00:15:10 -07:00
maybe_last_price : Option < f64 > ,
2023-03-11 14:50:22 -08:00
) -> Vec < Candle > {
2023-03-13 22:31:00 -07:00
let empty_candle = Candle ::create_empty_candle ( market . name . clone ( ) , Resolution ::R1m ) ;
2023-03-11 14:50:22 -08:00
let minutes = ( et - st ) . num_minutes ( ) ;
let mut candles = vec! [ empty_candle ; minutes as usize ] ;
let mut fills_iter = fills . iter_mut ( ) . peekable ( ) ;
2023-05-19 17:16:37 -07:00
let mut start_time = st ;
2023-03-11 14:50:22 -08:00
let mut end_time = start_time + Duration ::minutes ( 1 ) ;
2023-03-13 22:31:00 -07:00
let mut last_price = match maybe_last_price {
Some ( p ) = > p ,
2023-03-13 22:34:23 -07:00
None = > {
2023-05-19 17:16:37 -07:00
let first = fills_iter . peek ( ) . unwrap ( ) ;
2023-03-13 22:31:00 -07:00
let ( price , _ ) =
calculate_fill_price_and_size ( * * first , market . base_decimals , market . quote_decimals ) ;
price
2023-03-13 22:34:23 -07:00
}
2023-03-13 22:31:00 -07:00
} ;
2023-03-11 14:50:22 -08:00
for i in 0 .. candles . len ( ) {
candles [ i ] . open = last_price ;
candles [ i ] . close = last_price ;
candles [ i ] . low = last_price ;
candles [ i ] . high = last_price ;
while matches! ( fills_iter . peek ( ) , Some ( f ) if f . time < end_time ) {
let fill = fills_iter . next ( ) . unwrap ( ) ;
2023-06-01 07:30:54 -07:00
println! ( " adding fill from {} " , fill . time ) ;
2023-03-11 14:50:22 -08:00
let ( price , volume ) =
calculate_fill_price_and_size ( * fill , market . base_decimals , market . quote_decimals ) ;
candles [ i ] . close = price ;
2023-05-14 00:15:10 -07:00
candles [ i ] . low = f64_min ( price , candles [ i ] . low ) ;
candles [ i ] . high = f64_max ( price , candles [ i ] . high ) ;
2023-03-11 14:50:22 -08:00
candles [ i ] . volume + = volume ;
last_price = price ;
}
candles [ i ] . start_time = start_time ;
candles [ i ] . end_time = end_time ;
2023-06-01 07:30:54 -07:00
candles [ i ] . complete = matches! ( fills_iter . peek ( ) , Some ( f ) if f . time > end_time ) | | end_time < Utc ::now ( ) - Duration ::days ( 1 ) ;
if candles [ i ] . complete {
println! ( " candle {} complete with end time {} " , i , end_time ) ;
} else {
let peeked_fill = fills_iter . peek ( ) ;
match peeked_fill {
Some ( f ) = > println! ( " candle {} incomplete, peeked fill was at {} and end time was {} " , i , f . time , end_time ) ,
None = > { }
}
}
2023-03-11 14:50:22 -08:00
start_time = end_time ;
2023-05-19 17:16:37 -07:00
end_time + = Duration ::minutes ( 1 ) ;
2023-03-11 14:50:22 -08:00
}
candles
}
2023-05-19 17:15:13 -07:00
2023-05-31 22:58:57 -07:00
fn filter_redundant_candles ( existing_candles : Vec < Candle > , mut candles : Vec < Candle > ) -> Vec < Candle > {
candles . retain ( | c | {
! existing_candles . contains ( c )
} ) ;
println! ( " trimmed: {:?} " , candles . len ( ) ) ;
// println!("{:?}", candles.last());
println! ( " candles: {:?} " , existing_candles . len ( ) ) ;
// println!("{:?}", existing_candles.last());
candles
}
2023-05-19 17:15:13 -07:00
/// Goes from the earliest fill to the most recent. Will mark candles as complete if there are missing gaps of fills between the start and end.
pub async fn backfill_batch_1m_candles (
pool : & Pool ,
market : & MarketInfo ,
) -> anyhow ::Result < Vec < Candle > > {
let market_name = & market . name ;
let market_address = & market . address ;
let mut candles = vec! [ ] ;
let earliest_fill = fetch_earliest_fill ( pool , & market . address ) . await ? ;
if earliest_fill . is_none ( ) {
2023-05-31 06:32:34 -07:00
debug! ( " No fills found for: {:?} " , & market_name ) ;
2023-05-19 17:15:13 -07:00
return Ok ( candles ) ;
}
let mut start_time = earliest_fill
. unwrap ( )
. time
. duration_trunc ( Duration ::minutes ( 1 ) ) ? ;
while start_time < Utc ::now ( ) {
let end_time = min (
start_time + day ( ) ,
Utc ::now ( ) . duration_trunc ( Duration ::minutes ( 1 ) ) ? ,
) ;
let mut fills = fetch_fills_from ( pool , market_address , start_time , end_time ) . await ? ;
2023-05-19 17:16:37 -07:00
if ! fills . is_empty ( ) {
2023-05-19 17:15:13 -07:00
let mut minute_candles =
combine_fills_into_1m_candles ( & mut fills , market , start_time , end_time , None ) ;
candles . append ( & mut minute_candles ) ;
}
start_time + = day ( )
}
Ok ( candles )
}