2023-03-13 22:31:00 -07:00
|
|
|
use chrono::{DateTime, Duration, DurationRound, Utc};
|
2023-05-14 00:15:10 -07:00
|
|
|
use deadpool_postgres::Pool;
|
2023-05-31 06:32:34 -07:00
|
|
|
use log::debug;
|
2023-06-14 23:50:24 -07:00
|
|
|
use std::cmp::{max, min};
|
2023-07-26 18:25:19 -07:00
|
|
|
use strum::IntoEnumIterator;
|
2023-03-12 00:13:57 -08:00
|
|
|
|
|
|
|
use crate::{
|
2023-07-26 18:25:19 -07:00
|
|
|
database::{
|
|
|
|
fetch::{
|
|
|
|
fetch_candles_from, fetch_earliest_candles,
|
|
|
|
fetch_latest_finished_candle,
|
|
|
|
},
|
|
|
|
insert::build_candles_upsert_statement,
|
|
|
|
},
|
2023-03-13 09:51:30 -07:00
|
|
|
structs::{
|
|
|
|
candle::Candle,
|
|
|
|
resolution::{day, Resolution},
|
|
|
|
},
|
2023-07-26 18:25:19 -07:00
|
|
|
utils::{f64_max, f64_min, AnyhowWrap},
|
2023-03-12 00:13:57 -08:00
|
|
|
};
|
|
|
|
|
|
|
|
pub async fn batch_higher_order_candles(
|
2023-05-14 00:15:10 -07:00
|
|
|
pool: &Pool,
|
2023-03-12 18:03:39 -07:00
|
|
|
market_name: &str,
|
2023-03-12 00:13:57 -08:00
|
|
|
resolution: Resolution,
|
|
|
|
) -> anyhow::Result<Vec<Candle>> {
|
2023-05-14 00:15:10 -07:00
|
|
|
let latest_candle = fetch_latest_finished_candle(pool, market_name, resolution).await?;
|
2023-03-12 00:13:57 -08:00
|
|
|
|
|
|
|
match latest_candle {
|
|
|
|
Some(candle) => {
|
|
|
|
let start_time = candle.end_time;
|
2023-03-12 16:32:12 -07:00
|
|
|
let end_time = start_time + day();
|
2023-03-12 00:13:57 -08:00
|
|
|
let mut constituent_candles = fetch_candles_from(
|
2023-05-14 00:15:10 -07:00
|
|
|
pool,
|
2023-03-12 18:03:39 -07:00
|
|
|
market_name,
|
2023-03-12 00:13:57 -08:00
|
|
|
resolution.get_constituent_resolution(),
|
|
|
|
start_time,
|
|
|
|
end_time,
|
|
|
|
)
|
|
|
|
.await?;
|
2023-05-19 17:16:37 -07:00
|
|
|
if constituent_candles.is_empty() {
|
2023-03-12 00:13:57 -08:00
|
|
|
return Ok(Vec::new());
|
|
|
|
}
|
2023-07-26 18:25:19 -07:00
|
|
|
let combined_candles =
|
|
|
|
combine_into_higher_order_candles(&mut constituent_candles, resolution, start_time);
|
2023-03-12 00:13:57 -08:00
|
|
|
Ok(combined_candles)
|
|
|
|
}
|
|
|
|
None => {
|
2023-03-27 12:01:48 -07:00
|
|
|
let mut constituent_candles =
|
2023-05-14 00:15:10 -07:00
|
|
|
fetch_earliest_candles(pool, market_name, resolution.get_constituent_resolution())
|
2023-03-12 22:03:58 -07:00
|
|
|
.await?;
|
2023-05-19 17:16:37 -07:00
|
|
|
if constituent_candles.is_empty() {
|
2023-05-31 06:32:34 -07:00
|
|
|
debug!(
|
|
|
|
"Batching {}, but no candles found for: {:?}, {}",
|
|
|
|
resolution,
|
|
|
|
market_name,
|
|
|
|
resolution.get_constituent_resolution()
|
|
|
|
);
|
2023-03-12 00:13:57 -08:00
|
|
|
return Ok(Vec::new());
|
|
|
|
}
|
2023-03-27 12:01:48 -07:00
|
|
|
let start_time = constituent_candles[0].start_time.duration_trunc(day())?;
|
2023-03-12 00:13:57 -08:00
|
|
|
|
2023-05-19 17:16:37 -07:00
|
|
|
if constituent_candles.is_empty() {
|
2023-03-12 00:13:57 -08:00
|
|
|
return Ok(Vec::new());
|
|
|
|
}
|
|
|
|
|
2023-07-26 18:25:19 -07:00
|
|
|
let combined_candles =
|
|
|
|
combine_into_higher_order_candles(&mut constituent_candles, resolution, start_time);
|
2023-03-12 00:13:57 -08:00
|
|
|
|
2023-03-27 12:01:48 -07:00
|
|
|
Ok(trim_candles(
|
|
|
|
combined_candles,
|
|
|
|
constituent_candles[0].start_time,
|
|
|
|
))
|
2023-03-12 00:13:57 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn combine_into_higher_order_candles(
|
2023-07-26 18:25:19 -07:00
|
|
|
constituent_candles: &Vec<Candle>,
|
2023-03-12 00:13:57 -08:00
|
|
|
target_resolution: Resolution,
|
|
|
|
st: DateTime<Utc>,
|
|
|
|
) -> Vec<Candle> {
|
2023-05-31 06:32:34 -07:00
|
|
|
debug!("combining for target_resolution: {}", target_resolution);
|
2023-03-12 00:13:57 -08:00
|
|
|
|
|
|
|
let duration = target_resolution.get_duration();
|
|
|
|
|
2023-03-12 22:03:58 -07:00
|
|
|
let empty_candle = Candle::create_empty_candle(
|
|
|
|
constituent_candles[0].market_name.clone(),
|
|
|
|
target_resolution,
|
|
|
|
);
|
2023-03-13 22:31:00 -07:00
|
|
|
let now = Utc::now().duration_trunc(Duration::minutes(1)).unwrap();
|
2023-06-14 23:50:24 -07:00
|
|
|
let candle_window = min(now - st, day());
|
2023-03-27 12:01:48 -07:00
|
|
|
let num_candles = max(
|
|
|
|
1,
|
|
|
|
(candle_window.num_minutes() / duration.num_minutes()) as usize + 1,
|
|
|
|
);
|
2023-03-12 00:13:57 -08:00
|
|
|
|
2023-03-13 22:31:00 -07:00
|
|
|
let mut combined_candles = vec![empty_candle; num_candles];
|
2023-03-12 00:13:57 -08:00
|
|
|
|
2023-07-26 18:25:19 -07:00
|
|
|
let mut last_close = constituent_candles[0].close;
|
|
|
|
let mut con_iter = constituent_candles.iter().peekable();
|
2023-05-19 17:16:37 -07:00
|
|
|
let mut start_time = st;
|
2023-03-12 00:13:57 -08:00
|
|
|
let mut end_time = start_time + duration;
|
|
|
|
|
|
|
|
for i in 0..combined_candles.len() {
|
2023-07-26 18:25:19 -07:00
|
|
|
combined_candles[i].open = last_close;
|
|
|
|
combined_candles[i].low = last_close;
|
|
|
|
combined_candles[i].close = last_close;
|
|
|
|
combined_candles[i].high = last_close;
|
2023-03-12 00:13:57 -08:00
|
|
|
|
|
|
|
while matches!(con_iter.peek(), Some(c) if c.end_time <= end_time) {
|
|
|
|
let unit_candle = con_iter.next().unwrap();
|
2023-05-14 00:15:10 -07:00
|
|
|
combined_candles[i].high = f64_max(combined_candles[i].high, unit_candle.high);
|
|
|
|
combined_candles[i].low = f64_min(combined_candles[i].low, unit_candle.low);
|
2023-03-12 00:13:57 -08:00
|
|
|
combined_candles[i].close = unit_candle.close;
|
|
|
|
combined_candles[i].volume += unit_candle.volume;
|
|
|
|
combined_candles[i].complete = unit_candle.complete;
|
|
|
|
combined_candles[i].end_time = unit_candle.end_time;
|
|
|
|
}
|
|
|
|
|
|
|
|
combined_candles[i].start_time = start_time;
|
|
|
|
combined_candles[i].end_time = end_time;
|
|
|
|
|
|
|
|
start_time = end_time;
|
2023-05-19 17:16:37 -07:00
|
|
|
end_time += duration;
|
2023-03-12 00:13:57 -08:00
|
|
|
|
2023-07-26 18:25:19 -07:00
|
|
|
last_close = combined_candles[i].close;
|
2023-03-12 00:13:57 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
combined_candles
|
|
|
|
}
|
|
|
|
|
2023-03-27 12:01:48 -07:00
|
|
|
fn trim_candles(mut c: Vec<Candle>, start_time: DateTime<Utc>) -> Vec<Candle> {
|
2023-03-12 00:13:57 -08:00
|
|
|
let mut i = 0;
|
|
|
|
while i < c.len() {
|
2023-03-27 12:01:48 -07:00
|
|
|
if c[i].end_time <= start_time {
|
2023-03-12 00:13:57 -08:00
|
|
|
c.remove(i);
|
|
|
|
} else {
|
|
|
|
i += 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
c
|
|
|
|
}
|
2023-05-19 17:15:13 -07:00
|
|
|
|
|
|
|
pub async fn backfill_batch_higher_order_candles(
|
|
|
|
pool: &Pool,
|
|
|
|
market_name: &str,
|
2023-07-26 18:25:19 -07:00
|
|
|
) -> anyhow::Result<()> {
|
|
|
|
let earliest_candles = fetch_earliest_candles(pool, market_name, Resolution::R1m).await?;
|
|
|
|
let mut start_time = earliest_candles[0].start_time.duration_trunc(day())?;
|
|
|
|
while start_time < Utc::now() {
|
|
|
|
let mut candles = vec![];
|
|
|
|
let mut constituent_candles = fetch_candles_from(
|
|
|
|
pool,
|
|
|
|
market_name,
|
|
|
|
Resolution::R1m,
|
|
|
|
start_time,
|
|
|
|
start_time + day(),
|
|
|
|
)
|
|
|
|
.await?;
|
|
|
|
|
|
|
|
for resolution in Resolution::iter() {
|
|
|
|
if resolution == Resolution::R1m {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
let mut combined_candles =
|
|
|
|
combine_into_higher_order_candles(&mut constituent_candles, resolution, start_time);
|
|
|
|
candles.append(&mut combined_candles);
|
|
|
|
}
|
|
|
|
|
|
|
|
let upsert_statement = build_candles_upsert_statement(&candles);
|
|
|
|
let client = pool.get().await.unwrap();
|
|
|
|
client
|
|
|
|
.execute(&upsert_statement, &[])
|
|
|
|
.await
|
|
|
|
.map_err_anyhow()?;
|
|
|
|
// println!("{:?} {:?} done", market_name, start_time);
|
|
|
|
start_time += day();
|
2023-05-19 17:15:13 -07:00
|
|
|
}
|
|
|
|
|
2023-07-26 18:25:19 -07:00
|
|
|
Ok(())
|
2023-05-19 17:15:13 -07:00
|
|
|
}
|