Fetch historical prices
This commit is contained in:
parent
da31fbdb8e
commit
a59d7c9998
|
@ -54,6 +54,8 @@ sha2 = "0.9.5"
|
|||
lazy_static = "1.4.0"
|
||||
rustyline = "8.2.0"
|
||||
clap = "3.0.0-beta.2"
|
||||
chrono = "0.4.19"
|
||||
reqwest = { version = "0.11.4", features = ["json", "rustls-tls"], default-features = false }
|
||||
|
||||
# librustzcash synced to 35023ed8ca2fb1061e78fd740b640d4eefcc5edd
|
||||
|
||||
|
|
140
src/db.rs
140
src/db.rs
|
@ -1,5 +1,4 @@
|
|||
use crate::chain::{Nf, NfRef};
|
||||
use crate::db::migration::{get_schema_version, update_schema_version};
|
||||
use crate::taddr::{derive_tkeys, BIP44_PATH};
|
||||
use crate::transaction::{Contact, TransactionInfo};
|
||||
use crate::{CTree, Witness, NETWORK};
|
||||
|
@ -10,6 +9,7 @@ use zcash_primitives::consensus::{NetworkUpgrade, Parameters};
|
|||
use zcash_primitives::merkle_tree::IncrementalWitness;
|
||||
use zcash_primitives::sapling::{Diversifier, Node, Note, Rseed, SaplingIvk};
|
||||
use zcash_primitives::zip32::{DiversifierIndex, ExtendedFullViewingKey};
|
||||
use chrono::NaiveDateTime;
|
||||
|
||||
mod migration;
|
||||
|
||||
|
@ -72,112 +72,7 @@ impl DbAdapter {
|
|||
}
|
||||
|
||||
pub fn init_db(&self) -> anyhow::Result<()> {
|
||||
self.connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_version (
|
||||
id INTEGER PRIMARY KEY NOT NULL,
|
||||
version INTEGER NOT NULL)",
|
||||
NO_PARAMS,
|
||||
)?;
|
||||
|
||||
let version = get_schema_version(&self.connection)?;
|
||||
|
||||
if version < 1 {
|
||||
self.connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS accounts (
|
||||
id_account INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
seed TEXT,
|
||||
sk TEXT,
|
||||
ivk TEXT NOT NULL UNIQUE,
|
||||
address TEXT NOT NULL)",
|
||||
NO_PARAMS,
|
||||
)?;
|
||||
|
||||
self.connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS blocks (
|
||||
height INTEGER PRIMARY KEY,
|
||||
hash BLOB NOT NULL,
|
||||
timestamp INTEGER NOT NULL,
|
||||
sapling_tree BLOB NOT NULL)",
|
||||
NO_PARAMS,
|
||||
)?;
|
||||
|
||||
self.connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS transactions (
|
||||
id_tx INTEGER PRIMARY KEY,
|
||||
account INTEGER NOT NULL,
|
||||
txid BLOB NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
timestamp INTEGER NOT NULL,
|
||||
value INTEGER NOT NULL,
|
||||
address TEXT,
|
||||
memo TEXT,
|
||||
tx_index INTEGER,
|
||||
CONSTRAINT tx_account UNIQUE (height, tx_index, account))",
|
||||
NO_PARAMS,
|
||||
)?;
|
||||
|
||||
self.connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS received_notes (
|
||||
id_note INTEGER PRIMARY KEY,
|
||||
account INTEGER NOT NULL,
|
||||
position INTEGER NOT NULL,
|
||||
tx INTEGER NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
output_index INTEGER NOT NULL,
|
||||
diversifier BLOB NOT NULL,
|
||||
value INTEGER NOT NULL,
|
||||
rcm BLOB NOT NULL,
|
||||
nf BLOB NOT NULL UNIQUE,
|
||||
spent INTEGER,
|
||||
CONSTRAINT tx_output UNIQUE (tx, output_index))",
|
||||
NO_PARAMS,
|
||||
)?;
|
||||
|
||||
self.connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS sapling_witnesses (
|
||||
id_witness INTEGER PRIMARY KEY,
|
||||
note INTEGER NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
witness BLOB NOT NULL,
|
||||
CONSTRAINT witness_height UNIQUE (note, height))",
|
||||
NO_PARAMS,
|
||||
)?;
|
||||
|
||||
self.connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS diversifiers (
|
||||
account INTEGER PRIMARY KEY NOT NULL,
|
||||
diversifier_index BLOB NOT NULL)",
|
||||
NO_PARAMS,
|
||||
)?;
|
||||
|
||||
self.connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS taddrs (
|
||||
account INTEGER PRIMARY KEY NOT NULL,
|
||||
sk TEXT NOT NULL,
|
||||
address TEXT NOT NULL)",
|
||||
NO_PARAMS,
|
||||
)?;
|
||||
}
|
||||
|
||||
if version < 2 {
|
||||
self.connection
|
||||
.execute("ALTER TABLE received_notes ADD excluded BOOL", NO_PARAMS)?;
|
||||
}
|
||||
|
||||
if version < 3 {
|
||||
self.connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS contacts (
|
||||
account INTEGER NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
address TEXT NOT NULL,
|
||||
PRIMARY KEY (account, address))",
|
||||
NO_PARAMS,
|
||||
)?;
|
||||
}
|
||||
|
||||
update_schema_version(&self.connection, 3)?;
|
||||
|
||||
migration::init_db(&self.connection)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
@ -745,6 +640,37 @@ impl DbAdapter {
|
|||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_missing_prices_timestamp(&self, currency: &str) -> anyhow::Result<Vec<i64>> {
|
||||
let mut statement = self.connection.prepare(
|
||||
"WITH t AS (SELECT timestamp, timestamp/86400 AS day FROM transactions), p AS (SELECT price, timestamp/86400 AS day FROM historical_prices WHERE currency = ?1) \
|
||||
SELECT t.timestamp FROM t LEFT JOIN p ON t.day = p.day WHERE p.price IS NULL")?;
|
||||
let res = statement.query_map(params![currency], |row| {
|
||||
let timestamp: i64 = row.get(0)?;
|
||||
Ok(timestamp)
|
||||
})?;
|
||||
let mut timestamps: Vec<i64> = vec![];
|
||||
for ts in res {
|
||||
let ts = NaiveDateTime::from_timestamp(ts?, 0);
|
||||
let ts_date = ts.date().and_hms(0, 0, 0); // at midnight
|
||||
timestamps.push(ts_date.timestamp());
|
||||
}
|
||||
timestamps.sort();
|
||||
timestamps.dedup();
|
||||
Ok(timestamps)
|
||||
}
|
||||
|
||||
pub fn store_historical_prices(&mut self, prices: Vec<(i64, f64)>, currency: &str) -> anyhow::Result<()> {
|
||||
let db_transaction = self.connection.transaction()?;
|
||||
{
|
||||
let mut statement = db_transaction.prepare("INSERT INTO historical_prices(timestamp, price, currency) VALUES (?1, ?2, ?3)")?;
|
||||
for (ts, px) in prices {
|
||||
statement.execute(params![ts, px, currency])?;
|
||||
}
|
||||
}
|
||||
db_transaction.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
@ -19,3 +19,125 @@ pub fn update_schema_version(connection: &Connection, version: u32) -> anyhow::R
|
|||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn init_db(connection: &Connection) -> anyhow::Result<()> {
|
||||
connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_version (
|
||||
id INTEGER PRIMARY KEY NOT NULL,
|
||||
version INTEGER NOT NULL)",
|
||||
NO_PARAMS,
|
||||
)?;
|
||||
|
||||
let version = get_schema_version(&connection)?;
|
||||
|
||||
if version < 1 {
|
||||
connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS accounts (
|
||||
id_account INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
seed TEXT,
|
||||
sk TEXT,
|
||||
ivk TEXT NOT NULL UNIQUE,
|
||||
address TEXT NOT NULL)",
|
||||
NO_PARAMS,
|
||||
)?;
|
||||
|
||||
connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS blocks (
|
||||
height INTEGER PRIMARY KEY,
|
||||
hash BLOB NOT NULL,
|
||||
timestamp INTEGER NOT NULL,
|
||||
sapling_tree BLOB NOT NULL)",
|
||||
NO_PARAMS,
|
||||
)?;
|
||||
|
||||
connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS transactions (
|
||||
id_tx INTEGER PRIMARY KEY,
|
||||
account INTEGER NOT NULL,
|
||||
txid BLOB NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
timestamp INTEGER NOT NULL,
|
||||
value INTEGER NOT NULL,
|
||||
address TEXT,
|
||||
memo TEXT,
|
||||
tx_index INTEGER,
|
||||
CONSTRAINT tx_account UNIQUE (height, tx_index, account))",
|
||||
NO_PARAMS,
|
||||
)?;
|
||||
|
||||
connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS received_notes (
|
||||
id_note INTEGER PRIMARY KEY,
|
||||
account INTEGER NOT NULL,
|
||||
position INTEGER NOT NULL,
|
||||
tx INTEGER NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
output_index INTEGER NOT NULL,
|
||||
diversifier BLOB NOT NULL,
|
||||
value INTEGER NOT NULL,
|
||||
rcm BLOB NOT NULL,
|
||||
nf BLOB NOT NULL UNIQUE,
|
||||
spent INTEGER,
|
||||
CONSTRAINT tx_output UNIQUE (tx, output_index))",
|
||||
NO_PARAMS,
|
||||
)?;
|
||||
|
||||
connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS sapling_witnesses (
|
||||
id_witness INTEGER PRIMARY KEY,
|
||||
note INTEGER NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
witness BLOB NOT NULL,
|
||||
CONSTRAINT witness_height UNIQUE (note, height))",
|
||||
NO_PARAMS,
|
||||
)?;
|
||||
|
||||
connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS diversifiers (
|
||||
account INTEGER PRIMARY KEY NOT NULL,
|
||||
diversifier_index BLOB NOT NULL)",
|
||||
NO_PARAMS,
|
||||
)?;
|
||||
|
||||
connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS taddrs (
|
||||
account INTEGER PRIMARY KEY NOT NULL,
|
||||
sk TEXT NOT NULL,
|
||||
address TEXT NOT NULL)",
|
||||
NO_PARAMS,
|
||||
)?;
|
||||
}
|
||||
|
||||
if version < 2 {
|
||||
connection
|
||||
.execute("ALTER TABLE received_notes ADD excluded BOOL", NO_PARAMS)?;
|
||||
}
|
||||
|
||||
if version < 3 {
|
||||
connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS contacts (
|
||||
account INTEGER NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
address TEXT NOT NULL,
|
||||
PRIMARY KEY (account, address))",
|
||||
NO_PARAMS,
|
||||
)?;
|
||||
}
|
||||
|
||||
if version < 4 {
|
||||
connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS historical_prices (
|
||||
currency TEXT NOT NULL,
|
||||
timestamp INTEGER NOT NULL,
|
||||
price REAL NOT NULL,
|
||||
PRIMARY KEY (currency, timestamp))",
|
||||
NO_PARAMS,
|
||||
)?;
|
||||
}
|
||||
|
||||
update_schema_version(&connection, 4)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
@ -29,6 +29,7 @@ mod taddr;
|
|||
mod transaction;
|
||||
mod pay;
|
||||
mod wallet;
|
||||
mod prices;
|
||||
|
||||
pub use crate::builder::advance_tree;
|
||||
pub use crate::chain::{
|
||||
|
|
|
@ -0,0 +1,55 @@
|
|||
use chrono::NaiveDateTime;
|
||||
use std::collections::HashMap;
|
||||
|
||||
const DAY_SEC: i64 = 24*3600;
|
||||
|
||||
pub async fn retrieve_historical_prices(timestamps: &[i64], currency: &str) -> anyhow::Result<Vec<(i64, f64)>> {
|
||||
if timestamps.is_empty() { return Ok(Vec::new()); }
|
||||
let mut timestamps_map: HashMap<i64, Option<f64>> = HashMap::new();
|
||||
for ts in timestamps {
|
||||
timestamps_map.insert(*ts, None);
|
||||
}
|
||||
let client = reqwest::Client::new();
|
||||
let start = timestamps.first().unwrap();
|
||||
let end = timestamps.last().unwrap() + DAY_SEC;
|
||||
println!("{}", end);
|
||||
let url = "https://api.coingecko.com/api/v3/coins/zcash/market_chart/range";
|
||||
let params = [("from", start.to_string()), ("to", end.to_string()), ("vs_currency", currency.to_string())];
|
||||
let req = client.get(url).query(¶ms);
|
||||
println!("{:?}", req);
|
||||
let res = req.send().await?;
|
||||
let r: serde_json::Value = res.json().await?;
|
||||
let prices = r["prices"].as_array().unwrap();
|
||||
for p in prices.iter() {
|
||||
let p = p.as_array().unwrap();
|
||||
let ts = p[0].as_i64().unwrap() / 1000;
|
||||
let px = p[1].as_f64().unwrap();
|
||||
// rounded to daily
|
||||
let date = NaiveDateTime::from_timestamp(ts, 0).date().and_hms(0, 0, 0);
|
||||
let ts = date.timestamp();
|
||||
println!("{} - {}", date, px);
|
||||
if let Some(None) = timestamps_map.get(&ts) {
|
||||
timestamps_map.insert(ts, Some(px));
|
||||
}
|
||||
}
|
||||
let prices: Vec<_> = timestamps_map.iter().map(|(k, v)| {
|
||||
(*k, v.expect(&format!("missing price for ts {}", *k)))
|
||||
}).collect();
|
||||
Ok(prices)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::DbAdapter;
|
||||
use crate::db::DEFAULT_DB_PATH;
|
||||
use crate::prices::retrieve_historical_prices;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test() {
|
||||
let currency = "EUR";
|
||||
let mut db = DbAdapter::new(DEFAULT_DB_PATH).unwrap();
|
||||
let ts = db.get_missing_prices_timestamp("USD").unwrap();
|
||||
let prices = retrieve_historical_prices(&ts, currency).await.unwrap();
|
||||
db.store_historical_prices(prices, currency).unwrap();
|
||||
}
|
||||
}
|
|
@ -28,6 +28,7 @@ use std::convert::TryFrom;
|
|||
use std::str::FromStr;
|
||||
use crate::db::SpendableNote;
|
||||
use crate::pay::{ColdTxBuilder, Tx};
|
||||
use crate::prices::retrieve_historical_prices;
|
||||
|
||||
const DEFAULT_CHUNK_SIZE: u32 = 100_000;
|
||||
|
||||
|
@ -375,6 +376,15 @@ impl Wallet {
|
|||
}
|
||||
Ok(recipients)
|
||||
}
|
||||
|
||||
pub async fn sync_historical_prices(&mut self, currency: &str) -> anyhow::Result<u32> {
|
||||
let ts = self.db.get_missing_prices_timestamp(currency)?;
|
||||
if !ts.is_empty() {
|
||||
let prices = retrieve_historical_prices(&ts, currency).await?;
|
||||
self.db.store_historical_prices(prices, currency)?;
|
||||
}
|
||||
Ok(ts.len() as u32)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
Loading…
Reference in New Issue