diff --git a/Cargo.toml b/Cargo.toml index 12d4702..c4b3cc5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,8 @@ sha2 = "0.9.5" lazy_static = "1.4.0" rustyline = "8.2.0" clap = "3.0.0-beta.2" +chrono = "0.4.19" +reqwest = { version = "0.11.4", features = ["json", "rustls-tls"], default-features = false } # librustzcash synced to 35023ed8ca2fb1061e78fd740b640d4eefcc5edd diff --git a/src/db.rs b/src/db.rs index ebda27c..3f600e9 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,5 +1,4 @@ use crate::chain::{Nf, NfRef}; -use crate::db::migration::{get_schema_version, update_schema_version}; use crate::taddr::{derive_tkeys, BIP44_PATH}; use crate::transaction::{Contact, TransactionInfo}; use crate::{CTree, Witness, NETWORK}; @@ -10,6 +9,7 @@ use zcash_primitives::consensus::{NetworkUpgrade, Parameters}; use zcash_primitives::merkle_tree::IncrementalWitness; use zcash_primitives::sapling::{Diversifier, Node, Note, Rseed, SaplingIvk}; use zcash_primitives::zip32::{DiversifierIndex, ExtendedFullViewingKey}; +use chrono::NaiveDateTime; mod migration; @@ -72,112 +72,7 @@ impl DbAdapter { } pub fn init_db(&self) -> anyhow::Result<()> { - self.connection.execute( - "CREATE TABLE IF NOT EXISTS schema_version ( - id INTEGER PRIMARY KEY NOT NULL, - version INTEGER NOT NULL)", - NO_PARAMS, - )?; - - let version = get_schema_version(&self.connection)?; - - if version < 1 { - self.connection.execute( - "CREATE TABLE IF NOT EXISTS accounts ( - id_account INTEGER PRIMARY KEY, - name TEXT NOT NULL, - seed TEXT, - sk TEXT, - ivk TEXT NOT NULL UNIQUE, - address TEXT NOT NULL)", - NO_PARAMS, - )?; - - self.connection.execute( - "CREATE TABLE IF NOT EXISTS blocks ( - height INTEGER PRIMARY KEY, - hash BLOB NOT NULL, - timestamp INTEGER NOT NULL, - sapling_tree BLOB NOT NULL)", - NO_PARAMS, - )?; - - self.connection.execute( - "CREATE TABLE IF NOT EXISTS transactions ( - id_tx INTEGER PRIMARY KEY, - account INTEGER NOT NULL, - txid BLOB NOT NULL, - height INTEGER NOT NULL, - timestamp INTEGER NOT NULL, - value INTEGER NOT NULL, - address TEXT, - memo TEXT, - tx_index INTEGER, - CONSTRAINT tx_account UNIQUE (height, tx_index, account))", - NO_PARAMS, - )?; - - self.connection.execute( - "CREATE TABLE IF NOT EXISTS received_notes ( - id_note INTEGER PRIMARY KEY, - account INTEGER NOT NULL, - position INTEGER NOT NULL, - tx INTEGER NOT NULL, - height INTEGER NOT NULL, - output_index INTEGER NOT NULL, - diversifier BLOB NOT NULL, - value INTEGER NOT NULL, - rcm BLOB NOT NULL, - nf BLOB NOT NULL UNIQUE, - spent INTEGER, - CONSTRAINT tx_output UNIQUE (tx, output_index))", - NO_PARAMS, - )?; - - self.connection.execute( - "CREATE TABLE IF NOT EXISTS sapling_witnesses ( - id_witness INTEGER PRIMARY KEY, - note INTEGER NOT NULL, - height INTEGER NOT NULL, - witness BLOB NOT NULL, - CONSTRAINT witness_height UNIQUE (note, height))", - NO_PARAMS, - )?; - - self.connection.execute( - "CREATE TABLE IF NOT EXISTS diversifiers ( - account INTEGER PRIMARY KEY NOT NULL, - diversifier_index BLOB NOT NULL)", - NO_PARAMS, - )?; - - self.connection.execute( - "CREATE TABLE IF NOT EXISTS taddrs ( - account INTEGER PRIMARY KEY NOT NULL, - sk TEXT NOT NULL, - address TEXT NOT NULL)", - NO_PARAMS, - )?; - } - - if version < 2 { - self.connection - .execute("ALTER TABLE received_notes ADD excluded BOOL", NO_PARAMS)?; - } - - if version < 3 { - self.connection.execute( - "CREATE TABLE IF NOT EXISTS contacts ( - account INTEGER NOT NULL, - name TEXT NOT NULL, - address TEXT NOT NULL, - PRIMARY KEY (account, address))", - NO_PARAMS, - )?; - } - - update_schema_version(&self.connection, 3)?; - + migration::init_db(&self.connection)?; Ok(()) } @@ -745,6 +640,37 @@ impl DbAdapter { } Ok(()) } + + pub fn get_missing_prices_timestamp(&self, currency: &str) -> anyhow::Result> { + let mut statement = self.connection.prepare( + "WITH t AS (SELECT timestamp, timestamp/86400 AS day FROM transactions), p AS (SELECT price, timestamp/86400 AS day FROM historical_prices WHERE currency = ?1) \ + SELECT t.timestamp FROM t LEFT JOIN p ON t.day = p.day WHERE p.price IS NULL")?; + let res = statement.query_map(params![currency], |row| { + let timestamp: i64 = row.get(0)?; + Ok(timestamp) + })?; + let mut timestamps: Vec = vec![]; + for ts in res { + let ts = NaiveDateTime::from_timestamp(ts?, 0); + let ts_date = ts.date().and_hms(0, 0, 0); // at midnight + timestamps.push(ts_date.timestamp()); + } + timestamps.sort(); + timestamps.dedup(); + Ok(timestamps) + } + + pub fn store_historical_prices(&mut self, prices: Vec<(i64, f64)>, currency: &str) -> anyhow::Result<()> { + let db_transaction = self.connection.transaction()?; + { + let mut statement = db_transaction.prepare("INSERT INTO historical_prices(timestamp, price, currency) VALUES (?1, ?2, ?3)")?; + for (ts, px) in prices { + statement.execute(params![ts, px, currency])?; + } + } + db_transaction.commit()?; + Ok(()) + } } #[cfg(test)] diff --git a/src/db/migration.rs b/src/db/migration.rs index f02c896..6114fac 100644 --- a/src/db/migration.rs +++ b/src/db/migration.rs @@ -19,3 +19,125 @@ pub fn update_schema_version(connection: &Connection, version: u32) -> anyhow::R )?; Ok(()) } + +pub fn init_db(connection: &Connection) -> anyhow::Result<()> { + connection.execute( + "CREATE TABLE IF NOT EXISTS schema_version ( + id INTEGER PRIMARY KEY NOT NULL, + version INTEGER NOT NULL)", + NO_PARAMS, + )?; + + let version = get_schema_version(&connection)?; + + if version < 1 { + connection.execute( + "CREATE TABLE IF NOT EXISTS accounts ( + id_account INTEGER PRIMARY KEY, + name TEXT NOT NULL, + seed TEXT, + sk TEXT, + ivk TEXT NOT NULL UNIQUE, + address TEXT NOT NULL)", + NO_PARAMS, + )?; + + connection.execute( + "CREATE TABLE IF NOT EXISTS blocks ( + height INTEGER PRIMARY KEY, + hash BLOB NOT NULL, + timestamp INTEGER NOT NULL, + sapling_tree BLOB NOT NULL)", + NO_PARAMS, + )?; + + connection.execute( + "CREATE TABLE IF NOT EXISTS transactions ( + id_tx INTEGER PRIMARY KEY, + account INTEGER NOT NULL, + txid BLOB NOT NULL, + height INTEGER NOT NULL, + timestamp INTEGER NOT NULL, + value INTEGER NOT NULL, + address TEXT, + memo TEXT, + tx_index INTEGER, + CONSTRAINT tx_account UNIQUE (height, tx_index, account))", + NO_PARAMS, + )?; + + connection.execute( + "CREATE TABLE IF NOT EXISTS received_notes ( + id_note INTEGER PRIMARY KEY, + account INTEGER NOT NULL, + position INTEGER NOT NULL, + tx INTEGER NOT NULL, + height INTEGER NOT NULL, + output_index INTEGER NOT NULL, + diversifier BLOB NOT NULL, + value INTEGER NOT NULL, + rcm BLOB NOT NULL, + nf BLOB NOT NULL UNIQUE, + spent INTEGER, + CONSTRAINT tx_output UNIQUE (tx, output_index))", + NO_PARAMS, + )?; + + connection.execute( + "CREATE TABLE IF NOT EXISTS sapling_witnesses ( + id_witness INTEGER PRIMARY KEY, + note INTEGER NOT NULL, + height INTEGER NOT NULL, + witness BLOB NOT NULL, + CONSTRAINT witness_height UNIQUE (note, height))", + NO_PARAMS, + )?; + + connection.execute( + "CREATE TABLE IF NOT EXISTS diversifiers ( + account INTEGER PRIMARY KEY NOT NULL, + diversifier_index BLOB NOT NULL)", + NO_PARAMS, + )?; + + connection.execute( + "CREATE TABLE IF NOT EXISTS taddrs ( + account INTEGER PRIMARY KEY NOT NULL, + sk TEXT NOT NULL, + address TEXT NOT NULL)", + NO_PARAMS, + )?; + } + + if version < 2 { + connection + .execute("ALTER TABLE received_notes ADD excluded BOOL", NO_PARAMS)?; + } + + if version < 3 { + connection.execute( + "CREATE TABLE IF NOT EXISTS contacts ( + account INTEGER NOT NULL, + name TEXT NOT NULL, + address TEXT NOT NULL, + PRIMARY KEY (account, address))", + NO_PARAMS, + )?; + } + + if version < 4 { + connection.execute( + "CREATE TABLE IF NOT EXISTS historical_prices ( + currency TEXT NOT NULL, + timestamp INTEGER NOT NULL, + price REAL NOT NULL, + PRIMARY KEY (currency, timestamp))", + NO_PARAMS, + )?; + } + + update_schema_version(&connection, 4)?; + + Ok(()) +} + diff --git a/src/lib.rs b/src/lib.rs index d0e726d..476c219 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,6 +29,7 @@ mod taddr; mod transaction; mod pay; mod wallet; +mod prices; pub use crate::builder::advance_tree; pub use crate::chain::{ diff --git a/src/prices.rs b/src/prices.rs new file mode 100644 index 0000000..7af600b --- /dev/null +++ b/src/prices.rs @@ -0,0 +1,55 @@ +use chrono::NaiveDateTime; +use std::collections::HashMap; + +const DAY_SEC: i64 = 24*3600; + +pub async fn retrieve_historical_prices(timestamps: &[i64], currency: &str) -> anyhow::Result> { + if timestamps.is_empty() { return Ok(Vec::new()); } + let mut timestamps_map: HashMap> = HashMap::new(); + for ts in timestamps { + timestamps_map.insert(*ts, None); + } + let client = reqwest::Client::new(); + let start = timestamps.first().unwrap(); + let end = timestamps.last().unwrap() + DAY_SEC; + println!("{}", end); + let url = "https://api.coingecko.com/api/v3/coins/zcash/market_chart/range"; + let params = [("from", start.to_string()), ("to", end.to_string()), ("vs_currency", currency.to_string())]; + let req = client.get(url).query(¶ms); + println!("{:?}", req); + let res = req.send().await?; + let r: serde_json::Value = res.json().await?; + let prices = r["prices"].as_array().unwrap(); + for p in prices.iter() { + let p = p.as_array().unwrap(); + let ts = p[0].as_i64().unwrap() / 1000; + let px = p[1].as_f64().unwrap(); + // rounded to daily + let date = NaiveDateTime::from_timestamp(ts, 0).date().and_hms(0, 0, 0); + let ts = date.timestamp(); + println!("{} - {}", date, px); + if let Some(None) = timestamps_map.get(&ts) { + timestamps_map.insert(ts, Some(px)); + } + } + let prices: Vec<_> = timestamps_map.iter().map(|(k, v)| { + (*k, v.expect(&format!("missing price for ts {}", *k))) + }).collect(); + Ok(prices) +} + +#[cfg(test)] +mod tests { + use crate::DbAdapter; + use crate::db::DEFAULT_DB_PATH; + use crate::prices::retrieve_historical_prices; + + #[tokio::test] + async fn test() { + let currency = "EUR"; + let mut db = DbAdapter::new(DEFAULT_DB_PATH).unwrap(); + let ts = db.get_missing_prices_timestamp("USD").unwrap(); + let prices = retrieve_historical_prices(&ts, currency).await.unwrap(); + db.store_historical_prices(prices, currency).unwrap(); + } +} \ No newline at end of file diff --git a/src/wallet.rs b/src/wallet.rs index 178de2c..aa46b9e 100644 --- a/src/wallet.rs +++ b/src/wallet.rs @@ -28,6 +28,7 @@ use std::convert::TryFrom; use std::str::FromStr; use crate::db::SpendableNote; use crate::pay::{ColdTxBuilder, Tx}; +use crate::prices::retrieve_historical_prices; const DEFAULT_CHUNK_SIZE: u32 = 100_000; @@ -375,6 +376,15 @@ impl Wallet { } Ok(recipients) } + + pub async fn sync_historical_prices(&mut self, currency: &str) -> anyhow::Result { + let ts = self.db.get_missing_prices_timestamp(currency)?; + if !ts.is_empty() { + let prices = retrieve_historical_prices(&ts, currency).await?; + self.db.store_historical_prices(prices, currency)?; + } + Ok(ts.len() as u32) + } } #[cfg(test)]