Fetch historical prices

Cargo.toml

@@ -54,6 +54,8 @@ sha2 = "0.9.5"
 lazy_static = "1.4.0"
 rustyline = "8.2.0"
 clap = "3.0.0-beta.2"
+chrono = "0.4.19"
+reqwest = { version = "0.11.4", features = ["json", "rustls-tls"], default-features = false }
 
 # librustzcash synced to 35023ed8ca2fb1061e78fd740b640d4eefcc5edd
 

src/db.rs

@@ -1,5 +1,4 @@
 use crate::chain::{Nf, NfRef};
-use crate::db::migration::{get_schema_version, update_schema_version};
 use crate::taddr::{derive_tkeys, BIP44_PATH};
 use crate::transaction::{Contact, TransactionInfo};
 use crate::{CTree, Witness, NETWORK};
@@ -10,6 +9,7 @@ use zcash_primitives::consensus::{NetworkUpgrade, Parameters};
 use zcash_primitives::merkle_tree::IncrementalWitness;
 use zcash_primitives::sapling::{Diversifier, Node, Note, Rseed, SaplingIvk};
 use zcash_primitives::zip32::{DiversifierIndex, ExtendedFullViewingKey};
+use chrono::NaiveDateTime;
 
 mod migration;
 
@@ -72,112 +72,7 @@ impl DbAdapter {
     }
 
     pub fn init_db(&self) -> anyhow::Result<()> {
-        self.connection.execute(
+        migration::init_db(&self.connection)?;
-            "CREATE TABLE IF NOT EXISTS schema_version (
-            id INTEGER PRIMARY KEY NOT NULL,
-            version INTEGER NOT NULL)",
-            NO_PARAMS,
-        )?;
-
-        let version = get_schema_version(&self.connection)?;
-
-        if version < 1 {
-            self.connection.execute(
-                "CREATE TABLE IF NOT EXISTS accounts (
-            id_account INTEGER PRIMARY KEY,
-            name TEXT NOT NULL,
-            seed TEXT,
-            sk TEXT,
-            ivk TEXT NOT NULL UNIQUE,
-            address TEXT NOT NULL)",
-                NO_PARAMS,
-            )?;
-
-            self.connection.execute(
-                "CREATE TABLE IF NOT EXISTS blocks (
-            height INTEGER PRIMARY KEY,
-            hash BLOB NOT NULL,
-            timestamp INTEGER NOT NULL,
-            sapling_tree BLOB NOT NULL)",
-                NO_PARAMS,
-            )?;
-
-            self.connection.execute(
-                "CREATE TABLE IF NOT EXISTS transactions (
-            id_tx INTEGER PRIMARY KEY,
-            account INTEGER NOT NULL,
-            txid BLOB NOT NULL,
-            height INTEGER NOT NULL,
-            timestamp INTEGER NOT NULL,
-            value INTEGER NOT NULL,
-            address TEXT,
-            memo TEXT,
-            tx_index INTEGER,
-            CONSTRAINT tx_account UNIQUE (height, tx_index, account))",
-                NO_PARAMS,
-            )?;
-
-            self.connection.execute(
-                "CREATE TABLE IF NOT EXISTS received_notes (
-            id_note INTEGER PRIMARY KEY,
-            account INTEGER NOT NULL,
-            position INTEGER NOT NULL,
-            tx INTEGER NOT NULL,
-            height INTEGER NOT NULL,
-            output_index INTEGER NOT NULL,
-            diversifier BLOB NOT NULL,
-            value INTEGER NOT NULL,
-            rcm BLOB NOT NULL,
-            nf BLOB NOT NULL UNIQUE,
-            spent INTEGER,
-            CONSTRAINT tx_output UNIQUE (tx, output_index))",
-                NO_PARAMS,
-            )?;
-
-            self.connection.execute(
-                "CREATE TABLE IF NOT EXISTS sapling_witnesses (
-            id_witness INTEGER PRIMARY KEY,
-            note INTEGER NOT NULL,
-            height INTEGER NOT NULL,
-            witness BLOB NOT NULL,
-            CONSTRAINT witness_height UNIQUE (note, height))",
-                NO_PARAMS,
-            )?;
-
-            self.connection.execute(
-                "CREATE TABLE IF NOT EXISTS diversifiers (
-            account INTEGER PRIMARY KEY NOT NULL,
-            diversifier_index BLOB NOT NULL)",
-                NO_PARAMS,
-            )?;
-
-            self.connection.execute(
-                "CREATE TABLE IF NOT EXISTS taddrs (
-            account INTEGER PRIMARY KEY NOT NULL,
-            sk TEXT NOT NULL,
-            address TEXT NOT NULL)",
-                NO_PARAMS,
-            )?;
-        }
-
-        if version < 2 {
-            self.connection
-                .execute("ALTER TABLE received_notes ADD excluded BOOL", NO_PARAMS)?;
-        }
-
-        if version < 3 {
-            self.connection.execute(
-                "CREATE TABLE IF NOT EXISTS contacts (
-                account INTEGER NOT NULL,
-                name TEXT NOT NULL,
-                address TEXT NOT NULL,
-                PRIMARY KEY (account, address))",
-                NO_PARAMS,
-            )?;
-        }
-
-        update_schema_version(&self.connection, 3)?;
-
         Ok(())
     }
 
@@ -745,6 +640,37 @@ impl DbAdapter {
         }
         Ok(())
     }
+
+    pub fn get_missing_prices_timestamp(&self, currency: &str) -> anyhow::Result<Vec<i64>> {
+        let mut statement = self.connection.prepare(
+            "WITH t AS (SELECT timestamp, timestamp/86400 AS day FROM transactions), p AS (SELECT price, timestamp/86400 AS day FROM historical_prices WHERE currency = ?1) \
+                SELECT t.timestamp FROM t LEFT JOIN p ON t.day = p.day WHERE p.price IS NULL")?;
+        let res = statement.query_map(params![currency], |row| {
+            let timestamp: i64 = row.get(0)?;
+            Ok(timestamp)
+        })?;
+        let mut timestamps: Vec<i64> = vec![];
+        for ts in res {
+            let ts = NaiveDateTime::from_timestamp(ts?, 0);
+            let ts_date = ts.date().and_hms(0, 0, 0); // at midnight
+            timestamps.push(ts_date.timestamp());
+        }
+        timestamps.sort();
+        timestamps.dedup();
+        Ok(timestamps)
+    }
+
+    pub fn store_historical_prices(&mut self, prices: Vec<(i64, f64)>, currency: &str) -> anyhow::Result<()> {
+        let db_transaction = self.connection.transaction()?;
+        {
+            let mut statement = db_transaction.prepare("INSERT INTO historical_prices(timestamp, price, currency) VALUES (?1, ?2, ?3)")?;
+            for (ts, px) in prices {
+                statement.execute(params![ts, px, currency])?;
+            }
+        }
+        db_transaction.commit()?;
+        Ok(())
+    }
 }
 
 #[cfg(test)]

src/db/migration.rs

@@ -19,3 +19,125 @@ pub fn update_schema_version(connection: &Connection, version: u32) -> anyhow::R
     )?;
     Ok(())
 }
+
+pub fn init_db(connection: &Connection) -> anyhow::Result<()> {
+    connection.execute(
+        "CREATE TABLE IF NOT EXISTS schema_version (
+            id INTEGER PRIMARY KEY NOT NULL,
+            version INTEGER NOT NULL)",
+        NO_PARAMS,
+    )?;
+
+    let version = get_schema_version(&connection)?;
+
+    if version < 1 {
+        connection.execute(
+            "CREATE TABLE IF NOT EXISTS accounts (
+            id_account INTEGER PRIMARY KEY,
+            name TEXT NOT NULL,
+            seed TEXT,
+            sk TEXT,
+            ivk TEXT NOT NULL UNIQUE,
+            address TEXT NOT NULL)",
+            NO_PARAMS,
+        )?;
+
+        connection.execute(
+            "CREATE TABLE IF NOT EXISTS blocks (
+            height INTEGER PRIMARY KEY,
+            hash BLOB NOT NULL,
+            timestamp INTEGER NOT NULL,
+            sapling_tree BLOB NOT NULL)",
+            NO_PARAMS,
+        )?;
+
+        connection.execute(
+            "CREATE TABLE IF NOT EXISTS transactions (
+            id_tx INTEGER PRIMARY KEY,
+            account INTEGER NOT NULL,
+            txid BLOB NOT NULL,
+            height INTEGER NOT NULL,
+            timestamp INTEGER NOT NULL,
+            value INTEGER NOT NULL,
+            address TEXT,
+            memo TEXT,
+            tx_index INTEGER,
+            CONSTRAINT tx_account UNIQUE (height, tx_index, account))",
+            NO_PARAMS,
+        )?;
+
+        connection.execute(
+            "CREATE TABLE IF NOT EXISTS received_notes (
+            id_note INTEGER PRIMARY KEY,
+            account INTEGER NOT NULL,
+            position INTEGER NOT NULL,
+            tx INTEGER NOT NULL,
+            height INTEGER NOT NULL,
+            output_index INTEGER NOT NULL,
+            diversifier BLOB NOT NULL,
+            value INTEGER NOT NULL,
+            rcm BLOB NOT NULL,
+            nf BLOB NOT NULL UNIQUE,
+            spent INTEGER,
+            CONSTRAINT tx_output UNIQUE (tx, output_index))",
+            NO_PARAMS,
+        )?;
+
+        connection.execute(
+            "CREATE TABLE IF NOT EXISTS sapling_witnesses (
+            id_witness INTEGER PRIMARY KEY,
+            note INTEGER NOT NULL,
+            height INTEGER NOT NULL,
+            witness BLOB NOT NULL,
+            CONSTRAINT witness_height UNIQUE (note, height))",
+            NO_PARAMS,
+        )?;
+
+        connection.execute(
+            "CREATE TABLE IF NOT EXISTS diversifiers (
+            account INTEGER PRIMARY KEY NOT NULL,
+            diversifier_index BLOB NOT NULL)",
+            NO_PARAMS,
+        )?;
+
+        connection.execute(
+            "CREATE TABLE IF NOT EXISTS taddrs (
+            account INTEGER PRIMARY KEY NOT NULL,
+            sk TEXT NOT NULL,
+            address TEXT NOT NULL)",
+            NO_PARAMS,
+        )?;
+    }
+
+    if version < 2 {
+        connection
+            .execute("ALTER TABLE received_notes ADD excluded BOOL", NO_PARAMS)?;
+    }
+
+    if version < 3 {
+        connection.execute(
+            "CREATE TABLE IF NOT EXISTS contacts (
+                account INTEGER NOT NULL,
+                name TEXT NOT NULL,
+                address TEXT NOT NULL,
+                PRIMARY KEY (account, address))",
+            NO_PARAMS,
+        )?;
+    }
+
+    if version < 4 {
+        connection.execute(
+            "CREATE TABLE IF NOT EXISTS historical_prices (
+                currency TEXT NOT NULL,
+                timestamp INTEGER NOT NULL,
+                price REAL NOT NULL,
+                PRIMARY KEY (currency, timestamp))",
+            NO_PARAMS,
+        )?;
+    }
+
+    update_schema_version(&connection, 4)?;
+
+    Ok(())
+}
+

src/lib.rs

@@ -29,6 +29,7 @@ mod taddr;
 mod transaction;
 mod pay;
 mod wallet;
+mod prices;
 
 pub use crate::builder::advance_tree;
 pub use crate::chain::{

src/prices.rs

@@ -0,0 +1,55 @@
+use chrono::NaiveDateTime;
+use std::collections::HashMap;
+
+const DAY_SEC: i64 = 24*3600;
+
+pub async fn retrieve_historical_prices(timestamps: &[i64], currency: &str) -> anyhow::Result<Vec<(i64, f64)>> {
+    if timestamps.is_empty() { return Ok(Vec::new()); }
+    let mut timestamps_map: HashMap<i64, Option<f64>> = HashMap::new();
+    for ts in timestamps {
+        timestamps_map.insert(*ts, None);
+    }
+    let client = reqwest::Client::new();
+    let start = timestamps.first().unwrap();
+    let end = timestamps.last().unwrap() + DAY_SEC;
+    println!("{}", end);
+    let url = "https://api.coingecko.com/api/v3/coins/zcash/market_chart/range";
+    let params = [("from", start.to_string()), ("to", end.to_string()), ("vs_currency", currency.to_string())];
+    let req = client.get(url).query(&params);
+    println!("{:?}", req);
+    let res = req.send().await?;
+    let r: serde_json::Value = res.json().await?;
+    let prices = r["prices"].as_array().unwrap();
+    for p in prices.iter() {
+        let p = p.as_array().unwrap();
+        let ts = p[0].as_i64().unwrap() / 1000;
+        let px = p[1].as_f64().unwrap();
+        // rounded to daily
+        let date = NaiveDateTime::from_timestamp(ts, 0).date().and_hms(0, 0, 0);
+        let ts = date.timestamp();
+        println!("{} - {}", date, px);
+        if let Some(None) = timestamps_map.get(&ts) {
+            timestamps_map.insert(ts, Some(px));
+        }
+    }
+    let prices: Vec<_> = timestamps_map.iter().map(|(k, v)| {
+        (*k, v.expect(&format!("missing price for ts {}", *k)))
+    }).collect();
+    Ok(prices)
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::DbAdapter;
+    use crate::db::DEFAULT_DB_PATH;
+    use crate::prices::retrieve_historical_prices;
+
+    #[tokio::test]
+    async fn test() {
+        let currency = "EUR";
+        let mut db = DbAdapter::new(DEFAULT_DB_PATH).unwrap();
+        let ts = db.get_missing_prices_timestamp("USD").unwrap();
+        let prices = retrieve_historical_prices(&ts, currency).await.unwrap();
+        db.store_historical_prices(prices, currency).unwrap();
+    }
+}

src/wallet.rs

@@ -28,6 +28,7 @@ use std::convert::TryFrom;
 use std::str::FromStr;
 use crate::db::SpendableNote;
 use crate::pay::{ColdTxBuilder, Tx};
+use crate::prices::retrieve_historical_prices;
 
 const DEFAULT_CHUNK_SIZE: u32 = 100_000;
 
@@ -375,6 +376,15 @@ impl Wallet {
         }
         Ok(recipients)
     }
+
+    pub async fn sync_historical_prices(&mut self, currency: &str) -> anyhow::Result<u32> {
+        let ts = self.db.get_missing_prices_timestamp(currency)?;
+        if !ts.is_empty() {
+            let prices = retrieve_historical_prices(&ts, currency).await?;
+            self.db.store_historical_prices(prices, currency)?;
+        }
+        Ok(ts.len() as u32)
+    }
 }
 
 #[cfg(test)]