diff --git a/src/db.rs b/src/db.rs index 3b09675..76c95c5 100644 --- a/src/db.rs +++ b/src/db.rs @@ -7,6 +7,11 @@ use zcash_primitives::merkle_tree::IncrementalWitness; use zcash_primitives::sapling::{Diversifier, Node, Note, Rseed}; use zcash_primitives::zip32::{ExtendedFullViewingKey, DiversifierIndex}; use crate::taddr::{derive_tkeys, BIP44_PATH}; +use crate::db::migration::{get_schema_version, update_schema_version}; +use crate::transaction::TransactionInfo; +use zcash_primitives::memo::Memo; + +mod migration; #[allow(dead_code)] pub const DEFAULT_DB_PATH: &str = "zec.db"; @@ -40,6 +45,16 @@ impl DbAdapter { } pub fn init_db(&self) -> anyhow::Result<()> { + self.connection.execute( + "CREATE TABLE IF NOT EXISTS schema_version ( + id INTEGER PRIMARY KEY NOT NULL, + version INTEGER NOT NULL)", + NO_PARAMS, + )?; + + let version = get_schema_version(&self.connection)?; + if version == 1 { return Ok(()); } + self.connection.execute( "CREATE TABLE IF NOT EXISTS accounts ( id_account INTEGER PRIMARY KEY, @@ -64,11 +79,14 @@ impl DbAdapter { "CREATE TABLE IF NOT EXISTS transactions ( id_tx INTEGER PRIMARY KEY, account INTEGER NOT NULL, - txid BLOB NOT NULL UNIQUE, + txid BLOB NOT NULL, height INTEGER NOT NULL, timestamp INTEGER NOT NULL, value INTEGER NOT NULL, - tx_index INTEGER)", + address TEXT, + memo TEXT, + tx_index INTEGER, + CONSTRAINT tx_account UNIQUE (height, tx_index, account))", NO_PARAMS, )?; @@ -114,6 +132,8 @@ impl DbAdapter { NO_PARAMS, )?; + update_schema_version(&self.connection, 1)?; + Ok(()) } @@ -166,6 +186,18 @@ impl DbAdapter { Ok(()) } + pub fn get_txhash(&self, id_tx: u32) -> anyhow::Result<(u32, u32, Vec)> { + let (account, height, tx_hash) = self.connection.query_row( + "SELECT account, height, txid FROM transactions WHERE id_tx = ?1", params![id_tx], + |row| { + let account: u32 = row.get(0)?; + let height: u32 = row.get(1)?; + let tx_hash: Vec = row.get(2)?; + Ok((account, height, tx_hash)) + })?; + Ok((account, height, tx_hash)) + } + pub fn store_block( &self, height: u32, @@ -247,6 +279,19 @@ impl DbAdapter { Ok(()) } + pub fn store_tx_metadata(&self, id_tx: u32, tx_info: &TransactionInfo) -> anyhow::Result<()> { + let memo = match &tx_info.memo { + Memo::Empty => "".to_string(), + Memo::Text(text) => text.to_string(), + Memo::Future(_) => "Unrecognized".to_string(), + Memo::Arbitrary(_) => "Unrecognized".to_string(), + }; + self.connection.execute( + "UPDATE transactions SET address = ?1, memo = ?2 WHERE id_tx = ?3", params![tx_info.address, &memo, id_tx] + )?; + Ok(()) + } + pub fn add_value(&self, id_tx: u32, value: i64) -> anyhow::Result<()> { self.connection.execute( "UPDATE transactions SET value = value + ?2 WHERE id_tx = ?1", @@ -367,10 +412,14 @@ impl DbAdapter { Ok(nfs) } - pub fn get_nullifier_amounts(&self, account: u32) -> anyhow::Result, u64>> { + pub fn get_nullifier_amounts(&self, account: u32, unspent_only: bool) -> anyhow::Result, u64>> { + let mut sql = "SELECT value, nf FROM received_notes WHERE account = ?1".to_string(); + if unspent_only { + sql += "AND (spent IS NULL OR spent = 0)"; + } let mut statement = self .connection - .prepare("SELECT value, nf FROM received_notes WHERE account = ?1 AND (spent IS NULL OR spent = 0)")?; + .prepare(&sql)?; let nfs_res = statement.query_map(params![account], |row| { let amount: i64 = row.get(0)?; let nf: Vec = row.get(1)?; diff --git a/src/db/migration.rs b/src/db/migration.rs new file mode 100644 index 0000000..bfbdbbb --- /dev/null +++ b/src/db/migration.rs @@ -0,0 +1,14 @@ +use rusqlite::{NO_PARAMS, Connection, OptionalExtension, params}; + +pub fn get_schema_version(connection: &Connection) -> anyhow::Result { + let version: Option = connection.query_row("SELECT version FROM schema_version WHERE id = 1", NO_PARAMS, + |row| row.get(0)).optional()?; + Ok(version.unwrap_or(0)) +} + +pub fn update_schema_version(connection: &Connection, version: u32) -> anyhow::Result<()> { + connection.execute("INSERT INTO schema_version(id, version) VALUES (1, ?1) \ + ON CONFLICT (id) DO UPDATE SET version = excluded.version", params![version])?; + Ok(()) +} + diff --git a/src/lib.rs b/src/lib.rs index d3af4ba..933d123 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,8 +6,9 @@ pub mod lw_rpc; pub const NETWORK: Network = Network::MainNetwork; // Mainnet -pub const LWD_URL: &str = "https://mainnet.lightwalletd.com:9067"; +// pub const LWD_URL: &str = "https://mainnet.lightwalletd.com:9067"; // pub const LWD_URL: &str = "https://lwdv3.zecwallet.co"; +pub const LWD_URL: &str = "http://lwd.hanh.me:9067"; // Testnet // pub const LWD_URL: &str = "https://testnet.lightwalletd.com:9067"; @@ -23,6 +24,7 @@ mod mempool; mod print; mod scan; mod taddr; +mod transaction; mod wallet; pub use crate::builder::advance_tree; diff --git a/src/main/warp_cli.rs b/src/main/warp_cli.rs index 91ee378..d103a05 100644 --- a/src/main/warp_cli.rs +++ b/src/main/warp_cli.rs @@ -3,6 +3,7 @@ use rand::rngs::OsRng; use rand::RngCore; use sync::{DbAdapter, Wallet, ChainError, Witness, print_witness2, LWD_URL}; use rusqlite::NO_PARAMS; +use zcash_client_backend::data_api::wallet::ANCHOR_OFFSET; const DB_NAME: &str = "zec.db"; @@ -24,7 +25,7 @@ async fn test() -> anyhow::Result<()> { }; let wallet = Wallet::new(DB_NAME, LWD_URL); wallet.new_account_with_key("test", &seed).unwrap(); - let res = wallet.sync(progress).await; + let res = wallet.sync(true, ANCHOR_OFFSET, progress).await; if let Err(err) = res { if let Some(_) = err.downcast_ref::() { println!("REORG"); @@ -33,11 +34,11 @@ async fn test() -> anyhow::Result<()> { panic!(err); } } - let tx_id = wallet - .send_payment(1, &address, 50000, u64::max_value(), 2, move |progress| { println!("{}", progress.cur()); }) - .await - .unwrap(); - println!("TXID = {}", tx_id); + // let tx_id = wallet + // .send_payment(1, &address, 50000, u64::max_value(), 2, move |progress| { println!("{}", progress.cur()); }) + // .await + // .unwrap(); + // println!("TXID = {}", tx_id); Ok(()) } diff --git a/src/mempool.rs b/src/mempool.rs index 54945e3..ff74cf4 100644 --- a/src/mempool.rs +++ b/src/mempool.rs @@ -85,7 +85,7 @@ impl MemPool { fn clear(&mut self) -> anyhow::Result<()> { let db = DbAdapter::new(&self.db_path)?; self.height = BlockHeight::from_u32(0); - self.nfs = db.get_nullifier_amounts(self.account)?; + self.nfs = db.get_nullifier_amounts(self.account, true)?; self.transactions.clear(); self.balance = 0; Ok(()) diff --git a/src/scan.rs b/src/scan.rs index e251709..9fd6c68 100644 --- a/src/scan.rs +++ b/src/scan.rs @@ -19,6 +19,7 @@ use zcash_primitives::sapling::Node; use zcash_primitives::zip32::ExtendedFullViewingKey; use std::panic; use std::collections::HashMap; +use crate::transaction::retrieve_tx_info; pub async fn scan_all(fvks: &[ExtendedFullViewingKey]) -> anyhow::Result<()> { let fvks: HashMap<_, _> = fvks.iter().enumerate().map(|(i, fvk)| @@ -72,6 +73,7 @@ pub type ProgressCallback = Arc>; pub async fn sync_async( chunk_size: u32, + get_tx: bool, db_path: &str, target_height_offset: u32, progress_callback: ProgressCallback, @@ -103,8 +105,9 @@ pub async fn sync_async( let (downloader_tx, mut download_rx) = mpsc::channel::>(2); let (processor_tx, mut processor_rx) = mpsc::channel::(1); + let ld_url2 = ld_url.clone(); let downloader = tokio::spawn(async move { - let mut client = connect_lightwalletd(&ld_url).await?; + let mut client = connect_lightwalletd(&ld_url2).await?; while let Some(range) = download_rx.recv().await { log::info!("+ {:?}", range); let blocks = download_chain(&mut client, range.start, range.end, prev_hash).await?; @@ -128,6 +131,7 @@ pub async fn sync_async( let processor = tokio::spawn(async move { let db = DbAdapter::new(&db_path)?; let mut nfs = db.get_nullifiers()?; + let mut new_tx_ids: Vec = vec![]; let (mut tree, mut witnesses) = db.get_tree()?; let mut bp = BlockProcessor::new(&tree, &witnesses); @@ -168,6 +172,7 @@ pub async fn sync_async( b.compact_block.time, n.tx_index as u32, )?; + new_tx_ids.push(id_tx); let id_note = db.store_received_note( &ReceivedNote { account: n.account, @@ -205,6 +210,7 @@ pub async fn sync_async( b.compact_block.time, tx_index as u32, )?; + new_tx_ids.push(id_tx); db.add_value(id_tx, -(note_value as i64))?; } } @@ -258,6 +264,10 @@ pub async fn sync_async( } } + if get_tx && !new_tx_ids.is_empty() { + retrieve_tx_info(&new_tx_ids, &ld_url, &db_path).await?; + } + let callback = progress_callback.lock().await; callback(end_height); log::debug!("Witnesses {}", witnesses.len()); diff --git a/src/transaction.rs b/src/transaction.rs new file mode 100644 index 0000000..d45b826 --- /dev/null +++ b/src/transaction.rs @@ -0,0 +1,104 @@ +use crate::{CompactTxStreamerClient, TxFilter, DbAdapter, NETWORK, connect_lightwalletd}; +use tonic::transport::Channel; +use tonic::Request; +use zcash_primitives::transaction::Transaction; +use zcash_primitives::sapling::note_encryption::{try_sapling_note_decryption, try_sapling_output_recovery}; +use zcash_primitives::consensus::{BlockHeight, Parameters}; +use zcash_client_backend::encoding::{decode_extended_full_viewing_key, encode_payment_address}; +use zcash_primitives::memo::{MemoBytes, Memo}; +use std::convert::TryFrom; +use std::collections::HashMap; + +#[derive(Debug)] +pub struct TransactionInfo { + pub address: String, + pub memo: Memo, + amount: i64, + pub fee: u64, +} + +pub async fn decode_transaction(client: &mut CompactTxStreamerClient, + nfs: &HashMap, u64>, + fvk: &str, + tx_hash: &[u8], + height: u32) -> anyhow::Result { + let fvk = decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &fvk)?.unwrap(); + let ivk = fvk.fvk.vk.ivk(); + let ovk = fvk.fvk.ovk; + + let tx_filter = TxFilter { + block: None, + index: 0, + hash: tx_hash.to_vec(), // only hash is supported + }; + let raw_tx = client.get_transaction(Request::new(tx_filter)).await?.into_inner(); + let tx = Transaction::read(&*raw_tx.data)?; + + let height = BlockHeight::from_u32(height); + let mut amount = 0i64; + let mut address = String::new(); + for spend in tx.shielded_spends.iter() { + let nf = spend.nullifier.to_vec(); + if let Some(&v) = nfs.get(&nf) { + amount -= v as i64; + } + } + + let mut tx_memo = MemoBytes::empty(); + for output in tx.shielded_outputs.iter() { + if let Some((note, pa, memo)) = try_sapling_note_decryption(&NETWORK, height, &ivk, output) { + amount += note.value as i64; // change or self transfer + if address.is_empty() { + address = encode_payment_address(NETWORK.hrp_sapling_payment_address(), &pa); + tx_memo = memo; + } + } + else if let Some((_note, pa, memo)) = try_sapling_output_recovery(&NETWORK, height, &ovk, &output) { + address = encode_payment_address(NETWORK.hrp_sapling_payment_address(), &pa); + tx_memo = memo; + } + } + + let fee = u64::from(tx.value_balance); + + let tx_info = TransactionInfo { + address, + memo: Memo::try_from(tx_memo)?, + amount, + fee + }; + + Ok(tx_info) +} + +pub async fn retrieve_tx_info(tx_ids: &[u32], ld_url: &str, db_path: &str) -> anyhow::Result<()> { + let mut client = connect_lightwalletd(ld_url).await?; + let db = DbAdapter::new(db_path)?; + for &id_tx in tx_ids.iter() { + let (account, height, tx_hash) = db.get_txhash(id_tx)?; + let nfs = db.get_nullifier_amounts(account, false)?; + let fvk = db.get_ivk(account)?; + let tx_info = decode_transaction(&mut client, &nfs, &fvk, &tx_hash, height).await?; + db.store_tx_metadata(id_tx, &tx_info)?; + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::{connect_lightwalletd, LWD_URL, DbAdapter}; + use crate::transaction::decode_transaction; + + #[tokio::test] + async fn test_decode_transaction() { + let tx_hash = hex::decode("b47da170329dc311b98892eac23e83025f8bb3ce10bb07535698c91fb37e1e54").unwrap(); + let mut client = connect_lightwalletd(LWD_URL).await.unwrap(); + let db = DbAdapter::new("./zec.db").unwrap(); + let account = 1; + let nfs = db.get_nullifier_amounts(account, false).unwrap(); + let fvk = db.get_ivk(account).unwrap(); + let tx_info = decode_transaction(&mut client, &nfs, &fvk, &tx_hash, 1313212).await.unwrap(); + println!("{:?}", tx_info); + } +} diff --git a/src/wallet.rs b/src/wallet.rs index d3fcc3e..65a724b 100644 --- a/src/wallet.rs +++ b/src/wallet.rs @@ -20,6 +20,8 @@ use zcash_primitives::transaction::components::Amount; use zcash_primitives::zip32::ExtendedFullViewingKey; use zcash_proofs::prover::LocalTxProver; use crate::taddr::{get_taddr_balance, shield_taddr}; +use zcash_primitives::memo::Memo; +use std::str::FromStr; const DEFAULT_CHUNK_SIZE: u32 = 100_000; @@ -102,13 +104,14 @@ impl Wallet { } async fn scan_async( + get_tx: bool, db_path: &str, chunk_size: u32, target_height_offset: u32, progress_callback: ProgressCallback, ld_url: &str ) -> anyhow::Result<()> { - crate::scan::sync_async(chunk_size, db_path, target_height_offset, progress_callback, ld_url).await + crate::scan::sync_async(chunk_size, get_tx, db_path, target_height_offset, progress_callback, ld_url).await } pub async fn get_latest_height(&self) -> anyhow::Result { @@ -119,21 +122,25 @@ impl Wallet { // Not a method in order to avoid locking the instance pub async fn sync_ex( + get_tx: bool, + anchor_offset: u32, db_path: &str, progress_callback: impl Fn(u32) + Send + 'static, ld_url: &str ) -> anyhow::Result<()> { let cb = Arc::new(Mutex::new(progress_callback)); - Self::scan_async(db_path, DEFAULT_CHUNK_SIZE, 10, cb.clone(), ld_url).await?; - Self::scan_async(db_path, DEFAULT_CHUNK_SIZE, 0, cb.clone(), ld_url).await?; + Self::scan_async(get_tx, db_path, DEFAULT_CHUNK_SIZE, anchor_offset, cb.clone(), ld_url).await?; + Self::scan_async(get_tx, db_path, DEFAULT_CHUNK_SIZE, 0, cb.clone(), ld_url).await?; Ok(()) } pub async fn sync( &self, + get_tx: bool, + anchor_offset: u32, progress_callback: impl Fn(u32) + Send + 'static, ) -> anyhow::Result<()> { - Self::sync_ex(&self.db_path, progress_callback, &self.ld_url).await + Self::sync_ex(get_tx, anchor_offset, &self.db_path, progress_callback, &self.ld_url).await } pub async fn skip_to_last_height(&self) -> anyhow::Result<()> { @@ -164,6 +171,7 @@ impl Wallet { account: u32, to_address: &str, amount: u64, + memo: &str, max_amount_per_note: u64, anchor_offset: u32, progress_callback: impl Fn(Progress) + Send + 'static, @@ -232,7 +240,7 @@ impl Wallet { let note_amount = target_amount.min(max_amount_per_note); match &to_addr { RecipientAddress::Shielded(pa) => { - builder.add_sapling_output(Some(ovk), pa.clone(), note_amount, None) + builder.add_sapling_output(Some(ovk), pa.clone(), note_amount, Some(Memo::from_str(memo)?.into())) } RecipientAddress::Transparent(t_address) => { builder.add_transparent_output(&t_address, note_amount)