diff --git a/Cargo.toml b/Cargo.toml index d1fd338..e4e67a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,19 +37,31 @@ rayon = "1.5.1" byteorder = "1.4.3" tiny-bip39 = "0.8" rand = "0.8.4" -rusqlite = "^0.25.3" +rusqlite = { version = "^0.24", features = ["bundled"] } + +#[dependencies.zcash_client_backend] +#git = "https://github.com/zcash/librustzcash.git" +#rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb" +# +#[dependencies.zcash_primitives] +#git = "https://github.com/zcash/librustzcash.git" +#rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb" +# +#[dependencies.zcash_proofs] +#git = "https://github.com/zcash/librustzcash.git" +#rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb" [dependencies.zcash_client_backend] -git = "https://github.com/zcash/librustzcash.git" -rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb" +path = "/home/hanh/projects/librustzcash/zcash_client_backend" [dependencies.zcash_primitives] -git = "https://github.com/zcash/librustzcash.git" -rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb" +path = "/home/hanh/projects/librustzcash/zcash_primitives" [dependencies.zcash_proofs] -git = "https://github.com/zcash/librustzcash.git" -rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb" +path = "/home/hanh/projects/librustzcash/zcash_proofs" + +[dependencies.zcash_params] +path = "../zcash_params" [build-dependencies] tonic-build = "0.4.2" diff --git a/src/builder.rs b/src/builder.rs index f5f9ec9..382f5bc 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -17,13 +17,13 @@ struct CTreeBuilder { prev_tree: CTree, next_tree: CTree, start: usize, + total_len: usize, depth: usize, offset: Option, } impl Builder for CTreeBuilder { fn collect(&mut self, commitments: &[Node], _context: &()) -> usize { - // assert!(!commitments.is_empty() || self.left.is_some() || self.right.is_some()); assert!(self.right.is_none() || self.left.is_some()); // R can't be set without L let offset: Option; @@ -37,26 +37,30 @@ impl Builder for CTreeBuilder { m = commitments.len(); }; - let n = if self.depth == 0 { - if m % 2 == 0 { - self.next_tree.left = Some(*Self::get(commitments, m - 2, &offset)); - self.next_tree.right = Some(*Self::get(commitments, m - 1, &offset)); - m - 2 - } else { - self.next_tree.left = Some(*Self::get(commitments, m - 1, &offset)); - self.next_tree.right = None; - m - 1 + let n = + if self.total_len > 0 { + if self.depth == 0 { + if m % 2 == 0 { + self.next_tree.left = Some(*Self::get(commitments, m - 2, &offset)); + self.next_tree.right = Some(*Self::get(commitments, m - 1, &offset)); + m - 2 + } else { + self.next_tree.left = Some(*Self::get(commitments, m - 1, &offset)); + self.next_tree.right = None; + m - 1 + } + } else { + if m % 2 == 0 { + self.next_tree.parents.push(None); + m + } else { + let last_node = Self::get(commitments, m - 1, &offset); + self.next_tree.parents.push(Some(*last_node)); + m - 1 + } + } } - } else { - if m % 2 == 0 { - self.next_tree.parents.push(None); - m - } else { - let last_node = Self::get(commitments, m - 1, &offset); - self.next_tree.parents.push(Some(*last_node)); - m - 1 - } - }; + else { 0 }; assert_eq!(n % 2, 0); self.offset = offset; @@ -93,12 +97,16 @@ impl Builder for CTreeBuilder { } fn finalize(self, _context: &()) -> CTree { - self.next_tree + if self.total_len > 0 { + self.next_tree + } else { + self.prev_tree + } } } impl CTreeBuilder { - fn new(prev_tree: CTree) -> CTreeBuilder { + fn new(prev_tree: CTree, len: usize) -> CTreeBuilder { let start = prev_tree.get_position(); CTreeBuilder { left: prev_tree.left, @@ -106,6 +114,7 @@ impl CTreeBuilder { prev_tree, next_tree: CTree::new(), start, + total_len: len, depth: 0, offset: None, } @@ -133,25 +142,13 @@ impl CTreeBuilder { Self::get_opt(commitments, index, offset).unwrap() } - fn adjusted_start(&self, prev: &Option, depth: usize) -> usize { - if depth != 0 && prev.is_some() { + fn adjusted_start(&self, prev: &Option, _depth: usize) -> usize { + if prev.is_some() { self.start - 1 } else { self.start } } - - fn clone_trimmed(&self, mut depth: usize) -> CTree { - if depth == 0 { - return CTree::new() - } - let mut tree = self.next_tree.clone(); - while depth > 0 && depth <= tree.parents.len() && tree.parents[depth - 1].is_none() { - depth -= 1; - } - tree.parents.truncate(depth); - tree - } } fn combine_level(commitments: &mut [Node], offset: Option, n: usize, depth: usize) -> usize { @@ -218,6 +215,12 @@ impl Builder for WitnessBuilder { } } + // println!("D {}", depth); + // println!("O {:?}", offset.map(|r| hex::encode(r.repr))); + // println!("R {:?}", right.map(|r| hex::encode(r.repr))); + // for c in commitments.iter() { + // println!("{}", hex::encode(c.repr)); + // } let p1 = self.p + 1; let has_p1 = p1 >= context.adjusted_start(&right, depth) && p1 < context.start + commitments.len(); if has_p1 { @@ -244,32 +247,41 @@ impl Builder for WitnessBuilder { } fn finalize(mut self, context: &CTreeBuilder) -> Witness { - let tree = &self.witness.tree; - let mut num_filled = self.witness.filled.len(); - - if self.witness.position + 1 == context.next_tree.get_position() { + if context.total_len == 0 { self.witness.cursor = CTree::new(); - } - else { - let mut depth = 0; - loop { - let is_none = if depth == 0 { // check if this level is occupied - tree.right.is_none() - } else { - depth > tree.parents.len() || tree.parents[depth - 1].is_none() - }; - if is_none { - if num_filled > 0 { - num_filled -= 1; // we filled it - } else { - break - } - } - depth += 1; - // loop terminates because we are eventually going to run out of ancestors and filled - } - self.witness.cursor = context.clone_trimmed(depth - 1); + let mut final_position = context.prev_tree.get_position() as u32; + let mut witness_position = self.witness.tree.get_position() as u32; + assert_ne!(witness_position, 0); + witness_position = witness_position - 1; + + // look for first not equal bit in MSB order + final_position = final_position.reverse_bits(); + witness_position = witness_position.reverse_bits(); + let mut bit: i32 = 31; + // reverse bits because it is easier to do in LSB + // it should not underflow because these numbers are not equal + while bit >= 0 { + if final_position & 1 != witness_position & 1 { + break; + } + final_position >>= 1; + witness_position >>= 1; + bit -= 1; + } + // look for the first bit set in final_position after + final_position >>= 1; + bit -= 1; + while bit >= 0 { + if final_position & 1 == 1 { + break; + } + final_position >>= 1; + bit -= 1; + } + if bit >= 0 { + self.witness.cursor = context.prev_tree.clone_trimmed(bit as usize) + } } self.witness } @@ -281,10 +293,7 @@ pub fn advance_tree( prev_witnesses: &[Witness], mut commitments: &mut [Node], ) -> (CTree, Vec) { - if commitments.is_empty() { - return (prev_tree, prev_witnesses.to_vec()); - } - let mut builder = CTreeBuilder::new(prev_tree); + let mut builder = CTreeBuilder::new(prev_tree, commitments.len()); let mut witness_builders: Vec<_> = prev_witnesses .iter() .map(|witness| WitnessBuilder::new(&builder, witness.clone(), commitments.len())) @@ -295,11 +304,11 @@ pub fn advance_tree( b.collect(commitments, &builder); } let nn = combine_level(commitments, builder.offset, n, builder.depth); - commitments = &mut commitments[0..nn]; builder.up(); for b in witness_builders.iter_mut() { b.up(); } + commitments = &mut commitments[0..nn]; } let witnesses = witness_builders @@ -318,24 +327,35 @@ mod tests { use zcash_primitives::merkle_tree::{CommitmentTree, IncrementalWitness}; use zcash_primitives::sapling::Node; use crate::chain::DecryptedNote; + use crate::print::{print_witness, print_witness2, print_tree, print_ctree}; #[test] fn test_advance_tree() { - const NUM_NODES: usize = 1000; - const NUM_CHUNKS: usize = 50; - const WITNESS_PERCENT: f64 = 1.0; // percentage of notes that are ours - let witness_freq = (100.0 / WITNESS_PERCENT) as usize; + for num_nodes in 1..=10 { + for num_chunks in 1..=10 { + test_advance_tree_helper(num_nodes, num_chunks, 100.0); + } + } + + test_advance_tree_helper(100, 50, 1.0); + // test_advance_tree_helper(2, 10, 100.0); + // test_advance_tree_helper(1, 40, 100.0); + // test_advance_tree_helper(10, 2, 100.0); + } + + fn test_advance_tree_helper(num_nodes: usize, num_chunks: usize, witness_percent: f64) { + let witness_freq = (100.0 / witness_percent) as usize; let mut tree1: CommitmentTree = CommitmentTree::empty(); let mut tree2 = CTree::new(); let mut ws: Vec> = vec![]; let mut ws2: Vec = vec![]; - for i in 0..NUM_CHUNKS { + for i in 0..num_chunks { println!("{}", i); let mut nodes: Vec<_> = vec![]; - for j in 0..NUM_NODES { + for j in 0..num_nodes { let mut bb = [0u8; 32]; - let v = i * NUM_NODES + j; + let v = i * num_nodes + j; bb[0..8].copy_from_slice(&v.to_be_bytes()); let node = Node::new(bb); tree1.append(node).unwrap(); @@ -343,7 +363,7 @@ mod tests { w.append(node).unwrap(); } if v % witness_freq == 0 { - // if v == 499 { + // if v == 0 { let w = IncrementalWitness::from_tree(&tree1); ws.push(w); ws2.push(Witness::new(v, 0, None)); @@ -356,6 +376,13 @@ mod tests { ws2 = new_witnesses; } + // Push an empty block + // It will calculate the tail of the tree + // This step is required at the end of a series of chunks + let (new_tree, new_witnesses) = advance_tree(tree2, &ws2, &mut []); + tree2 = new_tree; + ws2 = new_witnesses; + // check final state let mut bb1: Vec = vec![]; tree1.write(&mut bb1).unwrap(); @@ -364,6 +391,11 @@ mod tests { tree2.write(&mut bb2).unwrap(); let equal = bb1.as_slice() == bb2.as_slice(); + if !equal { + println!("FAILED FINAL STATE"); + print_tree(&tree1); + print_ctree(&tree2); + } println!("# witnesses = {}", ws.len()); @@ -378,6 +410,16 @@ mod tests { if bb1.as_slice() != bb2.as_slice() { failed_index = Some(i); + println!("FAILED AT {}", i); + if let Some(ref c) = w1.cursor { + print_tree(c); + } + else { println!("NONE"); } + + println!("GOOD"); + print_witness(&w1); + println!("BAD"); + print_witness2(&w2); } } diff --git a/src/chain.rs b/src/chain.rs index 5356929..ce6f206 100644 --- a/src/chain.rs +++ b/src/chain.rs @@ -40,7 +40,7 @@ pub async fn download_chain( ) -> anyhow::Result> { let mut cbs: Vec = Vec::new(); let mut s = start_height + 1; - while s < end_height { + while s <= end_height { let e = (s + MAX_CHUNK - 1).min(end_height); let range = BlockRange { start: Some(BlockId { @@ -68,10 +68,14 @@ pub struct DecryptNode { fvks: Vec, } +#[derive(Eq, Hash, PartialEq)] +pub struct Nf(pub [u8; 32]); + pub struct DecryptedBlock { pub height: u32, pub notes: Vec, pub count_outputs: u32, + pub spends: Vec, } #[derive(Clone)] @@ -79,7 +83,7 @@ pub struct DecryptedNote { pub ivk: ExtendedFullViewingKey, pub note: Note, pub pa: PaymentAddress, - pub position: usize, + pub position_in_block: usize, pub height: u32, pub txid: Vec, @@ -90,8 +94,15 @@ pub struct DecryptedNote { fn decrypt_notes(block: &CompactBlock, fvks: &[ExtendedFullViewingKey]) -> DecryptedBlock { let height = BlockHeight::from_u32(block.height as u32); let mut count_outputs = 0u32; + let mut spends: Vec = vec![]; let mut notes: Vec = vec![]; for (tx_index, vtx) in block.vtx.iter().enumerate() { + for cs in vtx.spends.iter() { + let mut nf = [0u8; 32]; + nf.copy_from_slice(&cs.nf); + spends.push(Nf(nf)); + } + for (output_index, co) in vtx.outputs.iter().enumerate() { let mut cmu = [0u8; 32]; cmu.copy_from_slice(&co.cmu); @@ -113,7 +124,7 @@ fn decrypt_notes(block: &CompactBlock, fvks: &[ExtendedFullViewingKey]) -> Decry ivk: fvk.clone(), note, pa, - position: count_outputs as usize, + position_in_block: count_outputs as usize, height: block.height as u32, tx_index, txid: vtx.hash.clone(), @@ -126,6 +137,7 @@ fn decrypt_notes(block: &CompactBlock, fvks: &[ExtendedFullViewingKey]) -> Decry } DecryptedBlock { height: block.height as u32, + spends, notes, count_outputs, } @@ -160,6 +172,15 @@ async fn get_tree_state(client: &mut CompactTxStreamerClient, height: u rep.tree } +pub async fn send_transaction(client: &mut CompactTxStreamerClient, raw_tx: &[u8], height: u32) -> anyhow::Result { + let raw_tx = RawTransaction { + data: raw_tx.to_vec(), + height: height as u64 + }; + let rep = client.send_transaction(Request::new(raw_tx)).await?.into_inner(); + Ok(rep.error_message) +} + /* Using the IncrementalWitness */ #[allow(dead_code)] fn calculate_tree_state_v1( @@ -187,7 +208,7 @@ fn calculate_tree_state_v1( w.append(node).unwrap(); } if let Some(nn) = n { - if i == nn.position { + if i == nn.position_in_block { let w = IncrementalWitness::from_tree(&tree_state); witnesses.push(w); n = notes.next(); @@ -226,7 +247,7 @@ pub fn calculate_tree_state_v2(cbs: &[CompactBlock], blocks: &[DecryptedBlock]) nodes.push(node); if let Some(nn) = n { - if i == nn.position { + if i == nn.position_in_block { positions.push(p); n = notes.next(); } diff --git a/src/commitment.rs b/src/commitment.rs index 267bd6a..73916ec 100644 --- a/src/commitment.rs +++ b/src/commitment.rs @@ -174,6 +174,15 @@ impl CTree { p } + pub fn clone_trimmed(&self, depth: usize) -> CTree { + let mut tree = self.clone(); + tree.parents.truncate(depth); + if let Some(None) = tree.parents.last() { // Remove trailing None + tree.parents.truncate(depth - 1); + } + tree + } + pub fn to_commitment_tree(&self) -> CommitmentTree { let mut bb: Vec = vec![]; self.write(&mut bb).unwrap(); diff --git a/src/db.rs b/src/db.rs index 0bb7291..09c0848 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,5 +1,11 @@ -use rusqlite::{Connection, params, OptionalExtension}; +use rusqlite::{Connection, params, OptionalExtension, NO_PARAMS}; use crate::{Witness, CTree}; +use zcash_primitives::sapling::{Note, Diversifier, Rseed, Node}; +use zcash_primitives::zip32::ExtendedFullViewingKey; +use zcash_primitives::merkle_tree::IncrementalWitness; +use std::collections::HashMap; +use crate::chain::Nf; +use zcash_primitives::consensus::{Parameters, NetworkUpgrade}; pub struct DbAdapter { connection: Connection, @@ -12,9 +18,13 @@ pub struct ReceivedNote { pub value: u64, pub rcm: Vec, pub nf: Vec, - pub is_change: bool, - pub memo: Vec, - pub spent: bool, + pub spent: Option, +} + +pub struct SpendableNote { + pub note: Note, + pub diversifier: Diversifier, + pub witness: IncrementalWitness, } impl DbAdapter { @@ -26,16 +36,22 @@ impl DbAdapter { } pub fn init_db(&self) -> anyhow::Result<()> { + self.connection.execute("CREATE TABLE IF NOT EXISTS accounts ( + id_account INTEGER PRIMARY KEY, + sk TEXT NOT NULL UNIQUE, + ivk TEXT NOT NULL, + address TEXT NOT NULL)", NO_PARAMS)?; + self.connection.execute("CREATE TABLE IF NOT EXISTS blocks ( height INTEGER PRIMARY KEY, hash BLOB NOT NULL, - sapling_tree BLOB NOT NULL)", [])?; + sapling_tree BLOB NOT NULL)", NO_PARAMS)?; self.connection.execute("CREATE TABLE IF NOT EXISTS transactions ( id_tx INTEGER PRIMARY KEY, txid BLOB NOT NULL UNIQUE, height INTEGER, - tx_index INTEGER)", [])?; + tx_index INTEGER)", NO_PARAMS)?; self.connection.execute("CREATE TABLE IF NOT EXISTS received_notes ( id_note INTEGER PRIMARY KEY, @@ -47,24 +63,25 @@ impl DbAdapter { value INTEGER NOT NULL, rcm BLOB NOT NULL, nf BLOB NOT NULL UNIQUE, - is_change INTEGER NOT NULL, - memo BLOB, spent INTEGER, - FOREIGN KEY (tx) REFERENCES transactions(id_tx), - FOREIGN KEY (spent) REFERENCES transactions(id_tx), - CONSTRAINT tx_output UNIQUE (tx, output_index))", [])?; + CONSTRAINT tx_output UNIQUE (tx, output_index))", NO_PARAMS)?; self.connection.execute("CREATE TABLE IF NOT EXISTS sapling_witnesses ( id_witness INTEGER PRIMARY KEY, note INTEGER NOT NULL, height INTEGER NOT NULL, witness BLOB NOT NULL, - FOREIGN KEY (note) REFERENCES received_notes(id_note), - CONSTRAINT witness_height UNIQUE (note, height))", [])?; + CONSTRAINT witness_height UNIQUE (note, height))", NO_PARAMS)?; Ok(()) } + pub fn store_account(&self, sk: &str, ivk: &str, address: &str) -> anyhow::Result<()> { + self.connection.execute("INSERT INTO accounts(sk, ivk, address) VALUES (?1, ?2, ?3) + ON CONFLICT DO NOTHING", params![sk, ivk, address])?; + Ok(()) + } + pub fn trim_to_height(&mut self, height: u32) -> anyhow::Result<()> { let tx = self.connection.transaction()?; tx.execute("DELETE FROM blocks WHERE height >= ?1", params![height])?; @@ -77,53 +94,70 @@ impl DbAdapter { } pub fn store_block(&self, height: u32, hash: &[u8], tree: &CTree) -> anyhow::Result<()> { + log::info!("+block"); let mut bb: Vec = vec![]; tree.write(&mut bb)?; self.connection.execute("INSERT INTO blocks(height, hash, sapling_tree) VALUES (?1, ?2, ?3) ON CONFLICT DO NOTHING", params![height, hash, &bb])?; + log::info!("-block"); Ok(()) } pub fn store_transaction(&self, txid: &[u8], height: u32, tx_index: u32) -> anyhow::Result { + log::info!("+transaction"); self.connection.execute("INSERT INTO transactions(txid, height, tx_index) VALUES (?1, ?2, ?3) ON CONFLICT DO NOTHING", params![txid, height, tx_index])?; let id_tx: u32 = self.connection.query_row("SELECT id_tx FROM transactions WHERE txid = ?1", params![txid], |row| row.get(0))?; + log::info!("-transaction {}", id_tx); Ok(id_tx) } pub fn store_received_note(&self, note: &ReceivedNote, id_tx: u32, position: usize) -> anyhow::Result { - self.connection.execute("INSERT INTO received_notes(tx, height, position, output_index, diversifier, value, rcm, nf, is_change, memo, spent) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) - ON CONFLICT DO NOTHING", params![id_tx, note.height, position, note.output_index, note.diversifier, note.value, note.rcm, note.nf, note.is_change, note.memo, note.spent])?; + log::info!("+received_note {}", id_tx); + self.connection.execute("INSERT INTO received_notes(tx, height, position, output_index, diversifier, value, rcm, nf, spent) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) + ON CONFLICT DO NOTHING", params![id_tx, note.height, position as u32, note.output_index, note.diversifier, note.value as i64, note.rcm, note.nf, note.spent])?; let id_note: u32 = self.connection.query_row("SELECT id_note FROM received_notes WHERE tx = ?1 AND output_index = ?2", params![id_tx, note.output_index], |row| row.get(0))?; + log::info!("-received_note"); Ok(id_note) } pub fn store_witnesses(&self, witness: &Witness, height: u32, id_note: u32) -> anyhow::Result<()> { + log::info!("+witnesses"); let mut bb: Vec = vec![]; witness.write(&mut bb)?; - println!("{} {}", height, id_note); self.connection.execute("INSERT INTO sapling_witnesses(note, height, witness) VALUES (?1, ?2, ?3) ON CONFLICT DO NOTHING", params![id_note, height, bb])?; + log::info!("-witnesses"); Ok(()) } pub fn get_balance(&self) -> anyhow::Result { - let balance: u64 = self.connection.query_row("SELECT SUM(value) FROM received_notes WHERE spent = 0", [], |row| row.get(0))?; - Ok(balance) + let balance: Option = self.connection.query_row("SELECT SUM(value) FROM received_notes WHERE spent IS NULL", NO_PARAMS, |row| row.get(0))?; + Ok(balance.unwrap_or(0) as u64) } - pub fn get_last_height(&self) -> anyhow::Result> { - let height: Option = self.connection.query_row("SELECT MAX(height) FROM blocks", [], |row| row.get(0)).optional()?; + pub fn get_last_sync_height(&self) -> anyhow::Result> { + let height: Option = self.connection.query_row("SELECT MAX(height) FROM blocks", NO_PARAMS, |row| row.get(0))?; + Ok(height) + } + + pub fn get_db_height(&self) -> anyhow::Result { + let height: u32 = self.get_last_sync_height()?.unwrap_or_else(|| { + crate::NETWORK + .activation_height(NetworkUpgrade::Sapling) + .unwrap() + .into() + }); Ok(height) } pub fn get_tree(&self) -> anyhow::Result<(CTree, Vec)> { let res = self.connection.query_row( "SELECT height, sapling_tree FROM blocks WHERE height = (SELECT MAX(height) FROM blocks)", - [], |row| { + NO_PARAMS, |row| { let height: u32 = row.get(0)?; let tree: Vec = row.get(1)?; Ok((height, tree)) @@ -148,6 +182,89 @@ impl DbAdapter { None => (CTree::new(), vec![]) }) } + + pub fn get_nullifiers(&self) -> anyhow::Result> { + let mut statement = self.connection.prepare( + "SELECT id_note, nf FROM received_notes WHERE spent = 0")?; + let nfs_res = statement.query_map(NO_PARAMS, |row| { + let id_note: u32 = row.get(0)?; + let nf_vec: Vec = row.get(1)?; + let mut nf = [0u8; 32]; + nf.clone_from_slice(&nf_vec); + Ok((id_note, nf)) + })?; + let mut nfs: HashMap = HashMap::new(); + for n in nfs_res { + let n = n?; + nfs.insert(Nf(n.1), n.0); + } + + Ok(nfs) + } + + pub fn get_spendable_notes(&self, anchor_height: u32, fvk: &ExtendedFullViewingKey) -> anyhow::Result> { + let mut statement = self.connection.prepare( + "SELECT diversifier, value, rcm, witness FROM received_notes r, sapling_witnesses w WHERE spent IS NULL + AND w.height = ( + SELECT MAX(height) FROM sapling_witnesses WHERE height <= ?1 + ) AND r.id_note = w.note")?; + let notes = statement.query_map(params![anchor_height], |row| { + let diversifier: Vec = row.get(0)?; + let value: i64 = row.get(1)?; + let rcm: Vec = row.get(2)?; + let witness: Vec = row.get(3)?; + + let mut diversifer_bytes = [0u8; 11]; + diversifer_bytes.copy_from_slice(&diversifier); + let diversifier = Diversifier(diversifer_bytes); + let mut rcm_bytes = [0u8; 32]; + rcm_bytes.copy_from_slice(&rcm); + let rcm = jubjub::Fr::from_bytes(&rcm_bytes).unwrap(); + let rseed = Rseed::BeforeZip212(rcm); + let witness = IncrementalWitness::::read(&*witness).unwrap(); + + let pa = fvk.fvk.vk.to_payment_address(diversifier).unwrap(); + let note = pa.create_note(value as u64, rseed).unwrap(); + Ok(SpendableNote { + note, + diversifier, + witness + }) + })?; + let mut spendable_notes: Vec = vec![]; + for n in notes { + spendable_notes.push(n?); + } + + Ok(spendable_notes) + } + + pub fn mark_spent(&self, id: u32, height: u32) -> anyhow::Result<()> { + log::info!("+mark_spent"); + self.connection.execute("UPDATE received_notes SET spent = ?1 WHERE id_note = ?2", params![height, id])?; + log::info!("-mark_spent"); + Ok(()) + } + + pub fn get_sk(&self, account: u32) -> anyhow::Result { + log::info!("+get_sk"); + let ivk = self.connection.query_row("SELECT sk FROM accounts WHERE id_account = ?1", params![account], |row | { + let ivk: String = row.get(0)?; + Ok(ivk) + })?; + log::info!("-get_sk"); + Ok(ivk) + } + + pub fn get_ivk(&self, account: u32) -> anyhow::Result { + log::info!("+get_ivk"); + let ivk = self.connection.query_row("SELECT ivk FROM accounts WHERE id_account = ?1", params![account], |row | { + let ivk: String = row.get(0)?; + Ok(ivk) + })?; + log::info!("-get_ivk"); + Ok(ivk) + } } #[cfg(test)] @@ -172,9 +289,7 @@ mod tests { value: 0, rcm: vec![], nf: vec![], - is_change: false, - memo: vec![], - spent: false + spent: None }, id_tx, 5).unwrap(); let witness = Witness { position: 10, diff --git a/src/lib.rs b/src/lib.rs index c4872ca..3aff65c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,7 +3,7 @@ use zcash_primitives::consensus::Network; #[path = "generated/cash.z.wallet.sdk.rpc.rs"] pub mod lw_rpc; -pub const NETWORK: Network = Network::MainNetwork; +pub const NETWORK: Network = Network::TestNetwork; mod builder; mod chain; @@ -12,6 +12,7 @@ mod scan; mod key; mod db; mod wallet; +mod print; pub use crate::builder::advance_tree; pub use crate::chain::{ @@ -24,3 +25,4 @@ pub use crate::lw_rpc::*; pub use crate::scan::{scan_all, sync_async, latest_height}; pub use crate::key::{get_secret_key, get_address, get_viewing_key}; pub use crate::db::DbAdapter; +pub use crate::wallet::{Wallet, DEFAULT_ACCOUNT}; diff --git a/src/main/warp_cli.rs b/src/main/warp_cli.rs index 1b3b1ac..f0530d5 100644 --- a/src/main/warp_cli.rs +++ b/src/main/warp_cli.rs @@ -1,40 +1,59 @@ -use sync::{sync_async, DbAdapter}; +use sync::{DbAdapter, Wallet, DEFAULT_ACCOUNT}; use bip39::{Language, Mnemonic}; use rand::rngs::OsRng; use rand::RngCore; const DB_NAME: &str = "zec.db"; +fn init() { + let db = DbAdapter::new(DB_NAME).unwrap(); + db.init_db().unwrap(); +} + #[tokio::main] #[allow(dead_code)] async fn test() -> anyhow::Result<()> { dotenv::dotenv().unwrap(); env_logger::init(); - let ivk = dotenv::var("IVK").unwrap(); - { - let db = DbAdapter::new(DB_NAME)?; - db.init_db()?; - } - sync_async(&ivk, 50000, DB_NAME, |height| { + let seed = dotenv::var("SEED").unwrap(); + let address = dotenv::var("ADDRESS").unwrap(); + let progress = |height| { log::info!("Height = {}", height); - }).await?; + }; + let wallet = Wallet::new(DB_NAME); + wallet.new_account_with_seed(&seed).unwrap(); + wallet.sync(DEFAULT_ACCOUNT, progress).await.unwrap(); + let tx_id = wallet.send_payment(DEFAULT_ACCOUNT, &address, 1000).await.unwrap(); + println!("TXID = {}", tx_id); Ok(()) } +#[allow(dead_code)] +fn test_make_wallet() { + let mut entropy = [0u8; 32]; + OsRng.fill_bytes(&mut entropy); + let mnemonic = Mnemonic::from_entropy(&entropy, Language::English).unwrap(); + let phrase = mnemonic.phrase(); + println!("Seed Phrase: {}", phrase); +} + #[allow(dead_code)] fn test_rewind() { let mut db = DbAdapter::new(DB_NAME).unwrap(); - db.trim_to_height(1_250_000).unwrap(); + db.trim_to_height(1460000).unwrap(); +} + +fn test_get_balance() { + let db = DbAdapter::new(DB_NAME).unwrap(); + let balance = db.get_balance().unwrap(); + println!("Balance = {}", (balance as f64) / 100_000_000.0); } fn main() { + init(); // test_rewind(); test().unwrap(); - // let mut entropy = [0u8; 32]; - // OsRng.fill_bytes(&mut entropy); - // let mnemonic = Mnemonic::from_entropy(&entropy, Language::English).unwrap(); - // let phrase = mnemonic.phrase(); - // println!("Seed Phrase: {}", phrase); + test_get_balance(); } diff --git a/src/print.rs b/src/print.rs new file mode 100644 index 0000000..90958a2 --- /dev/null +++ b/src/print.rs @@ -0,0 +1,56 @@ +use zcash_primitives::sapling::Node; +use zcash_primitives::merkle_tree::{CommitmentTree, IncrementalWitness}; +use crate::{Witness, CTree}; + +#[allow(dead_code)] +pub fn print_node(n: &Node) { + println!("{:?}", hex::encode(n.repr)); +} + +#[allow(dead_code)] +pub fn print_tree(t: &CommitmentTree) { + println!("{:?}", t.left.map(|n| hex::encode(n.repr))); + println!("{:?}", t.right.map(|n| hex::encode(n.repr))); + for p in t.parents.iter() { + println!("{:?}", p.map(|n| hex::encode(n.repr))); + } +} + +#[allow(dead_code)] +pub fn print_witness(w: &IncrementalWitness) { + println!("Tree"); + print_tree(&w.tree); + println!("Filled"); + for n in w.filled.iter() { + print_node(n); + } + println!("Cursor"); + w.cursor.as_ref().map(|c| print_tree(c)); +} + +pub fn print_ctree(t: &CTree) { + println!("Tree"); + println!("{:?}", t.left.map(|n| hex::encode(n.repr))); + println!("{:?}", t.right.map(|n| hex::encode(n.repr))); + for p in t.parents.iter() { + println!("{:?}", p.map(|n| hex::encode(n.repr))); + } +} + +#[allow(dead_code)] +pub fn print_witness2(w: &Witness) { + let t = &w.tree; + print_ctree(t); + println!("Filled"); + for n in w.filled.iter() { + print_node(n); + } + let t = &w.cursor; + println!("Cursor"); + println!("{:?}", t.left.map(|n| hex::encode(n.repr))); + println!("{:?}", t.right.map(|n| hex::encode(n.repr))); + for p in t.parents.iter() { + println!("{:?}", p.map(|n| hex::encode(n.repr))); + } + +} \ No newline at end of file diff --git a/src/scan.rs b/src/scan.rs index 9d2c92e..f5547e3 100644 --- a/src/scan.rs +++ b/src/scan.rs @@ -10,6 +10,9 @@ use log::info; use crate::db::{DbAdapter, ReceivedNote}; use ff::PrimeField; use zcash_primitives::zip32::ExtendedFullViewingKey; +use crate::chain::Nf; +use std::sync::Arc; +use tokio::sync::Mutex; pub async fn scan_all(fvks: &[ExtendedFullViewingKey]) -> anyhow::Result<()> { let decrypter = DecryptNode::new(fvks.to_vec()); @@ -35,7 +38,7 @@ pub async fn scan_all(fvks: &[ExtendedFullViewingKey]) -> anyhow::Result<()> { info!("# Witnesses {}", witnesses.len()); for w in witnesses.iter() { let mut bb: Vec = vec![]; - w.write(&mut bb).unwrap(); + w.write(&mut bb)?; log::debug!("{}", hex::encode(&bb)); } @@ -45,6 +48,10 @@ pub async fn scan_all(fvks: &[ExtendedFullViewingKey]) -> anyhow::Result<()> { } struct Blocks(Vec); +struct BlockMetadata { + height: u32, + hash: [u8; 32], +} impl std::fmt::Debug for Blocks { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -52,69 +59,76 @@ impl std::fmt::Debug for Blocks { } } -fn get_db_height(db_path: &str) -> anyhow::Result { - let db = DbAdapter::new(db_path).unwrap(); - let height: u32 = db.get_last_height()?.unwrap_or_else(|| { - crate::NETWORK - .activation_height(NetworkUpgrade::Sapling) - .unwrap() - .into() - }); - Ok(height) -} +pub type ProgressCallback = Arc>; -pub async fn sync_async(ivk: &str, chunk_size: u32, db_path: &str, progress_callback: impl Fn(u32) + Send + 'static) -> anyhow::Result<()> { +pub async fn sync_async(ivk: &str, chunk_size: u32, db_path: &str, target_height_offset: u32, progress_callback: ProgressCallback) -> anyhow::Result<()> { let db_path = db_path.to_string(); let fvk = decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &ivk) - .unwrap() - .unwrap(); + ?.ok_or_else(|| anyhow::anyhow!("Invalid key"))?; let decrypter = DecryptNode::new(vec![fvk]); let mut client = connect_lightwalletd().await?; - let start_height = get_db_height(&db_path)?; + let start_height = { + let db = DbAdapter::new(&db_path)?; + db.get_db_height()? + }; let end_height = get_latest_height(&mut client).await?; + let end_height = (end_height - target_height_offset).max(start_height); let (downloader_tx, mut download_rx) = mpsc::channel::>(2); - let (processor_tx, mut processor_rx) = mpsc::channel::(2); + let (processor_tx, mut processor_rx) = mpsc::channel::(1); let (completed_tx, mut completed_rx) = mpsc::channel::<()>(1); - tokio::spawn(async move { - let mut client = connect_lightwalletd().await.unwrap(); + let downloader = tokio::spawn(async move { + let mut client = connect_lightwalletd().await?; while let Some(range) = download_rx.recv().await { - log::info!("{:?}", range); - let blocks = download_chain(&mut client, range.start, range.end).await.unwrap(); + log::info!("+ {:?}", range); + let blocks = download_chain(&mut client, range.start, range.end).await?; + log::info!("- {:?}", range); let b = Blocks(blocks); - processor_tx.send(b).await.unwrap(); + processor_tx.send(b).await?; } log::info!("download completed"); drop(processor_tx); - // Ok::<_, anyhow::Error>(()) + Ok::<_, anyhow::Error>(()) }); - tokio::spawn(async move { - let db = DbAdapter::new(&db_path).unwrap(); - let (mut tree, mut witnesses) = db.get_tree().unwrap(); - let mut pos = tree.get_position(); - // let mut tree = CTree::new(); - // let mut witnesses: Vec = vec![]; + let proc_callback = progress_callback.clone(); + + let processor = tokio::spawn(async move { + let db = DbAdapter::new(&db_path)?; + let mut nfs = db.get_nullifiers()?; + + let (mut tree, mut witnesses) = db.get_tree()?; + let mut absolute_position_at_block_start = tree.get_position(); + let mut last_block: Option = None; while let Some(blocks) = processor_rx.recv().await { log::info!("{:?}", blocks); if blocks.0.is_empty() { continue } let dec_blocks = decrypter.decrypt_blocks(&blocks.0); for b in dec_blocks.iter() { + for nf in b.spends.iter() { + if let Some(&id) = nfs.get(nf) { + println!("NF FOUND {} {}", id, b.height); + db.mark_spent(id, b.height)?; + } + } if !b.notes.is_empty() { log::info!("{} {}", b.height, b.notes.len()); + for nf in b.spends.iter() { + println!("{}", hex::encode(nf.0)); + } } for n in b.notes.iter() { - let p = pos + n.position; + let p = absolute_position_at_block_start + n.position_in_block; let note = &n.note; - let id_tx = db.store_transaction(&n.txid, n.height, n.tx_index as u32).unwrap(); + let id_tx = db.store_transaction(&n.txid, n.height, n.tx_index as u32)?; let rcm = note.rcm().to_repr(); - let nf = note.nf(&n.ivk.fvk.vk, n.position as u64); + let nf = note.nf(&n.ivk.fvk.vk, p as u64); let id_note = db.store_received_note(&ReceivedNote { height: n.height, @@ -123,15 +137,14 @@ pub async fn sync_async(ivk: &str, chunk_size: u32, db_path: &str, progress_call value: note.value, rcm: rcm.to_vec(), nf: nf.0.to_vec(), - is_change: false, // TODO: it's change the ovk matches too - memo: vec![], - spent: false - }, id_tx, n.position).unwrap(); + spent: None + }, id_tx, n.position_in_block)?; + nfs.insert(Nf(nf.0), id_note); let w = Witness::new(p as usize, id_note, Some(n.clone())); witnesses.push(w); } - pos += b.count_outputs as usize; + absolute_position_at_block_start += b.count_outputs as usize; } let mut nodes: Vec = vec![]; @@ -150,19 +163,37 @@ pub async fn sync_async(ivk: &str, chunk_size: u32, db_path: &str, progress_call tree = new_tree; witnesses = new_witnesses; - let last_block = blocks.0.last().unwrap(); - let last_height = last_block.height as u32; - db.store_block(last_height, &last_block.hash, &tree).unwrap(); - for w in witnesses.iter() { - db.store_witnesses(w, last_height, w.id_note).unwrap(); + if let Some(block) = blocks.0.last() { + let mut hash = [0u8; 32]; + hash.copy_from_slice(&block.hash); + last_block = Some(BlockMetadata { + height: block.height as u32, + hash, + }); } - - progress_callback(blocks.0[0].height as u32); + let callback = proc_callback.lock().await; + callback(blocks.0[0].height as u32); } - progress_callback(end_height); + // Finalize scan + let (new_tree, new_witnesses) = advance_tree(tree, &witnesses, &mut []); + tree = new_tree; + witnesses = new_witnesses; + + if let Some(last_block) = last_block { + let last_height = last_block.height; + db.store_block(last_height, &last_block.hash, &tree)?; + for w in witnesses.iter() { + db.store_witnesses(w, last_height, w.id_note)?; + } + } + + // let callback = progress_callback.lock()?; + // callback(end_height); log::info!("Witnesses {}", witnesses.len()); drop(completed_tx); + + Ok::<_, anyhow::Error>(()) }); let mut height = start_height; @@ -181,10 +212,20 @@ pub async fn sync_async(ivk: &str, chunk_size: u32, db_path: &str, progress_call completed_rx.recv().await; log::info!("completed"); + let res = tokio::try_join!(downloader, processor); + match res { + Ok((a, b)) => { + if let Err(err) = a { log::info!("Downloader error = {}", err) } + if let Err(err) = b { log::info!("Processor error = {}", err) } + }, + Err(err) => log::info!("Sync error = {}", err), + } + Ok(()) } -pub async fn latest_height() -> u32 { - let mut client = connect_lightwalletd().await.unwrap(); - get_latest_height(&mut client).await.unwrap() +pub async fn latest_height() -> anyhow::Result { + let mut client = connect_lightwalletd().await?; + let height = get_latest_height(&mut client).await?; + Ok(height) } diff --git a/src/wallet.rs b/src/wallet.rs new file mode 100644 index 0000000..d963a38 --- /dev/null +++ b/src/wallet.rs @@ -0,0 +1,150 @@ +use crate::{NETWORK, get_latest_height, connect_lightwalletd, DbAdapter, get_secret_key, get_viewing_key, get_address}; +use zcash_client_backend::address::RecipientAddress; +use zcash_primitives::transaction::components::Amount; +use zcash_primitives::transaction::builder::Builder; +use zcash_client_backend::encoding::decode_extended_spending_key; +use zcash_primitives::consensus::{Parameters, BlockHeight, BranchId}; +use zcash_primitives::zip32::ExtendedFullViewingKey; +use zcash_client_backend::data_api::wallet::ANCHOR_OFFSET; +use rand::prelude::SliceRandom; +use rand::rngs::OsRng; +use zcash_proofs::prover::LocalTxProver; +use crate::chain::send_transaction; +use zcash_params::{SPEND_PARAMS, OUTPUT_PARAMS}; +use std::sync::Arc; +use tokio::sync::Mutex; +use crate::scan::ProgressCallback; +use zcash_primitives::transaction::components::amount::DEFAULT_FEE; + +pub const DEFAULT_ACCOUNT: u32 = 1; +const DEFAULT_CHUNK_SIZE: u32 = 100_000; + +pub struct Wallet { + db_path: String, + db: DbAdapter, + prover: LocalTxProver, +} + +impl Wallet { + pub fn new(db_path: &str) -> Wallet { + let prover = LocalTxProver::from_bytes(SPEND_PARAMS, OUTPUT_PARAMS); + let db = DbAdapter::new(db_path).unwrap(); + db.init_db().unwrap(); + Wallet { + db_path: db_path.to_string(), + db, + prover, + } + } + + pub fn new_account_with_seed(&self, seed: &str) -> anyhow::Result<()> { + let sk = get_secret_key(&seed).unwrap(); + let vk = get_viewing_key(&sk).unwrap(); + let pa = get_address(&vk).unwrap(); + self.db.store_account(&sk, &vk, &pa)?; + Ok(()) + } + + async fn scan_async(&self, ivk: &str, chunk_size: u32, target_height_offset: u32, progress_callback: ProgressCallback) -> anyhow::Result<()> { + crate::scan::sync_async(ivk, chunk_size, &self.db_path, target_height_offset, progress_callback).await + } + + pub async fn get_latest_height() -> anyhow::Result { + let mut client = connect_lightwalletd().await?; + let last_height = get_latest_height(&mut client).await?; + Ok(last_height) + } + + pub async fn sync(&self, account: u32, progress_callback: impl Fn(u32) + Send + 'static) -> anyhow::Result<()> { + let ivk = self.db.get_ivk(account)?; + let cb = Arc::new(Mutex::new(progress_callback)); + self.scan_async(&ivk, DEFAULT_CHUNK_SIZE, 10, cb.clone()).await?; + self.scan_async(&ivk, DEFAULT_CHUNK_SIZE, 0, cb.clone()).await?; + Ok(()) + } + + pub fn get_balance(&self) -> anyhow::Result { + self.db.get_balance() + } + + pub async fn send_payment(&self, account: u32, to_address: &str, amount: u64) -> anyhow::Result { + let secret_key = self.db.get_sk(account)?; + let to_addr = RecipientAddress::decode(&NETWORK, to_address) + .ok_or(anyhow::anyhow!("Invalid address"))?; + let target_amount = Amount::from_u64(amount).unwrap(); + let skey = decode_extended_spending_key(NETWORK.hrp_sapling_extended_spending_key(), &secret_key)?.unwrap(); + let extfvk = ExtendedFullViewingKey::from(&skey); + let (_, change_address) = extfvk.default_address().unwrap(); + let ovk = extfvk.fvk.ovk; + let last_height = Self::get_latest_height().await?; + let mut builder = Builder::new(NETWORK, BlockHeight::from_u32(last_height)); + let anchor_height = self.db.get_last_sync_height()?.ok_or_else(|| anyhow::anyhow!("No spendable notes"))?; + let anchor_height = anchor_height.min(last_height - ANCHOR_OFFSET); + log::info!("Anchor = {}", anchor_height); + let mut notes = self.db.get_spendable_notes(anchor_height, &extfvk)?; + notes.shuffle(&mut OsRng); + log::info!("Spendable notes = {}", notes.len()); + + let mut amount = target_amount; + amount += DEFAULT_FEE; + for n in notes { + if amount.is_positive() { + let a = amount.min(Amount::from_u64(n.note.value).unwrap()); + amount -= a; + let merkle_path = n.witness.path().unwrap(); + builder.add_sapling_spend(skey.clone(), n.diversifier, n.note.clone(), merkle_path)?; + } + } + if amount.is_positive() { + anyhow::bail!("Not enough balance") + } + + builder.send_change_to(Some(ovk), change_address); + match to_addr { + RecipientAddress::Shielded(pa) => builder.add_sapling_output(Some(ovk), pa, target_amount, None), + RecipientAddress::Transparent(t_address) => builder.add_transparent_output(&t_address, target_amount), + }?; + + let consensus_branch_id = BranchId::for_height(&NETWORK, BlockHeight::from_u32(last_height)); + let (tx, _) = builder.build(consensus_branch_id, &self.prover)?; + let mut raw_tx: Vec = vec![]; + tx.write(&mut raw_tx)?; + + let mut client = connect_lightwalletd().await?; + let tx_id = send_transaction(&mut client, &raw_tx, last_height).await?; + Ok(tx_id) + } +} + +#[cfg(test)] +mod tests { + use crate::{get_secret_key, get_viewing_key, get_address}; + use crate::wallet::Wallet; + + #[tokio::test] + async fn test_wallet_seed() { + dotenv::dotenv().unwrap(); + env_logger::init(); + + let seed = dotenv::var("SEED").unwrap(); + let wallet = Wallet::new("zec.db"); + wallet.new_account_with_seed(&seed).unwrap(); + } + + #[tokio::test] + async fn test_payment() { + dotenv::dotenv().unwrap(); + env_logger::init(); + + let seed = dotenv::var("SEED").unwrap(); + let sk = get_secret_key(&seed).unwrap(); + let vk = get_viewing_key(&sk).unwrap(); + println!("{}", vk); + let pa = get_address(&vk).unwrap(); + println!("{}", pa); + let wallet = Wallet::new("zec.db"); + + let tx_id = wallet.send_payment(1, &pa, 1000).await.unwrap(); + println!("TXID = {}", tx_id); + } +} \ No newline at end of file