bug fixes

This commit is contained in:
Hanh 2021-06-24 20:08:20 +08:00
parent 572195d8df
commit 915769c47e
10 changed files with 639 additions and 172 deletions

View File

@ -37,19 +37,31 @@ rayon = "1.5.1"
byteorder = "1.4.3"
tiny-bip39 = "0.8"
rand = "0.8.4"
rusqlite = "^0.25.3"
rusqlite = { version = "^0.24", features = ["bundled"] }
#[dependencies.zcash_client_backend]
#git = "https://github.com/zcash/librustzcash.git"
#rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb"
#
#[dependencies.zcash_primitives]
#git = "https://github.com/zcash/librustzcash.git"
#rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb"
#
#[dependencies.zcash_proofs]
#git = "https://github.com/zcash/librustzcash.git"
#rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb"
[dependencies.zcash_client_backend]
git = "https://github.com/zcash/librustzcash.git"
rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb"
path = "/home/hanh/projects/librustzcash/zcash_client_backend"
[dependencies.zcash_primitives]
git = "https://github.com/zcash/librustzcash.git"
rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb"
path = "/home/hanh/projects/librustzcash/zcash_primitives"
[dependencies.zcash_proofs]
git = "https://github.com/zcash/librustzcash.git"
rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb"
path = "/home/hanh/projects/librustzcash/zcash_proofs"
[dependencies.zcash_params]
path = "../zcash_params"
[build-dependencies]
tonic-build = "0.4.2"

View File

@ -17,13 +17,13 @@ struct CTreeBuilder {
prev_tree: CTree,
next_tree: CTree,
start: usize,
total_len: usize,
depth: usize,
offset: Option<Node>,
}
impl Builder<CTree, ()> for CTreeBuilder {
fn collect(&mut self, commitments: &[Node], _context: &()) -> usize {
// assert!(!commitments.is_empty() || self.left.is_some() || self.right.is_some());
assert!(self.right.is_none() || self.left.is_some()); // R can't be set without L
let offset: Option<Node>;
@ -37,26 +37,30 @@ impl Builder<CTree, ()> for CTreeBuilder {
m = commitments.len();
};
let n = if self.depth == 0 {
if m % 2 == 0 {
self.next_tree.left = Some(*Self::get(commitments, m - 2, &offset));
self.next_tree.right = Some(*Self::get(commitments, m - 1, &offset));
m - 2
} else {
self.next_tree.left = Some(*Self::get(commitments, m - 1, &offset));
self.next_tree.right = None;
m - 1
let n =
if self.total_len > 0 {
if self.depth == 0 {
if m % 2 == 0 {
self.next_tree.left = Some(*Self::get(commitments, m - 2, &offset));
self.next_tree.right = Some(*Self::get(commitments, m - 1, &offset));
m - 2
} else {
self.next_tree.left = Some(*Self::get(commitments, m - 1, &offset));
self.next_tree.right = None;
m - 1
}
} else {
if m % 2 == 0 {
self.next_tree.parents.push(None);
m
} else {
let last_node = Self::get(commitments, m - 1, &offset);
self.next_tree.parents.push(Some(*last_node));
m - 1
}
}
}
} else {
if m % 2 == 0 {
self.next_tree.parents.push(None);
m
} else {
let last_node = Self::get(commitments, m - 1, &offset);
self.next_tree.parents.push(Some(*last_node));
m - 1
}
};
else { 0 };
assert_eq!(n % 2, 0);
self.offset = offset;
@ -93,12 +97,16 @@ impl Builder<CTree, ()> for CTreeBuilder {
}
fn finalize(self, _context: &()) -> CTree {
self.next_tree
if self.total_len > 0 {
self.next_tree
} else {
self.prev_tree
}
}
}
impl CTreeBuilder {
fn new(prev_tree: CTree) -> CTreeBuilder {
fn new(prev_tree: CTree, len: usize) -> CTreeBuilder {
let start = prev_tree.get_position();
CTreeBuilder {
left: prev_tree.left,
@ -106,6 +114,7 @@ impl CTreeBuilder {
prev_tree,
next_tree: CTree::new(),
start,
total_len: len,
depth: 0,
offset: None,
}
@ -133,25 +142,13 @@ impl CTreeBuilder {
Self::get_opt(commitments, index, offset).unwrap()
}
fn adjusted_start(&self, prev: &Option<Node>, depth: usize) -> usize {
if depth != 0 && prev.is_some() {
fn adjusted_start(&self, prev: &Option<Node>, _depth: usize) -> usize {
if prev.is_some() {
self.start - 1
} else {
self.start
}
}
fn clone_trimmed(&self, mut depth: usize) -> CTree {
if depth == 0 {
return CTree::new()
}
let mut tree = self.next_tree.clone();
while depth > 0 && depth <= tree.parents.len() && tree.parents[depth - 1].is_none() {
depth -= 1;
}
tree.parents.truncate(depth);
tree
}
}
fn combine_level(commitments: &mut [Node], offset: Option<Node>, n: usize, depth: usize) -> usize {
@ -218,6 +215,12 @@ impl Builder<Witness, CTreeBuilder> for WitnessBuilder {
}
}
// println!("D {}", depth);
// println!("O {:?}", offset.map(|r| hex::encode(r.repr)));
// println!("R {:?}", right.map(|r| hex::encode(r.repr)));
// for c in commitments.iter() {
// println!("{}", hex::encode(c.repr));
// }
let p1 = self.p + 1;
let has_p1 = p1 >= context.adjusted_start(&right, depth) && p1 < context.start + commitments.len();
if has_p1 {
@ -244,32 +247,41 @@ impl Builder<Witness, CTreeBuilder> for WitnessBuilder {
}
fn finalize(mut self, context: &CTreeBuilder) -> Witness {
let tree = &self.witness.tree;
let mut num_filled = self.witness.filled.len();
if self.witness.position + 1 == context.next_tree.get_position() {
if context.total_len == 0 {
self.witness.cursor = CTree::new();
}
else {
let mut depth = 0;
loop {
let is_none = if depth == 0 { // check if this level is occupied
tree.right.is_none()
} else {
depth > tree.parents.len() || tree.parents[depth - 1].is_none()
};
if is_none {
if num_filled > 0 {
num_filled -= 1; // we filled it
} else {
break
}
}
depth += 1;
// loop terminates because we are eventually going to run out of ancestors and filled
}
self.witness.cursor = context.clone_trimmed(depth - 1);
let mut final_position = context.prev_tree.get_position() as u32;
let mut witness_position = self.witness.tree.get_position() as u32;
assert_ne!(witness_position, 0);
witness_position = witness_position - 1;
// look for first not equal bit in MSB order
final_position = final_position.reverse_bits();
witness_position = witness_position.reverse_bits();
let mut bit: i32 = 31;
// reverse bits because it is easier to do in LSB
// it should not underflow because these numbers are not equal
while bit >= 0 {
if final_position & 1 != witness_position & 1 {
break;
}
final_position >>= 1;
witness_position >>= 1;
bit -= 1;
}
// look for the first bit set in final_position after
final_position >>= 1;
bit -= 1;
while bit >= 0 {
if final_position & 1 == 1 {
break;
}
final_position >>= 1;
bit -= 1;
}
if bit >= 0 {
self.witness.cursor = context.prev_tree.clone_trimmed(bit as usize)
}
}
self.witness
}
@ -281,10 +293,7 @@ pub fn advance_tree(
prev_witnesses: &[Witness],
mut commitments: &mut [Node],
) -> (CTree, Vec<Witness>) {
if commitments.is_empty() {
return (prev_tree, prev_witnesses.to_vec());
}
let mut builder = CTreeBuilder::new(prev_tree);
let mut builder = CTreeBuilder::new(prev_tree, commitments.len());
let mut witness_builders: Vec<_> = prev_witnesses
.iter()
.map(|witness| WitnessBuilder::new(&builder, witness.clone(), commitments.len()))
@ -295,11 +304,11 @@ pub fn advance_tree(
b.collect(commitments, &builder);
}
let nn = combine_level(commitments, builder.offset, n, builder.depth);
commitments = &mut commitments[0..nn];
builder.up();
for b in witness_builders.iter_mut() {
b.up();
}
commitments = &mut commitments[0..nn];
}
let witnesses = witness_builders
@ -318,24 +327,35 @@ mod tests {
use zcash_primitives::merkle_tree::{CommitmentTree, IncrementalWitness};
use zcash_primitives::sapling::Node;
use crate::chain::DecryptedNote;
use crate::print::{print_witness, print_witness2, print_tree, print_ctree};
#[test]
fn test_advance_tree() {
const NUM_NODES: usize = 1000;
const NUM_CHUNKS: usize = 50;
const WITNESS_PERCENT: f64 = 1.0; // percentage of notes that are ours
let witness_freq = (100.0 / WITNESS_PERCENT) as usize;
for num_nodes in 1..=10 {
for num_chunks in 1..=10 {
test_advance_tree_helper(num_nodes, num_chunks, 100.0);
}
}
test_advance_tree_helper(100, 50, 1.0);
// test_advance_tree_helper(2, 10, 100.0);
// test_advance_tree_helper(1, 40, 100.0);
// test_advance_tree_helper(10, 2, 100.0);
}
fn test_advance_tree_helper(num_nodes: usize, num_chunks: usize, witness_percent: f64) {
let witness_freq = (100.0 / witness_percent) as usize;
let mut tree1: CommitmentTree<Node> = CommitmentTree::empty();
let mut tree2 = CTree::new();
let mut ws: Vec<IncrementalWitness<Node>> = vec![];
let mut ws2: Vec<Witness> = vec![];
for i in 0..NUM_CHUNKS {
for i in 0..num_chunks {
println!("{}", i);
let mut nodes: Vec<_> = vec![];
for j in 0..NUM_NODES {
for j in 0..num_nodes {
let mut bb = [0u8; 32];
let v = i * NUM_NODES + j;
let v = i * num_nodes + j;
bb[0..8].copy_from_slice(&v.to_be_bytes());
let node = Node::new(bb);
tree1.append(node).unwrap();
@ -343,7 +363,7 @@ mod tests {
w.append(node).unwrap();
}
if v % witness_freq == 0 {
// if v == 499 {
// if v == 0 {
let w = IncrementalWitness::from_tree(&tree1);
ws.push(w);
ws2.push(Witness::new(v, 0, None));
@ -356,6 +376,13 @@ mod tests {
ws2 = new_witnesses;
}
// Push an empty block
// It will calculate the tail of the tree
// This step is required at the end of a series of chunks
let (new_tree, new_witnesses) = advance_tree(tree2, &ws2, &mut []);
tree2 = new_tree;
ws2 = new_witnesses;
// check final state
let mut bb1: Vec<u8> = vec![];
tree1.write(&mut bb1).unwrap();
@ -364,6 +391,11 @@ mod tests {
tree2.write(&mut bb2).unwrap();
let equal = bb1.as_slice() == bb2.as_slice();
if !equal {
println!("FAILED FINAL STATE");
print_tree(&tree1);
print_ctree(&tree2);
}
println!("# witnesses = {}", ws.len());
@ -378,6 +410,16 @@ mod tests {
if bb1.as_slice() != bb2.as_slice() {
failed_index = Some(i);
println!("FAILED AT {}", i);
if let Some(ref c) = w1.cursor {
print_tree(c);
}
else { println!("NONE"); }
println!("GOOD");
print_witness(&w1);
println!("BAD");
print_witness2(&w2);
}
}

View File

@ -40,7 +40,7 @@ pub async fn download_chain(
) -> anyhow::Result<Vec<CompactBlock>> {
let mut cbs: Vec<CompactBlock> = Vec::new();
let mut s = start_height + 1;
while s < end_height {
while s <= end_height {
let e = (s + MAX_CHUNK - 1).min(end_height);
let range = BlockRange {
start: Some(BlockId {
@ -68,10 +68,14 @@ pub struct DecryptNode {
fvks: Vec<ExtendedFullViewingKey>,
}
#[derive(Eq, Hash, PartialEq)]
pub struct Nf(pub [u8; 32]);
pub struct DecryptedBlock {
pub height: u32,
pub notes: Vec<DecryptedNote>,
pub count_outputs: u32,
pub spends: Vec<Nf>,
}
#[derive(Clone)]
@ -79,7 +83,7 @@ pub struct DecryptedNote {
pub ivk: ExtendedFullViewingKey,
pub note: Note,
pub pa: PaymentAddress,
pub position: usize,
pub position_in_block: usize,
pub height: u32,
pub txid: Vec<u8>,
@ -90,8 +94,15 @@ pub struct DecryptedNote {
fn decrypt_notes(block: &CompactBlock, fvks: &[ExtendedFullViewingKey]) -> DecryptedBlock {
let height = BlockHeight::from_u32(block.height as u32);
let mut count_outputs = 0u32;
let mut spends: Vec<Nf> = vec![];
let mut notes: Vec<DecryptedNote> = vec![];
for (tx_index, vtx) in block.vtx.iter().enumerate() {
for cs in vtx.spends.iter() {
let mut nf = [0u8; 32];
nf.copy_from_slice(&cs.nf);
spends.push(Nf(nf));
}
for (output_index, co) in vtx.outputs.iter().enumerate() {
let mut cmu = [0u8; 32];
cmu.copy_from_slice(&co.cmu);
@ -113,7 +124,7 @@ fn decrypt_notes(block: &CompactBlock, fvks: &[ExtendedFullViewingKey]) -> Decry
ivk: fvk.clone(),
note,
pa,
position: count_outputs as usize,
position_in_block: count_outputs as usize,
height: block.height as u32,
tx_index,
txid: vtx.hash.clone(),
@ -126,6 +137,7 @@ fn decrypt_notes(block: &CompactBlock, fvks: &[ExtendedFullViewingKey]) -> Decry
}
DecryptedBlock {
height: block.height as u32,
spends,
notes,
count_outputs,
}
@ -160,6 +172,15 @@ async fn get_tree_state(client: &mut CompactTxStreamerClient<Channel>, height: u
rep.tree
}
pub async fn send_transaction(client: &mut CompactTxStreamerClient<Channel>, raw_tx: &[u8], height: u32) -> anyhow::Result<String> {
let raw_tx = RawTransaction {
data: raw_tx.to_vec(),
height: height as u64
};
let rep = client.send_transaction(Request::new(raw_tx)).await?.into_inner();
Ok(rep.error_message)
}
/* Using the IncrementalWitness */
#[allow(dead_code)]
fn calculate_tree_state_v1(
@ -187,7 +208,7 @@ fn calculate_tree_state_v1(
w.append(node).unwrap();
}
if let Some(nn) = n {
if i == nn.position {
if i == nn.position_in_block {
let w = IncrementalWitness::from_tree(&tree_state);
witnesses.push(w);
n = notes.next();
@ -226,7 +247,7 @@ pub fn calculate_tree_state_v2(cbs: &[CompactBlock], blocks: &[DecryptedBlock])
nodes.push(node);
if let Some(nn) = n {
if i == nn.position {
if i == nn.position_in_block {
positions.push(p);
n = notes.next();
}

View File

@ -174,6 +174,15 @@ impl CTree {
p
}
pub fn clone_trimmed(&self, depth: usize) -> CTree {
let mut tree = self.clone();
tree.parents.truncate(depth);
if let Some(None) = tree.parents.last() { // Remove trailing None
tree.parents.truncate(depth - 1);
}
tree
}
pub fn to_commitment_tree(&self) -> CommitmentTree<Node> {
let mut bb: Vec<u8> = vec![];
self.write(&mut bb).unwrap();

165
src/db.rs
View File

@ -1,5 +1,11 @@
use rusqlite::{Connection, params, OptionalExtension};
use rusqlite::{Connection, params, OptionalExtension, NO_PARAMS};
use crate::{Witness, CTree};
use zcash_primitives::sapling::{Note, Diversifier, Rseed, Node};
use zcash_primitives::zip32::ExtendedFullViewingKey;
use zcash_primitives::merkle_tree::IncrementalWitness;
use std::collections::HashMap;
use crate::chain::Nf;
use zcash_primitives::consensus::{Parameters, NetworkUpgrade};
pub struct DbAdapter {
connection: Connection,
@ -12,9 +18,13 @@ pub struct ReceivedNote {
pub value: u64,
pub rcm: Vec<u8>,
pub nf: Vec<u8>,
pub is_change: bool,
pub memo: Vec<u8>,
pub spent: bool,
pub spent: Option<u32>,
}
pub struct SpendableNote {
pub note: Note,
pub diversifier: Diversifier,
pub witness: IncrementalWitness<Node>,
}
impl DbAdapter {
@ -26,16 +36,22 @@ impl DbAdapter {
}
pub fn init_db(&self) -> anyhow::Result<()> {
self.connection.execute("CREATE TABLE IF NOT EXISTS accounts (
id_account INTEGER PRIMARY KEY,
sk TEXT NOT NULL UNIQUE,
ivk TEXT NOT NULL,
address TEXT NOT NULL)", NO_PARAMS)?;
self.connection.execute("CREATE TABLE IF NOT EXISTS blocks (
height INTEGER PRIMARY KEY,
hash BLOB NOT NULL,
sapling_tree BLOB NOT NULL)", [])?;
sapling_tree BLOB NOT NULL)", NO_PARAMS)?;
self.connection.execute("CREATE TABLE IF NOT EXISTS transactions (
id_tx INTEGER PRIMARY KEY,
txid BLOB NOT NULL UNIQUE,
height INTEGER,
tx_index INTEGER)", [])?;
tx_index INTEGER)", NO_PARAMS)?;
self.connection.execute("CREATE TABLE IF NOT EXISTS received_notes (
id_note INTEGER PRIMARY KEY,
@ -47,24 +63,25 @@ impl DbAdapter {
value INTEGER NOT NULL,
rcm BLOB NOT NULL,
nf BLOB NOT NULL UNIQUE,
is_change INTEGER NOT NULL,
memo BLOB,
spent INTEGER,
FOREIGN KEY (tx) REFERENCES transactions(id_tx),
FOREIGN KEY (spent) REFERENCES transactions(id_tx),
CONSTRAINT tx_output UNIQUE (tx, output_index))", [])?;
CONSTRAINT tx_output UNIQUE (tx, output_index))", NO_PARAMS)?;
self.connection.execute("CREATE TABLE IF NOT EXISTS sapling_witnesses (
id_witness INTEGER PRIMARY KEY,
note INTEGER NOT NULL,
height INTEGER NOT NULL,
witness BLOB NOT NULL,
FOREIGN KEY (note) REFERENCES received_notes(id_note),
CONSTRAINT witness_height UNIQUE (note, height))", [])?;
CONSTRAINT witness_height UNIQUE (note, height))", NO_PARAMS)?;
Ok(())
}
pub fn store_account(&self, sk: &str, ivk: &str, address: &str) -> anyhow::Result<()> {
self.connection.execute("INSERT INTO accounts(sk, ivk, address) VALUES (?1, ?2, ?3)
ON CONFLICT DO NOTHING", params![sk, ivk, address])?;
Ok(())
}
pub fn trim_to_height(&mut self, height: u32) -> anyhow::Result<()> {
let tx = self.connection.transaction()?;
tx.execute("DELETE FROM blocks WHERE height >= ?1", params![height])?;
@ -77,53 +94,70 @@ impl DbAdapter {
}
pub fn store_block(&self, height: u32, hash: &[u8], tree: &CTree) -> anyhow::Result<()> {
log::info!("+block");
let mut bb: Vec<u8> = vec![];
tree.write(&mut bb)?;
self.connection.execute("INSERT INTO blocks(height, hash, sapling_tree)
VALUES (?1, ?2, ?3)
ON CONFLICT DO NOTHING", params![height, hash, &bb])?;
log::info!("-block");
Ok(())
}
pub fn store_transaction(&self, txid: &[u8], height: u32, tx_index: u32) -> anyhow::Result<u32> {
log::info!("+transaction");
self.connection.execute("INSERT INTO transactions(txid, height, tx_index)
VALUES (?1, ?2, ?3)
ON CONFLICT DO NOTHING", params![txid, height, tx_index])?;
let id_tx: u32 = self.connection.query_row("SELECT id_tx FROM transactions WHERE txid = ?1", params![txid], |row| row.get(0))?;
log::info!("-transaction {}", id_tx);
Ok(id_tx)
}
pub fn store_received_note(&self, note: &ReceivedNote, id_tx: u32, position: usize) -> anyhow::Result<u32> {
self.connection.execute("INSERT INTO received_notes(tx, height, position, output_index, diversifier, value, rcm, nf, is_change, memo, spent)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)
ON CONFLICT DO NOTHING", params![id_tx, note.height, position, note.output_index, note.diversifier, note.value, note.rcm, note.nf, note.is_change, note.memo, note.spent])?;
log::info!("+received_note {}", id_tx);
self.connection.execute("INSERT INTO received_notes(tx, height, position, output_index, diversifier, value, rcm, nf, spent)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
ON CONFLICT DO NOTHING", params![id_tx, note.height, position as u32, note.output_index, note.diversifier, note.value as i64, note.rcm, note.nf, note.spent])?;
let id_note: u32 = self.connection.query_row("SELECT id_note FROM received_notes WHERE tx = ?1 AND output_index = ?2", params![id_tx, note.output_index], |row| row.get(0))?;
log::info!("-received_note");
Ok(id_note)
}
pub fn store_witnesses(&self, witness: &Witness, height: u32, id_note: u32) -> anyhow::Result<()> {
log::info!("+witnesses");
let mut bb: Vec<u8> = vec![];
witness.write(&mut bb)?;
println!("{} {}", height, id_note);
self.connection.execute("INSERT INTO sapling_witnesses(note, height, witness) VALUES (?1, ?2, ?3)
ON CONFLICT DO NOTHING", params![id_note, height, bb])?;
log::info!("-witnesses");
Ok(())
}
pub fn get_balance(&self) -> anyhow::Result<u64> {
let balance: u64 = self.connection.query_row("SELECT SUM(value) FROM received_notes WHERE spent = 0", [], |row| row.get(0))?;
Ok(balance)
let balance: Option<i64> = self.connection.query_row("SELECT SUM(value) FROM received_notes WHERE spent IS NULL", NO_PARAMS, |row| row.get(0))?;
Ok(balance.unwrap_or(0) as u64)
}
pub fn get_last_height(&self) -> anyhow::Result<Option<u32>> {
let height: Option<u32> = self.connection.query_row("SELECT MAX(height) FROM blocks", [], |row| row.get(0)).optional()?;
pub fn get_last_sync_height(&self) -> anyhow::Result<Option<u32>> {
let height: Option<u32> = self.connection.query_row("SELECT MAX(height) FROM blocks", NO_PARAMS, |row| row.get(0))?;
Ok(height)
}
pub fn get_db_height(&self) -> anyhow::Result<u32> {
let height: u32 = self.get_last_sync_height()?.unwrap_or_else(|| {
crate::NETWORK
.activation_height(NetworkUpgrade::Sapling)
.unwrap()
.into()
});
Ok(height)
}
pub fn get_tree(&self) -> anyhow::Result<(CTree, Vec<Witness>)> {
let res = self.connection.query_row(
"SELECT height, sapling_tree FROM blocks WHERE height = (SELECT MAX(height) FROM blocks)",
[], |row| {
NO_PARAMS, |row| {
let height: u32 = row.get(0)?;
let tree: Vec<u8> = row.get(1)?;
Ok((height, tree))
@ -148,6 +182,89 @@ impl DbAdapter {
None => (CTree::new(), vec![])
})
}
pub fn get_nullifiers(&self) -> anyhow::Result<HashMap<Nf, u32>> {
let mut statement = self.connection.prepare(
"SELECT id_note, nf FROM received_notes WHERE spent = 0")?;
let nfs_res = statement.query_map(NO_PARAMS, |row| {
let id_note: u32 = row.get(0)?;
let nf_vec: Vec<u8> = row.get(1)?;
let mut nf = [0u8; 32];
nf.clone_from_slice(&nf_vec);
Ok((id_note, nf))
})?;
let mut nfs: HashMap<Nf, u32> = HashMap::new();
for n in nfs_res {
let n = n?;
nfs.insert(Nf(n.1), n.0);
}
Ok(nfs)
}
pub fn get_spendable_notes(&self, anchor_height: u32, fvk: &ExtendedFullViewingKey) -> anyhow::Result<Vec<SpendableNote>> {
let mut statement = self.connection.prepare(
"SELECT diversifier, value, rcm, witness FROM received_notes r, sapling_witnesses w WHERE spent IS NULL
AND w.height = (
SELECT MAX(height) FROM sapling_witnesses WHERE height <= ?1
) AND r.id_note = w.note")?;
let notes = statement.query_map(params![anchor_height], |row| {
let diversifier: Vec<u8> = row.get(0)?;
let value: i64 = row.get(1)?;
let rcm: Vec<u8> = row.get(2)?;
let witness: Vec<u8> = row.get(3)?;
let mut diversifer_bytes = [0u8; 11];
diversifer_bytes.copy_from_slice(&diversifier);
let diversifier = Diversifier(diversifer_bytes);
let mut rcm_bytes = [0u8; 32];
rcm_bytes.copy_from_slice(&rcm);
let rcm = jubjub::Fr::from_bytes(&rcm_bytes).unwrap();
let rseed = Rseed::BeforeZip212(rcm);
let witness = IncrementalWitness::<Node>::read(&*witness).unwrap();
let pa = fvk.fvk.vk.to_payment_address(diversifier).unwrap();
let note = pa.create_note(value as u64, rseed).unwrap();
Ok(SpendableNote {
note,
diversifier,
witness
})
})?;
let mut spendable_notes: Vec<SpendableNote> = vec![];
for n in notes {
spendable_notes.push(n?);
}
Ok(spendable_notes)
}
pub fn mark_spent(&self, id: u32, height: u32) -> anyhow::Result<()> {
log::info!("+mark_spent");
self.connection.execute("UPDATE received_notes SET spent = ?1 WHERE id_note = ?2", params![height, id])?;
log::info!("-mark_spent");
Ok(())
}
pub fn get_sk(&self, account: u32) -> anyhow::Result<String> {
log::info!("+get_sk");
let ivk = self.connection.query_row("SELECT sk FROM accounts WHERE id_account = ?1", params![account], |row | {
let ivk: String = row.get(0)?;
Ok(ivk)
})?;
log::info!("-get_sk");
Ok(ivk)
}
pub fn get_ivk(&self, account: u32) -> anyhow::Result<String> {
log::info!("+get_ivk");
let ivk = self.connection.query_row("SELECT ivk FROM accounts WHERE id_account = ?1", params![account], |row | {
let ivk: String = row.get(0)?;
Ok(ivk)
})?;
log::info!("-get_ivk");
Ok(ivk)
}
}
#[cfg(test)]
@ -172,9 +289,7 @@ mod tests {
value: 0,
rcm: vec![],
nf: vec![],
is_change: false,
memo: vec![],
spent: false
spent: None
}, id_tx, 5).unwrap();
let witness = Witness {
position: 10,

View File

@ -3,7 +3,7 @@ use zcash_primitives::consensus::Network;
#[path = "generated/cash.z.wallet.sdk.rpc.rs"]
pub mod lw_rpc;
pub const NETWORK: Network = Network::MainNetwork;
pub const NETWORK: Network = Network::TestNetwork;
mod builder;
mod chain;
@ -12,6 +12,7 @@ mod scan;
mod key;
mod db;
mod wallet;
mod print;
pub use crate::builder::advance_tree;
pub use crate::chain::{
@ -24,3 +25,4 @@ pub use crate::lw_rpc::*;
pub use crate::scan::{scan_all, sync_async, latest_height};
pub use crate::key::{get_secret_key, get_address, get_viewing_key};
pub use crate::db::DbAdapter;
pub use crate::wallet::{Wallet, DEFAULT_ACCOUNT};

View File

@ -1,40 +1,59 @@
use sync::{sync_async, DbAdapter};
use sync::{DbAdapter, Wallet, DEFAULT_ACCOUNT};
use bip39::{Language, Mnemonic};
use rand::rngs::OsRng;
use rand::RngCore;
const DB_NAME: &str = "zec.db";
fn init() {
let db = DbAdapter::new(DB_NAME).unwrap();
db.init_db().unwrap();
}
#[tokio::main]
#[allow(dead_code)]
async fn test() -> anyhow::Result<()> {
dotenv::dotenv().unwrap();
env_logger::init();
let ivk = dotenv::var("IVK").unwrap();
{
let db = DbAdapter::new(DB_NAME)?;
db.init_db()?;
}
sync_async(&ivk, 50000, DB_NAME, |height| {
let seed = dotenv::var("SEED").unwrap();
let address = dotenv::var("ADDRESS").unwrap();
let progress = |height| {
log::info!("Height = {}", height);
}).await?;
};
let wallet = Wallet::new(DB_NAME);
wallet.new_account_with_seed(&seed).unwrap();
wallet.sync(DEFAULT_ACCOUNT, progress).await.unwrap();
let tx_id = wallet.send_payment(DEFAULT_ACCOUNT, &address, 1000).await.unwrap();
println!("TXID = {}", tx_id);
Ok(())
}
#[allow(dead_code)]
fn test_make_wallet() {
let mut entropy = [0u8; 32];
OsRng.fill_bytes(&mut entropy);
let mnemonic = Mnemonic::from_entropy(&entropy, Language::English).unwrap();
let phrase = mnemonic.phrase();
println!("Seed Phrase: {}", phrase);
}
#[allow(dead_code)]
fn test_rewind() {
let mut db = DbAdapter::new(DB_NAME).unwrap();
db.trim_to_height(1_250_000).unwrap();
db.trim_to_height(1460000).unwrap();
}
fn test_get_balance() {
let db = DbAdapter::new(DB_NAME).unwrap();
let balance = db.get_balance().unwrap();
println!("Balance = {}", (balance as f64) / 100_000_000.0);
}
fn main() {
init();
// test_rewind();
test().unwrap();
// let mut entropy = [0u8; 32];
// OsRng.fill_bytes(&mut entropy);
// let mnemonic = Mnemonic::from_entropy(&entropy, Language::English).unwrap();
// let phrase = mnemonic.phrase();
// println!("Seed Phrase: {}", phrase);
test_get_balance();
}

56
src/print.rs Normal file
View File

@ -0,0 +1,56 @@
use zcash_primitives::sapling::Node;
use zcash_primitives::merkle_tree::{CommitmentTree, IncrementalWitness};
use crate::{Witness, CTree};
#[allow(dead_code)]
pub fn print_node(n: &Node) {
println!("{:?}", hex::encode(n.repr));
}
#[allow(dead_code)]
pub fn print_tree(t: &CommitmentTree<Node>) {
println!("{:?}", t.left.map(|n| hex::encode(n.repr)));
println!("{:?}", t.right.map(|n| hex::encode(n.repr)));
for p in t.parents.iter() {
println!("{:?}", p.map(|n| hex::encode(n.repr)));
}
}
#[allow(dead_code)]
pub fn print_witness(w: &IncrementalWitness<Node>) {
println!("Tree");
print_tree(&w.tree);
println!("Filled");
for n in w.filled.iter() {
print_node(n);
}
println!("Cursor");
w.cursor.as_ref().map(|c| print_tree(c));
}
pub fn print_ctree(t: &CTree) {
println!("Tree");
println!("{:?}", t.left.map(|n| hex::encode(n.repr)));
println!("{:?}", t.right.map(|n| hex::encode(n.repr)));
for p in t.parents.iter() {
println!("{:?}", p.map(|n| hex::encode(n.repr)));
}
}
#[allow(dead_code)]
pub fn print_witness2(w: &Witness) {
let t = &w.tree;
print_ctree(t);
println!("Filled");
for n in w.filled.iter() {
print_node(n);
}
let t = &w.cursor;
println!("Cursor");
println!("{:?}", t.left.map(|n| hex::encode(n.repr)));
println!("{:?}", t.right.map(|n| hex::encode(n.repr)));
for p in t.parents.iter() {
println!("{:?}", p.map(|n| hex::encode(n.repr)));
}
}

View File

@ -10,6 +10,9 @@ use log::info;
use crate::db::{DbAdapter, ReceivedNote};
use ff::PrimeField;
use zcash_primitives::zip32::ExtendedFullViewingKey;
use crate::chain::Nf;
use std::sync::Arc;
use tokio::sync::Mutex;
pub async fn scan_all(fvks: &[ExtendedFullViewingKey]) -> anyhow::Result<()> {
let decrypter = DecryptNode::new(fvks.to_vec());
@ -35,7 +38,7 @@ pub async fn scan_all(fvks: &[ExtendedFullViewingKey]) -> anyhow::Result<()> {
info!("# Witnesses {}", witnesses.len());
for w in witnesses.iter() {
let mut bb: Vec<u8> = vec![];
w.write(&mut bb).unwrap();
w.write(&mut bb)?;
log::debug!("{}", hex::encode(&bb));
}
@ -45,6 +48,10 @@ pub async fn scan_all(fvks: &[ExtendedFullViewingKey]) -> anyhow::Result<()> {
}
struct Blocks(Vec<CompactBlock>);
struct BlockMetadata {
height: u32,
hash: [u8; 32],
}
impl std::fmt::Debug for Blocks {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@ -52,69 +59,76 @@ impl std::fmt::Debug for Blocks {
}
}
fn get_db_height(db_path: &str) -> anyhow::Result<u32> {
let db = DbAdapter::new(db_path).unwrap();
let height: u32 = db.get_last_height()?.unwrap_or_else(|| {
crate::NETWORK
.activation_height(NetworkUpgrade::Sapling)
.unwrap()
.into()
});
Ok(height)
}
pub type ProgressCallback = Arc<Mutex<dyn Fn(u32) + Send>>;
pub async fn sync_async(ivk: &str, chunk_size: u32, db_path: &str, progress_callback: impl Fn(u32) + Send + 'static) -> anyhow::Result<()> {
pub async fn sync_async(ivk: &str, chunk_size: u32, db_path: &str, target_height_offset: u32, progress_callback: ProgressCallback) -> anyhow::Result<()> {
let db_path = db_path.to_string();
let fvk =
decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &ivk)
.unwrap()
.unwrap();
?.ok_or_else(|| anyhow::anyhow!("Invalid key"))?;
let decrypter = DecryptNode::new(vec![fvk]);
let mut client = connect_lightwalletd().await?;
let start_height = get_db_height(&db_path)?;
let start_height = {
let db = DbAdapter::new(&db_path)?;
db.get_db_height()?
};
let end_height = get_latest_height(&mut client).await?;
let end_height = (end_height - target_height_offset).max(start_height);
let (downloader_tx, mut download_rx) = mpsc::channel::<Range<u32>>(2);
let (processor_tx, mut processor_rx) = mpsc::channel::<Blocks>(2);
let (processor_tx, mut processor_rx) = mpsc::channel::<Blocks>(1);
let (completed_tx, mut completed_rx) = mpsc::channel::<()>(1);
tokio::spawn(async move {
let mut client = connect_lightwalletd().await.unwrap();
let downloader = tokio::spawn(async move {
let mut client = connect_lightwalletd().await?;
while let Some(range) = download_rx.recv().await {
log::info!("{:?}", range);
let blocks = download_chain(&mut client, range.start, range.end).await.unwrap();
log::info!("+ {:?}", range);
let blocks = download_chain(&mut client, range.start, range.end).await?;
log::info!("- {:?}", range);
let b = Blocks(blocks);
processor_tx.send(b).await.unwrap();
processor_tx.send(b).await?;
}
log::info!("download completed");
drop(processor_tx);
// Ok::<_, anyhow::Error>(())
Ok::<_, anyhow::Error>(())
});
tokio::spawn(async move {
let db = DbAdapter::new(&db_path).unwrap();
let (mut tree, mut witnesses) = db.get_tree().unwrap();
let mut pos = tree.get_position();
// let mut tree = CTree::new();
// let mut witnesses: Vec<Witness> = vec![];
let proc_callback = progress_callback.clone();
let processor = tokio::spawn(async move {
let db = DbAdapter::new(&db_path)?;
let mut nfs = db.get_nullifiers()?;
let (mut tree, mut witnesses) = db.get_tree()?;
let mut absolute_position_at_block_start = tree.get_position();
let mut last_block: Option<BlockMetadata> = None;
while let Some(blocks) = processor_rx.recv().await {
log::info!("{:?}", blocks);
if blocks.0.is_empty() { continue }
let dec_blocks = decrypter.decrypt_blocks(&blocks.0);
for b in dec_blocks.iter() {
for nf in b.spends.iter() {
if let Some(&id) = nfs.get(nf) {
println!("NF FOUND {} {}", id, b.height);
db.mark_spent(id, b.height)?;
}
}
if !b.notes.is_empty() {
log::info!("{} {}", b.height, b.notes.len());
for nf in b.spends.iter() {
println!("{}", hex::encode(nf.0));
}
}
for n in b.notes.iter() {
let p = pos + n.position;
let p = absolute_position_at_block_start + n.position_in_block;
let note = &n.note;
let id_tx = db.store_transaction(&n.txid, n.height, n.tx_index as u32).unwrap();
let id_tx = db.store_transaction(&n.txid, n.height, n.tx_index as u32)?;
let rcm = note.rcm().to_repr();
let nf = note.nf(&n.ivk.fvk.vk, n.position as u64);
let nf = note.nf(&n.ivk.fvk.vk, p as u64);
let id_note = db.store_received_note(&ReceivedNote {
height: n.height,
@ -123,15 +137,14 @@ pub async fn sync_async(ivk: &str, chunk_size: u32, db_path: &str, progress_call
value: note.value,
rcm: rcm.to_vec(),
nf: nf.0.to_vec(),
is_change: false, // TODO: it's change the ovk matches too
memo: vec![],
spent: false
}, id_tx, n.position).unwrap();
spent: None
}, id_tx, n.position_in_block)?;
nfs.insert(Nf(nf.0), id_note);
let w = Witness::new(p as usize, id_note, Some(n.clone()));
witnesses.push(w);
}
pos += b.count_outputs as usize;
absolute_position_at_block_start += b.count_outputs as usize;
}
let mut nodes: Vec<Node> = vec![];
@ -150,19 +163,37 @@ pub async fn sync_async(ivk: &str, chunk_size: u32, db_path: &str, progress_call
tree = new_tree;
witnesses = new_witnesses;
let last_block = blocks.0.last().unwrap();
let last_height = last_block.height as u32;
db.store_block(last_height, &last_block.hash, &tree).unwrap();
for w in witnesses.iter() {
db.store_witnesses(w, last_height, w.id_note).unwrap();
if let Some(block) = blocks.0.last() {
let mut hash = [0u8; 32];
hash.copy_from_slice(&block.hash);
last_block = Some(BlockMetadata {
height: block.height as u32,
hash,
});
}
progress_callback(blocks.0[0].height as u32);
let callback = proc_callback.lock().await;
callback(blocks.0[0].height as u32);
}
progress_callback(end_height);
// Finalize scan
let (new_tree, new_witnesses) = advance_tree(tree, &witnesses, &mut []);
tree = new_tree;
witnesses = new_witnesses;
if let Some(last_block) = last_block {
let last_height = last_block.height;
db.store_block(last_height, &last_block.hash, &tree)?;
for w in witnesses.iter() {
db.store_witnesses(w, last_height, w.id_note)?;
}
}
// let callback = progress_callback.lock()?;
// callback(end_height);
log::info!("Witnesses {}", witnesses.len());
drop(completed_tx);
Ok::<_, anyhow::Error>(())
});
let mut height = start_height;
@ -181,10 +212,20 @@ pub async fn sync_async(ivk: &str, chunk_size: u32, db_path: &str, progress_call
completed_rx.recv().await;
log::info!("completed");
let res = tokio::try_join!(downloader, processor);
match res {
Ok((a, b)) => {
if let Err(err) = a { log::info!("Downloader error = {}", err) }
if let Err(err) = b { log::info!("Processor error = {}", err) }
},
Err(err) => log::info!("Sync error = {}", err),
}
Ok(())
}
pub async fn latest_height() -> u32 {
let mut client = connect_lightwalletd().await.unwrap();
get_latest_height(&mut client).await.unwrap()
pub async fn latest_height() -> anyhow::Result<u32> {
let mut client = connect_lightwalletd().await?;
let height = get_latest_height(&mut client).await?;
Ok(height)
}

150
src/wallet.rs Normal file
View File

@ -0,0 +1,150 @@
use crate::{NETWORK, get_latest_height, connect_lightwalletd, DbAdapter, get_secret_key, get_viewing_key, get_address};
use zcash_client_backend::address::RecipientAddress;
use zcash_primitives::transaction::components::Amount;
use zcash_primitives::transaction::builder::Builder;
use zcash_client_backend::encoding::decode_extended_spending_key;
use zcash_primitives::consensus::{Parameters, BlockHeight, BranchId};
use zcash_primitives::zip32::ExtendedFullViewingKey;
use zcash_client_backend::data_api::wallet::ANCHOR_OFFSET;
use rand::prelude::SliceRandom;
use rand::rngs::OsRng;
use zcash_proofs::prover::LocalTxProver;
use crate::chain::send_transaction;
use zcash_params::{SPEND_PARAMS, OUTPUT_PARAMS};
use std::sync::Arc;
use tokio::sync::Mutex;
use crate::scan::ProgressCallback;
use zcash_primitives::transaction::components::amount::DEFAULT_FEE;
pub const DEFAULT_ACCOUNT: u32 = 1;
const DEFAULT_CHUNK_SIZE: u32 = 100_000;
pub struct Wallet {
db_path: String,
db: DbAdapter,
prover: LocalTxProver,
}
impl Wallet {
pub fn new(db_path: &str) -> Wallet {
let prover = LocalTxProver::from_bytes(SPEND_PARAMS, OUTPUT_PARAMS);
let db = DbAdapter::new(db_path).unwrap();
db.init_db().unwrap();
Wallet {
db_path: db_path.to_string(),
db,
prover,
}
}
pub fn new_account_with_seed(&self, seed: &str) -> anyhow::Result<()> {
let sk = get_secret_key(&seed).unwrap();
let vk = get_viewing_key(&sk).unwrap();
let pa = get_address(&vk).unwrap();
self.db.store_account(&sk, &vk, &pa)?;
Ok(())
}
async fn scan_async(&self, ivk: &str, chunk_size: u32, target_height_offset: u32, progress_callback: ProgressCallback) -> anyhow::Result<()> {
crate::scan::sync_async(ivk, chunk_size, &self.db_path, target_height_offset, progress_callback).await
}
pub async fn get_latest_height() -> anyhow::Result<u32> {
let mut client = connect_lightwalletd().await?;
let last_height = get_latest_height(&mut client).await?;
Ok(last_height)
}
pub async fn sync(&self, account: u32, progress_callback: impl Fn(u32) + Send + 'static) -> anyhow::Result<()> {
let ivk = self.db.get_ivk(account)?;
let cb = Arc::new(Mutex::new(progress_callback));
self.scan_async(&ivk, DEFAULT_CHUNK_SIZE, 10, cb.clone()).await?;
self.scan_async(&ivk, DEFAULT_CHUNK_SIZE, 0, cb.clone()).await?;
Ok(())
}
pub fn get_balance(&self) -> anyhow::Result<u64> {
self.db.get_balance()
}
pub async fn send_payment(&self, account: u32, to_address: &str, amount: u64) -> anyhow::Result<String> {
let secret_key = self.db.get_sk(account)?;
let to_addr = RecipientAddress::decode(&NETWORK, to_address)
.ok_or(anyhow::anyhow!("Invalid address"))?;
let target_amount = Amount::from_u64(amount).unwrap();
let skey = decode_extended_spending_key(NETWORK.hrp_sapling_extended_spending_key(), &secret_key)?.unwrap();
let extfvk = ExtendedFullViewingKey::from(&skey);
let (_, change_address) = extfvk.default_address().unwrap();
let ovk = extfvk.fvk.ovk;
let last_height = Self::get_latest_height().await?;
let mut builder = Builder::new(NETWORK, BlockHeight::from_u32(last_height));
let anchor_height = self.db.get_last_sync_height()?.ok_or_else(|| anyhow::anyhow!("No spendable notes"))?;
let anchor_height = anchor_height.min(last_height - ANCHOR_OFFSET);
log::info!("Anchor = {}", anchor_height);
let mut notes = self.db.get_spendable_notes(anchor_height, &extfvk)?;
notes.shuffle(&mut OsRng);
log::info!("Spendable notes = {}", notes.len());
let mut amount = target_amount;
amount += DEFAULT_FEE;
for n in notes {
if amount.is_positive() {
let a = amount.min(Amount::from_u64(n.note.value).unwrap());
amount -= a;
let merkle_path = n.witness.path().unwrap();
builder.add_sapling_spend(skey.clone(), n.diversifier, n.note.clone(), merkle_path)?;
}
}
if amount.is_positive() {
anyhow::bail!("Not enough balance")
}
builder.send_change_to(Some(ovk), change_address);
match to_addr {
RecipientAddress::Shielded(pa) => builder.add_sapling_output(Some(ovk), pa, target_amount, None),
RecipientAddress::Transparent(t_address) => builder.add_transparent_output(&t_address, target_amount),
}?;
let consensus_branch_id = BranchId::for_height(&NETWORK, BlockHeight::from_u32(last_height));
let (tx, _) = builder.build(consensus_branch_id, &self.prover)?;
let mut raw_tx: Vec<u8> = vec![];
tx.write(&mut raw_tx)?;
let mut client = connect_lightwalletd().await?;
let tx_id = send_transaction(&mut client, &raw_tx, last_height).await?;
Ok(tx_id)
}
}
#[cfg(test)]
mod tests {
use crate::{get_secret_key, get_viewing_key, get_address};
use crate::wallet::Wallet;
#[tokio::test]
async fn test_wallet_seed() {
dotenv::dotenv().unwrap();
env_logger::init();
let seed = dotenv::var("SEED").unwrap();
let wallet = Wallet::new("zec.db");
wallet.new_account_with_seed(&seed).unwrap();
}
#[tokio::test]
async fn test_payment() {
dotenv::dotenv().unwrap();
env_logger::init();
let seed = dotenv::var("SEED").unwrap();
let sk = get_secret_key(&seed).unwrap();
let vk = get_viewing_key(&sk).unwrap();
println!("{}", vk);
let pa = get_address(&vk).unwrap();
println!("{}", pa);
let wallet = Wallet::new("zec.db");
let tx_id = wallet.send_payment(1, &pa, 1000).await.unwrap();
println!("TXID = {}", tx_id);
}
}