diff --git a/.gitignore b/.gitignore index c874056..a435e8c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ .idea/ src/generated/*.rs docs/_site/ +*.db diff --git a/Cargo.lock b/Cargo.lock index 3b3046f..c5794e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,12 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "ahash" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" + [[package]] name = "aho-corasick" version = "0.7.18" @@ -127,9 +133,9 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" [[package]] name = "bech32" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c7f7096bc256f5e5cb960f60dfc4f4ef979ca65abe7fb9d5a4f77150d3783d4" +checksum = "cf9ff0bbfd639f15c74af777d81383cf53efb7c93613f6cab67c6c11e05bbf8b" [[package]] name = "bellman" @@ -638,11 +644,24 @@ dependencies = [ [[package]] name = "equihash" version = "0.1.0" +source = "git+https://github.com/zcash/librustzcash.git?rev=d50bb12a97da768dc8f3ee39b81f84262103e6eb#d50bb12a97da768dc8f3ee39b81f84262103e6eb" dependencies = [ "blake2b_simd", "byteorder", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "ff" version = "0.9.0" @@ -893,6 +912,18 @@ name = "hashbrown" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashlink" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d99cf782f0dc4372d26846bec3de7804ceb5df083c2d4462c0b8d2330e894fa8" +dependencies = [ + "hashbrown", +] [[package]] name = "heck" @@ -1064,6 +1095,17 @@ version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6" +[[package]] +name = "libsqlite3-sys" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d31059f22935e6c31830db5249ba2b7ecd54fd73a9909286f0a67aa55c2fbd" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "log" version = "0.4.14" @@ -1294,6 +1336,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + [[package]] name = "plotters" version = "0.3.1" @@ -1577,6 +1625,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "rusqlite" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38ee71cbab2c827ec0ac24e76f82eca723cee92c509a65f67dee393c25112" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "memchr", + "smallvec", + "time 0.2.27", +] + [[package]] name = "rustc_version" version = "0.2.3" @@ -1783,6 +1847,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527" +[[package]] +name = "smallvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" + [[package]] name = "socket2" version = "0.4.0" @@ -1907,6 +1977,7 @@ dependencies = [ "tonic", "tonic-build", "zcash_client_backend", + "zcash_client_sqlite", "zcash_primitives", "zcash_proofs", ] @@ -2252,6 +2323,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.3" @@ -2415,6 +2492,7 @@ checksum = "9fc79f4a1e39857fc00c3f662cbf2651c771f00e9c15fe2abc341806bd46bd71" [[package]] name = "zcash_client_backend" version = "0.5.0" +source = "git+https://github.com/zcash/librustzcash.git?rev=d50bb12a97da768dc8f3ee39b81f84262103e6eb#d50bb12a97da768dc8f3ee39b81f84262103e6eb" dependencies = [ "base64", "bech32", @@ -2435,9 +2513,28 @@ dependencies = [ "zcash_primitives", ] +[[package]] +name = "zcash_client_sqlite" +version = "0.3.0" +source = "git+https://github.com/zcash/librustzcash.git?rev=d50bb12a97da768dc8f3ee39b81f84262103e6eb#d50bb12a97da768dc8f3ee39b81f84262103e6eb" +dependencies = [ + "bech32", + "bs58", + "ff", + "group", + "jubjub", + "protobuf", + "rand_core", + "rusqlite", + "time 0.2.27", + "zcash_client_backend", + "zcash_primitives", +] + [[package]] name = "zcash_note_encryption" version = "0.0.0" +source = "git+https://github.com/zcash/librustzcash.git?rev=d50bb12a97da768dc8f3ee39b81f84262103e6eb#d50bb12a97da768dc8f3ee39b81f84262103e6eb" dependencies = [ "blake2b_simd", "byteorder", @@ -2451,6 +2548,7 @@ dependencies = [ [[package]] name = "zcash_primitives" version = "0.5.0" +source = "git+https://github.com/zcash/librustzcash.git?rev=d50bb12a97da768dc8f3ee39b81f84262103e6eb#d50bb12a97da768dc8f3ee39b81f84262103e6eb" dependencies = [ "aes", "bitvec 0.20.4", @@ -2478,6 +2576,7 @@ dependencies = [ [[package]] name = "zcash_proofs" version = "0.5.0" +source = "git+https://github.com/zcash/librustzcash.git?rev=d50bb12a97da768dc8f3ee39b81f84262103e6eb#d50bb12a97da768dc8f3ee39b81f84262103e6eb" dependencies = [ "bellman", "blake2b_simd", diff --git a/Cargo.toml b/Cargo.toml index 3e29124..d1fd338 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,10 @@ edition = "2018" name = "scan_all" harness = false +[[bin]] +name = "warp-cli" +path = "src/main/warp_cli.rs" + [dependencies] dotenv = "0.15.0" env_logger = "0.8.4" @@ -31,15 +35,21 @@ tonic = {version = "0.4.3", features = ["tls", "tls-roots"]} prost = "0.7" rayon = "1.5.1" byteorder = "1.4.3" +tiny-bip39 = "0.8" +rand = "0.8.4" +rusqlite = "^0.25.3" [dependencies.zcash_client_backend] -path = "../librustzcash/zcash_client_backend" +git = "https://github.com/zcash/librustzcash.git" +rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb" [dependencies.zcash_primitives] -path = "../librustzcash/zcash_primitives" +git = "https://github.com/zcash/librustzcash.git" +rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb" [dependencies.zcash_proofs] -path = "../librustzcash/zcash_proofs" +git = "https://github.com/zcash/librustzcash.git" +rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb" [build-dependencies] tonic-build = "0.4.2" diff --git a/benches/scan_all.rs b/benches/scan_all.rs index f9a153e..e5ab403 100644 --- a/benches/scan_all.rs +++ b/benches/scan_all.rs @@ -14,13 +14,12 @@ fn scan(c: &mut Criterion) { .unwrap() .unwrap(); - let ivk = fvk.fvk.vk.ivk(); - let ivks = &vec![ivk]; + let fvks = &vec![fvk]; c.bench_function("scan all", |b| { b.iter(|| { let r = Runtime::new().unwrap(); - r.block_on(scan_all(ivks.clone().as_slice())).unwrap(); + r.block_on(scan_all(fvks.clone().as_slice())).unwrap(); }); }); } diff --git a/src/builder.rs b/src/builder.rs index 83c8c50..f5f9ec9 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -315,16 +315,15 @@ pub fn advance_tree( mod tests { use crate::builder::advance_tree; use crate::commitment::{CTree, Witness}; - use crate::print::{print_tree, print_witness}; use zcash_primitives::merkle_tree::{CommitmentTree, IncrementalWitness}; use zcash_primitives::sapling::Node; + use crate::chain::DecryptedNote; #[test] fn test_advance_tree() { const NUM_NODES: usize = 1000; const NUM_CHUNKS: usize = 50; const WITNESS_PERCENT: f64 = 1.0; // percentage of notes that are ours - const DEBUG_PRINT: bool = true; let witness_freq = (100.0 / WITNESS_PERCENT) as usize; let mut tree1: CommitmentTree = CommitmentTree::empty(); @@ -347,7 +346,7 @@ mod tests { // if v == 499 { let w = IncrementalWitness::from_tree(&tree1); ws.push(w); - ws2.push(Witness::new(v)); + ws2.push(Witness::new(v, 0, None)); } nodes.push(node); } @@ -382,48 +381,6 @@ mod tests { } } - if DEBUG_PRINT && (failed_index.is_some() || !equal) { - let i = failed_index.unwrap(); - println!("FAILED AT {}", i); - print_witness(&ws[i]); - - // println!("-----"); - // println!("Final-----"); - // - // println!("{:?}", tree2.left.map(|n| hex::encode(n.repr))); - // println!("{:?}", tree2.right.map(|n| hex::encode(n.repr))); - // for p in tree2.parents.iter() { - // println!("{:?}", p.map(|n| hex::encode(n.repr))); - // } - // println!("-----"); - - // println!("{:?}", tree1.left.map(|n| hex::encode(n.repr))); - // println!("{:?}", tree1.right.map(|n| hex::encode(n.repr))); - // for p in tree1.parents.iter() { - // println!("{:?}", p.map(|n| hex::encode(n.repr))); - // } - println!("----- {}", ws2[i].position); - let tree2 = &ws2[i].tree; - println!("{:?}", tree2.left.map(|n| hex::encode(n.repr))); - println!("{:?}", tree2.right.map(|n| hex::encode(n.repr))); - for p in tree2.parents.iter() { - println!("{:?}", p.map(|n| hex::encode(n.repr))); - } - println!("-----"); - let filled2 = &ws2[i].filled; - println!("Filled"); - for f in filled2.iter() { - println!("{:?}", hex::encode(f.repr)); - } - println!("Cursor"); - let cursor2 = &ws2[i].cursor; - println!("{:?}", cursor2.left.map(|n| hex::encode(n.repr))); - println!("{:?}", cursor2.right.map(|n| hex::encode(n.repr))); - for p in cursor2.parents.iter() { - println!("{:?}", p.map(|n| hex::encode(n.repr))); - } - - assert!(false); - } + assert!(equal && failed_index.is_none()); } } diff --git a/src/ca.pem b/src/ca.pem new file mode 100644 index 0000000..b2e43c9 --- /dev/null +++ b/src/ca.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSjCCAjKgAwIBAgIQRK+wgNajJ7qJMDmGLvhAazANBgkqhkiG9w0BAQUFADA/ +MSQwIgYDVQQKExtEaWdpdGFsIFNpZ25hdHVyZSBUcnVzdCBDby4xFzAVBgNVBAMT +DkRTVCBSb290IENBIFgzMB4XDTAwMDkzMDIxMTIxOVoXDTIxMDkzMDE0MDExNVow +PzEkMCIGA1UEChMbRGlnaXRhbCBTaWduYXR1cmUgVHJ1c3QgQ28uMRcwFQYDVQQD +Ew5EU1QgUm9vdCBDQSBYMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +AN+v6ZdQCINXtMxiZfaQguzH0yxrMMpb7NnDfcdAwRgUi+DoM3ZJKuM/IUmTrE4O +rz5Iy2Xu/NMhD2XSKtkyj4zl93ewEnu1lcCJo6m67XMuegwGMoOifooUMM0RoOEq +OLl5CjH9UL2AZd+3UWODyOKIYepLYYHsUmu5ouJLGiifSKOeDNoJjj4XLh7dIN9b +xiqKqy69cK3FCxolkHRyxXtqqzTWMIn/5WgTe1QLyNau7Fqckh49ZLOMxt+/yUFw +7BZy1SbsOFU5Q9D8/RhcQPGX69Wam40dutolucbY38EVAjqr2m7xPi71XAicPNaD +aeQQmxkqtilX4+U9m5/wAl0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNV +HQ8BAf8EBAMCAQYwHQYDVR0OBBYEFMSnsaR7LHH62+FLkHX/xBVghYkQMA0GCSqG +SIb3DQEBBQUAA4IBAQCjGiybFwBcqR7uKGY3Or+Dxz9LwwmglSBd49lZRNI+DT69 +ikugdB/OEIKcdBodfpga3csTS7MgROSR6cz8faXbauX+5v3gTt23ADq1cEmv8uXr +AvHRAosZy5Q6XkjEGB5YGV8eAlrwDPGxrancWYaLbumR9YbK+rlmM6pZW87ipxZz +R8srzJmwN0jP41ZL9c8PDHIyh8bwRLtTcm1D9SZImlJnt1ir/md2cXjbDaJWFBM5 +JDGFoqgCWjBH4d1QB7wCCZAA62RjYJsWvIjJEubSfZGL+T0yjWW06XyxV3bqxbYo +Ob8VZRzI9neWagqNdwvYkQsEjgfbKbYK7p2CNTUQ +-----END CERTIFICATE----- diff --git a/src/chain.rs b/src/chain.rs index 9650ad8..5356929 100644 --- a/src/chain.rs +++ b/src/chain.rs @@ -1,21 +1,26 @@ use crate::lw_rpc::compact_tx_streamer_client::CompactTxStreamerClient; use crate::lw_rpc::*; -use crate::NETWORK; +use crate::{NETWORK, advance_tree}; use ff::PrimeField; use group::GroupEncoding; use rayon::prelude::*; -use tonic::transport::Channel; +use tonic::transport::{Channel, Certificate, ClientTlsConfig}; use tonic::Request; -use zcash_primitives::consensus::BlockHeight; +use zcash_primitives::consensus::{BlockHeight, Parameters, NetworkUpgrade}; use zcash_primitives::merkle_tree::{CommitmentTree, IncrementalWitness}; use zcash_primitives::sapling::note_encryption::try_sapling_compact_note_decryption; -use zcash_primitives::sapling::{Node, Note, SaplingIvk}; +use zcash_primitives::sapling::{Node, Note, PaymentAddress}; use zcash_primitives::transaction::components::sapling::CompactOutputDescription; -use crate::commitment::{CTree, Witness, NotePosition}; +use crate::commitment::{CTree, Witness}; use std::time::Instant; use log::info; +use zcash_client_backend::encoding::decode_extended_full_viewing_key; +use zcash_primitives::zip32::ExtendedFullViewingKey; const MAX_CHUNK: u32 = 50000; +// pub const LWD_URL: &str = "https://mainnet.lightwalletd.com:9067"; +// pub const LWD_URL: &str = "http://lwd.hanh.me:9067"; +// pub const LWD_URL: &str = "https://lwdv3.zecwallet.co"; pub const LWD_URL: &str = "http://127.0.0.1:9067"; pub async fn get_latest_height( @@ -60,7 +65,7 @@ pub async fn download_chain( } pub struct DecryptNode { - ivks: Vec, + fvks: Vec, } pub struct DecryptedBlock { @@ -69,17 +74,25 @@ pub struct DecryptedBlock { pub count_outputs: u32, } +#[derive(Clone)] pub struct DecryptedNote { + pub ivk: ExtendedFullViewingKey, pub note: Note, - pub position: u32, + pub pa: PaymentAddress, + pub position: usize, + + pub height: u32, + pub txid: Vec, + pub tx_index: usize, + pub output_index: usize, } -fn decrypt_notes(block: &CompactBlock, ivks: &[SaplingIvk]) -> DecryptedBlock { +fn decrypt_notes(block: &CompactBlock, fvks: &[ExtendedFullViewingKey]) -> DecryptedBlock { let height = BlockHeight::from_u32(block.height as u32); let mut count_outputs = 0u32; let mut notes: Vec = vec![]; - for vtx in block.vtx.iter() { - for co in vtx.outputs.iter() { + for (tx_index, vtx) in block.vtx.iter().enumerate() { + for (output_index, co) in vtx.outputs.iter().enumerate() { let mut cmu = [0u8; 32]; cmu.copy_from_slice(&co.cmu); let cmu = bls12_381::Scalar::from_repr(cmu).unwrap(); @@ -91,13 +104,20 @@ fn decrypt_notes(block: &CompactBlock, ivks: &[SaplingIvk]) -> DecryptedBlock { cmu, enc_ciphertext: co.ciphertext.to_vec(), }; - for ivk in ivks.iter() { - if let Some((note, _pa)) = + for fvk in fvks.iter() { + let ivk = &fvk.fvk.vk.ivk(); + if let Some((note, pa)) = try_sapling_compact_note_decryption(&NETWORK, height, ivk, &od) { notes.push(DecryptedNote { + ivk: fvk.clone(), note, - position: count_outputs, + pa, + position: count_outputs as usize, + height: block.height as u32, + tx_index, + txid: vtx.hash.clone(), + output_index, }); } } @@ -112,14 +132,14 @@ fn decrypt_notes(block: &CompactBlock, ivks: &[SaplingIvk]) -> DecryptedBlock { } impl DecryptNode { - pub fn new(ivks: Vec) -> DecryptNode { - DecryptNode { ivks } + pub fn new(fvks: Vec) -> DecryptNode { + DecryptNode { fvks } } pub fn decrypt_blocks(&self, blocks: &[CompactBlock]) -> Vec { let mut decrypted_blocks: Vec = blocks .par_iter() - .map(|b| decrypt_notes(b, &self.ivks)) + .map(|b| decrypt_notes(b, &self.fvks)) .collect(); decrypted_blocks.sort_by(|a, b| a.height.cmp(&b.height)); decrypted_blocks @@ -156,7 +176,7 @@ fn calculate_tree_state_v1( } // skip before height let mut notes = block.notes.iter(); let mut n = notes.next(); - let mut i = 0u32; + let mut i = 0usize; for tx in cb.vtx.iter() { for co in tx.outputs.iter() { let mut cmu = [0u8; 32]; @@ -192,9 +212,12 @@ pub fn calculate_tree_state_v2(cbs: &[CompactBlock], blocks: &[DecryptedBlock]) let start = Instant::now(); for (cb, block) in cbs.iter().zip(blocks) { assert_eq!(cb.height as u32, block.height); + if !block.notes.is_empty() { + println!("{} {}", block.height, block.notes.len()); + } let mut notes = block.notes.iter(); let mut n = notes.next(); - let mut i = 0u32; + let mut i = 0usize; for tx in cb.vtx.iter() { for co in tx.outputs.iter() { let mut cmu = [0u8; 32]; @@ -215,13 +238,57 @@ pub fn calculate_tree_state_v2(cbs: &[CompactBlock], blocks: &[DecryptedBlock]) } info!("Build CMU list: {} ms - {} nodes", start.elapsed().as_millis(), nodes.len()); - let start = Instant::now(); - let n = nodes.len(); - let mut positions: Vec<_> = positions.iter().map(|&p| NotePosition::new(p, n)).collect(); - let _frontier = CTree::calc_state(nodes, &mut positions, None); - let witnesses: Vec<_> = positions.iter().map(|p| p.witness.clone()).collect(); + let witnesses: Vec<_> = positions.iter().map(|p| Witness::new(*p, 0, None)).collect(); + let (_, new_witnesses) = advance_tree(CTree::new(), &witnesses, &mut nodes); info!("Tree State & Witnesses: {} ms", start.elapsed().as_millis()); - witnesses + new_witnesses +} + +pub async fn connect_lightwalletd() -> anyhow::Result> { + let mut channel = tonic::transport::Channel::from_shared(LWD_URL)?; + if LWD_URL.starts_with("https") { + let pem = include_bytes!("ca.pem"); + let ca = Certificate::from_pem(pem); + let tls = ClientTlsConfig::new().ca_certificate(ca); + channel = channel.tls_config(tls)?; + } + let client = CompactTxStreamerClient::connect(channel).await?; + Ok(client) +} + +pub async fn sync(ivk: &str) -> anyhow::Result<()> { + let fvk = + decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &ivk) + .unwrap() + .unwrap(); + let decrypter = DecryptNode::new(vec![fvk]); + let mut client = connect_lightwalletd().await?; + let start_height: u32 = crate::NETWORK + .activation_height(NetworkUpgrade::Sapling) + .unwrap() + .into(); + let end_height = get_latest_height(&mut client).await?; + + let start = Instant::now(); + let cbs = download_chain(&mut client, start_height, end_height).await?; + eprintln!("Download chain: {} ms", start.elapsed().as_millis()); + + let start = Instant::now(); + let blocks = decrypter.decrypt_blocks(&cbs); + eprintln!("Decrypt Notes: {} ms", start.elapsed().as_millis()); + + let start = Instant::now(); + let witnesses = calculate_tree_state_v2(&cbs, &blocks); + eprintln!("Tree State & Witnesses: {} ms", start.elapsed().as_millis()); + + eprintln!("# Witnesses {}", witnesses.len()); + for w in witnesses.iter() { + let mut bb: Vec = vec![]; + w.write(&mut bb).unwrap(); + log::info!("{}", hex::encode(&bb)); + } + + Ok(()) } #[cfg(test)] @@ -247,14 +314,13 @@ mod tests { #[tokio::test] async fn test_download_chain() -> anyhow::Result<()> { dotenv::dotenv().unwrap(); - let ivk = dotenv::var("IVK").unwrap(); + let fvk = dotenv::var("FVK").unwrap(); let fvk = - decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &ivk) + decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &fvk) .unwrap() .unwrap(); - let ivk = fvk.fvk.vk.ivk(); - let decrypter = DecryptNode::new(vec![ivk]); + let decrypter = DecryptNode::new(vec![fvk]); let mut client = CompactTxStreamerClient::connect(LWD_URL).await?; let start_height: u32 = crate::NETWORK .activation_height(NetworkUpgrade::Sapling) @@ -282,13 +348,15 @@ mod tests { // let witnesses = calculate_tree_state(&cbs, &blocks, 0, tree_state); + let start = Instant::now(); let witnesses = calculate_tree_state_v2(&cbs, &blocks); + eprintln!("Tree State & Witnesses: {} ms", start.elapsed().as_millis()); eprintln!("# Witnesses {}", witnesses.len()); for w in witnesses.iter() { let mut bb: Vec = vec![]; w.write(&mut bb).unwrap(); - eprintln!("{}", hex::encode(&bb)); + log::info!("{}", hex::encode(&bb)); } Ok(()) diff --git a/src/commitment.rs b/src/commitment.rs index 2bb11da..267bd6a 100644 --- a/src/commitment.rs +++ b/src/commitment.rs @@ -1,10 +1,9 @@ -use crate::path::MerklePath; use byteorder::WriteBytesExt; -use rayon::prelude::*; -use std::io::Write; -use zcash_primitives::merkle_tree::Hashable; +use std::io::{Write, Read}; +use zcash_primitives::merkle_tree::{Hashable, CommitmentTree}; use zcash_primitives::sapling::Node; use zcash_primitives::serialize::{Optional, Vector}; +use crate::chain::DecryptedNote; /* Same behavior and structure as CommitmentTree from librustzcash @@ -81,18 +80,41 @@ pub struct Witness { pub tree: CTree, // commitment tree at the moment the witness is created: immutable pub filled: Vec, // as more nodes are added, levels get filled up: won't change anymore pub cursor: CTree, // partial tree which still updates when nodes are added + + // not used for decryption but identifies the witness + pub id_note: u32, + pub note: Option, } impl Witness { - pub fn new(position: usize) -> Witness { + pub fn new(position: usize, id_note: u32, note: Option) -> Witness { Witness { position, + id_note, + note, tree: CTree::new(), filled: vec![], cursor: CTree::new(), } } + pub fn read(position: usize, id_note: u32, mut reader: R) -> std::io::Result { + let tree = CTree::read(&mut reader)?; + let filled = Vector::read(&mut reader, |r| Node::read(r))?; + let cursor = Optional::read(&mut reader, |r| CTree::read(r))?; + + let witness = Witness { + position, + id_note, + tree, + filled, + cursor: cursor.unwrap_or_else(CTree::new), + note: None, + }; + + Ok(witness) + } + pub fn write(&self, mut writer: W) -> std::io::Result<()> { self.tree.write(&mut writer)?; Vector::write(&mut writer, &self.filled, |w, n| n.write(w))?; @@ -106,191 +128,7 @@ impl Witness { } } -pub struct NotePosition { - p0: usize, - p: usize, - p2: usize, - c: usize, - pub witness: Witness, -} - -fn collect( - tree: &mut CTree, - mut p: usize, - depth: usize, - commitments: &[Node], - offset: usize, - cursor: bool, -) -> usize { - // println!("--> {} {} {}", depth, p, offset); - if p < offset { - return p; - } - if depth == 0 { - if p % 2 == 0 { - tree.left = Some(commitments[p - offset]); - } else { - tree.left = Some(commitments[p - 1 - offset]); - tree.right = Some(commitments[p - offset]); - p -= 1; - } - } else { - // the rest gets combined as a binary tree - if p % 2 != 0 { - tree.parents.push(Some(commitments[p - 1 - offset])); - } else if (cursor && p != offset) || !cursor && (p != 0 || offset != 0) { - tree.parents.push(None); - } - } - p -} - -impl NotePosition { - pub fn new(position: usize, count: usize) -> NotePosition { - let c = cursor_start_position(position, count); - NotePosition { - p0: position, - p: position, - p2: count - 1, - c, - witness: Witness::new(position), - } - } - - pub fn reset(&mut self, count: usize) { - let c = cursor_start_position(self.p0, count); - self.p = self.p0; - self.p2 = count - 1; - self.c = c; - self.witness.cursor = CTree::new(); - } - - fn collect(&mut self, depth: usize, commitments: &[Node], offset: usize) { - let count = commitments.len(); - let p = self.p; - - self.p = collect(&mut self.witness.tree, p, depth, commitments, offset, false); - - if p % 2 == 0 && p + 1 >= offset && p + 1 - offset < commitments.len() { - let filler = commitments[p + 1 - offset]; - self.witness.filled.push(filler); - } - - let c = self.c - offset; - let cursor_commitments = &commitments[c..count]; - // println!("c> {} {} {}", c, count, depth); - // println!("> {} {}", self.p2, self.c); - if !cursor_commitments.is_empty() { - let p2 = collect( - &mut self.witness.cursor, - self.p2, - depth, - cursor_commitments, - offset + c, - true, - ); - self.p2 = (p2 - self.c) / 2 + self.c / 2; - // println!("+ {} {}", self.p2, self.c); - } - - self.p /= 2; - self.c /= 2; - } -} - -fn cursor_start_position(mut position: usize, mut count: usize) -> usize { - assert!(position < count); - // same logic as filler - let mut depth = 0; - loop { - if position % 2 == 0 { - if position + 1 < count { - position += 1; - } else { - break; - } - } - - position /= 2; - count /= 2; - depth += 1; - } - (position + 1) << depth -} - impl CTree { - pub fn calc_state( - mut commitments: Vec, - positions: &mut [NotePosition], - prev_frontier: Option, - ) -> CTree { - let mut n = commitments.len(); - assert_ne!(n, 0); - - let prev_count = prev_frontier - .as_ref() - .map(|f| f.get_position()) - .unwrap_or(0); - let count = prev_count + n; - let mut last_path = prev_frontier - .as_ref() - .map(|f| MerklePath::new(f.left, f.right)); - let mut frontier = NotePosition::new(count - 1, count); - let mut offset = prev_count; - - for p in positions.iter_mut() { - p.reset(count); - } - - let mut depth = 0usize; - while n + offset > 0 { - if offset % 2 == 1 { - // start is not aligned - let mut lp = last_path.take().unwrap(); - let node = lp.get(); // prepend the last node from the previous run - if n > 0 { - lp.set(commitments[0]); // put the right node into the path - } - last_path = Some(lp); - commitments.insert(0, node); - n += 1; - offset -= 1; - } - let commitment_slice = &commitments[0..n]; - frontier.collect(depth, commitment_slice, offset); - - for p in positions.iter_mut() { - p.collect(depth, commitment_slice, offset); - } - - let nn = n / 2; - let next_level: Vec<_> = (0..nn) - .into_par_iter() - .map(|i| Node::combine(depth, &commitments[2 * i], &commitments[2 * i + 1])) - .collect(); - commitments[0..nn].copy_from_slice(&next_level); - - if let Some(mut lp) = last_path.take() { - lp.up( - depth, - prev_frontier - .as_ref() - .unwrap() - .parents - .get(depth) - .unwrap_or(&None), - ); - last_path = Some(lp); - } - - depth += 1; - n = nn; - offset /= 2; - } - - frontier.witness.tree - } - pub fn new() -> CTree { CTree { left: None, @@ -299,11 +137,7 @@ impl CTree { } } - pub fn is_empty(&self) -> bool { - self.left.is_none() && self.right.is_none() - } - - pub(crate) fn write(&self, mut writer: W) -> std::io::Result<()> { + pub fn write(&self, mut writer: W) -> std::io::Result<()> { Optional::write(&mut writer, &self.left, |w, n| n.write(w))?; Optional::write(&mut writer, &self.right, |w, n| n.write(w))?; Vector::write(&mut writer, &self.parents, |w, e| { @@ -311,7 +145,19 @@ impl CTree { }) } - pub(crate) fn get_position(&self) -> usize { + pub fn read(mut reader: R) -> std::io::Result { + let left = Optional::read(&mut reader, |r| Node::read(r))?; + let right = Optional::read(&mut reader, |r| Node::read(r))?; + let parents = Vector::read(&mut reader, |r| Optional::read(r, |r| Node::read(r)))?; + + Ok(CTree { + left, + right, + parents, + }) + } + + pub fn get_position(&self) -> usize { let mut p = 0usize; for parent in self.parents.iter().rev() { if parent.is_some() { @@ -327,137 +173,10 @@ impl CTree { } p } -} -#[cfg(test)] -mod tests { - use crate::commitment::{cursor_start_position, CTree, NotePosition}; - #[allow(unused_imports)] - use crate::print::{print_tree, print_witness}; - use std::time::Instant; - use zcash_primitives::merkle_tree::{CommitmentTree, IncrementalWitness}; - use zcash_primitives::sapling::Node; - - /* - Build incremental witnesses with both methods and compare their binary serialization - */ - #[test] - fn test_calc_witnesses() { - const NUM_CHUNKS: usize = 3; - const NUM_NODES: usize = 20; // number of notes - const WITNESS_PERCENT: usize = 1; // percentage of notes that are ours - const DEBUG_PRINT: bool = true; - - let _witness_freq = 100000 / WITNESS_PERCENT; - let mut tree1: CommitmentTree = CommitmentTree::empty(); - let mut tree2: Option = None; - let mut witnesses: Vec> = vec![]; - let mut all_positions: Vec = vec![]; - - for c in 0..NUM_CHUNKS { - let mut positions: Vec = vec![]; - let mut nodes: Vec = vec![]; - for i in 1..=NUM_NODES { - let mut bb = [0u8; 32]; - bb[0..8].copy_from_slice(&i.to_be_bytes()); - let node = Node::new(bb); - - tree1.append(node).unwrap(); - - for w in witnesses.iter_mut() { - w.append(node).unwrap(); - } - - // if i % witness_freq == 0 { - if c == 0 && i == 1 { - let w = IncrementalWitness::::from_tree(&tree1); - witnesses.push(w); - positions.push((i - 1 + c * NUM_NODES) as usize); - } - - nodes.push(node); - } - - let start = Instant::now(); - let n = nodes.len(); - let mut positions: Vec<_> = positions - .iter() - .map(|&p| NotePosition::new(p, n + c * NUM_NODES)) - .collect(); - all_positions.append(&mut positions); - tree2 = Some(CTree::calc_state(nodes, &mut all_positions, tree2)); - eprintln!( - "Update State & Witnesses: {} ms", - start.elapsed().as_millis() - ); - } - let tree2 = tree2.unwrap(); - - println!("# witnesses = {}", all_positions.len()); - - for (_i, (w, p)) in witnesses.iter().zip(&all_positions).enumerate() { - let mut bb1: Vec = vec![]; - w.write(&mut bb1).unwrap(); - - let mut bb2: Vec = vec![]; - p.witness.write(&mut bb2).unwrap(); - - // assert_eq!(bb1.as_slice(), bb2.as_slice(), "failed at {}", i); - } - - let mut bb1: Vec = vec![]; - tree1.write(&mut bb1).unwrap(); - - let mut bb2: Vec = vec![]; - tree2.write(&mut bb2).unwrap(); - - assert_eq!(bb1.as_slice(), bb2.as_slice(), "tree states not equal"); - - if DEBUG_PRINT { - let slot = 0usize; - print_witness(&witnesses[slot]); - - println!("+++++"); - println!("Tree"); - let t = &all_positions[slot].witness.tree; - println!("{:?}", t.left.map(|n| hex::encode(n.repr))); - println!("{:?}", t.right.map(|n| hex::encode(n.repr))); - for p in t.parents.iter() { - println!("{:?}", p.map(|n| hex::encode(n.repr))); - } - println!("Filled"); - for f in all_positions[slot].witness.filled.iter() { - println!("{:?}", hex::encode(f.repr)); - } - println!("Cursor"); - let t = &all_positions[slot].witness.cursor; - println!("{:?}", t.left.map(|n| hex::encode(n.repr))); - println!("{:?}", t.right.map(|n| hex::encode(n.repr))); - for p in t.parents.iter() { - println!("{:?}", p.map(|n| hex::encode(n.repr))); - } - println!("===="); - - // println!("{:?}", tree1.left.map(|n| hex::encode(n.repr))); - // println!("{:?}", tree1.right.map(|n| hex::encode(n.repr))); - // for p in tree1.parents.iter() { - // println!("{:?}", p.map(|n| hex::encode(n.repr))); - // } - // - // println!("-----"); - // - // println!("{:?}", tree2.left.map(|n| hex::encode(n.repr))); - // println!("{:?}", tree2.right.map(|n| hex::encode(n.repr))); - // for p in tree2.parents.iter() { - // println!("{:?}", p.map(|n| hex::encode(n.repr))); - // } - } - } - - #[test] - fn test_cursor() { - // println!("{}", cursor_start_position(8, 14)); - println!("{}", cursor_start_position(9, 14)); - // println!("{}", cursor_start_position(10, 14)); + pub fn to_commitment_tree(&self) -> CommitmentTree { + let mut bb: Vec = vec![]; + self.write(&mut bb).unwrap(); + CommitmentTree::::read(&*bb).unwrap() } } diff --git a/src/db.rs b/src/db.rs new file mode 100644 index 0000000..0bb7291 --- /dev/null +++ b/src/db.rs @@ -0,0 +1,196 @@ +use rusqlite::{Connection, params, OptionalExtension}; +use crate::{Witness, CTree}; + +pub struct DbAdapter { + connection: Connection, +} + +pub struct ReceivedNote { + pub height: u32, + pub output_index: u32, + pub diversifier: Vec, + pub value: u64, + pub rcm: Vec, + pub nf: Vec, + pub is_change: bool, + pub memo: Vec, + pub spent: bool, +} + +impl DbAdapter { + pub fn new(db_path: &str) -> anyhow::Result { + let connection = Connection::open(db_path)?; + Ok(DbAdapter { + connection, + }) + } + + pub fn init_db(&self) -> anyhow::Result<()> { + self.connection.execute("CREATE TABLE IF NOT EXISTS blocks ( + height INTEGER PRIMARY KEY, + hash BLOB NOT NULL, + sapling_tree BLOB NOT NULL)", [])?; + + self.connection.execute("CREATE TABLE IF NOT EXISTS transactions ( + id_tx INTEGER PRIMARY KEY, + txid BLOB NOT NULL UNIQUE, + height INTEGER, + tx_index INTEGER)", [])?; + + self.connection.execute("CREATE TABLE IF NOT EXISTS received_notes ( + id_note INTEGER PRIMARY KEY, + position INTEGER NOT NULL, + tx INTEGER NOT NULL, + height INTEGER NOT NULL, + output_index INTEGER NOT NULL, + diversifier BLOB NOT NULL, + value INTEGER NOT NULL, + rcm BLOB NOT NULL, + nf BLOB NOT NULL UNIQUE, + is_change INTEGER NOT NULL, + memo BLOB, + spent INTEGER, + FOREIGN KEY (tx) REFERENCES transactions(id_tx), + FOREIGN KEY (spent) REFERENCES transactions(id_tx), + CONSTRAINT tx_output UNIQUE (tx, output_index))", [])?; + + self.connection.execute("CREATE TABLE IF NOT EXISTS sapling_witnesses ( + id_witness INTEGER PRIMARY KEY, + note INTEGER NOT NULL, + height INTEGER NOT NULL, + witness BLOB NOT NULL, + FOREIGN KEY (note) REFERENCES received_notes(id_note), + CONSTRAINT witness_height UNIQUE (note, height))", [])?; + + Ok(()) + } + + pub fn trim_to_height(&mut self, height: u32) -> anyhow::Result<()> { + let tx = self.connection.transaction()?; + tx.execute("DELETE FROM blocks WHERE height >= ?1", params![height])?; + tx.execute("DELETE FROM sapling_witnesses WHERE height >= ?1", params![height])?; + tx.execute("DELETE FROM received_notes WHERE height >= ?1", params![height])?; + tx.execute("DELETE FROM transactions WHERE height >= ?1", params![height])?; + tx.commit()?; + + Ok(()) + } + + pub fn store_block(&self, height: u32, hash: &[u8], tree: &CTree) -> anyhow::Result<()> { + let mut bb: Vec = vec![]; + tree.write(&mut bb)?; + self.connection.execute("INSERT INTO blocks(height, hash, sapling_tree) + VALUES (?1, ?2, ?3) + ON CONFLICT DO NOTHING", params![height, hash, &bb])?; + Ok(()) + } + + pub fn store_transaction(&self, txid: &[u8], height: u32, tx_index: u32) -> anyhow::Result { + self.connection.execute("INSERT INTO transactions(txid, height, tx_index) + VALUES (?1, ?2, ?3) + ON CONFLICT DO NOTHING", params![txid, height, tx_index])?; + let id_tx: u32 = self.connection.query_row("SELECT id_tx FROM transactions WHERE txid = ?1", params![txid], |row| row.get(0))?; + Ok(id_tx) + } + + pub fn store_received_note(&self, note: &ReceivedNote, id_tx: u32, position: usize) -> anyhow::Result { + self.connection.execute("INSERT INTO received_notes(tx, height, position, output_index, diversifier, value, rcm, nf, is_change, memo, spent) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) + ON CONFLICT DO NOTHING", params![id_tx, note.height, position, note.output_index, note.diversifier, note.value, note.rcm, note.nf, note.is_change, note.memo, note.spent])?; + let id_note: u32 = self.connection.query_row("SELECT id_note FROM received_notes WHERE tx = ?1 AND output_index = ?2", params![id_tx, note.output_index], |row| row.get(0))?; + Ok(id_note) + } + + pub fn store_witnesses(&self, witness: &Witness, height: u32, id_note: u32) -> anyhow::Result<()> { + let mut bb: Vec = vec![]; + witness.write(&mut bb)?; + println!("{} {}", height, id_note); + self.connection.execute("INSERT INTO sapling_witnesses(note, height, witness) VALUES (?1, ?2, ?3) + ON CONFLICT DO NOTHING", params![id_note, height, bb])?; + Ok(()) + } + + pub fn get_balance(&self) -> anyhow::Result { + let balance: u64 = self.connection.query_row("SELECT SUM(value) FROM received_notes WHERE spent = 0", [], |row| row.get(0))?; + Ok(balance) + } + + pub fn get_last_height(&self) -> anyhow::Result> { + let height: Option = self.connection.query_row("SELECT MAX(height) FROM blocks", [], |row| row.get(0)).optional()?; + Ok(height) + } + + pub fn get_tree(&self) -> anyhow::Result<(CTree, Vec)> { + let res = self.connection.query_row( + "SELECT height, sapling_tree FROM blocks WHERE height = (SELECT MAX(height) FROM blocks)", + [], |row| { + let height: u32 = row.get(0)?; + let tree: Vec = row.get(1)?; + Ok((height, tree)) + }).optional()?; + Ok(match res { + Some((height, tree)) => { + let tree = CTree::read(&*tree)?; + let mut statement = self.connection.prepare( + "SELECT id_note, position, witness FROM sapling_witnesses w, received_notes n WHERE w.height = ?1 AND w.note = n.id_note")?; + let ws = statement.query_map(params![height], |row| { + let id_note: u32 = row.get(0)?; + let position: u32 = row.get(1)?; + let witness: Vec = row.get(2)?; + Ok(Witness::read(position as usize, id_note, &*witness).unwrap()) + })?; + let mut witnesses: Vec = vec![]; + for w in ws { + witnesses.push(w?); + } + (tree, witnesses) + }, + None => (CTree::new(), vec![]) + }) + } +} + +#[cfg(test)] +mod tests { + use crate::db::{DbAdapter, ReceivedNote}; + use crate::{Witness, CTree}; + + const DB_PATH: &str = "zec.db"; + + #[test] + fn test_db() { + let mut db = DbAdapter::new(DB_PATH).unwrap(); + db.init_db().unwrap(); + db.trim_to_height(0).unwrap(); + + db.store_block(1, &[0u8; 32], &CTree::new()).unwrap(); + let id_tx = db.store_transaction(&[0; 32], 1, 20).unwrap(); + db.store_received_note(&ReceivedNote { + height: 1, + output_index: 0, + diversifier: vec![], + value: 0, + rcm: vec![], + nf: vec![], + is_change: false, + memo: vec![], + spent: false + }, id_tx, 5).unwrap(); + let witness = Witness { + position: 10, + id_note: 0, + note: None, + tree: CTree::new(), + filled: vec![], + cursor: CTree::new(), + }; + db.store_witnesses(&witness, 1000, 1).unwrap(); + } + + #[test] + fn test_balance() { + let db = DbAdapter::new(DB_PATH).unwrap(); + let balance = db.get_balance().unwrap(); + println!("{}", balance); + } +} diff --git a/src/key.rs b/src/key.rs new file mode 100644 index 0000000..8f565f6 --- /dev/null +++ b/src/key.rs @@ -0,0 +1,35 @@ +use bip39::{Language, Mnemonic, Seed}; +use zcash_primitives::zip32::{ExtendedSpendingKey, ExtendedFullViewingKey, ChildIndex}; +use crate::NETWORK; +use zcash_primitives::consensus::Parameters; +use zcash_client_backend::encoding::{encode_extended_spending_key, encode_extended_full_viewing_key, encode_payment_address, decode_extended_spending_key, decode_extended_full_viewing_key}; +use anyhow::anyhow; + +pub fn get_secret_key(seed: &str) -> anyhow::Result { + let mnemonic = Mnemonic::from_phrase(&seed, Language::English)?; + let seed = Seed::new(&mnemonic, ""); + let master = ExtendedSpendingKey::master(seed.as_bytes()); + let path = [ + ChildIndex::Hardened(32), + ChildIndex::Hardened(NETWORK.coin_type()), + ChildIndex::Hardened(0), + ]; + let extsk = ExtendedSpendingKey::from_path(&master, &path); + let spending_key = encode_extended_spending_key(NETWORK.hrp_sapling_extended_spending_key(), &extsk); + + Ok(spending_key) +} + +pub fn get_viewing_key(secret_key: &str) -> anyhow::Result { + let extsk = decode_extended_spending_key(NETWORK.hrp_sapling_extended_spending_key(), secret_key)?.ok_or(anyhow!("Invalid Secret Key"))?; + let fvk = ExtendedFullViewingKey::from(&extsk); + let viewing_key = encode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &fvk); + Ok(viewing_key) +} + +pub fn get_address(viewing_key: &str) -> anyhow::Result { + let fvk = decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &viewing_key)?.ok_or(anyhow!("Invalid Viewing Key"))?; + let (_, payment_address) = fvk.default_address().unwrap(); + let address = encode_payment_address(NETWORK.hrp_sapling_payment_address(), &payment_address); + Ok(address) +} diff --git a/src/lib.rs b/src/lib.rs index d6aaf3a..c4872ca 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,16 +5,22 @@ pub mod lw_rpc; pub const NETWORK: Network = Network::MainNetwork; -mod print; +mod builder; mod chain; -mod path; mod commitment; mod scan; -mod builder; +mod key; +mod db; +mod wallet; -pub use crate::chain::{LWD_URL, get_latest_height, download_chain, calculate_tree_state_v2, DecryptNode}; -pub use crate::commitment::{NotePosition, Witness, CTree}; pub use crate::builder::advance_tree; +pub use crate::chain::{ + calculate_tree_state_v2, connect_lightwalletd, download_chain, get_latest_height, sync, + DecryptNode, LWD_URL, +}; +pub use crate::commitment::{CTree, Witness}; pub use crate::lw_rpc::compact_tx_streamer_client::CompactTxStreamerClient; pub use crate::lw_rpc::*; -pub use crate::scan::scan_all; +pub use crate::scan::{scan_all, sync_async, latest_height}; +pub use crate::key::{get_secret_key, get_address, get_viewing_key}; +pub use crate::db::DbAdapter; diff --git a/src/main.rs b/src/main.rs index 5a49dc7..82bd34e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,9 +16,8 @@ async fn main_scan() { decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &ivk) .unwrap() .unwrap(); - let ivk = fvk.fvk.vk.ivk(); - scan_all(&vec![ivk]).await.unwrap(); + scan_all(&vec![fvk]).await.unwrap(); } #[allow(dead_code)] @@ -48,7 +47,7 @@ fn test_increasing_notes() { if v % witness_freq == 0 { // let w = IncrementalWitness::from_tree(&tree1); // ws.push(w); - ws2.push(Witness::new(v)); + ws2.push(Witness::new(v, 0, None)); } nodes.push(node); } @@ -89,7 +88,7 @@ fn test_increasing_gap(run_normal: bool, run_warp: bool) { let w = IncrementalWitness::from_tree(&tree1); ws.push(w); } - ws2.push(Witness::new(pos)); + ws2.push(Witness::new(pos, 0, None)); nodes.push(node); pos += 1; } diff --git a/src/main/warp_cli.rs b/src/main/warp_cli.rs new file mode 100644 index 0000000..1b3b1ac --- /dev/null +++ b/src/main/warp_cli.rs @@ -0,0 +1,40 @@ +use sync::{sync_async, DbAdapter}; +use bip39::{Language, Mnemonic}; +use rand::rngs::OsRng; +use rand::RngCore; + +const DB_NAME: &str = "zec.db"; + +#[tokio::main] +#[allow(dead_code)] +async fn test() -> anyhow::Result<()> { + dotenv::dotenv().unwrap(); + env_logger::init(); + + let ivk = dotenv::var("IVK").unwrap(); + { + let db = DbAdapter::new(DB_NAME)?; + db.init_db()?; + } + sync_async(&ivk, 50000, DB_NAME, |height| { + log::info!("Height = {}", height); + }).await?; + + Ok(()) +} + +#[allow(dead_code)] +fn test_rewind() { + let mut db = DbAdapter::new(DB_NAME).unwrap(); + db.trim_to_height(1_250_000).unwrap(); +} + +fn main() { + // test_rewind(); + test().unwrap(); + // let mut entropy = [0u8; 32]; + // OsRng.fill_bytes(&mut entropy); + // let mnemonic = Mnemonic::from_entropy(&entropy, Language::English).unwrap(); + // let phrase = mnemonic.phrase(); + // println!("Seed Phrase: {}", phrase); +} diff --git a/src/path.rs b/src/path.rs deleted file mode 100644 index 6a286ae..0000000 --- a/src/path.rs +++ /dev/null @@ -1,37 +0,0 @@ -use zcash_primitives::merkle_tree::Hashable; -use zcash_primitives::sapling::Node; - -pub struct MerklePath { - left: Option, - right: Option, -} - -impl MerklePath { - pub fn new(left: Option, right: Option) -> Self { - MerklePath { left, right } - } - - pub fn get(&self) -> Node { - self.left.unwrap() // shouldn't call if empty - } - - pub fn set(&mut self, right: Node) { - assert!(self.left.is_some()); - self.right = Some(right); - } - - pub fn up(&mut self, depth: usize, parent: &Option) { - let node = if self.left.is_some() && self.right.is_some() { - Some(Node::combine(depth, &self.left.unwrap(), &self.right.unwrap())) - } else { - None - }; - if parent.is_some() { - self.left = *parent; - self.right = node; - } else { - self.left = node; - self.right = None; - } - } -} diff --git a/src/print.rs b/src/print.rs deleted file mode 100644 index 975a520..0000000 --- a/src/print.rs +++ /dev/null @@ -1,28 +0,0 @@ -use zcash_primitives::sapling::Node; -use zcash_primitives::merkle_tree::{CommitmentTree, IncrementalWitness}; - -#[allow(dead_code)] -pub fn print_node(n: &Node) { - println!("{:?}", hex::encode(n.repr)); -} - -#[allow(dead_code)] -pub fn print_tree(t: &CommitmentTree) { - println!("{:?}", t.left.map(|n| hex::encode(n.repr))); - println!("{:?}", t.right.map(|n| hex::encode(n.repr))); - for p in t.parents.iter() { - println!("{:?}", p.map(|n| hex::encode(n.repr))); - } -} - -#[allow(dead_code)] -pub fn print_witness(w: &IncrementalWitness) { - println!("Tree"); - print_tree(&w.tree); - println!("Filled"); - for n in w.filled.iter() { - print_node(n); - } - println!("Cursor"); - w.cursor.as_ref().map(|c| print_tree(c)); -} diff --git a/src/scan.rs b/src/scan.rs index 06559f8..9d2c92e 100644 --- a/src/scan.rs +++ b/src/scan.rs @@ -1,12 +1,18 @@ -use zcash_primitives::sapling::SaplingIvk; +use zcash_primitives::sapling::Node; use crate::lw_rpc::compact_tx_streamer_client::CompactTxStreamerClient; -use crate::{DecryptNode, LWD_URL, get_latest_height, download_chain, calculate_tree_state_v2}; +use crate::{DecryptNode, LWD_URL, get_latest_height, download_chain, calculate_tree_state_v2, CompactBlock, NETWORK, connect_lightwalletd, Witness, advance_tree}; use zcash_primitives::consensus::{NetworkUpgrade, Parameters}; +use zcash_client_backend::encoding::decode_extended_full_viewing_key; +use tokio::sync::mpsc; use std::time::Instant; +use std::ops::Range; use log::info; +use crate::db::{DbAdapter, ReceivedNote}; +use ff::PrimeField; +use zcash_primitives::zip32::ExtendedFullViewingKey; -pub async fn scan_all(ivks: &[SaplingIvk]) -> anyhow::Result<()> { - let decrypter = DecryptNode::new(ivks.to_vec()); +pub async fn scan_all(fvks: &[ExtendedFullViewingKey]) -> anyhow::Result<()> { + let decrypter = DecryptNode::new(fvks.to_vec()); let total_start = Instant::now(); let mut client = CompactTxStreamerClient::connect(LWD_URL).await?; @@ -38,3 +44,147 @@ pub async fn scan_all(ivks: &[SaplingIvk]) -> anyhow::Result<()> { Ok(()) } +struct Blocks(Vec); + +impl std::fmt::Debug for Blocks { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Blocks of len {}", self.0.len()) + } +} + +fn get_db_height(db_path: &str) -> anyhow::Result { + let db = DbAdapter::new(db_path).unwrap(); + let height: u32 = db.get_last_height()?.unwrap_or_else(|| { + crate::NETWORK + .activation_height(NetworkUpgrade::Sapling) + .unwrap() + .into() + }); + Ok(height) +} + +pub async fn sync_async(ivk: &str, chunk_size: u32, db_path: &str, progress_callback: impl Fn(u32) + Send + 'static) -> anyhow::Result<()> { + let db_path = db_path.to_string(); + let fvk = + decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &ivk) + .unwrap() + .unwrap(); + let decrypter = DecryptNode::new(vec![fvk]); + + let mut client = connect_lightwalletd().await?; + let start_height = get_db_height(&db_path)?; + let end_height = get_latest_height(&mut client).await?; + + let (downloader_tx, mut download_rx) = mpsc::channel::>(2); + let (processor_tx, mut processor_rx) = mpsc::channel::(2); + let (completed_tx, mut completed_rx) = mpsc::channel::<()>(1); + + tokio::spawn(async move { + let mut client = connect_lightwalletd().await.unwrap(); + while let Some(range) = download_rx.recv().await { + log::info!("{:?}", range); + let blocks = download_chain(&mut client, range.start, range.end).await.unwrap(); + let b = Blocks(blocks); + processor_tx.send(b).await.unwrap(); + } + log::info!("download completed"); + drop(processor_tx); + + // Ok::<_, anyhow::Error>(()) + }); + + tokio::spawn(async move { + let db = DbAdapter::new(&db_path).unwrap(); + let (mut tree, mut witnesses) = db.get_tree().unwrap(); + let mut pos = tree.get_position(); + // let mut tree = CTree::new(); + // let mut witnesses: Vec = vec![]; + while let Some(blocks) = processor_rx.recv().await { + log::info!("{:?}", blocks); + if blocks.0.is_empty() { continue } + + let dec_blocks = decrypter.decrypt_blocks(&blocks.0); + for b in dec_blocks.iter() { + if !b.notes.is_empty() { + log::info!("{} {}", b.height, b.notes.len()); + } + for n in b.notes.iter() { + let p = pos + n.position; + + let note = &n.note; + let id_tx = db.store_transaction(&n.txid, n.height, n.tx_index as u32).unwrap(); + let rcm = note.rcm().to_repr(); + let nf = note.nf(&n.ivk.fvk.vk, n.position as u64); + + let id_note = db.store_received_note(&ReceivedNote { + height: n.height, + output_index: n.output_index as u32, + diversifier: n.pa.diversifier().0.to_vec(), + value: note.value, + rcm: rcm.to_vec(), + nf: nf.0.to_vec(), + is_change: false, // TODO: it's change the ovk matches too + memo: vec![], + spent: false + }, id_tx, n.position).unwrap(); + + let w = Witness::new(p as usize, id_note, Some(n.clone())); + witnesses.push(w); + } + pos += b.count_outputs as usize; + } + + let mut nodes: Vec = vec![]; + for cb in blocks.0.iter() { + for tx in cb.vtx.iter() { + for co in tx.outputs.iter() { + let mut cmu = [0u8; 32]; + cmu.copy_from_slice(&co.cmu); + let node = Node::new(cmu); + nodes.push(node); + } + } + } + + let (new_tree, new_witnesses) = advance_tree(tree, &witnesses, &mut nodes); + tree = new_tree; + witnesses = new_witnesses; + + let last_block = blocks.0.last().unwrap(); + let last_height = last_block.height as u32; + db.store_block(last_height, &last_block.hash, &tree).unwrap(); + for w in witnesses.iter() { + db.store_witnesses(w, last_height, w.id_note).unwrap(); + } + + progress_callback(blocks.0[0].height as u32); + } + + progress_callback(end_height); + log::info!("Witnesses {}", witnesses.len()); + drop(completed_tx); + }); + + let mut height = start_height; + while height < end_height { + let s = height; + let e = (height + chunk_size).min(end_height); + let range = s..e; + + downloader_tx.send(range).await?; + + height = e; + } + drop(downloader_tx); + log::info!("req downloading completed"); + + completed_rx.recv().await; + log::info!("completed"); + + Ok(()) +} + +pub async fn latest_height() -> u32 { + let mut client = connect_lightwalletd().await.unwrap(); + get_latest_height(&mut client).await.unwrap() +}