diff --git a/Cargo.lock b/Cargo.lock index 908399d..3b3046f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -250,6 +250,18 @@ dependencies = [ "sha2", ] +[[package]] +name = "bstr" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90682c8d613ad3373e66de8c6411e0ae2ab2571e879d2efbf73558cc66f21279" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.7.0" @@ -268,6 +280,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" +[[package]] +name = "cast" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57cdfa5d50aad6cb4d44dcab6101a7f79925bd59d82ca42f38a9856a28865374" +dependencies = [ + "rustc_version 0.3.3", +] + [[package]] name = "cc" version = "1.0.68" @@ -308,6 +329,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "bitflags", + "textwrap", + "unicode-width", +] + [[package]] name = "const_fn" version = "0.4.8" @@ -354,6 +386,42 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "criterion" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab327ed7354547cc2ef43cbe20ef68b988e70b4b593cbd66a2a61733123a3d23" +dependencies = [ + "atty", + "cast", + "clap", + "criterion-plot", + "csv", + "itertools 0.10.1", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e022feadec601fba1649cfa83586381a4ad31c6bf3a9ab7d408118b05dd9889d" +dependencies = [ + "cast", + "itertools 0.9.0", +] + [[package]] name = "crossbeam" version = "0.7.3" @@ -485,6 +553,28 @@ dependencies = [ "crypto_api", ] +[[package]] +name = "csv" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +dependencies = [ + "bstr", + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + [[package]] name = "digest" version = "0.9.0" @@ -532,10 +622,22 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +[[package]] +name = "env_logger" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + [[package]] name = "equihash" version = "0.1.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=d50bb12a97da768dc8f3ee39b81f84262103e6eb#d50bb12a97da768dc8f3ee39b81f84262103e6eb" dependencies = [ "blake2b_simd", "byteorder", @@ -780,6 +882,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62aca2aba2d62b4a7f5b33f3712cb1b0692779a56fb510499d5c0aa594daeaf3" + [[package]] name = "hashbrown" version = "0.9.1" @@ -844,6 +952,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.9" @@ -887,6 +1001,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69ddb889f9d0d08a67338271fa9b62996bc788c7796a5c18cf057420aaed5eaf" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.7" @@ -1086,6 +1209,12 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "opaque-debug" version = "0.3.0" @@ -1114,6 +1243,15 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + [[package]] name = "petgraph" version = "0.5.1" @@ -1156,6 +1294,34 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "plotters" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a3fd9ec30b9749ce28cd91f255d569591cdf937fe280c312143e3c4bad6f2a" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07fffcddc1cb3a1de753caa4e4df03b79922ba43cf882acc1bdd7e8df9f4590" + +[[package]] +name = "plotters-svg" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b38a02e23bd9604b842a812063aec4ef702b57989c37b655254bb61c471ad211" +dependencies = [ + "plotters-backend", +] + [[package]] name = "ppv-lite86" version = "0.2.10" @@ -1201,7 +1367,7 @@ checksum = "32d3ebd75ac2679c2af3a92246639f9fcc8a442ee420719cc4fe195b98dd5fa3" dependencies = [ "bytes", "heck", - "itertools", + "itertools 0.9.0", "log", "multimap", "petgraph", @@ -1218,7 +1384,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "169a15f3008ecb5160cba7d37bcd690a7601b6d30cfb87a117d45e59d52af5d4" dependencies = [ "anyhow", - "itertools", + "itertools 0.9.0", "proc-macro2", "quote", "syn", @@ -1375,6 +1541,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" + [[package]] name = "regex-syntax" version = "0.6.25" @@ -1405,24 +1577,22 @@ dependencies = [ "winapi", ] -[[package]] -name = "ripemd160" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eca4ecc81b7f313189bf73ce724400a07da2a6dac19588b03c8bd76a2dcc251" -dependencies = [ - "block-buffer", - "digest", - "opaque-debug", -] - [[package]] name = "rustc_version" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" dependencies = [ - "semver", + "semver 0.9.0", +] + +[[package]] +name = "rustc_version" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" +dependencies = [ + "semver 0.11.0", ] [[package]] @@ -1456,6 +1626,15 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.19" @@ -1482,24 +1661,6 @@ dependencies = [ "untrusted", ] -[[package]] -name = "secp256k1" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee5070fdc6f26ca5be6dcfc3d07c76fdb974a63a8b246b459854274145f5a258" -dependencies = [ - "secp256k1-sys", -] - -[[package]] -name = "secp256k1-sys" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67e4b6455ee49f5901c8985b88f98fb0a0e1d90a6661f5a03f4888bd987dad29" -dependencies = [ - "cc", -] - [[package]] name = "security-framework" version = "2.3.1" @@ -1529,7 +1690,16 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "semver-parser", + "semver-parser 0.7.0", +] + +[[package]] +name = "semver" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" +dependencies = [ + "semver-parser 0.10.2", ] [[package]] @@ -1538,6 +1708,15 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "semver-parser" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" +dependencies = [ + "pest", +] + [[package]] name = "serde" version = "1.0.126" @@ -1547,6 +1726,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_cbor" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e18acfa2f90e8b735b2836ab8d538de304cbb6729a7360729ea5a895d15a622" +dependencies = [ + "half", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.126" @@ -1632,7 +1821,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" dependencies = [ "discard", - "rustc_version", + "rustc_version 0.2.3", "stdweb-derive", "stdweb-internal-macros", "stdweb-internal-runtime", @@ -1697,8 +1886,11 @@ version = "0.1.0" dependencies = [ "anyhow", "bls12_381", + "byteorder", "bytes", + "criterion", "dotenv", + "env_logger", "ff", "flexi_logger", "futures 0.3.15", @@ -1739,6 +1931,24 @@ dependencies = [ "winapi", ] +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + [[package]] name = "thiserror" version = "1.0.25" @@ -1808,6 +2018,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tokio" version = "1.7.0" @@ -2002,12 +2222,24 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" +[[package]] +name = "ucd-trie" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + [[package]] name = "unicode-segmentation" version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + [[package]] name = "unicode-xid" version = "0.2.2" @@ -2026,6 +2258,17 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + [[package]] name = "want" version = "0.3.0" @@ -2142,6 +2385,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -2163,7 +2415,6 @@ checksum = "9fc79f4a1e39857fc00c3f662cbf2651c771f00e9c15fe2abc341806bd46bd71" [[package]] name = "zcash_client_backend" version = "0.5.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=d50bb12a97da768dc8f3ee39b81f84262103e6eb#d50bb12a97da768dc8f3ee39b81f84262103e6eb" dependencies = [ "base64", "bech32", @@ -2187,7 +2438,6 @@ dependencies = [ [[package]] name = "zcash_note_encryption" version = "0.0.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=d50bb12a97da768dc8f3ee39b81f84262103e6eb#d50bb12a97da768dc8f3ee39b81f84262103e6eb" dependencies = [ "blake2b_simd", "byteorder", @@ -2201,7 +2451,6 @@ dependencies = [ [[package]] name = "zcash_primitives" version = "0.5.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=d50bb12a97da768dc8f3ee39b81f84262103e6eb#d50bb12a97da768dc8f3ee39b81f84262103e6eb" dependencies = [ "aes", "bitvec 0.20.4", @@ -2221,8 +2470,6 @@ dependencies = [ "log", "rand", "rand_core", - "ripemd160", - "secp256k1", "sha2", "subtle", "zcash_note_encryption", @@ -2231,7 +2478,6 @@ dependencies = [ [[package]] name = "zcash_proofs" version = "0.5.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=d50bb12a97da768dc8f3ee39b81f84262103e6eb#d50bb12a97da768dc8f3ee39b81f84262103e6eb" dependencies = [ "bellman", "blake2b_simd", diff --git a/Cargo.toml b/Cargo.toml index 465089f..3e29124 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,8 +6,13 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[[bench]] +name = "scan_all" +harness = false + [dependencies] dotenv = "0.15.0" +env_logger = "0.8.4" anyhow = "1.0.40" log = "0.4.14" flexi_logger = {version="0.17.1", features = ["compress"]} @@ -25,19 +30,19 @@ futures = "0.3.15" tonic = {version = "0.4.3", features = ["tls", "tls-roots"]} prost = "0.7" rayon = "1.5.1" +byteorder = "1.4.3" [dependencies.zcash_client_backend] -git = "https://github.com/zcash/librustzcash.git" -rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb" +path = "../librustzcash/zcash_client_backend" [dependencies.zcash_primitives] -git = "https://github.com/zcash/librustzcash.git" -features = [ "transparent-inputs" ] -rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb" +path = "../librustzcash/zcash_primitives" [dependencies.zcash_proofs] -git = "https://github.com/zcash/librustzcash.git" -rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb" +path = "../librustzcash/zcash_proofs" [build-dependencies] tonic-build = "0.4.2" + +[dev-dependencies] +criterion = "0.3.4" diff --git a/benches/results.txt b/benches/results.txt new file mode 100644 index 0000000..1e46218 --- /dev/null +++ b/benches/results.txt @@ -0,0 +1,4 @@ +scan all time: [14.108 s 14.246 s 14.370 s] +Found 2 outliers among 10 measurements (20.00%) + 1 (10.00%) low mild + 1 (10.00%) high mild diff --git a/benches/scan_all.rs b/benches/scan_all.rs new file mode 100644 index 0000000..f9a153e --- /dev/null +++ b/benches/scan_all.rs @@ -0,0 +1,32 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +use tokio::runtime::Runtime; +use sync::{scan_all, NETWORK}; +use zcash_client_backend::encoding::decode_extended_full_viewing_key; +use zcash_primitives::consensus::Parameters; + +fn scan(c: &mut Criterion) { + dotenv::dotenv().unwrap(); + env_logger::init(); + + let ivk = dotenv::var("IVK").unwrap(); + let fvk = + decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &ivk) + .unwrap() + .unwrap(); + + let ivk = fvk.fvk.vk.ivk(); + let ivks = &vec![ivk]; + + c.bench_function("scan all", |b| { + b.iter(|| { + let r = Runtime::new().unwrap(); + r.block_on(scan_all(ivks.clone().as_slice())).unwrap(); + }); + }); +} + +criterion_group!( + name = benches; + config = Criterion::default().sample_size(10); + targets = scan); +criterion_main!(benches); diff --git a/out.log b/out.log new file mode 100644 index 0000000..dcab997 --- /dev/null +++ b/out.log @@ -0,0 +1,155 @@ + Compiling sync v0.1.0 (/home/hanh/projects/sync) +warning: unused import: `zcash_primitives::transaction::components::OutputDescription` + --> src/chain.rs:9:5 + | +9 | use zcash_primitives::transaction::components::OutputDescription; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `#[warn(unused_imports)]` on by default + +warning: unused import: `jubjub::Scalar` + --> src/chain.rs:10:5 + | +10 | use jubjub::Scalar; + | ^^^^^^^^^^^^^^ + +warning: unused import: `tokio::runtime::Runtime` + --> src/chain.rs:14:5 + | +14 | use tokio::runtime::Runtime; + | ^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused imports: `Arc`, `Mutex` + --> src/chain.rs:15:17 + | +15 | use std::sync::{Arc, Mutex}; + | ^^^ ^^^^^ + +warning: unused import: `tokio::task::JoinHandle` + --> src/chain.rs:16:5 + | +16 | use tokio::task::JoinHandle; + | ^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused import: `futures::future::JoinAll` + --> src/chain.rs:17:5 + | +17 | use futures::future::JoinAll; + | ^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused import: `tokio::runtime::Runtime` + --> src/chain.rs:104:9 + | +104 | use tokio::runtime::Runtime; + | ^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused import: `zcash_primitives::transaction::components::OutputDescription` + --> src/chain.rs:9:5 + | +9 | use zcash_primitives::transaction::components::OutputDescription; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `#[warn(unused_imports)]` on by default + +warning: unused import: `jubjub::Scalar` + --> src/chain.rs:10:5 + | +10 | use jubjub::Scalar; + | ^^^^^^^^^^^^^^ + +warning: unused import: `tokio::runtime::Runtime` + --> src/chain.rs:14:5 + | +14 | use tokio::runtime::Runtime; + | ^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused imports: `Arc`, `Mutex` + --> src/chain.rs:15:17 + | +15 | use std::sync::{Arc, Mutex}; + | ^^^ ^^^^^ + +warning: unused import: `tokio::task::JoinHandle` + --> src/chain.rs:16:5 + | +16 | use tokio::task::JoinHandle; + | ^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused import: `futures::future::JoinAll` + --> src/chain.rs:17:5 + | +17 | use futures::future::JoinAll; + | ^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: constant is never used: `MAX_CHUNK` + --> src/chain.rs:20:1 + | +20 | const MAX_CHUNK: u32 = 50000; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `#[warn(dead_code)]` on by default + +warning: function is never used: `get_latest_height` + --> src/chain.rs:22:14 + | +22 | pub async fn get_latest_height( + | ^^^^^^^^^^^^^^^^^ + +warning: function is never used: `download_chain` + --> src/chain.rs:32:14 + | +32 | pub async fn download_chain( + | ^^^^^^^^^^^^^^ + +warning: struct is never constructed: `DecryptNode` + --> src/chain.rs:55:8 + | +55 | struct DecryptNode { + | ^^^^^^^^^^^ + +warning: function is never used: `decrypt_notes` + --> src/chain.rs:59:4 + | +59 | fn decrypt_notes(block: &CompactBlock, ivks: &[SaplingIvk]) { + | ^^^^^^^^^^^^^ + +warning: associated function is never used: `new` + --> src/chain.rs:84:12 + | +84 | pub fn new(ivks: Vec) -> DecryptNode { + | ^^^ + +warning: associated function is never used: `decrypt_blocks` + --> src/chain.rs:89:12 + | +89 | pub fn decrypt_blocks(&self, blocks: &[CompactBlock]) { + | ^^^^^^^^^^^^^^ + +warning: 13 warnings emitted + +warning: 7 warnings emitted + + Finished release [optimized] target(s) in 5.46s + Running target/release/deps/sync-40a700cf0ceeec0a +419201 +469202 +519203 +569204 +619205 +669206 +719207 +769208 +819209 +869210 +919211 +969212 +1019213 +1069214 +1119215 +1169216 +1219217 +1269218 +Download chain: 5629 ms +Decrypt Notes: 5917 ms + Running target/release/deps/sync-0273dad5347eddbd + Doc-tests sync diff --git a/report/scan/base/benchmark.json b/report/scan/base/benchmark.json new file mode 100644 index 0000000..d466882 --- /dev/null +++ b/report/scan/base/benchmark.json @@ -0,0 +1 @@ +{"group_id":"scan all","function_id":null,"value_str":null,"throughput":null,"full_id":"scan all","directory_name":"scan all","title":"scan all"} \ No newline at end of file diff --git a/report/scan/base/estimates.json b/report/scan/base/estimates.json new file mode 100644 index 0000000..b563756 --- /dev/null +++ b/report/scan/base/estimates.json @@ -0,0 +1 @@ +{"mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":14108152718.8,"upper_bound":14370408508.0},"point_estimate":14246432770.5,"standard_error":66652427.41497482},"median":{"confidence_interval":{"confidence_level":0.95,"lower_bound":14168851704.0,"upper_bound":14373090215.5},"point_estimate":14240423154.5,"standard_error":54885063.868419364},"median_abs_dev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":31818705.17490506,"upper_bound":333947948.45060974},"point_estimate":120754458.69297981,"standard_error":75571123.53948104},"slope":null,"std_dev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":77029147.73519874,"upper_bound":313070155.1646717},"point_estimate":221212708.77470955,"standard_error":61920450.102123074}} \ No newline at end of file diff --git a/report/scan/base/raw.csv b/report/scan/base/raw.csv new file mode 100644 index 0000000..6517035 --- /dev/null +++ b/report/scan/base/raw.csv @@ -0,0 +1,11 @@ +group,function,value,throughput_num,throughput_type,sample_measured_value,unit,iteration_count +scan all,,,,,14435894614.0,ns,1 +scan all,,,,,14147390281.0,ns,1 +scan all,,,,,13760811376.0,ns,1 +scan all,,,,,14224292953.0,ns,1 +scan all,,,,,14175213190.0,ns,1 +scan all,,,,,14349106401.0,ns,1 +scan all,,,,,14256553356.0,ns,1 +scan all,,,,,14614466590.0,ns,1 +scan all,,,,,14190313127.0,ns,1 +scan all,,,,,14310285817.0,ns,1 diff --git a/report/scan/base/sample.json b/report/scan/base/sample.json new file mode 100644 index 0000000..34e79a6 --- /dev/null +++ b/report/scan/base/sample.json @@ -0,0 +1 @@ +{"sampling_mode":"Flat","iters":[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0],"times":[14435894614.0,14147390281.0,13760811376.0,14224292953.0,14175213190.0,14349106401.0,14256553356.0,14614466590.0,14190313127.0,14310285817.0]} \ No newline at end of file diff --git a/report/scan/base/tukey.json b/report/scan/base/tukey.json new file mode 100644 index 0000000..7c55da3 --- /dev/null +++ b/report/scan/base/tukey.json @@ -0,0 +1 @@ +[13697748932.0,13938368553.125,14580020876.125,14820640497.25] \ No newline at end of file diff --git a/report/scan/new/benchmark.json b/report/scan/new/benchmark.json new file mode 100644 index 0000000..d466882 --- /dev/null +++ b/report/scan/new/benchmark.json @@ -0,0 +1 @@ +{"group_id":"scan all","function_id":null,"value_str":null,"throughput":null,"full_id":"scan all","directory_name":"scan all","title":"scan all"} \ No newline at end of file diff --git a/report/scan/new/estimates.json b/report/scan/new/estimates.json new file mode 100644 index 0000000..b563756 --- /dev/null +++ b/report/scan/new/estimates.json @@ -0,0 +1 @@ +{"mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":14108152718.8,"upper_bound":14370408508.0},"point_estimate":14246432770.5,"standard_error":66652427.41497482},"median":{"confidence_interval":{"confidence_level":0.95,"lower_bound":14168851704.0,"upper_bound":14373090215.5},"point_estimate":14240423154.5,"standard_error":54885063.868419364},"median_abs_dev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":31818705.17490506,"upper_bound":333947948.45060974},"point_estimate":120754458.69297981,"standard_error":75571123.53948104},"slope":null,"std_dev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":77029147.73519874,"upper_bound":313070155.1646717},"point_estimate":221212708.77470955,"standard_error":61920450.102123074}} \ No newline at end of file diff --git a/report/scan/new/raw.csv b/report/scan/new/raw.csv new file mode 100644 index 0000000..6517035 --- /dev/null +++ b/report/scan/new/raw.csv @@ -0,0 +1,11 @@ +group,function,value,throughput_num,throughput_type,sample_measured_value,unit,iteration_count +scan all,,,,,14435894614.0,ns,1 +scan all,,,,,14147390281.0,ns,1 +scan all,,,,,13760811376.0,ns,1 +scan all,,,,,14224292953.0,ns,1 +scan all,,,,,14175213190.0,ns,1 +scan all,,,,,14349106401.0,ns,1 +scan all,,,,,14256553356.0,ns,1 +scan all,,,,,14614466590.0,ns,1 +scan all,,,,,14190313127.0,ns,1 +scan all,,,,,14310285817.0,ns,1 diff --git a/report/scan/new/sample.json b/report/scan/new/sample.json new file mode 100644 index 0000000..34e79a6 --- /dev/null +++ b/report/scan/new/sample.json @@ -0,0 +1 @@ +{"sampling_mode":"Flat","iters":[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0],"times":[14435894614.0,14147390281.0,13760811376.0,14224292953.0,14175213190.0,14349106401.0,14256553356.0,14614466590.0,14190313127.0,14310285817.0]} \ No newline at end of file diff --git a/report/scan/new/tukey.json b/report/scan/new/tukey.json new file mode 100644 index 0000000..7c55da3 --- /dev/null +++ b/report/scan/new/tukey.json @@ -0,0 +1 @@ +[13697748932.0,13938368553.125,14580020876.125,14820640497.25] \ No newline at end of file diff --git a/report/scan/report/MAD.svg b/report/scan/report/MAD.svg new file mode 100644 index 0000000..0040d70 --- /dev/null +++ b/report/scan/report/MAD.svg @@ -0,0 +1,92 @@ + + +scan all:MAD + + +Density (a.u.) + + +Average time (ms) + + + +0.001 + + + +0.002 + + + +0.003 + + + +0.004 + + + +0.005 + + + +0.006 + + + +0.007 + + + +0.008 + + + + +0 + + + +50 + + + +100 + + + +150 + + + +200 + + + +250 + + + +300 + + + +350 + + + + + + +Bootstrap distribution + + +Confidence interval + + +Point estimate + + + + + diff --git a/report/scan/report/SD.svg b/report/scan/report/SD.svg new file mode 100644 index 0000000..d6a27eb --- /dev/null +++ b/report/scan/report/SD.svg @@ -0,0 +1,80 @@ + + +scan all:SD + + +Density (a.u.) + + +Average time (ms) + + + +0.001 + + + +0.002 + + + +0.003 + + + +0.004 + + + +0.005 + + + +0.006 + + + +0.007 + + + +0.008 + + + + +100 + + + +150 + + + +200 + + + +250 + + + +300 + + + + + + +Bootstrap distribution + + +Confidence interval + + +Point estimate + + + + + diff --git a/report/scan/report/index.html b/report/scan/report/index.html new file mode 100644 index 0000000..5a68266 --- /dev/null +++ b/report/scan/report/index.html @@ -0,0 +1,193 @@ + + + + + + scan all - Criterion.rs + + + + +
+

scan all

+
+
+ + + + + + + +
+ + PDF of Slope + + + + Iteration Times + +
+
+
+
+

Additional Statistics:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Lower boundEstimateUpper bound
R²0.00641370.00917270.0068159
Mean14.108 s14.246 s14.370 s
Std. Dev.77.029 ms221.21 ms313.07 ms
Median14.169 s14.240 s14.373 s
MAD31.819 ms120.75 ms333.95 ms
+
+
+

Additional Plots:

+ +
+
+
+

Understanding this report:

+

The plot on the left displays the average time per iteration for this benchmark. The shaded region + shows the estimated probability of an iteration taking a certain amount of time, while the line + shows the mean. Click on the plot for a larger view showing the outliers.

+

The plot on the right shows the average time per iteration for the samples. Each point + represents one sample.

+

See the + documentation for more details on the additional statistics.

+
+
+
+ + + + \ No newline at end of file diff --git a/report/scan/report/iteration_times.svg b/report/scan/report/iteration_times.svg new file mode 100644 index 0000000..4b5578e --- /dev/null +++ b/report/scan/report/iteration_times.svg @@ -0,0 +1,119 @@ + + +scan all + + +Average Iteration Time (s) + + + + + + + + + + + + + + + + + + + + + + +13.8 + + + +13.9 + + + +14.0 + + + +14.1 + + + +14.2 + + + +14.3 + + + +14.4 + + + +14.5 + + + +14.6 + + + + +2 + + + +3 + + + +4 + + + +5 + + + +6 + + + +7 + + + +8 + + + +9 + + + +10 + + + +11 + + + + + + + + + + + + + +Sample + + + diff --git a/report/scan/report/iteration_times_small.svg b/report/scan/report/iteration_times_small.svg new file mode 100644 index 0000000..1d2a6b9 --- /dev/null +++ b/report/scan/report/iteration_times_small.svg @@ -0,0 +1,112 @@ + + +Average Iteration Time (s) + + + + + + + + + + + + + + + + + + + + + + +13.8 + + + +13.9 + + + +14.0 + + + +14.1 + + + +14.2 + + + +14.3 + + + +14.4 + + + +14.5 + + + +14.6 + + + + +2 + + + +3 + + + +4 + + + +5 + + + +6 + + + +7 + + + +8 + + + +9 + + + +10 + + + +11 + + + + + + + + + + + + + diff --git a/report/scan/report/mean.svg b/report/scan/report/mean.svg new file mode 100644 index 0000000..a9280d3 --- /dev/null +++ b/report/scan/report/mean.svg @@ -0,0 +1,76 @@ + + +scan all:mean + + +Density (a.u.) + + +Average time (s) + + + +1 + + + +2 + + + +3 + + + +4 + + + +5 + + + +6 + + + + +14.1 + + + +14.15 + + + +14.2 + + + +14.25 + + + +14.3 + + + +14.35 + + + + + + +Bootstrap distribution + + +Confidence interval + + +Point estimate + + + + + diff --git a/report/scan/report/median.svg b/report/scan/report/median.svg new file mode 100644 index 0000000..a012088 --- /dev/null +++ b/report/scan/report/median.svg @@ -0,0 +1,88 @@ + + +scan all:median + + +Density (a.u.) + + +Average time (s) + + + +1 + + + +2 + + + +3 + + + +4 + + + +5 + + + +6 + + + +7 + + + +8 + + + +9 + + + +10 + + + + +14.15 + + + +14.2 + + + +14.25 + + + +14.3 + + + +14.35 + + + + + + +Bootstrap distribution + + +Confidence interval + + +Point estimate + + + + + diff --git a/report/scan/report/pdf.svg b/report/scan/report/pdf.svg new file mode 100644 index 0000000..a469641 --- /dev/null +++ b/report/scan/report/pdf.svg @@ -0,0 +1,161 @@ + + +scan all + + +Iterations + + +Average Time (s) + + + +0.1 + + + +0.2 + + + +0.3 + + + +0.4 + + + +0.5 + + + +0.6 + + + +0.7 + + + +0.8 + + + +0.9 + + + + +13.4 + + + +13.6 + + + +13.8 + + + +14 + + + +14.2 + + + +14.4 + + + +14.6 + + + +14.8 + + + +15 + + + +Density (a.u.) + + + +0.2 + + + +0.4 + + + +0.6 + + + +0.8 + + + +1 + + + +1.2 + + + +1.4 + + + +1.6 + + + +1.8 + + + + + + + + + + + + + + + + + + + +PDF + + +Mean + + +"Clean" sample + + +Mild outliers + + +Severe outliers + + + + + + + diff --git a/report/scan/report/pdf_small.svg b/report/scan/report/pdf_small.svg new file mode 100644 index 0000000..85d9614 --- /dev/null +++ b/report/scan/report/pdf_small.svg @@ -0,0 +1,68 @@ + + +Density (a.u.) + + +Average Time (s) + + + +0.2 + + + +0.4 + + + +0.6 + + + +0.8 + + + +1 + + + +1.2 + + + +1.4 + + + +1.6 + + + +1.8 + + + +2 + + + + +13.5 + + + +14 + + + +14.5 + + + +15 + + + + + diff --git a/report/scan/report/typical.svg b/report/scan/report/typical.svg new file mode 100644 index 0000000..b558f20 --- /dev/null +++ b/report/scan/report/typical.svg @@ -0,0 +1,76 @@ + + +scan all:typical + + +Density (a.u.) + + +Average time (s) + + + +1 + + + +2 + + + +3 + + + +4 + + + +5 + + + +6 + + + + +14.1 + + + +14.15 + + + +14.2 + + + +14.25 + + + +14.3 + + + +14.35 + + + + + + +Bootstrap distribution + + +Confidence interval + + +Point estimate + + + + + diff --git a/src/chain.rs b/src/chain.rs index ceac863..1f81fb2 100644 --- a/src/chain.rs +++ b/src/chain.rs @@ -1,23 +1,22 @@ use crate::lw_rpc::compact_tx_streamer_client::CompactTxStreamerClient; use crate::lw_rpc::*; +use crate::NETWORK; +use ff::PrimeField; +use group::GroupEncoding; +use rayon::prelude::*; use tonic::transport::Channel; use tonic::Request; -use zcash_primitives::sapling::note_encryption::try_sapling_compact_note_decryption; -use crate::NETWORK; use zcash_primitives::consensus::BlockHeight; -use zcash_primitives::sapling::SaplingIvk; -use zcash_primitives::transaction::components::OutputDescription; -use jubjub::Scalar; -use group::GroupEncoding; -use ff::PrimeField; +use zcash_primitives::merkle_tree::{CommitmentTree, IncrementalWitness}; +use zcash_primitives::sapling::note_encryption::try_sapling_compact_note_decryption; +use zcash_primitives::sapling::{Node, Note, SaplingIvk}; use zcash_primitives::transaction::components::sapling::CompactOutputDescription; -use tokio::runtime::Runtime; -use std::sync::{Arc, Mutex}; -use tokio::task::JoinHandle; -use futures::future::JoinAll; -use rayon::prelude::*; +use crate::commitment::{CTree, Witness}; +use std::time::Instant; +use log::info; const MAX_CHUNK: u32 = 50000; +pub const LWD_URL: &str = "http://127.0.0.1:9067"; pub async fn get_latest_height( client: &mut CompactTxStreamerClient, @@ -32,18 +31,26 @@ pub async fn get_latest_height( pub async fn download_chain( client: &mut CompactTxStreamerClient, start_height: u32, - end_height: u32 + end_height: u32, ) -> anyhow::Result> { let mut cbs: Vec = Vec::new(); let mut s = start_height + 1; while s < end_height { - eprintln!("{}", s); - let e = (s + MAX_CHUNK).min(end_height); + let e = (s + MAX_CHUNK - 1).min(end_height); let range = BlockRange { - start: Some(BlockId { height: s as u64, hash: vec![] }), - end: Some(BlockId { height: e as u64, hash: vec![] }) + start: Some(BlockId { + height: s as u64, + hash: vec![], + }), + end: Some(BlockId { + height: e as u64, + hash: vec![], + }), }; - let mut block_stream = client.get_block_range(Request::new(range)).await?.into_inner(); + let mut block_stream = client + .get_block_range(Request::new(range)) + .await? + .into_inner(); while let Some(block) = block_stream.message().await? { cbs.push(block); } @@ -52,12 +59,25 @@ pub async fn download_chain( Ok(cbs) } -struct DecryptNode { +pub struct DecryptNode { ivks: Vec, } -fn decrypt_notes(block: &CompactBlock, ivks: &[SaplingIvk]) { +pub struct DecryptedBlock { + pub height: u32, + pub notes: Vec, + pub count_outputs: u32, +} + +pub struct DecryptedNote { + pub note: Note, + pub position: u32, +} + +fn decrypt_notes(block: &CompactBlock, ivks: &[SaplingIvk]) -> DecryptedBlock { let height = BlockHeight::from_u32(block.height as u32); + let mut count_outputs = 0u32; + let mut notes: Vec = vec![]; for vtx in block.vtx.iter() { for co in vtx.outputs.iter() { let mut cmu = [0u8; 32]; @@ -72,41 +92,151 @@ fn decrypt_notes(block: &CompactBlock, ivks: &[SaplingIvk]) { enc_ciphertext: co.ciphertext.to_vec(), }; for ivk in ivks.iter() { - if let Some((note, pa)) = try_sapling_compact_note_decryption(&NETWORK, height, ivk, &od) { - println!("{:?} {:?}", note, pa); + if let Some((note, _pa)) = + try_sapling_compact_note_decryption(&NETWORK, height, ivk, &od) + { + notes.push(DecryptedNote { + note, + position: count_outputs, + }); } } + count_outputs += 1; } } + DecryptedBlock { + height: block.height as u32, + notes, + count_outputs, + } } impl DecryptNode { pub fn new(ivks: Vec) -> DecryptNode { - DecryptNode { - ivks, + DecryptNode { ivks } + } + + pub fn decrypt_blocks(&self, blocks: &[CompactBlock]) -> Vec { + let mut decrypted_blocks: Vec = blocks + .par_iter() + .map(|b| decrypt_notes(b, &self.ivks)) + .collect(); + decrypted_blocks.sort_by(|a, b| a.height.cmp(&b.height)); + decrypted_blocks + } +} + +#[allow(dead_code)] +async fn get_tree_state(client: &mut CompactTxStreamerClient, height: u32) -> String { + let block_id = BlockId { + height: height as u64, + hash: vec![], + }; + let rep = client + .get_tree_state(Request::new(block_id)) + .await + .unwrap() + .into_inner(); + rep.tree +} + +/* Using the IncrementalWitness */ +#[allow(dead_code)] +fn calculate_tree_state_v1( + cbs: &[CompactBlock], + blocks: &[DecryptedBlock], + height: u32, + mut tree_state: CommitmentTree, +) -> Vec> { + let mut witnesses: Vec> = vec![]; + for (cb, block) in cbs.iter().zip(blocks) { + assert_eq!(cb.height as u32, block.height); + if block.height < height { + continue; + } // skip before height + let mut notes = block.notes.iter(); + let mut n = notes.next(); + let mut i = 0u32; + for tx in cb.vtx.iter() { + for co in tx.outputs.iter() { + let mut cmu = [0u8; 32]; + cmu.copy_from_slice(&co.cmu); + let node = Node::new(cmu); + tree_state.append(node).unwrap(); + for w in witnesses.iter_mut() { + w.append(node).unwrap(); + } + if let Some(nn) = n { + if i == nn.position { + let w = IncrementalWitness::from_tree(&tree_state); + witnesses.push(w); + n = notes.next(); + } + } + i += 1; + } } } - pub fn decrypt_blocks(&self, blocks: &[CompactBlock]) { - blocks.par_iter().for_each(|b| { - decrypt_notes(b, &self.ivks); - }); + // let mut bb: Vec = vec![]; + // tree_state.write(&mut bb).unwrap(); + // hex::encode(bb) + + witnesses +} + +pub fn calculate_tree_state_v2(cbs: &[CompactBlock], blocks: &[DecryptedBlock]) -> Vec { + let mut p = 0usize; + let mut nodes: Vec = vec![]; + let mut positions: Vec = vec![]; + + let start = Instant::now(); + for (cb, block) in cbs.iter().zip(blocks) { + assert_eq!(cb.height as u32, block.height); + let mut notes = block.notes.iter(); + let mut n = notes.next(); + let mut i = 0u32; + for tx in cb.vtx.iter() { + for co in tx.outputs.iter() { + let mut cmu = [0u8; 32]; + cmu.copy_from_slice(&co.cmu); + let node = Node::new(cmu); + nodes.push(node); + + if let Some(nn) = n { + if i == nn.position { + positions.push(p); + n = notes.next(); + } + } + i += 1; + p += 1; + } + } } + info!("Build CMU list: {} ms - {} nodes", start.elapsed().as_millis(), nodes.len()); + + let start = Instant::now(); + let (_tree, positions) = CTree::calc_state(&mut nodes, &positions); + let witnesses: Vec<_> = positions.iter().map(|p| p.witness.clone()).collect(); + info!("Tree State & Witnesses: {} ms", start.elapsed().as_millis()); + witnesses } #[cfg(test)] mod tests { - use crate::chain::{get_latest_height, download_chain, DecryptNode}; + #[allow(unused_imports)] + use crate::chain::{download_chain, get_latest_height, get_tree_state, calculate_tree_state_v1, calculate_tree_state_v2, DecryptNode}; use crate::lw_rpc::compact_tx_streamer_client::CompactTxStreamerClient; - use zcash_primitives::consensus::{Parameters, NetworkUpgrade}; - use zcash_client_backend::encoding::decode_extended_full_viewing_key; use crate::NETWORK; use dotenv; - use tokio::runtime::Runtime; use std::time::Instant; + use zcash_client_backend::encoding::decode_extended_full_viewing_key; + use zcash_primitives::consensus::{NetworkUpgrade, Parameters}; + use crate::chain::LWD_URL; #[tokio::test] async fn test_get_latest_height() -> anyhow::Result<()> { - let mut client = CompactTxStreamerClient::connect("http://127.0.0.1:9067").await?; + let mut client = CompactTxStreamerClient::connect(LWD_URL).await?; let height = get_latest_height(&mut client).await?; assert!(height > 1288000); Ok(()) @@ -117,11 +247,17 @@ mod tests { dotenv::dotenv().unwrap(); let ivk = dotenv::var("IVK").unwrap(); - let fvk = decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &ivk).unwrap().unwrap(); + let fvk = + decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &ivk) + .unwrap() + .unwrap(); let ivk = fvk.fvk.vk.ivk(); let decrypter = DecryptNode::new(vec![ivk]); - let mut client = CompactTxStreamerClient::connect("http://127.0.0.1:9067").await?; - let start_height: u32 = crate::NETWORK.activation_height(NetworkUpgrade::Sapling).unwrap().into(); + let mut client = CompactTxStreamerClient::connect(LWD_URL).await?; + let start_height: u32 = crate::NETWORK + .activation_height(NetworkUpgrade::Sapling) + .unwrap() + .into(); let end_height = get_latest_height(&mut client).await?; let start = Instant::now(); @@ -129,9 +265,30 @@ mod tests { eprintln!("Download chain: {} ms", start.elapsed().as_millis()); let start = Instant::now(); - decrypter.decrypt_blocks(&cbs); + let blocks = decrypter.decrypt_blocks(&cbs); eprintln!("Decrypt Notes: {} ms", start.elapsed().as_millis()); + // no need to calculate tree before the first note if we can + // get it from the server + // disabled because I want to see the performance of a complete scan + + // let first_block = blocks.iter().find(|b| !b.notes.is_empty()).unwrap(); + // let height = first_block.height - 1; + // let tree_state = get_tree_state(&mut client, height).await; + // let tree_state = hex::decode(tree_state).unwrap(); + // let tree_state = CommitmentTree::::read(&*tree_state).unwrap(); + + // let witnesses = calculate_tree_state(&cbs, &blocks, 0, tree_state); + + let witnesses = calculate_tree_state_v2(&cbs, &blocks); + + eprintln!("# Witnesses {}", witnesses.len()); + for w in witnesses.iter() { + let mut bb: Vec = vec![]; + w.write(&mut bb).unwrap(); + eprintln!("{}", hex::encode(&bb)); + } + Ok(()) } } diff --git a/src/commitment.rs b/src/commitment.rs new file mode 100644 index 0000000..3bab9e7 --- /dev/null +++ b/src/commitment.rs @@ -0,0 +1,292 @@ +use zcash_primitives::merkle_tree::Hashable; +use zcash_primitives::sapling::Node; +use std::io::Write; +use zcash_primitives::serialize::{Optional, Vector}; +use byteorder::WriteBytesExt; +use rayon::prelude::*; + +#[derive(Clone)] +pub struct CTree { + left: Option, + right: Option, + parents: Vec>, +} + +#[derive(Clone)] +pub struct Witness { + tree: CTree, // commitment tree at the moment the witness is created: immutable + filled: Vec, // as more nodes are added, levels get filled up: won't change anymore + cursor: CTree, // partial tree which still updates when nodes are added +} + +impl Witness { + pub fn new() -> Witness { + Witness { + tree: CTree::new(), + filled: vec![], + cursor: CTree::new(), + } + } + + pub fn write(&self, mut writer: W) -> std::io::Result<()> { + self.tree.write(&mut writer)?; + Vector::write(&mut writer, &self.filled, |w, n| n.write(w))?; + if self.cursor.left == None && self.cursor.right == None { + writer.write_u8(0)?; + } + else { + writer.write_u8(1)?; + self.cursor.write(writer)?; + }; + Ok(()) + } +} + +pub struct NotePosition { + p: usize, + p2: Option, + c: usize, + pub witness: Witness, + is_last: bool, +} + +fn collect(tree: &mut CTree, mut p: usize, depth: usize, commitments: &[Node]) -> usize { + if depth == 0 { + if p % 2 == 0 { + tree.left = Some(commitments[p]); + } else { + tree.left = Some(commitments[p - 1]); + tree.right = Some(commitments[p]); + p -= 1; + } + } else { + // the rest gets combined as a binary tree + if p % 2 != 0 { + tree.parents.push(Some(commitments[p - 1])); + } else if p != 0 { + tree.parents.push(None); + } + } + p +} + +impl NotePosition { + fn new(position: usize, count: usize) -> NotePosition { + let is_last = position == count - 1; + let c = if !is_last { + cursor_start_position(position, count) + } else { + 0 + }; + let cursor_length = count - c; + NotePosition { + p: position, + p2: if cursor_length > 0 { + Some(cursor_length - 1) + } else { + None + }, + c, + witness: Witness::new(), + is_last, + } + } + + fn collect(&mut self, depth: usize, commitments: &[Node]) { + let count = commitments.len(); + let p = self.p; + + self.p = collect(&mut self.witness.tree, p, depth, commitments); + + if !self.is_last { + if p % 2 == 0 && p + 1 < commitments.len() { + let filler = commitments[p + 1]; + self.witness.filled.push(filler); + } + } + + if let Some(ref mut p2) = self.p2 { + if !self.is_last { + let cursor_commitments = &commitments[self.c..count]; + *p2 = collect(&mut self.witness.cursor, *p2, depth, cursor_commitments); + } + *p2 /= 2; + } + self.p /= 2; + self.c /= 2; + } +} + +fn cursor_start_position(mut position: usize, mut count: usize) -> usize { + assert!(position < count); + // same logic as filler + let mut depth = 0; + loop { + if position % 2 == 0 { + if position + 1 < count { + position += 1; + } else { + break; + } + } + + position /= 2; + count /= 2; + depth += 1; + } + (position + 1) << depth +} + +impl CTree { + pub fn calc_state(commitments: &mut [Node], positions: &[usize]) -> (CTree, Vec) { + let mut n = commitments.len(); + let mut positions: Vec<_> = positions.iter().map(|&p| NotePosition::new(p, n)).collect(); + assert_ne!(n, 0); + + let mut depth = 0usize; + let mut frontier = NotePosition::new(n - 1, n); + while n > 0 { + let commitment_slice = &commitments[0..n]; + frontier.collect(depth, commitment_slice); + + for p in positions.iter_mut() { + p.collect(depth, commitment_slice); + } + + let nn = n / 2; + let next_level: Vec<_> = (0..nn).into_par_iter().map(|i| { + Node::combine(depth, &commitments[2 * i], &commitments[2 * i + 1]) + }).collect(); + commitments[0..nn].copy_from_slice(&next_level); + + depth += 1; + n = nn; + } + + (frontier.witness.tree, positions) + } + + fn new() -> CTree { + CTree { + left: None, + right: None, + parents: vec![], + } + } + + fn write(&self, mut writer: W) -> std::io::Result<()> { + Optional::write(&mut writer, &self.left, |w, n| n.write(w))?; + Optional::write(&mut writer, &self.right, |w, n| n.write(w))?; + Vector::write(&mut writer, &self.parents, |w, e| { + Optional::write(w, e, |w, n| n.write(w)) + }) + } +} + +#[cfg(test)] +mod tests { + use crate::commitment::{cursor_start_position, CTree}; + #[allow(unused_imports)] + use crate::print::{print_tree, print_witness}; + use std::time::Instant; + use zcash_primitives::merkle_tree::{CommitmentTree, IncrementalWitness}; + use zcash_primitives::sapling::Node; + + /* + Build incremental witnesses with both methods and compare their binary serialization + */ + #[test] + fn test_calc_witnesses() { + const NUM_NODES: u32 = 100000; // number of notes + const WITNESS_PERCENT: u32 = 1; // percentage of notes that are ours + const DEBUG_PRINT: bool = false; + + let witness_freq = 100 / WITNESS_PERCENT; + let mut tree1: CommitmentTree = CommitmentTree::empty(); + let mut nodes: Vec = vec![]; + let mut witnesses: Vec> = vec![]; + let mut positions: Vec = vec![]; + for i in 1..=NUM_NODES { + let mut bb = [0u8; 32]; + bb[0..4].copy_from_slice(&i.to_be_bytes()); + let node = Node::new(bb); + + tree1.append(node).unwrap(); + + for w in witnesses.iter_mut() { + w.append(node).unwrap(); + } + + if i % witness_freq == 0 { + let w = IncrementalWitness::::from_tree(&tree1); + witnesses.push(w); + positions.push((i - 1) as usize); + } + + nodes.push(node); + } + + let start = Instant::now(); + let (tree2, positions) = CTree::calc_state(&mut nodes, &positions); + eprintln!( + "Update State & Witnesses: {} ms", + start.elapsed().as_millis() + ); + + println!("# witnesses = {}", positions.len()); + + for (w, p) in witnesses.iter().zip(&positions) { + let mut bb1: Vec = vec![]; + w.write(&mut bb1).unwrap(); + + let mut bb2: Vec = vec![]; + p.witness.write(&mut bb2).unwrap(); + + assert_eq!(bb1.as_slice(), bb2.as_slice()); + } + + if DEBUG_PRINT { + print_witness(&witnesses[0]); + + println!("Tree"); + let t = &positions[0].witness.tree; + println!("{:?}", t.left.map(|n| hex::encode(n.repr))); + println!("{:?}", t.right.map(|n| hex::encode(n.repr))); + for p in t.parents.iter() { + println!("{:?}", p.map(|n| hex::encode(n.repr))); + } + println!("Filled"); + for f in positions[0].witness.filled.iter() { + println!("{:?}", hex::encode(f.repr)); + } + println!("Cursor"); + let t = &positions[0].witness.cursor; + println!("{:?}", t.left.map(|n| hex::encode(n.repr))); + println!("{:?}", t.right.map(|n| hex::encode(n.repr))); + for p in t.parents.iter() { + println!("{:?}", p.map(|n| hex::encode(n.repr))); + } + + println!("{:?}", tree1.left.map(|n| hex::encode(n.repr))); + println!("{:?}", tree1.right.map(|n| hex::encode(n.repr))); + for p in tree1.parents.iter() { + println!("{:?}", p.map(|n| hex::encode(n.repr))); + } + + println!("-----"); + + println!("{:?}", tree2.left.map(|n| hex::encode(n.repr))); + println!("{:?}", tree2.right.map(|n| hex::encode(n.repr))); + for p in tree2.parents.iter() { + println!("{:?}", p.map(|n| hex::encode(n.repr))); + } + } + } + + #[test] + fn test_cursor() { + // println!("{}", cursor_start_position(8, 14)); + println!("{}", cursor_start_position(9, 14)); + // println!("{}", cursor_start_position(10, 14)); + } +} diff --git a/src/lib.rs b/src/lib.rs index a8b4af9..b5cdf5b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,4 +5,12 @@ pub mod lw_rpc; pub const NETWORK: Network = Network::MainNetwork; +mod print; mod chain; +mod commitment; +mod scan; + +pub use crate::chain::{LWD_URL, get_latest_height, download_chain, calculate_tree_state_v2, DecryptNode}; +pub use crate::lw_rpc::compact_tx_streamer_client::CompactTxStreamerClient; +pub use crate::lw_rpc::*; +pub use crate::scan::scan_all; diff --git a/src/main.rs b/src/main.rs index 310c18b..f6c5049 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,18 @@ -#[tokio::main] -async fn main() -> anyhow::Result<()> { +use zcash_client_backend::encoding::decode_extended_full_viewing_key; +use sync::{NETWORK, scan_all}; +use zcash_primitives::consensus::Parameters; - Ok(()) +#[tokio::main] +async fn main() { + dotenv::dotenv().unwrap(); + env_logger::init(); + + let ivk = dotenv::var("IVK").unwrap(); + let fvk = + decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &ivk) + .unwrap() + .unwrap(); + let ivk = fvk.fvk.vk.ivk(); + + scan_all(&vec![ivk]).await.unwrap(); } diff --git a/src/print.rs b/src/print.rs new file mode 100644 index 0000000..975a520 --- /dev/null +++ b/src/print.rs @@ -0,0 +1,28 @@ +use zcash_primitives::sapling::Node; +use zcash_primitives::merkle_tree::{CommitmentTree, IncrementalWitness}; + +#[allow(dead_code)] +pub fn print_node(n: &Node) { + println!("{:?}", hex::encode(n.repr)); +} + +#[allow(dead_code)] +pub fn print_tree(t: &CommitmentTree) { + println!("{:?}", t.left.map(|n| hex::encode(n.repr))); + println!("{:?}", t.right.map(|n| hex::encode(n.repr))); + for p in t.parents.iter() { + println!("{:?}", p.map(|n| hex::encode(n.repr))); + } +} + +#[allow(dead_code)] +pub fn print_witness(w: &IncrementalWitness) { + println!("Tree"); + print_tree(&w.tree); + println!("Filled"); + for n in w.filled.iter() { + print_node(n); + } + println!("Cursor"); + w.cursor.as_ref().map(|c| print_tree(c)); +} diff --git a/src/scan.rs b/src/scan.rs new file mode 100644 index 0000000..06559f8 --- /dev/null +++ b/src/scan.rs @@ -0,0 +1,40 @@ +use zcash_primitives::sapling::SaplingIvk; +use crate::lw_rpc::compact_tx_streamer_client::CompactTxStreamerClient; +use crate::{DecryptNode, LWD_URL, get_latest_height, download_chain, calculate_tree_state_v2}; +use zcash_primitives::consensus::{NetworkUpgrade, Parameters}; +use std::time::Instant; +use log::info; + +pub async fn scan_all(ivks: &[SaplingIvk]) -> anyhow::Result<()> { + let decrypter = DecryptNode::new(ivks.to_vec()); + + let total_start = Instant::now(); + let mut client = CompactTxStreamerClient::connect(LWD_URL).await?; + let start_height: u32 = crate::NETWORK + .activation_height(NetworkUpgrade::Sapling) + .unwrap() + .into(); + let end_height = get_latest_height(&mut client).await?; + + let start = Instant::now(); + let cbs = download_chain(&mut client, start_height, end_height).await?; + info!("Download chain: {} ms", start.elapsed().as_millis()); + + let start = Instant::now(); + let blocks = decrypter.decrypt_blocks(&cbs); + info!("Decrypt Notes: {} ms", start.elapsed().as_millis()); + + let witnesses = calculate_tree_state_v2(&cbs, &blocks); + + info!("# Witnesses {}", witnesses.len()); + for w in witnesses.iter() { + let mut bb: Vec = vec![]; + w.write(&mut bb).unwrap(); + log::debug!("{}", hex::encode(&bb)); + } + + info!("Total: {} ms", total_start.elapsed().as_millis()); + + Ok(()) +} +