zecfoundation
/
zcash-sync
mirror of https://github.com/ZcashFoundation/zcash-sync.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

cleanup

This commit is contained in:
Hanh 2021-06-22 08:33:13 +08:00
parent 6cbd30c959
commit 572195d8df
16 changed files with 722 additions and 488 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -3,3 +3,4 @@
.idea/
src/generated/*.rs
docs/_site/
*.db

103
Cargo.lock generated
View File

@ -37,6 +37,12 @@ dependencies = [
"opaque-debug",
]
[[package]]
name = "ahash"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e"
[[package]]
name = "aho-corasick"
version = "0.7.18"
@ -127,9 +133,9 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
[[package]]
name = "bech32"
version = "0.8.0"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c7f7096bc256f5e5cb960f60dfc4f4ef979ca65abe7fb9d5a4f77150d3783d4"
checksum = "cf9ff0bbfd639f15c74af777d81383cf53efb7c93613f6cab67c6c11e05bbf8b"
[[package]]
name = "bellman"
@ -638,11 +644,24 @@ dependencies = [
[[package]]
name = "equihash"
version = "0.1.0"
source = "git+https://github.com/zcash/librustzcash.git?rev=d50bb12a97da768dc8f3ee39b81f84262103e6eb#d50bb12a97da768dc8f3ee39b81f84262103e6eb"
dependencies = [
"blake2b_simd",
"byteorder",
]
[[package]]
name = "fallible-iterator"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "ff"
version = "0.9.0"
@ -893,6 +912,18 @@ name = "hashbrown"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
dependencies = [
"ahash",
]
[[package]]
name = "hashlink"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d99cf782f0dc4372d26846bec3de7804ceb5df083c2d4462c0b8d2330e894fa8"
dependencies = [
"hashbrown",
]
[[package]]
name = "heck"
@ -1064,6 +1095,17 @@ version = "0.2.97"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6"
[[package]]
name = "libsqlite3-sys"
version = "0.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64d31059f22935e6c31830db5249ba2b7ecd54fd73a9909286f0a67aa55c2fbd"
dependencies = [
"cc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "log"
version = "0.4.14"
@ -1294,6 +1336,12 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "pkg-config"
version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
[[package]]
name = "plotters"
version = "0.3.1"
@ -1577,6 +1625,22 @@ dependencies = [
"winapi",
]
[[package]]
name = "rusqlite"
version = "0.24.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5f38ee71cbab2c827ec0ac24e76f82eca723cee92c509a65f67dee393c25112"
dependencies = [
"bitflags",
"fallible-iterator",
"fallible-streaming-iterator",
"hashlink",
"libsqlite3-sys",
"memchr",
"smallvec",
"time 0.2.27",
]
[[package]]
name = "rustc_version"
version = "0.2.3"
@ -1783,6 +1847,12 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527"
[[package]]
name = "smallvec"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
[[package]]
name = "socket2"
version = "0.4.0"
@ -1907,6 +1977,7 @@ dependencies = [
"tonic",
"tonic-build",
"zcash_client_backend",
"zcash_client_sqlite",
"zcash_primitives",
"zcash_proofs",
]
@ -2252,6 +2323,12 @@ version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "version_check"
version = "0.9.3"
@ -2415,6 +2492,7 @@ checksum = "9fc79f4a1e39857fc00c3f662cbf2651c771f00e9c15fe2abc341806bd46bd71"
[[package]]
name = "zcash_client_backend"
version = "0.5.0"
source = "git+https://github.com/zcash/librustzcash.git?rev=d50bb12a97da768dc8f3ee39b81f84262103e6eb#d50bb12a97da768dc8f3ee39b81f84262103e6eb"
dependencies = [
"base64",
"bech32",
@ -2435,9 +2513,28 @@ dependencies = [
"zcash_primitives",
]
[[package]]
name = "zcash_client_sqlite"
version = "0.3.0"
source = "git+https://github.com/zcash/librustzcash.git?rev=d50bb12a97da768dc8f3ee39b81f84262103e6eb#d50bb12a97da768dc8f3ee39b81f84262103e6eb"
dependencies = [
"bech32",
"bs58",
"ff",
"group",
"jubjub",
"protobuf",
"rand_core",
"rusqlite",
"time 0.2.27",
"zcash_client_backend",
"zcash_primitives",
]
[[package]]
name = "zcash_note_encryption"
version = "0.0.0"
source = "git+https://github.com/zcash/librustzcash.git?rev=d50bb12a97da768dc8f3ee39b81f84262103e6eb#d50bb12a97da768dc8f3ee39b81f84262103e6eb"
dependencies = [
"blake2b_simd",
"byteorder",
@ -2451,6 +2548,7 @@ dependencies = [
[[package]]
name = "zcash_primitives"
version = "0.5.0"
source = "git+https://github.com/zcash/librustzcash.git?rev=d50bb12a97da768dc8f3ee39b81f84262103e6eb#d50bb12a97da768dc8f3ee39b81f84262103e6eb"
dependencies = [
"aes",
"bitvec 0.20.4",
@ -2478,6 +2576,7 @@ dependencies = [
[[package]]
name = "zcash_proofs"
version = "0.5.0"
source = "git+https://github.com/zcash/librustzcash.git?rev=d50bb12a97da768dc8f3ee39b81f84262103e6eb#d50bb12a97da768dc8f3ee39b81f84262103e6eb"
dependencies = [
"bellman",
"blake2b_simd",

16
Cargo.toml
View File

@ -10,6 +10,10 @@ edition = "2018"
name = "scan_all"
harness = false
[[bin]]
name = "warp-cli"
path = "src/main/warp_cli.rs"
[dependencies]
dotenv = "0.15.0"
env_logger = "0.8.4"
@ -31,15 +35,21 @@ tonic = {version = "0.4.3", features = ["tls", "tls-roots"]}
prost = "0.7"
rayon = "1.5.1"
byteorder = "1.4.3"
tiny-bip39 = "0.8"
rand = "0.8.4"
rusqlite = "^0.25.3"
[dependencies.zcash_client_backend]
path = "../librustzcash/zcash_client_backend"
git = "https://github.com/zcash/librustzcash.git"
rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb"
[dependencies.zcash_primitives]
path = "../librustzcash/zcash_primitives"
git = "https://github.com/zcash/librustzcash.git"
rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb"
[dependencies.zcash_proofs]
path = "../librustzcash/zcash_proofs"
git = "https://github.com/zcash/librustzcash.git"
rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb"
[build-dependencies]
tonic-build = "0.4.2"

5
benches/scan_all.rs
View File

@ -14,13 +14,12 @@ fn scan(c: &mut Criterion) {
.unwrap()
.unwrap();
let ivk = fvk.fvk.vk.ivk();
let ivks = &vec![ivk];
let fvks = &vec![fvk];
c.bench_function("scan all", |b| {
b.iter(|| {
let r = Runtime::new().unwrap();
r.block_on(scan_all(ivks.clone().as_slice())).unwrap();
r.block_on(scan_all(fvks.clone().as_slice())).unwrap();
});
});
}

49
src/builder.rs
View File

@ -315,16 +315,15 @@ pub fn advance_tree(
mod tests {
use crate::builder::advance_tree;
use crate::commitment::{CTree, Witness};
use crate::print::{print_tree, print_witness};
use zcash_primitives::merkle_tree::{CommitmentTree, IncrementalWitness};
use zcash_primitives::sapling::Node;
use crate::chain::DecryptedNote;
#[test]
fn test_advance_tree() {
const NUM_NODES: usize = 1000;
const NUM_CHUNKS: usize = 50;
const WITNESS_PERCENT: f64 = 1.0; // percentage of notes that are ours
const DEBUG_PRINT: bool = true;
let witness_freq = (100.0 / WITNESS_PERCENT) as usize;
let mut tree1: CommitmentTree<Node> = CommitmentTree::empty();
@ -347,7 +346,7 @@ mod tests {
// if v == 499 {
let w = IncrementalWitness::from_tree(&tree1);
ws.push(w);
ws2.push(Witness::new(v));
ws2.push(Witness::new(v, 0, None));
}
nodes.push(node);
}
@ -382,48 +381,6 @@ mod tests {
}
}
if DEBUG_PRINT && (failed_index.is_some() || !equal) {
let i = failed_index.unwrap();
println!("FAILED AT {}", i);
print_witness(&ws[i]);
// println!("-----");
// println!("Final-----");
//
// println!("{:?}", tree2.left.map(|n| hex::encode(n.repr)));
// println!("{:?}", tree2.right.map(|n| hex::encode(n.repr)));
// for p in tree2.parents.iter() {
// println!("{:?}", p.map(|n| hex::encode(n.repr)));
// }
// println!("-----");
// println!("{:?}", tree1.left.map(|n| hex::encode(n.repr)));
// println!("{:?}", tree1.right.map(|n| hex::encode(n.repr)));
// for p in tree1.parents.iter() {
// println!("{:?}", p.map(|n| hex::encode(n.repr)));
// }
println!("----- {}", ws2[i].position);
let tree2 = &ws2[i].tree;
println!("{:?}", tree2.left.map(|n| hex::encode(n.repr)));
println!("{:?}", tree2.right.map(|n| hex::encode(n.repr)));
for p in tree2.parents.iter() {
println!("{:?}", p.map(|n| hex::encode(n.repr)));
}
println!("-----");
let filled2 = &ws2[i].filled;
println!("Filled");
for f in filled2.iter() {
println!("{:?}", hex::encode(f.repr));
}
println!("Cursor");
let cursor2 = &ws2[i].cursor;
println!("{:?}", cursor2.left.map(|n| hex::encode(n.repr)));
println!("{:?}", cursor2.right.map(|n| hex::encode(n.repr)));
for p in cursor2.parents.iter() {
println!("{:?}", p.map(|n| hex::encode(n.repr)));
}
assert!(false);
}
assert!(equal && failed_index.is_none());
}
}

20
src/ca.pem Normal file
View File

@ -0,0 +1,20 @@
-----BEGIN CERTIFICATE-----
MIIDSjCCAjKgAwIBAgIQRK+wgNajJ7qJMDmGLvhAazANBgkqhkiG9w0BAQUFADA/
MSQwIgYDVQQKExtEaWdpdGFsIFNpZ25hdHVyZSBUcnVzdCBDby4xFzAVBgNVBAMT
DkRTVCBSb290IENBIFgzMB4XDTAwMDkzMDIxMTIxOVoXDTIxMDkzMDE0MDExNVow
PzEkMCIGA1UEChMbRGlnaXRhbCBTaWduYXR1cmUgVHJ1c3QgQ28uMRcwFQYDVQQD
Ew5EU1QgUm9vdCBDQSBYMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
AN+v6ZdQCINXtMxiZfaQguzH0yxrMMpb7NnDfcdAwRgUi+DoM3ZJKuM/IUmTrE4O
rz5Iy2Xu/NMhD2XSKtkyj4zl93ewEnu1lcCJo6m67XMuegwGMoOifooUMM0RoOEq
OLl5CjH9UL2AZd+3UWODyOKIYepLYYHsUmu5ouJLGiifSKOeDNoJjj4XLh7dIN9b
xiqKqy69cK3FCxolkHRyxXtqqzTWMIn/5WgTe1QLyNau7Fqckh49ZLOMxt+/yUFw
7BZy1SbsOFU5Q9D8/RhcQPGX69Wam40dutolucbY38EVAjqr2m7xPi71XAicPNaD
aeQQmxkqtilX4+U9m5/wAl0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNV
HQ8BAf8EBAMCAQYwHQYDVR0OBBYEFMSnsaR7LHH62+FLkHX/xBVghYkQMA0GCSqG
SIb3DQEBBQUAA4IBAQCjGiybFwBcqR7uKGY3Or+Dxz9LwwmglSBd49lZRNI+DT69
ikugdB/OEIKcdBodfpga3csTS7MgROSR6cz8faXbauX+5v3gTt23ADq1cEmv8uXr
AvHRAosZy5Q6XkjEGB5YGV8eAlrwDPGxrancWYaLbumR9YbK+rlmM6pZW87ipxZz
R8srzJmwN0jP41ZL9c8PDHIyh8bwRLtTcm1D9SZImlJnt1ir/md2cXjbDaJWFBM5
JDGFoqgCWjBH4d1QB7wCCZAA62RjYJsWvIjJEubSfZGL+T0yjWW06XyxV3bqxbYo
Ob8VZRzI9neWagqNdwvYkQsEjgfbKbYK7p2CNTUQ
-----END CERTIFICATE-----

126
src/chain.rs
View File

@ -1,21 +1,26 @@
use crate::lw_rpc::compact_tx_streamer_client::CompactTxStreamerClient;
use crate::lw_rpc::*;
use crate::NETWORK;
use crate::{NETWORK, advance_tree};
use ff::PrimeField;
use group::GroupEncoding;
use rayon::prelude::*;
use tonic::transport::Channel;
use tonic::transport::{Channel, Certificate, ClientTlsConfig};
use tonic::Request;
use zcash_primitives::consensus::BlockHeight;
use zcash_primitives::consensus::{BlockHeight, Parameters, NetworkUpgrade};
use zcash_primitives::merkle_tree::{CommitmentTree, IncrementalWitness};
use zcash_primitives::sapling::note_encryption::try_sapling_compact_note_decryption;
use zcash_primitives::sapling::{Node, Note, SaplingIvk};
use zcash_primitives::sapling::{Node, Note, PaymentAddress};
use zcash_primitives::transaction::components::sapling::CompactOutputDescription;
use crate::commitment::{CTree, Witness, NotePosition};
use crate::commitment::{CTree, Witness};
use std::time::Instant;
use log::info;
use zcash_client_backend::encoding::decode_extended_full_viewing_key;
use zcash_primitives::zip32::ExtendedFullViewingKey;
const MAX_CHUNK: u32 = 50000;
// pub const LWD_URL: &str = "https://mainnet.lightwalletd.com:9067";
// pub const LWD_URL: &str = "http://lwd.hanh.me:9067";
// pub const LWD_URL: &str = "https://lwdv3.zecwallet.co";
pub const LWD_URL: &str = "http://127.0.0.1:9067";
pub async fn get_latest_height(
@ -60,7 +65,7 @@ pub async fn download_chain(
}
pub struct DecryptNode {
ivks: Vec<SaplingIvk>,
fvks: Vec<ExtendedFullViewingKey>,
}
pub struct DecryptedBlock {
@ -69,17 +74,25 @@ pub struct DecryptedBlock {
pub count_outputs: u32,
}
#[derive(Clone)]
pub struct DecryptedNote {
pub ivk: ExtendedFullViewingKey,
pub note: Note,
pub position: u32,
pub pa: PaymentAddress,
pub position: usize,
pub height: u32,
pub txid: Vec<u8>,
pub tx_index: usize,
pub output_index: usize,
}
fn decrypt_notes(block: &CompactBlock, ivks: &[SaplingIvk]) -> DecryptedBlock {
fn decrypt_notes(block: &CompactBlock, fvks: &[ExtendedFullViewingKey]) -> DecryptedBlock {
let height = BlockHeight::from_u32(block.height as u32);
let mut count_outputs = 0u32;
let mut notes: Vec<DecryptedNote> = vec![];
for vtx in block.vtx.iter() {
for co in vtx.outputs.iter() {
for (tx_index, vtx) in block.vtx.iter().enumerate() {
for (output_index, co) in vtx.outputs.iter().enumerate() {
let mut cmu = [0u8; 32];
cmu.copy_from_slice(&co.cmu);
let cmu = bls12_381::Scalar::from_repr(cmu).unwrap();
@ -91,13 +104,20 @@ fn decrypt_notes(block: &CompactBlock, ivks: &[SaplingIvk]) -> DecryptedBlock {
cmu,
enc_ciphertext: co.ciphertext.to_vec(),
};
for ivk in ivks.iter() {
if let Some((note, _pa)) =
for fvk in fvks.iter() {
let ivk = &fvk.fvk.vk.ivk();
if let Some((note, pa)) =
try_sapling_compact_note_decryption(&NETWORK, height, ivk, &od)
{
notes.push(DecryptedNote {
ivk: fvk.clone(),
note,
position: count_outputs,
pa,
position: count_outputs as usize,
height: block.height as u32,
tx_index,
txid: vtx.hash.clone(),
output_index,
});
}
}
@ -112,14 +132,14 @@ fn decrypt_notes(block: &CompactBlock, ivks: &[SaplingIvk]) -> DecryptedBlock {
}
impl DecryptNode {
pub fn new(ivks: Vec<SaplingIvk>) -> DecryptNode {
DecryptNode { ivks }
pub fn new(fvks: Vec<ExtendedFullViewingKey>) -> DecryptNode {
DecryptNode { fvks }
}
pub fn decrypt_blocks(&self, blocks: &[CompactBlock]) -> Vec<DecryptedBlock> {
let mut decrypted_blocks: Vec<DecryptedBlock> = blocks
.par_iter()
.map(|b| decrypt_notes(b, &self.ivks))
.map(|b| decrypt_notes(b, &self.fvks))
.collect();
decrypted_blocks.sort_by(|a, b| a.height.cmp(&b.height));
decrypted_blocks
@ -156,7 +176,7 @@ fn calculate_tree_state_v1(
} // skip before height
let mut notes = block.notes.iter();
let mut n = notes.next();
let mut i = 0u32;
let mut i = 0usize;
for tx in cb.vtx.iter() {
for co in tx.outputs.iter() {
let mut cmu = [0u8; 32];
@ -192,9 +212,12 @@ pub fn calculate_tree_state_v2(cbs: &[CompactBlock], blocks: &[DecryptedBlock])
let start = Instant::now();
for (cb, block) in cbs.iter().zip(blocks) {
assert_eq!(cb.height as u32, block.height);
if !block.notes.is_empty() {
println!("{} {}", block.height, block.notes.len());
}
let mut notes = block.notes.iter();
let mut n = notes.next();
let mut i = 0u32;
let mut i = 0usize;
for tx in cb.vtx.iter() {
for co in tx.outputs.iter() {
let mut cmu = [0u8; 32];
@ -215,13 +238,57 @@ pub fn calculate_tree_state_v2(cbs: &[CompactBlock], blocks: &[DecryptedBlock])
}
info!("Build CMU list: {} ms - {} nodes", start.elapsed().as_millis(), nodes.len());
let start = Instant::now();
let n = nodes.len();
let mut positions: Vec<_> = positions.iter().map(|&p| NotePosition::new(p, n)).collect();
let _frontier = CTree::calc_state(nodes, &mut positions, None);
let witnesses: Vec<_> = positions.iter().map(|p| p.witness.clone()).collect();
let witnesses: Vec<_> = positions.iter().map(|p| Witness::new(*p, 0, None)).collect();
let (_, new_witnesses) = advance_tree(CTree::new(), &witnesses, &mut nodes);
info!("Tree State & Witnesses: {} ms", start.elapsed().as_millis());
witnesses
new_witnesses
}
pub async fn connect_lightwalletd() -> anyhow::Result<CompactTxStreamerClient<Channel>> {
let mut channel = tonic::transport::Channel::from_shared(LWD_URL)?;
if LWD_URL.starts_with("https") {
let pem = include_bytes!("ca.pem");
let ca = Certificate::from_pem(pem);
let tls = ClientTlsConfig::new().ca_certificate(ca);
channel = channel.tls_config(tls)?;
}
let client = CompactTxStreamerClient::connect(channel).await?;
Ok(client)
}
pub async fn sync(ivk: &str) -> anyhow::Result<()> {
let fvk =
decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &ivk)
.unwrap()
.unwrap();
let decrypter = DecryptNode::new(vec![fvk]);
let mut client = connect_lightwalletd().await?;
let start_height: u32 = crate::NETWORK
.activation_height(NetworkUpgrade::Sapling)
.unwrap()
.into();
let end_height = get_latest_height(&mut client).await?;
let start = Instant::now();
let cbs = download_chain(&mut client, start_height, end_height).await?;
eprintln!("Download chain: {} ms", start.elapsed().as_millis());
let start = Instant::now();
let blocks = decrypter.decrypt_blocks(&cbs);
eprintln!("Decrypt Notes: {} ms", start.elapsed().as_millis());
let start = Instant::now();
let witnesses = calculate_tree_state_v2(&cbs, &blocks);
eprintln!("Tree State & Witnesses: {} ms", start.elapsed().as_millis());
eprintln!("# Witnesses {}", witnesses.len());
for w in witnesses.iter() {
let mut bb: Vec<u8> = vec![];
w.write(&mut bb).unwrap();
log::info!("{}", hex::encode(&bb));
}
Ok(())
}
#[cfg(test)]
@ -247,14 +314,13 @@ mod tests {
#[tokio::test]
async fn test_download_chain() -> anyhow::Result<()> {
dotenv::dotenv().unwrap();
let ivk = dotenv::var("IVK").unwrap();
let fvk = dotenv::var("FVK").unwrap();
let fvk =
decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &ivk)
decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &fvk)
.unwrap()
.unwrap();
let ivk = fvk.fvk.vk.ivk();
let decrypter = DecryptNode::new(vec![ivk]);
let decrypter = DecryptNode::new(vec![fvk]);
let mut client = CompactTxStreamerClient::connect(LWD_URL).await?;
let start_height: u32 = crate::NETWORK
.activation_height(NetworkUpgrade::Sapling)
@ -282,13 +348,15 @@ mod tests {
// let witnesses = calculate_tree_state(&cbs, &blocks, 0, tree_state);
let start = Instant::now();
let witnesses = calculate_tree_state_v2(&cbs, &blocks);
eprintln!("Tree State & Witnesses: {} ms", start.elapsed().as_millis());
eprintln!("# Witnesses {}", witnesses.len());
for w in witnesses.iter() {
let mut bb: Vec<u8> = vec![];
w.write(&mut bb).unwrap();
eprintln!("{}", hex::encode(&bb));
log::info!("{}", hex::encode(&bb));
}
Ok(())

371
src/commitment.rs
View File

@ -1,10 +1,9 @@
use crate::path::MerklePath;
use byteorder::WriteBytesExt;
use rayon::prelude::*;
use std::io::Write;
use zcash_primitives::merkle_tree::Hashable;
use std::io::{Write, Read};
use zcash_primitives::merkle_tree::{Hashable, CommitmentTree};
use zcash_primitives::sapling::Node;
use zcash_primitives::serialize::{Optional, Vector};
use crate::chain::DecryptedNote;
/*
Same behavior and structure as CommitmentTree<Node> from librustzcash
@ -81,18 +80,41 @@ pub struct Witness {
pub tree: CTree, // commitment tree at the moment the witness is created: immutable
pub filled: Vec<Node>, // as more nodes are added, levels get filled up: won't change anymore
pub cursor: CTree, // partial tree which still updates when nodes are added
// not used for decryption but identifies the witness
pub id_note: u32,
pub note: Option<DecryptedNote>,
}
impl Witness {
pub fn new(position: usize) -> Witness {
pub fn new(position: usize, id_note: u32, note: Option<DecryptedNote>) -> Witness {
Witness {
position,
id_note,
note,
tree: CTree::new(),
filled: vec![],
cursor: CTree::new(),
}
}
pub fn read<R: Read>(position: usize, id_note: u32, mut reader: R) -> std::io::Result<Self> {
let tree = CTree::read(&mut reader)?;
let filled = Vector::read(&mut reader, |r| Node::read(r))?;
let cursor = Optional::read(&mut reader, |r| CTree::read(r))?;
let witness = Witness {
position,
id_note,
tree,
filled,
cursor: cursor.unwrap_or_else(CTree::new),
note: None,
};
Ok(witness)
}
pub fn write<W: Write>(&self, mut writer: W) -> std::io::Result<()> {
self.tree.write(&mut writer)?;
Vector::write(&mut writer, &self.filled, |w, n| n.write(w))?;
@ -106,191 +128,7 @@ impl Witness {
}
}
pub struct NotePosition {
p0: usize,
p: usize,
p2: usize,
c: usize,
pub witness: Witness,
}
fn collect(
tree: &mut CTree,
mut p: usize,
depth: usize,
commitments: &[Node],
offset: usize,
cursor: bool,
) -> usize {
// println!("--> {} {} {}", depth, p, offset);
if p < offset {
return p;
}
if depth == 0 {
if p % 2 == 0 {
tree.left = Some(commitments[p - offset]);
} else {
tree.left = Some(commitments[p - 1 - offset]);
tree.right = Some(commitments[p - offset]);
p -= 1;
}
} else {
// the rest gets combined as a binary tree
if p % 2 != 0 {
tree.parents.push(Some(commitments[p - 1 - offset]));
} else if (cursor && p != offset) || !cursor && (p != 0 || offset != 0) {
tree.parents.push(None);
}
}
p
}
impl NotePosition {
pub fn new(position: usize, count: usize) -> NotePosition {
let c = cursor_start_position(position, count);
NotePosition {
p0: position,
p: position,
p2: count - 1,
c,
witness: Witness::new(position),
}
}
pub fn reset(&mut self, count: usize) {
let c = cursor_start_position(self.p0, count);
self.p = self.p0;
self.p2 = count - 1;
self.c = c;
self.witness.cursor = CTree::new();
}
fn collect(&mut self, depth: usize, commitments: &[Node], offset: usize) {
let count = commitments.len();
let p = self.p;
self.p = collect(&mut self.witness.tree, p, depth, commitments, offset, false);
if p % 2 == 0 && p + 1 >= offset && p + 1 - offset < commitments.len() {
let filler = commitments[p + 1 - offset];
self.witness.filled.push(filler);
}
let c = self.c - offset;
let cursor_commitments = &commitments[c..count];
// println!("c> {} {} {}", c, count, depth);
// println!("> {} {}", self.p2, self.c);
if !cursor_commitments.is_empty() {
let p2 = collect(
&mut self.witness.cursor,
self.p2,
depth,
cursor_commitments,
offset + c,
true,
);
self.p2 = (p2 - self.c) / 2 + self.c / 2;
// println!("+ {} {}", self.p2, self.c);
}
self.p /= 2;
self.c /= 2;
}
}
fn cursor_start_position(mut position: usize, mut count: usize) -> usize {
assert!(position < count);
// same logic as filler
let mut depth = 0;
loop {
if position % 2 == 0 {
if position + 1 < count {
position += 1;
} else {
break;
}
}
position /= 2;
count /= 2;
depth += 1;
}
(position + 1) << depth
}
impl CTree {
pub fn calc_state(
mut commitments: Vec<Node>,
positions: &mut [NotePosition],
prev_frontier: Option<CTree>,
) -> CTree {
let mut n = commitments.len();
assert_ne!(n, 0);
let prev_count = prev_frontier
.as_ref()
.map(|f| f.get_position())
.unwrap_or(0);
let count = prev_count + n;
let mut last_path = prev_frontier
.as_ref()
.map(|f| MerklePath::new(f.left, f.right));
let mut frontier = NotePosition::new(count - 1, count);
let mut offset = prev_count;
for p in positions.iter_mut() {
p.reset(count);
}
let mut depth = 0usize;
while n + offset > 0 {
if offset % 2 == 1 {
// start is not aligned
let mut lp = last_path.take().unwrap();
let node = lp.get(); // prepend the last node from the previous run
if n > 0 {
lp.set(commitments[0]); // put the right node into the path
}
last_path = Some(lp);
commitments.insert(0, node);
n += 1;
offset -= 1;
}
let commitment_slice = &commitments[0..n];
frontier.collect(depth, commitment_slice, offset);
for p in positions.iter_mut() {
p.collect(depth, commitment_slice, offset);
}
let nn = n / 2;
let next_level: Vec<_> = (0..nn)
.into_par_iter()
.map(|i| Node::combine(depth, &commitments[2 * i], &commitments[2 * i + 1]))
.collect();
commitments[0..nn].copy_from_slice(&next_level);
if let Some(mut lp) = last_path.take() {
lp.up(
depth,
prev_frontier
.as_ref()
.unwrap()
.parents
.get(depth)
.unwrap_or(&None),
);
last_path = Some(lp);
}
depth += 1;
n = nn;
offset /= 2;
}
frontier.witness.tree
}
pub fn new() -> CTree {
CTree {
left: None,
@ -299,11 +137,7 @@ impl CTree {
}
}
pub fn is_empty(&self) -> bool {
self.left.is_none() && self.right.is_none()
}
pub(crate) fn write<W: Write>(&self, mut writer: W) -> std::io::Result<()> {
pub fn write<W: Write>(&self, mut writer: W) -> std::io::Result<()> {
Optional::write(&mut writer, &self.left, |w, n| n.write(w))?;
Optional::write(&mut writer, &self.right, |w, n| n.write(w))?;
Vector::write(&mut writer, &self.parents, |w, e| {
@ -311,7 +145,19 @@ impl CTree {
})
}
pub(crate) fn get_position(&self) -> usize {
pub fn read<R: Read>(mut reader: R) -> std::io::Result<Self> {
let left = Optional::read(&mut reader, |r| Node::read(r))?;
let right = Optional::read(&mut reader, |r| Node::read(r))?;
let parents = Vector::read(&mut reader, |r| Optional::read(r, |r| Node::read(r)))?;
Ok(CTree {
left,
right,
parents,
})
}
pub fn get_position(&self) -> usize {
let mut p = 0usize;
for parent in self.parents.iter().rev() {
if parent.is_some() {
@ -327,137 +173,10 @@ impl CTree {
}
p
}
}
#[cfg(test)]
mod tests {
use crate::commitment::{cursor_start_position, CTree, NotePosition};
#[allow(unused_imports)]
use crate::print::{print_tree, print_witness};
use std::time::Instant;
use zcash_primitives::merkle_tree::{CommitmentTree, IncrementalWitness};
use zcash_primitives::sapling::Node;
/*
Build incremental witnesses with both methods and compare their binary serialization
*/
#[test]
fn test_calc_witnesses() {
const NUM_CHUNKS: usize = 3;
const NUM_NODES: usize = 20; // number of notes
const WITNESS_PERCENT: usize = 1; // percentage of notes that are ours
const DEBUG_PRINT: bool = true;
let _witness_freq = 100000 / WITNESS_PERCENT;
let mut tree1: CommitmentTree<Node> = CommitmentTree::empty();
let mut tree2: Option<CTree> = None;
let mut witnesses: Vec<IncrementalWitness<Node>> = vec![];
let mut all_positions: Vec<NotePosition> = vec![];
for c in 0..NUM_CHUNKS {
let mut positions: Vec<usize> = vec![];
let mut nodes: Vec<Node> = vec![];
for i in 1..=NUM_NODES {
let mut bb = [0u8; 32];
bb[0..8].copy_from_slice(&i.to_be_bytes());
let node = Node::new(bb);
tree1.append(node).unwrap();
for w in witnesses.iter_mut() {
w.append(node).unwrap();
}
// if i % witness_freq == 0 {
if c == 0 && i == 1 {
let w = IncrementalWitness::<Node>::from_tree(&tree1);
witnesses.push(w);
positions.push((i - 1 + c * NUM_NODES) as usize);
}
nodes.push(node);
}
let start = Instant::now();
let n = nodes.len();
let mut positions: Vec<_> = positions
.iter()
.map(|&p| NotePosition::new(p, n + c * NUM_NODES))
.collect();
all_positions.append(&mut positions);
tree2 = Some(CTree::calc_state(nodes, &mut all_positions, tree2));
eprintln!(
"Update State & Witnesses: {} ms",
start.elapsed().as_millis()
);
}
let tree2 = tree2.unwrap();
println!("# witnesses = {}", all_positions.len());
for (_i, (w, p)) in witnesses.iter().zip(&all_positions).enumerate() {
let mut bb1: Vec<u8> = vec![];
w.write(&mut bb1).unwrap();
let mut bb2: Vec<u8> = vec![];
p.witness.write(&mut bb2).unwrap();
// assert_eq!(bb1.as_slice(), bb2.as_slice(), "failed at {}", i);
}
let mut bb1: Vec<u8> = vec![];
tree1.write(&mut bb1).unwrap();
let mut bb2: Vec<u8> = vec![];
tree2.write(&mut bb2).unwrap();
assert_eq!(bb1.as_slice(), bb2.as_slice(), "tree states not equal");
if DEBUG_PRINT {
let slot = 0usize;
print_witness(&witnesses[slot]);
println!("+++++");
println!("Tree");
let t = &all_positions[slot].witness.tree;
println!("{:?}", t.left.map(|n| hex::encode(n.repr)));
println!("{:?}", t.right.map(|n| hex::encode(n.repr)));
for p in t.parents.iter() {
println!("{:?}", p.map(|n| hex::encode(n.repr)));
}
println!("Filled");
for f in all_positions[slot].witness.filled.iter() {
println!("{:?}", hex::encode(f.repr));
}
println!("Cursor");
let t = &all_positions[slot].witness.cursor;
println!("{:?}", t.left.map(|n| hex::encode(n.repr)));
println!("{:?}", t.right.map(|n| hex::encode(n.repr)));
for p in t.parents.iter() {
println!("{:?}", p.map(|n| hex::encode(n.repr)));
}
println!("====");
// println!("{:?}", tree1.left.map(|n| hex::encode(n.repr)));
// println!("{:?}", tree1.right.map(|n| hex::encode(n.repr)));
// for p in tree1.parents.iter() {
// println!("{:?}", p.map(|n| hex::encode(n.repr)));
// }
//
// println!("-----");
//
// println!("{:?}", tree2.left.map(|n| hex::encode(n.repr)));
// println!("{:?}", tree2.right.map(|n| hex::encode(n.repr)));
// for p in tree2.parents.iter() {
// println!("{:?}", p.map(|n| hex::encode(n.repr)));
// }
}
}
#[test]
fn test_cursor() {
// println!("{}", cursor_start_position(8, 14));
println!("{}", cursor_start_position(9, 14));
// println!("{}", cursor_start_position(10, 14));
pub fn to_commitment_tree(&self) -> CommitmentTree<Node> {
let mut bb: Vec<u8> = vec![];
self.write(&mut bb).unwrap();
CommitmentTree::<Node>::read(&*bb).unwrap()
}
}

196
src/db.rs Normal file
View File

@ -0,0 +1,196 @@
use rusqlite::{Connection, params, OptionalExtension};
use crate::{Witness, CTree};
pub struct DbAdapter {
connection: Connection,
}
pub struct ReceivedNote {
pub height: u32,
pub output_index: u32,
pub diversifier: Vec<u8>,
pub value: u64,
pub rcm: Vec<u8>,
pub nf: Vec<u8>,
pub is_change: bool,
pub memo: Vec<u8>,
pub spent: bool,
}
impl DbAdapter {
pub fn new(db_path: &str) -> anyhow::Result<DbAdapter> {
let connection = Connection::open(db_path)?;
Ok(DbAdapter {
connection,
})
}
pub fn init_db(&self) -> anyhow::Result<()> {
self.connection.execute("CREATE TABLE IF NOT EXISTS blocks (
height INTEGER PRIMARY KEY,
hash BLOB NOT NULL,
sapling_tree BLOB NOT NULL)", [])?;
self.connection.execute("CREATE TABLE IF NOT EXISTS transactions (
id_tx INTEGER PRIMARY KEY,
txid BLOB NOT NULL UNIQUE,
height INTEGER,
tx_index INTEGER)", [])?;
self.connection.execute("CREATE TABLE IF NOT EXISTS received_notes (
id_note INTEGER PRIMARY KEY,
position INTEGER NOT NULL,
tx INTEGER NOT NULL,
height INTEGER NOT NULL,
output_index INTEGER NOT NULL,
diversifier BLOB NOT NULL,
value INTEGER NOT NULL,
rcm BLOB NOT NULL,
nf BLOB NOT NULL UNIQUE,
is_change INTEGER NOT NULL,
memo BLOB,
spent INTEGER,
FOREIGN KEY (tx) REFERENCES transactions(id_tx),
FOREIGN KEY (spent) REFERENCES transactions(id_tx),
CONSTRAINT tx_output UNIQUE (tx, output_index))", [])?;
self.connection.execute("CREATE TABLE IF NOT EXISTS sapling_witnesses (
id_witness INTEGER PRIMARY KEY,
note INTEGER NOT NULL,
height INTEGER NOT NULL,
witness BLOB NOT NULL,
FOREIGN KEY (note) REFERENCES received_notes(id_note),
CONSTRAINT witness_height UNIQUE (note, height))", [])?;
Ok(())
}
pub fn trim_to_height(&mut self, height: u32) -> anyhow::Result<()> {
let tx = self.connection.transaction()?;
tx.execute("DELETE FROM blocks WHERE height >= ?1", params![height])?;
tx.execute("DELETE FROM sapling_witnesses WHERE height >= ?1", params![height])?;
tx.execute("DELETE FROM received_notes WHERE height >= ?1", params![height])?;
tx.execute("DELETE FROM transactions WHERE height >= ?1", params![height])?;
tx.commit()?;
Ok(())
}
pub fn store_block(&self, height: u32, hash: &[u8], tree: &CTree) -> anyhow::Result<()> {
let mut bb: Vec<u8> = vec![];
tree.write(&mut bb)?;
self.connection.execute("INSERT INTO blocks(height, hash, sapling_tree)
VALUES (?1, ?2, ?3)
ON CONFLICT DO NOTHING", params![height, hash, &bb])?;
Ok(())
}
pub fn store_transaction(&self, txid: &[u8], height: u32, tx_index: u32) -> anyhow::Result<u32> {
self.connection.execute("INSERT INTO transactions(txid, height, tx_index)
VALUES (?1, ?2, ?3)
ON CONFLICT DO NOTHING", params![txid, height, tx_index])?;
let id_tx: u32 = self.connection.query_row("SELECT id_tx FROM transactions WHERE txid = ?1", params![txid], |row| row.get(0))?;
Ok(id_tx)
}
pub fn store_received_note(&self, note: &ReceivedNote, id_tx: u32, position: usize) -> anyhow::Result<u32> {
self.connection.execute("INSERT INTO received_notes(tx, height, position, output_index, diversifier, value, rcm, nf, is_change, memo, spent)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)
ON CONFLICT DO NOTHING", params![id_tx, note.height, position, note.output_index, note.diversifier, note.value, note.rcm, note.nf, note.is_change, note.memo, note.spent])?;
let id_note: u32 = self.connection.query_row("SELECT id_note FROM received_notes WHERE tx = ?1 AND output_index = ?2", params![id_tx, note.output_index], |row| row.get(0))?;
Ok(id_note)
}
pub fn store_witnesses(&self, witness: &Witness, height: u32, id_note: u32) -> anyhow::Result<()> {
let mut bb: Vec<u8> = vec![];
witness.write(&mut bb)?;
println!("{} {}", height, id_note);
self.connection.execute("INSERT INTO sapling_witnesses(note, height, witness) VALUES (?1, ?2, ?3)
ON CONFLICT DO NOTHING", params![id_note, height, bb])?;
Ok(())
}
pub fn get_balance(&self) -> anyhow::Result<u64> {
let balance: u64 = self.connection.query_row("SELECT SUM(value) FROM received_notes WHERE spent = 0", [], |row| row.get(0))?;
Ok(balance)
}
pub fn get_last_height(&self) -> anyhow::Result<Option<u32>> {
let height: Option<u32> = self.connection.query_row("SELECT MAX(height) FROM blocks", [], |row| row.get(0)).optional()?;
Ok(height)
}
pub fn get_tree(&self) -> anyhow::Result<(CTree, Vec<Witness>)> {
let res = self.connection.query_row(
"SELECT height, sapling_tree FROM blocks WHERE height = (SELECT MAX(height) FROM blocks)",
[], |row| {
let height: u32 = row.get(0)?;
let tree: Vec<u8> = row.get(1)?;
Ok((height, tree))
}).optional()?;
Ok(match res {
Some((height, tree)) => {
let tree = CTree::read(&*tree)?;
let mut statement = self.connection.prepare(
"SELECT id_note, position, witness FROM sapling_witnesses w, received_notes n WHERE w.height = ?1 AND w.note = n.id_note")?;
let ws = statement.query_map(params![height], |row| {
let id_note: u32 = row.get(0)?;
let position: u32 = row.get(1)?;
let witness: Vec<u8> = row.get(2)?;
Ok(Witness::read(position as usize, id_note, &*witness).unwrap())
})?;
let mut witnesses: Vec<Witness> = vec![];
for w in ws {
witnesses.push(w?);
}
(tree, witnesses)
},
None => (CTree::new(), vec![])
})
}
}
#[cfg(test)]
mod tests {
use crate::db::{DbAdapter, ReceivedNote};
use crate::{Witness, CTree};
const DB_PATH: &str = "zec.db";
#[test]
fn test_db() {
let mut db = DbAdapter::new(DB_PATH).unwrap();
db.init_db().unwrap();
db.trim_to_height(0).unwrap();
db.store_block(1, &[0u8; 32], &CTree::new()).unwrap();
let id_tx = db.store_transaction(&[0; 32], 1, 20).unwrap();
db.store_received_note(&ReceivedNote {
height: 1,
output_index: 0,
diversifier: vec![],
value: 0,
rcm: vec![],
nf: vec![],
is_change: false,
memo: vec![],
spent: false
}, id_tx, 5).unwrap();
let witness = Witness {
position: 10,
id_note: 0,
note: None,
tree: CTree::new(),
filled: vec![],
cursor: CTree::new(),
};
db.store_witnesses(&witness, 1000, 1).unwrap();
}
#[test]
fn test_balance() {
let db = DbAdapter::new(DB_PATH).unwrap();
let balance = db.get_balance().unwrap();
println!("{}", balance);
}
}

35
src/key.rs Normal file
View File

@ -0,0 +1,35 @@
use bip39::{Language, Mnemonic, Seed};
use zcash_primitives::zip32::{ExtendedSpendingKey, ExtendedFullViewingKey, ChildIndex};
use crate::NETWORK;
use zcash_primitives::consensus::Parameters;
use zcash_client_backend::encoding::{encode_extended_spending_key, encode_extended_full_viewing_key, encode_payment_address, decode_extended_spending_key, decode_extended_full_viewing_key};
use anyhow::anyhow;
pub fn get_secret_key(seed: &str) -> anyhow::Result<String> {
let mnemonic = Mnemonic::from_phrase(&seed, Language::English)?;
let seed = Seed::new(&mnemonic, "");
let master = ExtendedSpendingKey::master(seed.as_bytes());
let path = [
ChildIndex::Hardened(32),
ChildIndex::Hardened(NETWORK.coin_type()),
ChildIndex::Hardened(0),
];
let extsk = ExtendedSpendingKey::from_path(&master, &path);
let spending_key = encode_extended_spending_key(NETWORK.hrp_sapling_extended_spending_key(), &extsk);
Ok(spending_key)
}
pub fn get_viewing_key(secret_key: &str) -> anyhow::Result<String> {
let extsk = decode_extended_spending_key(NETWORK.hrp_sapling_extended_spending_key(), secret_key)?.ok_or(anyhow!("Invalid Secret Key"))?;
let fvk = ExtendedFullViewingKey::from(&extsk);
let viewing_key = encode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &fvk);
Ok(viewing_key)
}
pub fn get_address(viewing_key: &str) -> anyhow::Result<String> {
let fvk = decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &viewing_key)?.ok_or(anyhow!("Invalid Viewing Key"))?;
let (_, payment_address) = fvk.default_address().unwrap();
let address = encode_payment_address(NETWORK.hrp_sapling_payment_address(), &payment_address);
Ok(address)
}

18
src/lib.rs
View File

@ -5,16 +5,22 @@ pub mod lw_rpc;
pub const NETWORK: Network = Network::MainNetwork;
mod print;
mod builder;
mod chain;
mod path;
mod commitment;
mod scan;
mod builder;
mod key;
mod db;
mod wallet;
pub use crate::chain::{LWD_URL, get_latest_height, download_chain, calculate_tree_state_v2, DecryptNode};
pub use crate::commitment::{NotePosition, Witness, CTree};
pub use crate::builder::advance_tree;
pub use crate::chain::{
calculate_tree_state_v2, connect_lightwalletd, download_chain, get_latest_height, sync,
DecryptNode, LWD_URL,
};
pub use crate::commitment::{CTree, Witness};
pub use crate::lw_rpc::compact_tx_streamer_client::CompactTxStreamerClient;
pub use crate::lw_rpc::*;
pub use crate::scan::scan_all;
pub use crate::scan::{scan_all, sync_async, latest_height};
pub use crate::key::{get_secret_key, get_address, get_viewing_key};
pub use crate::db::DbAdapter;

7
src/main.rs
View File

@ -16,9 +16,8 @@ async fn main_scan() {
decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &ivk)
.unwrap()
.unwrap();
let ivk = fvk.fvk.vk.ivk();
scan_all(&vec![ivk]).await.unwrap();
scan_all(&vec![fvk]).await.unwrap();
}
#[allow(dead_code)]
@ -48,7 +47,7 @@ fn test_increasing_notes() {
if v % witness_freq == 0 {
// let w = IncrementalWitness::from_tree(&tree1);
// ws.push(w);
ws2.push(Witness::new(v));
ws2.push(Witness::new(v, 0, None));
}
nodes.push(node);
}
@ -89,7 +88,7 @@ fn test_increasing_gap(run_normal: bool, run_warp: bool) {
let w = IncrementalWitness::from_tree(&tree1);
ws.push(w);
}
ws2.push(Witness::new(pos));
ws2.push(Witness::new(pos, 0, None));
nodes.push(node);
pos += 1;
}

40
src/main/warp_cli.rs Normal file
View File

@ -0,0 +1,40 @@
use sync::{sync_async, DbAdapter};
use bip39::{Language, Mnemonic};
use rand::rngs::OsRng;
use rand::RngCore;
const DB_NAME: &str = "zec.db";
#[tokio::main]
#[allow(dead_code)]
async fn test() -> anyhow::Result<()> {
dotenv::dotenv().unwrap();
env_logger::init();
let ivk = dotenv::var("IVK").unwrap();
{
let db = DbAdapter::new(DB_NAME)?;
db.init_db()?;
}
sync_async(&ivk, 50000, DB_NAME, |height| {
log::info!("Height = {}", height);
}).await?;
Ok(())
}
#[allow(dead_code)]
fn test_rewind() {
let mut db = DbAdapter::new(DB_NAME).unwrap();
db.trim_to_height(1_250_000).unwrap();
}
fn main() {
// test_rewind();
test().unwrap();
// let mut entropy = [0u8; 32];
// OsRng.fill_bytes(&mut entropy);
// let mnemonic = Mnemonic::from_entropy(&entropy, Language::English).unwrap();
// let phrase = mnemonic.phrase();
// println!("Seed Phrase: {}", phrase);
}

37
src/path.rs
View File

@ -1,37 +0,0 @@
use zcash_primitives::merkle_tree::Hashable;
use zcash_primitives::sapling::Node;
pub struct MerklePath {
left: Option<Node>,
right: Option<Node>,
}
impl MerklePath {
pub fn new(left: Option<Node>, right: Option<Node>) -> Self {
MerklePath { left, right }
}
pub fn get(&self) -> Node {
self.left.unwrap() // shouldn't call if empty
}
pub fn set(&mut self, right: Node) {
assert!(self.left.is_some());
self.right = Some(right);
}
pub fn up(&mut self, depth: usize, parent: &Option<Node>) {
let node = if self.left.is_some() && self.right.is_some() {
Some(Node::combine(depth, &self.left.unwrap(), &self.right.unwrap()))
} else {
None
};
if parent.is_some() {
self.left = *parent;
self.right = node;
} else {
self.left = node;
self.right = None;
}
}
}

28
src/print.rs
View File

@ -1,28 +0,0 @@
use zcash_primitives::sapling::Node;
use zcash_primitives::merkle_tree::{CommitmentTree, IncrementalWitness};
#[allow(dead_code)]
pub fn print_node(n: &Node) {
println!("{:?}", hex::encode(n.repr));
}
#[allow(dead_code)]
pub fn print_tree(t: &CommitmentTree<Node>) {
println!("{:?}", t.left.map(|n| hex::encode(n.repr)));
println!("{:?}", t.right.map(|n| hex::encode(n.repr)));
for p in t.parents.iter() {
println!("{:?}", p.map(|n| hex::encode(n.repr)));
}
}
#[allow(dead_code)]
pub fn print_witness(w: &IncrementalWitness<Node>) {
println!("Tree");
print_tree(&w.tree);
println!("Filled");
for n in w.filled.iter() {
print_node(n);
}
println!("Cursor");
w.cursor.as_ref().map(|c| print_tree(c));
}

158
src/scan.rs
View File

@ -1,12 +1,18 @@
use zcash_primitives::sapling::SaplingIvk;
use zcash_primitives::sapling::Node;
use crate::lw_rpc::compact_tx_streamer_client::CompactTxStreamerClient;
use crate::{DecryptNode, LWD_URL, get_latest_height, download_chain, calculate_tree_state_v2};
use crate::{DecryptNode, LWD_URL, get_latest_height, download_chain, calculate_tree_state_v2, CompactBlock, NETWORK, connect_lightwalletd, Witness, advance_tree};
use zcash_primitives::consensus::{NetworkUpgrade, Parameters};
use zcash_client_backend::encoding::decode_extended_full_viewing_key;
use tokio::sync::mpsc;
use std::time::Instant;
use std::ops::Range;
use log::info;
use crate::db::{DbAdapter, ReceivedNote};
use ff::PrimeField;
use zcash_primitives::zip32::ExtendedFullViewingKey;
pub async fn scan_all(ivks: &[SaplingIvk]) -> anyhow::Result<()> {
let decrypter = DecryptNode::new(ivks.to_vec());
pub async fn scan_all(fvks: &[ExtendedFullViewingKey]) -> anyhow::Result<()> {
let decrypter = DecryptNode::new(fvks.to_vec());
let total_start = Instant::now();
let mut client = CompactTxStreamerClient::connect(LWD_URL).await?;
@ -38,3 +44,147 @@ pub async fn scan_all(ivks: &[SaplingIvk]) -> anyhow::Result<()> {
Ok(())
}
struct Blocks(Vec<CompactBlock>);
impl std::fmt::Debug for Blocks {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Blocks of len {}", self.0.len())
}
}
fn get_db_height(db_path: &str) -> anyhow::Result<u32> {
let db = DbAdapter::new(db_path).unwrap();
let height: u32 = db.get_last_height()?.unwrap_or_else(|| {
crate::NETWORK
.activation_height(NetworkUpgrade::Sapling)
.unwrap()
.into()
});
Ok(height)
}
pub async fn sync_async(ivk: &str, chunk_size: u32, db_path: &str, progress_callback: impl Fn(u32) + Send + 'static) -> anyhow::Result<()> {
let db_path = db_path.to_string();
let fvk =
decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &ivk)
.unwrap()
.unwrap();
let decrypter = DecryptNode::new(vec![fvk]);
let mut client = connect_lightwalletd().await?;
let start_height = get_db_height(&db_path)?;
let end_height = get_latest_height(&mut client).await?;
let (downloader_tx, mut download_rx) = mpsc::channel::<Range<u32>>(2);
let (processor_tx, mut processor_rx) = mpsc::channel::<Blocks>(2);
let (completed_tx, mut completed_rx) = mpsc::channel::<()>(1);
tokio::spawn(async move {
let mut client = connect_lightwalletd().await.unwrap();
while let Some(range) = download_rx.recv().await {
log::info!("{:?}", range);
let blocks = download_chain(&mut client, range.start, range.end).await.unwrap();
let b = Blocks(blocks);
processor_tx.send(b).await.unwrap();
}
log::info!("download completed");
drop(processor_tx);
// Ok::<_, anyhow::Error>(())
});
tokio::spawn(async move {
let db = DbAdapter::new(&db_path).unwrap();
let (mut tree, mut witnesses) = db.get_tree().unwrap();
let mut pos = tree.get_position();
// let mut tree = CTree::new();
// let mut witnesses: Vec<Witness> = vec![];
while let Some(blocks) = processor_rx.recv().await {
log::info!("{:?}", blocks);
if blocks.0.is_empty() { continue }
let dec_blocks = decrypter.decrypt_blocks(&blocks.0);
for b in dec_blocks.iter() {
if !b.notes.is_empty() {
log::info!("{} {}", b.height, b.notes.len());
}
for n in b.notes.iter() {
let p = pos + n.position;
let note = &n.note;
let id_tx = db.store_transaction(&n.txid, n.height, n.tx_index as u32).unwrap();
let rcm = note.rcm().to_repr();
let nf = note.nf(&n.ivk.fvk.vk, n.position as u64);
let id_note = db.store_received_note(&ReceivedNote {
height: n.height,
output_index: n.output_index as u32,
diversifier: n.pa.diversifier().0.to_vec(),
value: note.value,
rcm: rcm.to_vec(),
nf: nf.0.to_vec(),
is_change: false, // TODO: it's change the ovk matches too
memo: vec![],
spent: false
}, id_tx, n.position).unwrap();
let w = Witness::new(p as usize, id_note, Some(n.clone()));
witnesses.push(w);
}
pos += b.count_outputs as usize;
}
let mut nodes: Vec<Node> = vec![];
for cb in blocks.0.iter() {
for tx in cb.vtx.iter() {
for co in tx.outputs.iter() {
let mut cmu = [0u8; 32];
cmu.copy_from_slice(&co.cmu);
let node = Node::new(cmu);
nodes.push(node);
}
}
}
let (new_tree, new_witnesses) = advance_tree(tree, &witnesses, &mut nodes);
tree = new_tree;
witnesses = new_witnesses;
let last_block = blocks.0.last().unwrap();
let last_height = last_block.height as u32;
db.store_block(last_height, &last_block.hash, &tree).unwrap();
for w in witnesses.iter() {
db.store_witnesses(w, last_height, w.id_note).unwrap();
}
progress_callback(blocks.0[0].height as u32);
}
progress_callback(end_height);
log::info!("Witnesses {}", witnesses.len());
drop(completed_tx);
});
let mut height = start_height;
while height < end_height {
let s = height;
let e = (height + chunk_size).min(end_height);
let range = s..e;
downloader_tx.send(range).await?;
height = e;
}
drop(downloader_tx);
log::info!("req downloading completed");
completed_rx.recv().await;
log::info!("completed");
Ok(())
}
pub async fn latest_height() -> u32 {
let mut client = connect_lightwalletd().await.unwrap();
get_latest_height(&mut client).await.unwrap()
}