From 088a4d1ef58e124b67dc8a596ba5b5179457d4d3 Mon Sep 17 00:00:00 2001 From: Hanh Date: Fri, 28 Oct 2022 21:02:34 +0800 Subject: [PATCH] Orchard warp sync --- Cargo.toml | 5 ++ src/chain.rs | 1 - src/db.rs | 58 ++++++++++++++----- src/db/migration.rs | 9 ++- src/lib.rs | 5 +- src/orchard.rs | 5 ++ src/orchard/hash.rs | 118 ++++++++++++++++++++++++++++++++++++++ src/orchard/note.rs | 96 +++++++++++++++++++++++++++++++ src/sapling/hash.rs | 2 +- src/sapling/note.rs | 3 +- src/scan.rs | 73 +++++++++++++++++------ src/sync.rs | 5 +- src/sync/trial_decrypt.rs | 27 ++++++--- 13 files changed, 357 insertions(+), 50 deletions(-) create mode 100644 src/orchard.rs create mode 100644 src/orchard/hash.rs create mode 100644 src/orchard/note.rs diff --git a/Cargo.toml b/Cargo.toml index 189a554..5c65683 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,6 +73,11 @@ chrono = "0.4.19" lazycell = "1.3.0" reqwest = { version = "0.11.4", features = ["json", "rustls-tls"], default-features = false } +# Halo +orchard = "0.3.0" +halo2_proofs = "0.2" +halo2_gadgets = "0.2" + bech32 = "0.8.1" rand_chacha = "0.3.1" blake2b_simd = "1.0.0" diff --git a/src/chain.rs b/src/chain.rs index a48c087..129beda 100644 --- a/src/chain.rs +++ b/src/chain.rs @@ -190,7 +190,6 @@ pub async fn download_chain( ph.copy_from_slice(&block.hash); prev_hash = Some(ph); for tx in block.vtx.iter_mut() { - tx.actions.clear(); // don't need Orchard actions let mut skipped = false; if tx.outputs.len() > max_cost as usize { for co in tx.outputs.iter_mut() { diff --git a/src/db.rs b/src/db.rs index 3f620be..7fb0a29 100644 --- a/src/db.rs +++ b/src/db.rs @@ -92,6 +92,11 @@ pub struct AccountBackup { pub t_addr: Option, } +pub struct AccountSeed { + pub id_account: u32, + pub seed: String, +} + pub fn wrap_query_no_rows(name: &'static str) -> impl Fn(rusqlite::Error) -> anyhow::Error { move |err: rusqlite::Error| match err { QueryReturnedNoRows => anyhow::anyhow!("Query {} returned no rows", name), @@ -232,6 +237,24 @@ impl DbAdapter { Ok(fvks) } + pub fn get_seeds(&self) -> anyhow::Result> { + let mut statement = self + .connection + .prepare("SELECT id_account, seed FROM accounts WHERE seed IS NOT NULL")?; + let rows = statement.query_map([], |row| { + let id_account: u32 = row.get(0)?; + let seed: String = row.get(1)?; + Ok(AccountSeed { + id_account, + seed}) + })?; + let mut accounts = vec![]; + for row in rows { + accounts.push(row?); + } + Ok(accounts) + } + pub fn trim_to_height(&mut self, height: u32) -> anyhow::Result { // snap height to an existing checkpoint let height = self.connection.query_row( @@ -426,16 +449,15 @@ impl DbAdapter { Ok(()) } - pub fn store_tree(height: u32, hash: &[u8], tree: &CTree, db_tx: &Connection, shielded_pool: &str) -> anyhow::Result<()> { - let mut bb: Vec = vec![]; - tree.write(&mut bb)?; - db_tx.execute(&format!("INSERT INTO blocks(height, hash, {pool}_tree, timestamp) VALUES (?1,?2,?3,0) ON CONFLICT DO UPDATE - SET {pool}_tree = excluded.{pool}_tree", pool = shielded_pool), params![height, hash, &bb])?; + pub fn store_block_timestamp(&self, height: u32, hash: &[u8], timestamp: u32) -> anyhow::Result<()> { + self.connection.execute("INSERT INTO blocks(height, hash, timestamp) VALUES (?1,?2,?3)", params![height, hash, timestamp])?; Ok(()) } - pub fn store_block_timestamp(&self, height: u32, timestamp: u32) -> anyhow::Result<()> { - self.connection.execute("UPDATE blocks SET timestamp = ?1 WHERE height = ?2", params![timestamp, height])?; + pub fn store_tree(height: u32, tree: &CTree, db_tx: &Connection, shielded_pool: &str) -> anyhow::Result<()> { + let mut bb: Vec = vec![]; + tree.write(&mut bb)?; + db_tx.execute(&format!("INSERT INTO {}_tree(height, tree) VALUES (?1,?2)", shielded_pool), params![height, &bb])?; Ok(()) } @@ -519,15 +541,21 @@ impl DbAdapter { } pub fn get_tree_by_name(&self, shielded_pool: &str) -> anyhow::Result { - let res = self.connection.query_row( - &format!("SELECT height, {}_tree FROM blocks WHERE height = (SELECT MAX(height) FROM blocks)", shielded_pool), + let height = self.connection.query_row( + "SELECT MAX(height) FROM blocks", [], |row| { - let height: u32 = row.get(0)?; - let tree: Vec = row.get(1)?; - Ok((height, tree)) - }).optional()?; - Ok(match res { - Some((height, tree)) => { + let height: Option = row.get(0)?; + Ok(height) + })?; + Ok(match height { + Some(height) => { + let tree = self.connection.query_row( + &format!("SELECT tree FROM {}_tree WHERE height = ?1", shielded_pool), + [height], |row| { + let tree: Vec = row.get(0)?; + Ok(tree) + })?; + let tree = sync::CTree::read(&*tree)?; let mut statement = self.connection.prepare( &format!("SELECT id_note, witness FROM {}_witnesses w, received_notes n WHERE w.height = ?1 AND w.note = n.id_note AND (n.spent IS NULL OR n.spent = 0)", shielded_pool))?; diff --git a/src/db/migration.rs b/src/db/migration.rs index f208b51..0c3d3ae 100644 --- a/src/db/migration.rs +++ b/src/db/migration.rs @@ -178,7 +178,14 @@ pub fn init_db(connection: &Connection) -> anyhow::Result<()> { } if version < 5 { - connection.execute("ALTER TABLE blocks ADD orchard_tree BLOB", [])?; + connection.execute("CREATE TABLE sapling_tree( + height INTEGER PRIMARY KEY, + tree BLOB NOT NULL)", [])?; + connection.execute("CREATE TABLE orchard_tree( + height INTEGER PRIMARY KEY, + tree BLOB NOT NULL)", [])?; + connection.execute("INSERT INTO sapling_tree SELECT height, sapling_tree FROM blocks", [])?; + connection.execute("ALTER TABLE blocks DROP sapling_tree", [])?; connection.execute("ALTER TABLE received_notes ADD rho BLOB", [])?; connection.execute( "CREATE TABLE IF NOT EXISTS orchard_witnesses ( diff --git a/src/lib.rs b/src/lib.rs index 979af63..51ab361 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -82,8 +82,9 @@ mod contact; mod db; mod fountain; mod hash; -pub(crate) mod sync; -pub mod sapling; +mod sync; +mod sapling; +mod orchard; mod key; mod key2; mod mempool; diff --git a/src/orchard.rs b/src/orchard.rs new file mode 100644 index 0000000..9ab2df0 --- /dev/null +++ b/src/orchard.rs @@ -0,0 +1,5 @@ +mod hash; +mod note; + +pub use note::{OrchardDecrypter, OrchardViewKey, DecryptedOrchardNote}; +pub use hash::OrchardHasher; diff --git a/src/orchard/hash.rs b/src/orchard/hash.rs new file mode 100644 index 0000000..1467c33 --- /dev/null +++ b/src/orchard/hash.rs @@ -0,0 +1,118 @@ +#![allow(non_snake_case)] + +use group::cofactor::CofactorCurveAffine; +use halo2_gadgets::sinsemilla::primitives::SINSEMILLA_S; +use halo2_proofs::arithmetic::{CurveAffine, CurveExt}; +use halo2_proofs::pasta::EpAffine; +use halo2_proofs::pasta::group::ff::PrimeField; +use halo2_proofs::pasta::group::Curve; +use halo2_proofs::pasta::pallas::{self, Affine, Point}; +use lazy_static::lazy_static; +use crate::Hash; +use crate::sync::{Hasher, Node}; + +pub const Q_PERSONALIZATION: &str = "z.cash:SinsemillaQ"; +pub const MERKLE_CRH_PERSONALIZATION: &str = "z.cash:Orchard-MerkleCRH"; + +lazy_static! { + pub static ref ORCHARD_ROOTS: Vec = { + let h = OrchardHasher::new(); + h.empty_roots(32) + }; +} + +#[derive(Clone)] +pub struct OrchardHasher { + Q: Point, +} + +impl OrchardHasher { + pub fn new() -> Self { + let Q: Point = + Point::hash_to_curve(Q_PERSONALIZATION)(MERKLE_CRH_PERSONALIZATION.as_bytes()); + OrchardHasher { Q } + } + + fn node_combine_inner(&self, depth: u8, left: &Node, right: &Node) -> Point { + let mut acc = self.Q; + let (S_x, S_y) = SINSEMILLA_S[depth as usize]; + let S_chunk = Affine::from_xy(S_x, S_y).unwrap(); + acc = (acc + S_chunk) + acc; // TODO Bail if + gives point at infinity? + + // Shift right by 1 bit and overwrite the 256th bit of left + let mut left = *left; + let mut right = *right; + left[31] |= (right[0] & 1) << 7; // move the first bit of right into 256th of left + for i in 0..32 { + // move by 1 bit to fill the missing 256th bit of left + let carry = if i < 31 { (right[i + 1] & 1) << 7 } else { 0 }; + right[i] = right[i] >> 1 | carry; + } + + // we have 255*2/10 = 51 chunks + let mut bit_offset = 0; + let mut byte_offset = 0; + for _ in 0..51 { + let mut v = if byte_offset < 31 { + left[byte_offset] as u16 | (left[byte_offset + 1] as u16) << 8 + } else if byte_offset == 31 { + left[31] as u16 | (right[0] as u16) << 8 + } else { + right[byte_offset - 32] as u16 | (right[byte_offset - 31] as u16) << 8 + }; + v = v >> bit_offset & 0x03FF; // keep 10 bits + let (S_x, S_y) = SINSEMILLA_S[v as usize]; + let S_chunk = Affine::from_xy(S_x, S_y).unwrap(); + acc = (acc + S_chunk) + acc; + bit_offset += 10; + if bit_offset >= 8 { + byte_offset += bit_offset / 8; + bit_offset %= 8; + } + } + acc + } + + pub fn empty_roots(&self, height: usize) -> Vec { + let mut roots = vec![]; + let mut cur = pallas::Base::from(2).to_repr(); + roots.push(cur); + for depth in 0..height { + cur = self.node_combine(depth as u8, &cur, &cur); + roots.push(cur); + } + roots + } +} + +impl Hasher for OrchardHasher { + type Extended = Point; + + fn uncommited_node() -> Node { + pallas::Base::from(2).to_repr() + } + + fn node_combine(&self, depth: u8, left: &Node, right: &Node) -> Node { + let acc = self.node_combine_inner(depth, left, right); + let p = acc + .to_affine() + .coordinates() + .map(|c| *c.x()) + .unwrap_or_else(pallas::Base::zero); + p.to_repr() + } + + fn node_combine_extended(&self, depth: u8, left: &Node, right: &Node) -> Self::Extended { + self.node_combine_inner(depth, left, right) + } + + fn normalize(&self, extended: &[Self::Extended]) -> Vec { + let mut hash_affine = vec![EpAffine::identity(); extended.len()]; + Point::batch_normalize(extended, &mut hash_affine); + hash_affine + .iter() + .map(|p| + p.coordinates().map(|c| *c.x()).unwrap_or_else(pallas::Base::zero).to_repr()) + .collect() + } +} diff --git a/src/orchard/note.rs b/src/orchard/note.rs new file mode 100644 index 0000000..ff95f43 --- /dev/null +++ b/src/orchard/note.rs @@ -0,0 +1,96 @@ +use orchard::note_encryption::OrchardDomain; +use zcash_primitives::consensus::{BlockHeight, Parameters}; +use crate::chain::Nf; +use crate::CompactTx; +use crate::db::ReceivedNote; +use crate::sync::{CompactOutputBytes, DecryptedNote, Node, OutputPosition, TrialDecrypter, ViewKey}; +use zcash_note_encryption; +use zcash_primitives::sapling::Nullifier; + +#[derive(Clone)] +pub struct OrchardViewKey { + pub account: u32, + pub fvk: orchard::keys::FullViewingKey, +} + +impl ViewKey for OrchardViewKey { + fn account(&self) -> u32 { + self.account + } + + fn ivk(&self) -> orchard::keys::IncomingViewingKey { + self.fvk.to_ivk(orchard::keys::Scope::External) + } +} + +pub struct DecryptedOrchardNote { + pub vk: OrchardViewKey, + pub note: orchard::Note, + pub pa: orchard::Address, + pub output_position: OutputPosition, + pub cmx: Node, +} + +impl DecryptedNote for DecryptedOrchardNote { + fn from_parts(vk: OrchardViewKey, note: orchard::Note, pa: orchard::Address, output_position: OutputPosition, cmx: Node) -> Self { + DecryptedOrchardNote { + vk, + note, + pa, + output_position, + cmx + } + } + + fn position(&self, block_offset: usize) -> usize { + block_offset + self.output_position.position_in_block + } + + fn cmx(&self) -> Node { + self.cmx + } + + fn to_received_note(&self, position: u64) -> ReceivedNote { + ReceivedNote { + account: self.vk.account, + height: self.output_position.height, + output_index: self.output_position.output_index as u32, + diversifier: self.pa.diversifier().as_array().to_vec(), + value: self.note.value().inner(), + rcm: self.note.rseed().as_bytes().to_vec(), + nf: self.note.nullifier(&self.vk.fvk).to_bytes().to_vec(), + rho: Some(self.note.rho().to_bytes().to_vec()), + spent: None + } + } +} + +#[derive(Clone)] +pub struct OrchardDecrypter { + pub network: N, +} + +impl OrchardDecrypter { + pub fn new(network: N) -> Self { + OrchardDecrypter { + network, + } + } +} + +impl TrialDecrypter for OrchardDecrypter { + fn domain(&self, _height: BlockHeight, cob: &CompactOutputBytes) -> OrchardDomain { + OrchardDomain::for_nullifier(orchard::note::Nullifier::from_bytes(&cob.nullifier).unwrap()) + } + + fn spends(&self, vtx: &CompactTx) -> Vec { + vtx.actions.iter().map(|co| { + let nf: [u8; 32] = co.nullifier.clone().try_into().unwrap(); + Nf(nf) + }).collect() + } + + fn outputs(&self, vtx: &CompactTx) -> Vec { + vtx.actions.iter().map(|co| co.into()).collect() + } +} diff --git a/src/sapling/hash.rs b/src/sapling/hash.rs index 2722421..a5355af 100644 --- a/src/sapling/hash.rs +++ b/src/sapling/hash.rs @@ -125,7 +125,7 @@ impl Hasher for SaplingHasher { } fn normalize(&self, extended: &[Self::Extended]) -> Vec { - let mut hash_affine: Vec = vec![AffinePoint::identity(); extended.len()]; + let mut hash_affine = vec![AffinePoint::identity(); extended.len()]; ExtendedPoint::batch_normalize(extended, &mut hash_affine); hash_affine .iter() diff --git a/src/sapling/note.rs b/src/sapling/note.rs index b6aa7b6..9fdd113 100644 --- a/src/sapling/note.rs +++ b/src/sapling/note.rs @@ -82,7 +82,7 @@ impl SaplingDecrypter { } impl TrialDecrypter, SaplingViewKey, DecryptedSaplingNote> for SaplingDecrypter { - fn domain(&self, height: BlockHeight) -> SaplingDomain { + fn domain(&self, height: BlockHeight, _cob: &CompactOutputBytes) -> SaplingDomain { SaplingDomain::::for_height(self.network.clone(), height) } @@ -97,4 +97,3 @@ impl TrialDecrypter, SaplingViewKey, Decrypt vtx.outputs.iter().map(|co| co.into()).collect() } } - diff --git a/src/scan.rs b/src/scan.rs index b7e5b2a..e76cd1c 100644 --- a/src/scan.rs +++ b/src/scan.rs @@ -4,7 +4,7 @@ use serde::Serialize; use std::cmp::Ordering; use crate::transaction::retrieve_tx_info; -use crate::{connect_lightwalletd, CompactBlock, CompactSaplingOutput, CompactTx, DbAdapterBuilder, chain}; +use crate::{connect_lightwalletd, CompactBlock, CompactSaplingOutput, CompactTx, DbAdapterBuilder, chain, AccountRec}; use crate::chain::{DecryptNode, download_chain}; use ff::PrimeField; @@ -14,15 +14,19 @@ use std::collections::HashMap; use std::panic; use std::sync::Arc; use std::time::Instant; +use bip39::{Language, Mnemonic}; +use orchard::keys::{FullViewingKey, SpendingKey}; +use orchard::note_encryption::OrchardDomain; use tokio::runtime::{Builder, Runtime}; use tokio::sync::mpsc; use tokio::sync::Mutex; use zcash_client_backend::encoding::decode_extended_full_viewing_key; use zcash_params::coin::{get_coin_chain, CoinType}; -use zcash_primitives::consensus::{Network, Parameters}; +use zcash_primitives::consensus::{Network, NetworkUpgrade, Parameters}; use zcash_primitives::sapling::{Node, Note}; use zcash_primitives::sapling::note_encryption::SaplingDomain; +use crate::orchard::{DecryptedOrchardNote, OrchardDecrypter, OrchardHasher, OrchardViewKey}; use crate::sapling::{DecryptedSaplingNote, SaplingDecrypter, SaplingHasher, SaplingViewKey}; use crate::sync::{CTree, Synchronizer, WarpProcessor}; @@ -61,6 +65,9 @@ pub struct TxIdHeight { type SaplingSynchronizer = Synchronizer, SaplingViewKey, DecryptedSaplingNote, SaplingDecrypter, SaplingHasher>; +type OrchardSynchronizer = Synchronizer, OrchardHasher>; + pub async fn sync_async<'a>( coin_type: CoinType, _chunk_size: u32, @@ -80,7 +87,7 @@ pub async fn sync_async<'a>( }; let mut client = connect_lightwalletd(&ld_url).await?; - let (start_height, prev_hash, sapling_vks) = { + let (start_height, prev_hash, sapling_vks, orchard_vks) = { let db = DbAdapter::new(coin_type, &db_path)?; let height = db.get_db_height()?; let hash = db.get_db_hash(height)?; @@ -92,7 +99,22 @@ pub async fn sync_async<'a>( ivk: ak.ivk.clone() } }).collect(); - (height, hash, sapling_vks) + let orchard_vks: Vec<_> = db.get_seeds()?.iter().map(|a| { + let mnemonic = Mnemonic::from_phrase(&a.seed, Language::English).unwrap(); + let sk = SpendingKey::from_zip32_seed( + mnemonic.entropy(), + network.coin_type(), + a.id_account, + ).unwrap(); + let fvk = FullViewingKey::from(&sk); + let vk = + OrchardViewKey { + account: a.id_account, + fvk, + }; + vk + }).collect(); + (height, hash, sapling_vks, orchard_vks) }; let end_height = get_latest_height(&mut client).await?; let end_height = (end_height - target_height_offset).max(start_height); @@ -111,25 +133,42 @@ pub async fn sync_async<'a>( let first_block = blocks.0.first().unwrap(); // cannot be empty because blocks are not log::info!("Height: {}", first_block.height); let last_block = blocks.0.last().unwrap(); + let last_hash: [u8; 32] = last_block.hash.clone().try_into().unwrap(); let last_height = last_block.height as u32; let last_timestamp = last_block.time; - let decrypter = SaplingDecrypter::new(network); - let warper = WarpProcessor::new(SaplingHasher::default()); - let mut sapling_synchronizer = SaplingSynchronizer::new( - decrypter, - warper, - sapling_vks.clone(), - db_builder.clone(), - "sapling".to_string(), - ); - sapling_synchronizer.initialize()?; - sapling_synchronizer.process(blocks.0)?; + // Sapling + { + let decrypter = SaplingDecrypter::new(network); + let warper = WarpProcessor::new(SaplingHasher::default()); + let mut synchronizer = SaplingSynchronizer::new( + decrypter, + warper, + sapling_vks.clone(), + db_builder.clone(), + "sapling".to_string(), + ); + synchronizer.initialize()?; + synchronizer.process(&blocks.0)?; + } - // TODO - Orchard + // Orchard + { + let decrypter = OrchardDecrypter::new(network); + let warper = WarpProcessor::new(OrchardHasher::new()); + let mut synchronizer = OrchardSynchronizer::new( + decrypter, + warper, + orchard_vks.clone(), + db_builder.clone(), + "orchard".to_string(), + ); + synchronizer.initialize()?; + synchronizer.process(&blocks.0)?; + } let db = db_builder.build()?; - db.store_block_timestamp(last_height, last_timestamp)?; + db.store_block_timestamp(last_height, &last_hash, last_timestamp)?; } Ok(()) diff --git a/src/sync.rs b/src/sync.rs index 923cf68..cd0214c 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -53,7 +53,6 @@ impl Result<()> { let db = self.db.build()?; let TreeCheckpoint { tree, witnesses } = db.get_tree_by_name(&self.shielded_pool)?; @@ -69,7 +68,7 @@ impl ) -> Result<()> { + pub fn process(&mut self, blocks: &[CompactBlock]) -> Result<()> { if blocks.is_empty() { return Ok(()) } let decrypter = self.decrypter.clone(); let decrypted_blocks: Vec<_> = blocks @@ -135,7 +134,7 @@ impl : Send + Sync { // Deep copy from protobuf message pub struct CompactOutputBytes { + pub nullifier: [u8; 32], pub epk: [u8; 32], pub cmx: [u8; 32], pub ciphertext: [u8; 52], @@ -59,6 +60,7 @@ pub struct CompactOutputBytes { impl From<&CompactSaplingOutput> for CompactOutputBytes { fn from(co: &CompactSaplingOutput) -> Self { CompactOutputBytes { + nullifier: [0u8; 32], epk: if co.epk.is_empty() { [0u8; 32] } else { co.epk.clone().try_into().unwrap() } , cmx: co.cmu.clone().try_into().unwrap(), // cannot be filtered out ciphertext: if co.ciphertext.is_empty() { [0u8; 52] } else { co.ciphertext.clone().try_into().unwrap() }, @@ -66,6 +68,18 @@ impl From<&CompactSaplingOutput> for CompactOutputBytes { } } +impl From<&CompactOrchardAction> for CompactOutputBytes { + fn from(co: &CompactOrchardAction) -> Self { + CompactOutputBytes { + nullifier: co.nullifier.clone().try_into().unwrap(), + epk: if co.ephemeral_key.is_empty() { [0u8; 32] } else { co.ephemeral_key.clone().try_into().unwrap() } , + cmx: co.cmx.clone().try_into().unwrap(), // cannot be filtered out + ciphertext: if co.ciphertext.is_empty() { [0u8; 52] } else { co.ciphertext.clone().try_into().unwrap() }, + } + } +} + + pub struct CompactShieldedOutput(CompactOutputBytes, OutputPosition); impl> ShieldedOutput @@ -95,17 +109,14 @@ pub trait TrialDecrypter D; + fn domain(&self, height: BlockHeight, cob: &CompactOutputBytes) -> D; fn spends(&self, vtx: &CompactTx) -> Vec; fn outputs(&self, vtx: &CompactTx) -> Vec; }