From 3e358bc1c95d7b0f4ce8f662ac1668865661aed1 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 3 Apr 2023 13:53:43 -0600 Subject: [PATCH] zcash_client_backend: Use `shardtree` for note commitments in block scanning. Also adds a skeleton `zcash_client_sqlite` implementation of `shardtree::ShardStore` and a skeleton migration for related database changes. --- Cargo.toml | 5 + zcash_client_backend/Cargo.toml | 1 + zcash_client_backend/src/data_api.rs | 155 ++++++++---- zcash_client_backend/src/data_api/chain.rs | 158 ++++-------- .../src/data_api/chain/error.rs | 12 + zcash_client_backend/src/data_api/error.rs | 33 ++- zcash_client_backend/src/data_api/wallet.rs | 144 +++++++---- .../src/data_api/wallet/input_selection.rs | 53 ++-- zcash_client_backend/src/wallet.rs | 16 +- zcash_client_backend/src/welding_rig.rs | 235 ++++++++++-------- zcash_client_sqlite/Cargo.toml | 1 + zcash_client_sqlite/src/chain.rs | 45 +++- zcash_client_sqlite/src/error.rs | 11 + zcash_client_sqlite/src/lib.rs | 234 +++++++++-------- zcash_client_sqlite/src/wallet.rs | 79 ++++-- zcash_client_sqlite/src/wallet/init.rs | 17 +- .../src/wallet/init/migrations.rs | 2 + .../init/migrations/shardtree_support.rs | 56 +++++ zcash_client_sqlite/src/wallet/sapling.rs | 222 +++++++---------- .../src/wallet/sapling/commitment_tree.rs | 123 +++++++++ zcash_primitives/src/merkle_tree.rs | 2 +- 21 files changed, 1015 insertions(+), 589 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs create mode 100644 zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs diff --git a/Cargo.toml b/Cargo.toml index 044d879e9..073970f92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,3 +17,8 @@ members = [ lto = true panic = 'abort' codegen-units = 1 + +[patch.crates-io] +incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "082109deacf8611ee7917732e19b56158bda96d5" } +shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "082109deacf8611ee7917732e19b56158bda96d5" } +orchard = { git = "https://github.com/zcash/orchard.git", rev = "5da41a6bbb44290e353ee4b38bcafe37ffe79ce8" } diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index eb6eb2c1d..7d0e46382 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -21,6 +21,7 @@ development = ["zcash_proofs"] [dependencies] incrementalmerkletree = { version = "0.4", features = ["legacy-api"] } +shardtree = "0.0" zcash_address = { version = "0.3", path = "../components/zcash_address" } zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } zcash_note_encryption = "0.4" diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 1b3dff2a7..80ad24f55 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -1,10 +1,12 @@ //! Interfaces for wallet data persistence & low-level wallet utilities. -use std::cmp; use std::collections::HashMap; use std::fmt::Debug; +use std::{cmp, ops::Range}; +use incrementalmerkletree::Retention; use secrecy::SecretVec; +use shardtree::{ShardStore, ShardTree, ShardTreeError}; use zcash_primitives::{ block::BlockHash, consensus::BlockHeight, @@ -29,6 +31,8 @@ pub mod chain; pub mod error; pub mod wallet; +pub const SAPLING_SHARD_HEIGHT: u8 = sapling::NOTE_COMMITMENT_TREE_DEPTH / 2; + pub enum NullifierQuery { Unspent, All, @@ -61,6 +65,30 @@ pub trait WalletRead { /// This will return `Ok(None)` if no block data is present in the database. fn block_height_extrema(&self) -> Result, Self::Error>; + /// Returns the height to which the wallet has been fully scanned. + /// + /// This is the height for which the wallet has fully trial-decrypted this and all preceding + /// blocks above the wallet's birthday height. Along with this height, this method returns + /// metadata describing the state of the wallet's note commitment trees as of the end of that + /// block. + fn fully_scanned_height( + &self, + ) -> Result, Self::Error>; + + /// Returns a vector of suggested scan ranges based upon the current wallet state. + /// + /// This method should only be used in cases where the [`CompactBlock`] data that will be made + /// available to `scan_cached_blocks` for the requested block ranges includes note commitment + /// tree size information for each block; or else the scan is likely to fail if notes belonging + /// to the wallet are detected. + /// + /// [`CompactBlock`]: crate::proto::compact_formats::CompactBlock + fn suggest_scan_ranges( + &self, + batch_size: usize, + limit: usize, + ) -> Result>, Self::Error>; + /// Returns the default target height (for the block in which a new /// transaction would be mined) and anchor height (to use for a new /// transaction), given the range of block heights that the backend @@ -165,19 +193,6 @@ pub trait WalletRead { /// Returns a transaction. fn get_transaction(&self, id_tx: Self::TxRef) -> Result; - /// Returns the note commitment tree at the specified block height. - fn get_commitment_tree( - &self, - block_height: BlockHeight, - ) -> Result, Self::Error>; - - /// Returns the incremental witnesses as of the specified block height. - #[allow(clippy::type_complexity)] - fn get_witnesses( - &self, - block_height: BlockHeight, - ) -> Result, Self::Error>; - /// Returns the nullifiers for notes that the wallet is tracking, along with their associated /// account IDs, that are either unspent or have not yet been confirmed as spent (in that a /// spending transaction known to the wallet has not yet been included in a block). @@ -236,12 +251,13 @@ pub trait WalletRead { /// decrypted and extracted from a [`CompactBlock`]. /// /// [`CompactBlock`]: crate::proto::compact_formats::CompactBlock -pub struct PrunedBlock<'a> { +pub struct PrunedBlock { pub block_height: BlockHeight, pub block_hash: BlockHash, pub block_time: u32, - pub commitment_tree: &'a sapling::CommitmentTree, - pub transactions: &'a Vec>, + pub transactions: Vec>, + pub sapling_commitment_tree_size: Option, + pub sapling_commitments: Vec<(sapling::Node, Retention)>, } /// A transaction that was detected during scanning of the blockchain, @@ -381,16 +397,14 @@ pub trait WalletWrite: WalletRead { account: AccountId, ) -> Result, Self::Error>; - /// Updates the state of the wallet database by persisting the provided - /// block information, along with the updated witness data that was - /// produced when scanning the block for transactions pertaining to - /// this wallet. + /// Updates the state of the wallet database by persisting the provided block information, + /// along with the note commitments that were detected when scanning the block for transactions + /// pertaining to this wallet. #[allow(clippy::type_complexity)] fn advance_by_block( &mut self, - block: &PrunedBlock, - updated_witnesses: &[(Self::NoteRef, sapling::IncrementalWitness)], - ) -> Result, Self::Error>; + block: PrunedBlock, + ) -> Result, Self::Error>; /// Caches a decrypted transaction in the persistent wallet store. fn store_decrypted_tx( @@ -424,10 +438,31 @@ pub trait WalletWrite: WalletRead { ) -> Result; } +pub trait WalletCommitmentTrees { + type Error; + type SaplingShardStore<'a>: ShardStore< + H = sapling::Node, + CheckpointId = BlockHeight, + Error = Self::Error, + >; + + fn with_sapling_tree_mut(&mut self, callback: F) -> Result + where + for<'a> F: FnMut( + &'a mut ShardTree< + Self::SaplingShardStore<'a>, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + SAPLING_SHARD_HEIGHT, + >, + ) -> Result, + E: From>; +} + #[cfg(feature = "test-dependencies")] pub mod testing { use secrecy::{ExposeSecret, SecretVec}; - use std::collections::HashMap; + use shardtree::{MemoryShardStore, ShardTree, ShardTreeError}; + use std::{collections::HashMap, convert::Infallible, ops::Range}; use zcash_primitives::{ block::BlockHash, @@ -449,11 +484,26 @@ pub mod testing { }; use super::{ - DecryptedTransaction, NullifierQuery, PrunedBlock, SentTransaction, WalletRead, WalletWrite, + chain, DecryptedTransaction, NullifierQuery, PrunedBlock, SentTransaction, + WalletCommitmentTrees, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, }; pub struct MockWalletDb { pub network: Network, + pub sapling_tree: ShardTree< + MemoryShardStore, + { SAPLING_SHARD_HEIGHT * 2 }, + SAPLING_SHARD_HEIGHT, + >, + } + + impl MockWalletDb { + pub fn new(network: Network) -> Self { + Self { + network, + sapling_tree: ShardTree::new(MemoryShardStore::empty(), 100), + } + } } impl WalletRead for MockWalletDb { @@ -465,6 +515,20 @@ pub mod testing { Ok(None) } + fn fully_scanned_height( + &self, + ) -> Result, Self::Error> { + Ok(None) + } + + fn suggest_scan_ranges( + &self, + _batch_size: usize, + _limit: usize, + ) -> Result>, Self::Error> { + Ok(vec![]) + } + fn get_min_unspent_height(&self) -> Result, Self::Error> { Ok(None) } @@ -524,21 +588,6 @@ pub mod testing { Err(()) } - fn get_commitment_tree( - &self, - _block_height: BlockHeight, - ) -> Result, Self::Error> { - Ok(None) - } - - #[allow(clippy::type_complexity)] - fn get_witnesses( - &self, - _block_height: BlockHeight, - ) -> Result, Self::Error> { - Ok(Vec::new()) - } - fn get_sapling_nullifiers( &self, _query: NullifierQuery, @@ -613,9 +662,8 @@ pub mod testing { #[allow(clippy::type_complexity)] fn advance_by_block( &mut self, - _block: &PrunedBlock, - _updated_witnesses: &[(Self::NoteRef, sapling::IncrementalWitness)], - ) -> Result, Self::Error> { + _block: PrunedBlock, + ) -> Result, Self::Error> { Ok(vec![]) } @@ -645,4 +693,23 @@ pub mod testing { Ok(0) } } + + impl WalletCommitmentTrees for MockWalletDb { + type Error = Infallible; + type SaplingShardStore<'a> = MemoryShardStore; + + fn with_sapling_tree_mut(&mut self, mut callback: F) -> Result + where + for<'a> F: FnMut( + &'a mut ShardTree< + Self::SaplingShardStore<'a>, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + SAPLING_SHARD_HEIGHT, + >, + ) -> Result, + E: From>, + { + callback(&mut self.sapling_tree) + } + } } diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 44736228d..ce0eb2a81 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -33,9 +33,7 @@ //! # fn test() -> Result<(), Error<(), Infallible, u32>> { //! let network = Network::TestNetwork; //! let block_source = chain_testing::MockBlockSource; -//! let mut db_data = testing::MockWalletDb { -//! network: Network::TestNetwork -//! }; +//! let mut db_data = testing::MockWalletDb::new(Network::TestNetwork); //! //! // 1) Download new CompactBlocks into block_source. //! @@ -79,7 +77,7 @@ //! // At this point, the cache and scanned data are locally consistent (though not //! // necessarily consistent with the latest chain tip - this would be discovered the //! // next time this codepath is executed after new blocks are received). -//! scan_cached_blocks(&network, &block_source, &mut db_data, None) +//! scan_cached_blocks(&network, &block_source, &mut db_data, None, None) //! # } //! # } //! ``` @@ -89,22 +87,34 @@ use std::convert::Infallible; use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight}, - sapling::{self, note_encryption::PreparedIncomingViewingKey, Nullifier}, + sapling::{self, note_encryption::PreparedIncomingViewingKey}, zip32::Scope, }; use crate::{ - data_api::{PrunedBlock, WalletWrite}, + data_api::{NullifierQuery, WalletWrite}, proto::compact_formats::CompactBlock, scan::BatchRunner, - wallet::WalletTx, welding_rig::{add_block_to_runner, scan_block_with_runner}, }; pub mod error; use error::{ChainError, Error}; -use super::NullifierQuery; +pub struct CommitmentTreeMeta { + sapling_tree_size: u64, + //TODO: orchard_tree_size: u64 +} + +impl CommitmentTreeMeta { + pub fn from_parts(sapling_tree_size: u64) -> Self { + Self { sapling_tree_size } + } + + pub fn sapling_tree_size(&self) -> u64 { + self.sapling_tree_size + } +} /// This trait provides sequential access to raw blockchain data via a callback-oriented /// API. @@ -212,6 +222,7 @@ pub fn scan_cached_blocks( params: &ParamsT, block_source: &BlockSourceT, data_db: &mut DbT, + from_height: Option, limit: Option, ) -> Result<(), Error> where @@ -219,12 +230,6 @@ where BlockSourceT: BlockSource, DbT: WalletWrite, { - // Recall where we synced up to previously. - let mut last_height = data_db - .block_height_extrema() - .map_err(Error::Wallet)? - .map(|(_, max)| max); - // Fetch the UnifiedFullViewingKeys we are tracking let ufvks = data_db .get_unified_full_viewing_keys() @@ -236,25 +241,8 @@ where .filter_map(|(account, ufvk)| ufvk.sapling().map(move |k| (account, k))) .collect(); - // Get the most recent CommitmentTree - let mut tree = last_height.map_or_else( - || Ok(sapling::CommitmentTree::empty()), - |h| { - data_db - .get_commitment_tree(h) - .map(|t| t.unwrap_or_else(sapling::CommitmentTree::empty)) - .map_err(Error::Wallet) - }, - )?; - - // Get most recent incremental witnesses for the notes we are tracking - let mut witnesses = last_height.map_or_else( - || Ok(vec![]), - |h| data_db.get_witnesses(h).map_err(Error::Wallet), - )?; - - // Get the nullifiers for the notes we are tracking - let mut nullifiers = data_db + // Get the nullifiers for the unspent notes we are tracking + let mut sapling_nullifiers = data_db .get_sapling_nullifiers(NullifierQuery::Unspent) .map_err(Error::Wallet)?; @@ -271,8 +259,19 @@ where .map(|(tag, ivk)| (tag, PreparedIncomingViewingKey::new(&ivk))), ); + // Start at either the provided height, or where we synced up to previously. + let (last_scanned_height, commitment_tree_meta) = from_height.map_or_else( + || { + data_db.fully_scanned_height().map_or_else( + |e| Err(Error::Wallet(e)), + |next| Ok(next.map_or_else(|| (None, None), |(h, m)| (Some(h), Some(m)))), + ) + }, + |h| Ok((Some(h), None)), + )?; + block_source.with_blocks::<_, DbT::Error, DbT::NoteRef>( - last_height, + last_scanned_height, limit, |block: CompactBlock| { add_block_to_runner(params, block, &mut batch_runner); @@ -283,90 +282,35 @@ where batch_runner.flush(); block_source.with_blocks::<_, DbT::Error, DbT::NoteRef>( - last_height, + last_scanned_height, limit, |block: CompactBlock| { - let current_height = block.height(); + let pruned_block = scan_block_with_runner( + params, + block, + &dfvks, + &sapling_nullifiers, + commitment_tree_meta.as_ref(), + Some(&mut batch_runner), + ) + .map_err(Error::Sync)?; - // Scanned blocks MUST be height-sequential. - if let Some(h) = last_height { - if current_height != (h + 1) { - return Err( - ChainError::block_height_discontinuity(h + 1, current_height).into(), - ); - } - } - - let block_hash = BlockHash::from_slice(&block.hash); - let block_time = block.time; - - let txs: Vec> = { - let mut witness_refs: Vec<_> = witnesses.iter_mut().map(|w| &mut w.1).collect(); - - scan_block_with_runner( - params, - block, - &dfvks, - &nullifiers, - &mut tree, - &mut witness_refs[..], - Some(&mut batch_runner), - ) - }; - - // Enforce that all roots match. This is slow, so only include in debug builds. - #[cfg(debug_assertions)] - { - let cur_root = tree.root(); - for row in &witnesses { - if row.1.root() != cur_root { - return Err( - ChainError::invalid_witness_anchor(current_height, row.0).into() - ); - } - } - for tx in &txs { - for output in tx.sapling_outputs.iter() { - if output.witness().root() != cur_root { - return Err(ChainError::invalid_new_witness_anchor( - current_height, - tx.txid, - output.index(), - output.witness().root(), - ) - .into()); - } - } - } - } - - let new_witnesses = data_db - .advance_by_block( - &(PrunedBlock { - block_height: current_height, - block_hash, - block_time, - commitment_tree: &tree, - transactions: &txs, - }), - &witnesses, - ) - .map_err(Error::Wallet)?; - - let spent_nf: Vec<&Nullifier> = txs + let spent_nf: Vec<&sapling::Nullifier> = pruned_block + .transactions .iter() .flat_map(|tx| tx.sapling_spends.iter().map(|spend| spend.nf())) .collect(); - nullifiers.retain(|(_, nf)| !spent_nf.contains(&nf)); - nullifiers.extend(txs.iter().flat_map(|tx| { + + sapling_nullifiers.retain(|(_, nf)| !spent_nf.contains(&nf)); + sapling_nullifiers.extend(pruned_block.transactions.iter().flat_map(|tx| { tx.sapling_outputs .iter() .map(|out| (out.account(), *out.nf())) })); - witnesses.extend(new_witnesses); - - last_height = Some(current_height); + data_db + .advance_by_block(pruned_block) + .map_err(Error::Wallet)?; Ok(()) }, diff --git a/zcash_client_backend/src/data_api/chain/error.rs b/zcash_client_backend/src/data_api/chain/error.rs index b35334c6a..b28380d13 100644 --- a/zcash_client_backend/src/data_api/chain/error.rs +++ b/zcash_client_backend/src/data_api/chain/error.rs @@ -5,6 +5,8 @@ use std::fmt::{self, Debug, Display}; use zcash_primitives::{consensus::BlockHeight, sapling, transaction::TxId}; +use crate::welding_rig::SyncError; + /// The underlying cause of a [`ChainError`]. #[derive(Copy, Clone, Debug)] pub enum Cause { @@ -142,6 +144,9 @@ pub enum Error { /// commitments that could not be reconciled with the note commitment tree(s) maintained by the /// wallet. Chain(ChainError), + + /// An error occorred in block scanning. + Sync(SyncError), } impl fmt::Display for Error { @@ -164,6 +169,13 @@ impl fmt::Display for Error { write!(f, "{}", err) } + Error::Sync(SyncError::SaplingTreeSizeUnknown(h)) => { + write!( + f, + "Sync failed due to missing Sapling note commitment tree size at height {}", + h + ) + } } } } diff --git a/zcash_client_backend/src/data_api/error.rs b/zcash_client_backend/src/data_api/error.rs index 0614a612d..db4ffb984 100644 --- a/zcash_client_backend/src/data_api/error.rs +++ b/zcash_client_backend/src/data_api/error.rs @@ -1,5 +1,6 @@ //! Types for wallet error handling. +use shardtree::ShardTreeError; use std::error; use std::fmt::{self, Debug, Display}; use zcash_primitives::{ @@ -20,10 +21,13 @@ use zcash_primitives::{legacy::TransparentAddress, zip32::DiversifierIndex}; /// Errors that can occur as a consequence of wallet operations. #[derive(Debug)] -pub enum Error { +pub enum Error { /// An error occurred retrieving data from the underlying data source DataSource(DataSourceError), + /// An error in computations involving the note commitment trees. + CommitmentTree(ShardTreeError), + /// An error in note selection NoteSelection(SelectionError), @@ -60,9 +64,10 @@ pub enum Error { ChildIndexOutOfRange(DiversifierIndex), } -impl fmt::Display for Error +impl fmt::Display for Error where DE: fmt::Display, + CE: fmt::Display, SE: fmt::Display, FE: fmt::Display, N: fmt::Display, @@ -76,6 +81,9 @@ where e ) } + Error::CommitmentTree(e) => { + write!(f, "An error occurred in querying or updating a note commitment tree: {}", e) + } Error::NoteSelection(e) => { write!(f, "Note selection encountered the following error: {}", e) } @@ -120,9 +128,10 @@ where } } -impl error::Error for Error +impl error::Error for Error where DE: Debug + Display + error::Error + 'static, + CE: Debug + Display + error::Error + 'static, SE: Debug + Display + error::Error + 'static, FE: Debug + Display + 'static, N: Debug + Display, @@ -130,6 +139,7 @@ where fn source(&self) -> Option<&(dyn error::Error + 'static)> { match &self { Error::DataSource(e) => Some(e), + Error::CommitmentTree(e) => Some(e), Error::NoteSelection(e) => Some(e), Error::Builder(e) => Some(e), _ => None, @@ -137,19 +147,19 @@ where } } -impl From> for Error { +impl From> for Error { fn from(e: builder::Error) -> Self { Error::Builder(e) } } -impl From for Error { +impl From for Error { fn from(e: BalanceError) -> Self { Error::BalanceError(e) } } -impl From> for Error { +impl From> for Error { fn from(e: InputSelectorError) -> Self { match e { InputSelectorError::DataSource(e) => Error::DataSource(e), @@ -161,18 +171,25 @@ impl From> for Error { available, required, }, + InputSelectorError::SyncRequired => Error::ScanRequired, } } } -impl From for Error { +impl From for Error { fn from(e: sapling::builder::Error) -> Self { Error::Builder(builder::Error::SaplingBuild(e)) } } -impl From for Error { +impl From for Error { fn from(e: transparent::builder::Error) -> Self { Error::Builder(builder::Error::TransparentBuild(e)) } } + +impl From> for Error { + fn from(e: ShardTreeError) -> Self { + Error::CommitmentTree(e) + } +} diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 160529dc6..b0930d966 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -1,8 +1,9 @@ use std::convert::Infallible; use std::fmt::Debug; +use shardtree::{ShardStore, ShardTree, ShardTreeError}; use zcash_primitives::{ - consensus::{self, NetworkUpgrade}, + consensus::{self, BlockHeight, NetworkUpgrade}, memo::MemoBytes, sapling::{ self, @@ -23,7 +24,8 @@ use crate::{ address::RecipientAddress, data_api::{ error::Error, wallet::input_selection::Proposal, DecryptedTransaction, PoolType, Recipient, - SentTransaction, SentTransactionOutput, WalletWrite, + SentTransaction, SentTransactionOutput, WalletCommitmentTrees, WalletRead, WalletWrite, + SAPLING_SHARD_HEIGHT, }, decrypt_transaction, fees::{self, ChangeValue, DustOutputPolicy}, @@ -122,7 +124,7 @@ where /// # Examples /// /// ``` -/// # #[cfg(feature = "test-dependencies")] +/// # #[cfg(all(feature = "test-dependencies", feature = "local-prover"))] /// # { /// use tempfile::NamedTempFile; /// use zcash_primitives::{ @@ -200,7 +202,8 @@ pub fn create_spend_to_address( ) -> Result< DbT::TxRef, Error< - DbT::Error, + ::Error, + ::Error, GreedyInputSelectorError, Infallible, DbT::NoteRef, @@ -208,7 +211,7 @@ pub fn create_spend_to_address( > where ParamsT: consensus::Parameters + Clone, - DbT: WalletWrite, + DbT: WalletWrite + WalletCommitmentTrees, DbT::NoteRef: Copy + Eq + Ord, { let req = zip321::TransactionRequest::new(vec![Payment { @@ -300,10 +303,16 @@ pub fn spend( min_confirmations: u32, ) -> Result< DbT::TxRef, - Error::Error, DbT::NoteRef>, + Error< + ::Error, + ::Error, + InputsT::Error, + ::Error, + DbT::NoteRef, + >, > where - DbT: WalletWrite, + DbT: WalletWrite + WalletCommitmentTrees, DbT::TxRef: Copy + Debug, DbT::NoteRef: Copy + Eq + Ord, ParamsT: consensus::Parameters + Clone, @@ -323,7 +332,16 @@ where min_confirmations, )?; - create_proposed_transaction(wallet_db, params, prover, usk, ovk_policy, proposal, None) + create_proposed_transaction( + wallet_db, + params, + prover, + usk, + ovk_policy, + proposal, + min_confirmations, + None, + ) } /// Select transaction inputs, compute fees, and construct a proposal for a transaction @@ -331,7 +349,7 @@ where /// [`create_proposed_transaction`]. #[allow(clippy::too_many_arguments)] #[allow(clippy::type_complexity)] -pub fn propose_transfer( +pub fn propose_transfer( wallet_db: &mut DbT, params: &ParamsT, spend_from_account: AccountId, @@ -340,7 +358,13 @@ pub fn propose_transfer( min_confirmations: u32, ) -> Result< Proposal, - Error::Error, DbT::NoteRef>, + Error< + DbT::Error, + CommitmentTreeErrT, + InputsT::Error, + ::Error, + DbT::NoteRef, + >, > where DbT: WalletWrite, @@ -348,20 +372,13 @@ where ParamsT: consensus::Parameters + Clone, InputsT: InputSelector, { - // Target the next block, assuming we are up-to-date. - let (target_height, anchor_height) = wallet_db - .get_target_and_anchor_heights(min_confirmations) - .map_err(Error::DataSource) - .and_then(|x| x.ok_or(Error::ScanRequired))?; - input_selector .propose_transaction( params, wallet_db, spend_from_account, - anchor_height, - target_height, request, + min_confirmations, ) .map_err(Error::from) } @@ -369,7 +386,7 @@ where #[cfg(feature = "transparent-inputs")] #[allow(clippy::too_many_arguments)] #[allow(clippy::type_complexity)] -pub fn propose_shielding( +pub fn propose_shielding( wallet_db: &mut DbT, params: &ParamsT, input_selector: &InputsT, @@ -378,7 +395,13 @@ pub fn propose_shielding( min_confirmations: u32, ) -> Result< Proposal, - Error::Error, DbT::NoteRef>, + Error< + DbT::Error, + CommitmentTreeErrT, + InputsT::Error, + ::Error, + DbT::NoteRef, + >, > where ParamsT: consensus::Parameters, @@ -386,19 +409,13 @@ where DbT::NoteRef: Copy + Eq + Ord, InputsT: InputSelector, { - let (target_height, latest_anchor) = wallet_db - .get_target_and_anchor_heights(min_confirmations) - .map_err(Error::DataSource) - .and_then(|x| x.ok_or(Error::ScanRequired))?; - input_selector .propose_shielding( params, wallet_db, shielding_threshold, from_addrs, - latest_anchor, - target_height, + min_confirmations, ) .map_err(Error::from) } @@ -417,10 +434,20 @@ pub fn create_proposed_transaction( usk: &UnifiedSpendingKey, ovk_policy: OvkPolicy, proposal: Proposal, + min_confirmations: u32, change_memo: Option, -) -> Result> +) -> Result< + DbT::TxRef, + Error< + ::Error, + ::Error, + InputsErrT, + FeeRuleT::Error, + DbT::NoteRef, + >, +> where - DbT: WalletWrite, + DbT: WalletWrite + WalletCommitmentTrees, DbT::TxRef: Copy + Debug, DbT::NoteRef: Copy + Eq + Ord, ParamsT: consensus::Parameters + Clone, @@ -459,14 +486,25 @@ where // Create the transaction. The type of the proposal ensures that there // are no possible transparent inputs, so we ignore those - let mut builder = Builder::new(params.clone(), proposal.target_height(), None); + let mut builder = Builder::new(params.clone(), proposal.min_target_height(), None); - for selected in proposal.sapling_inputs() { - let (note, key, merkle_path) = select_key_for_note(selected, usk.sapling(), &dfvk) + wallet_db.with_sapling_tree_mut::<_, _, Error<_, _, _, _, _>>(|sapling_tree| { + for selected in proposal.sapling_inputs() { + let (note, key, merkle_path) = select_key_for_note( + sapling_tree, + selected, + usk.sapling(), + &dfvk, + min_confirmations + .try_into() + .expect("min_confirmations should never be anywhere close to usize::MAX"), + )? .ok_or(Error::NoteMismatch(selected.note_id))?; - builder.add_sapling_spend(key, selected.diversifier, note, merkle_path)?; - } + builder.add_sapling_spend(key, selected.diversifier, note, merkle_path)?; + } + Ok(()) + })?; #[cfg(feature = "transparent-inputs")] let utxos = { @@ -577,7 +615,7 @@ where tx.sapling_bundle().and_then(|bundle| { try_sapling_note_decryption( params, - proposal.target_height(), + proposal.min_target_height(), &internal_ivk, &bundle.shielded_outputs()[output_index], ) @@ -672,11 +710,17 @@ pub fn shield_transparent_funds( min_confirmations: u32, ) -> Result< DbT::TxRef, - Error::Error, DbT::NoteRef>, + Error< + ::Error, + ::Error, + InputsT::Error, + ::Error, + DbT::NoteRef, + >, > where ParamsT: consensus::Parameters, - DbT: WalletWrite, + DbT: WalletWrite + WalletCommitmentTrees, DbT::NoteRef: Copy + Eq + Ord, InputsT: InputSelector, { @@ -696,17 +740,26 @@ where usk, OvkPolicy::Sender, proposal, + min_confirmations, Some(memo.clone()), ) } -fn select_key_for_note( +#[allow(clippy::type_complexity)] +fn select_key_for_note>( + commitment_tree: &mut ShardTree< + S, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + SAPLING_SHARD_HEIGHT, + >, selected: &ReceivedSaplingNote, extsk: &ExtendedSpendingKey, dfvk: &DiversifiableFullViewingKey, -) -> Option<(sapling::Note, ExtendedSpendingKey, sapling::MerklePath)> { - let merkle_path = selected.witness.path().expect("the tree is not empty"); - + checkpoint_depth: usize, +) -> Result< + Option<(sapling::Note, ExtendedSpendingKey, sapling::MerklePath)>, + ShardTreeError, +> { // Attempt to reconstruct the note being spent using both the internal and external dfvks // corresponding to the unified spending key, checking against the witness we are using // to spend the note that we've used the correct key. @@ -717,13 +770,16 @@ fn select_key_for_note( .diversified_change_address(selected.diversifier) .map(|addr| addr.create_note(selected.note_value.into(), selected.rseed)); - let expected_root = selected.witness.root(); - external_note + let expected_root = commitment_tree.root_at_checkpoint(checkpoint_depth)?; + let merkle_path = commitment_tree + .witness_caching(selected.note_commitment_tree_position, checkpoint_depth)?; + + Ok(external_note .filter(|n| expected_root == merkle_path.root(Node::from_cmu(&n.cmu()))) .map(|n| (n, extsk.clone(), merkle_path.clone())) .or_else(|| { internal_note .filter(|n| expected_root == merkle_path.root(Node::from_cmu(&n.cmu()))) .map(|n| (n, extsk.derive_internal(), merkle_path)) - }) + })) } diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 798b83450..403497c0d 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -35,6 +35,9 @@ pub enum InputSelectorError { /// Insufficient funds were available to satisfy the payment request that inputs were being /// selected to attempt to satisfy. InsufficientFunds { available: Amount, required: Amount }, + /// The data source does not have enough information to choose an expiry height + /// for the transaction. + SyncRequired, } impl fmt::Display for InputSelectorError { @@ -59,6 +62,7 @@ impl fmt::Display for InputSelectorError write!(f, "No chain data is available."), } } } @@ -71,7 +75,8 @@ pub struct Proposal { sapling_inputs: Vec>, balance: TransactionBalance, fee_rule: FeeRuleT, - target_height: BlockHeight, + min_target_height: BlockHeight, + min_anchor_height: BlockHeight, is_shielding: bool, } @@ -97,8 +102,19 @@ impl Proposal { &self.fee_rule } /// Returns the target height for which the proposal was prepared. - pub fn target_height(&self) -> BlockHeight { - self.target_height + /// + /// The chain must contain at least this many blocks in order for the proposal to + /// be executed. + pub fn min_target_height(&self) -> BlockHeight { + self.min_target_height + } + /// Returns the anchor height used in preparing the proposal. + /// + /// If, at the time that the proposal is executed, the anchor height required to satisfy + /// the minimum confirmation depth is less than this height, the proposal execution + /// API should return an error. + pub fn min_anchor_height(&self) -> BlockHeight { + self.min_anchor_height } /// Returns a flag indicating whether or not the proposed transaction /// is exclusively wallet-internal (if it does not involve any external @@ -146,9 +162,8 @@ pub trait InputSelector { params: &ParamsT, wallet_db: &Self::DataSource, account: AccountId, - anchor_height: BlockHeight, - target_height: BlockHeight, transaction_request: TransactionRequest, + min_confirmations: u32, ) -> Result< Proposal::DataSource as WalletRead>::NoteRef>, InputSelectorError<<::DataSource as WalletRead>::Error, Self::Error>, @@ -172,8 +187,7 @@ pub trait InputSelector { wallet_db: &Self::DataSource, shielding_threshold: NonNegativeAmount, source_addrs: &[TransparentAddress], - confirmed_height: BlockHeight, - target_height: BlockHeight, + min_confirmations: u32, ) -> Result< Proposal::DataSource as WalletRead>::NoteRef>, InputSelectorError<<::DataSource as WalletRead>::Error, Self::Error>, @@ -292,13 +306,18 @@ where params: &ParamsT, wallet_db: &Self::DataSource, account: AccountId, - anchor_height: BlockHeight, - target_height: BlockHeight, transaction_request: TransactionRequest, + min_confirmations: u32, ) -> Result, InputSelectorError> where ParamsT: consensus::Parameters, { + // Target the next block, assuming we are up-to-date. + let (target_height, anchor_height) = wallet_db + .get_target_and_anchor_heights(min_confirmations) + .map_err(InputSelectorError::DataSource) + .and_then(|x| x.ok_or(InputSelectorError::SyncRequired))?; + let mut transparent_outputs = vec![]; let mut sapling_outputs = vec![]; let mut output_total = Amount::zero(); @@ -362,7 +381,8 @@ where sapling_inputs, balance, fee_rule: (*self.change_strategy.fee_rule()).clone(), - target_height, + min_target_height: target_height, + min_anchor_height: anchor_height, is_shielding: false, }); } @@ -405,15 +425,19 @@ where wallet_db: &Self::DataSource, shielding_threshold: NonNegativeAmount, source_addrs: &[TransparentAddress], - confirmed_height: BlockHeight, - target_height: BlockHeight, + min_confirmations: u32, ) -> Result, InputSelectorError> where ParamsT: consensus::Parameters, { + let (target_height, latest_anchor) = wallet_db + .get_target_and_anchor_heights(min_confirmations) + .map_err(InputSelectorError::DataSource) + .and_then(|x| x.ok_or(InputSelectorError::SyncRequired))?; + let mut transparent_inputs: Vec = source_addrs .iter() - .map(|taddr| wallet_db.get_unspent_transparent_outputs(taddr, confirmed_height, &[])) + .map(|taddr| wallet_db.get_unspent_transparent_outputs(taddr, latest_anchor, &[])) .collect::>, _>>() .map_err(InputSelectorError::DataSource)? .into_iter() @@ -458,7 +482,8 @@ where sapling_inputs: vec![], balance, fee_rule: (*self.change_strategy.fee_rule()).clone(), - target_height, + min_target_height: target_height, + min_anchor_height: latest_anchor, is_shielding: true, }) } else { diff --git a/zcash_client_backend/src/wallet.rs b/zcash_client_backend/src/wallet.rs index ba58340b3..c702cfa73 100644 --- a/zcash_client_backend/src/wallet.rs +++ b/zcash_client_backend/src/wallet.rs @@ -1,6 +1,7 @@ //! Structs representing transaction data scanned from the block chain by a wallet or //! light client. +use incrementalmerkletree::Position; use zcash_note_encryption::EphemeralKeyBytes; use zcash_primitives::{ consensus::BlockHeight, @@ -117,7 +118,7 @@ pub struct WalletSaplingOutput { account: AccountId, note: sapling::Note, is_change: bool, - witness: sapling::IncrementalWitness, + note_commitment_tree_position: Position, nf: N, } @@ -131,7 +132,7 @@ impl WalletSaplingOutput { account: AccountId, note: sapling::Note, is_change: bool, - witness: sapling::IncrementalWitness, + note_commitment_tree_position: Position, nf: N, ) -> Self { Self { @@ -141,7 +142,7 @@ impl WalletSaplingOutput { account, note, is_change, - witness, + note_commitment_tree_position, nf, } } @@ -164,11 +165,8 @@ impl WalletSaplingOutput { pub fn is_change(&self) -> bool { self.is_change } - pub fn witness(&self) -> &sapling::IncrementalWitness { - &self.witness - } - pub fn witness_mut(&mut self) -> &mut sapling::IncrementalWitness { - &mut self.witness + pub fn note_commitment_tree_position(&self) -> Position { + self.note_commitment_tree_position } pub fn nf(&self) -> &N { &self.nf @@ -182,7 +180,7 @@ pub struct ReceivedSaplingNote { pub diversifier: sapling::Diversifier, pub note_value: Amount, pub rseed: sapling::Rseed, - pub witness: sapling::IncrementalWitness, + pub note_commitment_tree_position: Position, } impl sapling_fees::InputView for ReceivedSaplingNote { diff --git a/zcash_client_backend/src/welding_rig.rs b/zcash_client_backend/src/welding_rig.rs index 1906c133e..3266d4059 100644 --- a/zcash_client_backend/src/welding_rig.rs +++ b/zcash_client_backend/src/welding_rig.rs @@ -1,21 +1,27 @@ //! Tools for scanning a compact representation of the Zcash block chain. +//! +//! TODO: rename this module to `block_scanner` use std::collections::{HashMap, HashSet}; use std::convert::TryFrom; +use incrementalmerkletree::{Position, Retention}; use subtle::{ConditionallySelectable, ConstantTimeEq, CtOption}; use zcash_note_encryption::batch; +use zcash_primitives::consensus::BlockHeight; use zcash_primitives::{ consensus, sapling::{ self, note_encryption::{PreparedIncomingViewingKey, SaplingDomain}, - Node, Note, Nullifier, NullifierDerivingKey, SaplingIvk, + SaplingIvk, }, transaction::components::sapling::CompactOutputDescription, zip32::{sapling::DiversifiableFullViewingKey, AccountId, Scope}, }; +use crate::data_api::chain::CommitmentTreeMeta; +use crate::data_api::PrunedBlock; use crate::{ proto::compact_formats::CompactBlock, scan::{Batch, BatchRunner, Tasks}, @@ -56,16 +62,13 @@ pub trait ScanningKey { /// IVK-based implementations of this trait cannot successfully derive /// nullifiers, in which case `Self::Nf` should be set to the unit type /// and this function is a no-op. - fn sapling_nf( - key: &Self::SaplingNk, - note: &Note, - witness: &sapling::IncrementalWitness, - ) -> Self::Nf; + fn sapling_nf(key: &Self::SaplingNk, note: &sapling::Note, note_position: Position) + -> Self::Nf; } impl ScanningKey for DiversifiableFullViewingKey { type Scope = Scope; - type SaplingNk = NullifierDerivingKey; + type SaplingNk = sapling::NullifierDerivingKey; type SaplingKeys = [(Self::Scope, SaplingIvk, Self::SaplingNk); 2]; type Nf = sapling::Nullifier; @@ -84,16 +87,8 @@ impl ScanningKey for DiversifiableFullViewingKey { ] } - fn sapling_nf( - key: &Self::SaplingNk, - note: &Note, - witness: &sapling::IncrementalWitness, - ) -> Self::Nf { - note.nf( - key, - u64::try_from(witness.position()) - .expect("Sapling note commitment tree position must fit into a u64"), - ) + fn sapling_nf(key: &Self::SaplingNk, note: &sapling::Note, position: Position) -> Self::Nf { + note.nf(key, position.into()) } } @@ -111,7 +106,15 @@ impl ScanningKey for SaplingIvk { [((), self.clone(), ())] } - fn sapling_nf(_key: &Self::SaplingNk, _note: &Note, _witness: &sapling::IncrementalWitness) {} + fn sapling_nf(_key: &Self::SaplingNk, _note: &sapling::Note, _position: Position) {} +} + +/// Errors that can occur in block scanning. +#[derive(Debug)] +pub enum SyncError { + /// The size of the Sapling note commitment tree was not provided as part of a [`CompactBlock`] + /// being scanned, making it impossible to construct the nullifier for a detected note. + SaplingTreeSizeUnknown(BlockHeight), } /// Scans a [`CompactBlock`] with a set of [`ScanningKey`]s. @@ -141,17 +144,15 @@ pub fn scan_block( params: &P, block: CompactBlock, vks: &[(&AccountId, &K)], - nullifiers: &[(AccountId, Nullifier)], - tree: &mut sapling::CommitmentTree, - existing_witnesses: &mut [&mut sapling::IncrementalWitness], -) -> Vec> { + sapling_nullifiers: &[(AccountId, sapling::Nullifier)], + initial_commitment_tree_meta: Option<&CommitmentTreeMeta>, +) -> Result, SyncError> { scan_block_with_runner::<_, _, ()>( params, block, vks, - nullifiers, - tree, - existing_witnesses, + sapling_nullifiers, + initial_commitment_tree_meta, None, ) } @@ -202,21 +203,41 @@ pub(crate) fn scan_block_with_runner< params: &P, block: CompactBlock, vks: &[(&AccountId, &K)], - nullifiers: &[(AccountId, Nullifier)], - tree: &mut sapling::CommitmentTree, - existing_witnesses: &mut [&mut sapling::IncrementalWitness], + nullifiers: &[(AccountId, sapling::Nullifier)], + initial_commitment_tree_meta: Option<&CommitmentTreeMeta>, mut batch_runner: Option<&mut TaggedBatchRunner>, -) -> Vec> { +) -> Result, SyncError> { let mut wtxs: Vec> = vec![]; + let mut sapling_note_commitments: Vec<(sapling::Node, Retention)> = vec![]; let block_height = block.height(); let block_hash = block.hash(); + // It's possible to make progress without a Sapling tree position if we don't have any Sapling + // notes in the block, since we only use the position for constructing nullifiers for our own + // received notes. Thus, we allow it to be optional here, and only produce an error if we try + // to use it. `block.sapling_commitment_tree_size` is expected to be correct as of the end of + // the block, and we can't have a note of ours in a block with no outputs so treating the zero + // default value from the protobuf as `None` is always correct. + let mut sapling_tree_position = if block.sapling_commitment_tree_size == 0 { + initial_commitment_tree_meta.map(|m| (m.sapling_tree_size() + 1).into()) + } else { + let end_position_exclusive = Position::from(u64::from(block.sapling_commitment_tree_size)); + let output_count = block + .vtx + .iter() + .map(|tx| u64::try_from(tx.outputs.len()).unwrap()) + .sum(); + Some(end_position_exclusive - output_count) + }; + for tx in block.vtx.into_iter() { let txid = tx.txid(); let index = tx.index as usize; - // Check for spent notes - // The only step that is not constant-time is the filter() at the end. + // Check for spent notes. The only step that is not constant-time is + // the filter() at the end. + // TODO: However, this is O(|nullifiers| * |notes|); does using + // constant-time operations here really make sense? let shielded_spends: Vec<_> = tx .spends .into_iter() @@ -248,19 +269,8 @@ pub(crate) fn scan_block_with_runner< // Check for incoming notes while incrementing tree and witnesses let mut shielded_outputs: Vec> = vec![]; + let tx_outputs_len = u64::try_from(tx.outputs.len()).unwrap(); { - // Grab mutable references to new witnesses from previous transactions - // in this block so that we can update them. Scoped so we don't hold - // mutable references to wtxs for too long. - let mut block_witnesses: Vec<_> = wtxs - .iter_mut() - .flat_map(|tx| { - tx.sapling_outputs - .iter_mut() - .map(|output| output.witness_mut()) - }) - .collect(); - let decoded = &tx .outputs .into_iter() @@ -292,7 +302,7 @@ pub(crate) fn scan_block_with_runner< "The batch runner and scan_block must use the same set of IVKs.", ); - ((d_note.note, d_note.recipient), a, (*nk).clone()) + (d_note.note, a, (*nk).clone()) }) }) .collect() @@ -312,40 +322,21 @@ pub(crate) fn scan_block_with_runner< .map(PreparedIncomingViewingKey::new) .collect::>(); - batch::try_compact_note_decryption(&ivks, decoded) + batch::try_compact_note_decryption(&ivks, &decoded[..]) .into_iter() .map(|v| { - v.map(|(note_data, ivk_idx)| { + v.map(|((note, _), ivk_idx)| { let (account, _, nk) = &vks[ivk_idx]; - (note_data, *account, (*nk).clone()) + (note, *account, (*nk).clone()) }) }) .collect() }; for (index, ((_, output), dec_output)) in decoded.iter().zip(decrypted).enumerate() { - // Grab mutable references to new witnesses from previous outputs - // in this transaction so that we can update them. Scoped so we - // don't hold mutable references to shielded_outputs for too long. - let new_witnesses: Vec<_> = shielded_outputs - .iter_mut() - .map(|out| out.witness_mut()) - .collect(); - - // Increment tree and witnesses - let node = Node::from_cmu(&output.cmu); - for witness in &mut *existing_witnesses { - witness.append(node).unwrap(); - } - for witness in &mut block_witnesses { - witness.append(node).unwrap(); - } - for witness in new_witnesses { - witness.append(node).unwrap(); - } - tree.append(node).unwrap(); - - if let Some(((note, _), account, nk)) = dec_output { + // Collect block note commitments + let node = sapling::Node::from_cmu(&output.cmu); + if let Some((note, account, nk)) = dec_output { // A note is marked as "change" if the account that received it // also spent notes in the same transaction. This will catch, // for instance: @@ -353,8 +344,10 @@ pub(crate) fn scan_block_with_runner< // - Notes created by consolidation transactions. // - Notes sent from one account to itself. let is_change = spent_from_accounts.contains(&account); - let witness = sapling::IncrementalWitness::from_tree(tree.clone()); - let nf = K::sapling_nf(&nk, ¬e, &witness); + let note_commitment_tree_position = sapling_tree_position + .ok_or(SyncError::SaplingTreeSizeUnknown(block_height))? + + index.try_into().unwrap(); + let nf = K::sapling_nf(&nk, ¬e, note_commitment_tree_position); shielded_outputs.push(WalletSaplingOutput::from_parts( index, @@ -363,9 +356,33 @@ pub(crate) fn scan_block_with_runner< account, note, is_change, - witness, + note_commitment_tree_position, nf, - )) + )); + + sapling_note_commitments.push(( + node, + if index == decoded.len() - 1 { + Retention::Checkpoint { + id: block_height, + is_marked: true, + } + } else { + Retention::Marked + }, + )); + } else { + sapling_note_commitments.push(( + node, + if index == decoded.len() - 1 { + Retention::Checkpoint { + id: block_height, + is_marked: false, + } + } else { + Retention::Ephemeral + }, + )); } } } @@ -378,9 +395,22 @@ pub(crate) fn scan_block_with_runner< sapling_outputs: shielded_outputs, }); } + + sapling_tree_position = sapling_tree_position.map(|pos| pos + tx_outputs_len); } - wtxs + Ok(PrunedBlock { + block_height, + block_hash, + block_time: block.time, + transactions: wtxs, + sapling_commitment_tree_size: if block.sapling_commitment_tree_size == 0 { + None + } else { + Some(block.sapling_commitment_tree_size) + }, + sapling_commitments: sapling_note_commitments, + }) } #[cfg(test)] @@ -396,16 +426,18 @@ mod tests { constants::SPENDING_KEY_GENERATOR, memo::MemoBytes, sapling::{ + self, note_encryption::{sapling_note_encryption, PreparedIncomingViewingKey, SaplingDomain}, util::generate_random_rseed, value::NoteValue, - CommitmentTree, Note, Nullifier, SaplingIvk, + Nullifier, SaplingIvk, }, transaction::components::Amount, zip32::{AccountId, DiversifiableFullViewingKey, ExtendedSpendingKey}, }; use crate::{ + data_api::chain::CommitmentTreeMeta, proto::compact_formats::{ CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, }, @@ -455,13 +487,14 @@ mod tests { dfvk: &DiversifiableFullViewingKey, value: Amount, tx_after: bool, + initial_sapling_tree_size: u32, ) -> CompactBlock { let to = dfvk.default_address().1; // Create a fake Note for the account let mut rng = OsRng; let rseed = generate_random_rseed(&Network::TestNetwork, height, &mut rng); - let note = Note::from_parts(to, NoteValue::from_raw(value.into()), rseed); + let note = sapling::Note::from_parts(to, NoteValue::from_raw(value.into()), rseed); let encryptor = sapling_note_encryption::<_, Network>( Some(dfvk.fvk().ovk), note.clone(), @@ -514,6 +547,9 @@ mod tests { cb.vtx.push(tx); } + cb.sapling_commitment_tree_size = initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(); + cb } @@ -530,10 +566,10 @@ mod tests { &dfvk, Amount::from_u64(5).unwrap(), false, + 0, ); assert_eq!(cb.vtx.len(), 2); - let mut tree = CommitmentTree::empty(); let mut batch_runner = if scan_multithreaded { let mut runner = BatchRunner::<_, _, _, ()>::new( 10, @@ -551,15 +587,16 @@ mod tests { None }; - let txs = scan_block_with_runner( + let pruned_block = scan_block_with_runner( &Network::TestNetwork, cb, &[(&account, &dfvk)], &[], - &mut tree, - &mut [], + Some(&CommitmentTreeMeta::from_parts(0)), batch_runner.as_mut(), - ); + ) + .unwrap(); + let txs = pruned_block.transactions; assert_eq!(txs.len(), 1); let tx = &txs[0]; @@ -569,9 +606,6 @@ mod tests { assert_eq!(tx.sapling_outputs[0].index(), 0); assert_eq!(tx.sapling_outputs[0].account(), account); assert_eq!(tx.sapling_outputs[0].note().value().inner(), 5); - - // Check that the witness root matches - assert_eq!(tx.sapling_outputs[0].witness().root(), tree.root()); } go(false); @@ -591,10 +625,10 @@ mod tests { &dfvk, Amount::from_u64(5).unwrap(), true, + 0, ); assert_eq!(cb.vtx.len(), 3); - let mut tree = CommitmentTree::empty(); let mut batch_runner = if scan_multithreaded { let mut runner = BatchRunner::<_, _, _, ()>::new( 10, @@ -612,15 +646,16 @@ mod tests { None }; - let txs = scan_block_with_runner( + let pruned_block = scan_block_with_runner( &Network::TestNetwork, cb, &[(&AccountId::from(0), &dfvk)], &[], - &mut tree, - &mut [], + Some(&CommitmentTreeMeta::from_parts(0)), batch_runner.as_mut(), - ); + ) + .unwrap(); + let txs = pruned_block.transactions; assert_eq!(txs.len(), 1); let tx = &txs[0]; @@ -630,9 +665,6 @@ mod tests { assert_eq!(tx.sapling_outputs[0].index(), 0); assert_eq!(tx.sapling_outputs[0].account(), AccountId::from(0)); assert_eq!(tx.sapling_outputs[0].note().value().inner(), 5); - - // Check that the witness root matches - assert_eq!(tx.sapling_outputs[0].witness().root(), tree.root()); } go(false); @@ -646,19 +678,26 @@ mod tests { let nf = Nullifier([7; 32]); let account = AccountId::from(12); - let cb = fake_compact_block(1u32.into(), nf, &dfvk, Amount::from_u64(5).unwrap(), false); + let cb = fake_compact_block( + 1u32.into(), + nf, + &dfvk, + Amount::from_u64(5).unwrap(), + false, + 0, + ); assert_eq!(cb.vtx.len(), 2); let vks: Vec<(&AccountId, &SaplingIvk)> = vec![]; - let mut tree = CommitmentTree::empty(); - let txs = scan_block( + let pruned_block = scan_block( &Network::TestNetwork, cb, &vks[..], &[(account, nf)], - &mut tree, - &mut [], - ); + Some(&CommitmentTreeMeta::from_parts(0)), + ) + .unwrap(); + let txs = pruned_block.transactions; assert_eq!(txs.len(), 1); let tx = &txs[0]; diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index de71727b3..88585e0d2 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -15,6 +15,7 @@ rust-version = "1.65" [dependencies] incrementalmerkletree = { version = "0.4", features = ["legacy-api"] } +shardtree = { version = "0.0", features = ["legacy-api"] } zcash_client_backend = { version = "0.9", path = "../zcash_client_backend" } zcash_primitives = { version = "0.12", path = "../zcash_primitives", default-features = false } diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 81a0e028a..11e065f9e 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -314,6 +314,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(5).unwrap(), + 0, ); insert_into_cache(&db_cache, &cb); @@ -328,7 +329,7 @@ mod tests { assert_matches!(validate_chain_result, Ok(())); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Data-only chain should be valid validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); @@ -340,6 +341,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(7).unwrap(), + 1, ); insert_into_cache(&db_cache, &cb2); @@ -347,7 +349,7 @@ mod tests { validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Data-only chain should be valid validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); @@ -373,6 +375,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(5).unwrap(), + 0, ); let (cb2, _) = fake_compact_block( sapling_activation_height() + 1, @@ -380,12 +383,13 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(7).unwrap(), + 1, ); insert_into_cache(&db_cache, &cb); insert_into_cache(&db_cache, &cb2); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Data-only chain should be valid validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); @@ -397,6 +401,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(8).unwrap(), + 2, ); let (cb4, _) = fake_compact_block( sapling_activation_height() + 3, @@ -404,6 +409,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(3).unwrap(), + 3, ); insert_into_cache(&db_cache, &cb3); insert_into_cache(&db_cache, &cb4); @@ -434,6 +440,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(5).unwrap(), + 0, ); let (cb2, _) = fake_compact_block( sapling_activation_height() + 1, @@ -441,12 +448,13 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(7).unwrap(), + 1, ); insert_into_cache(&db_cache, &cb); insert_into_cache(&db_cache, &cb2); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Data-only chain should be valid validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); @@ -458,6 +466,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(8).unwrap(), + 2, ); let (cb4, _) = fake_compact_block( sapling_activation_height() + 3, @@ -465,6 +474,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(3).unwrap(), + 3, ); insert_into_cache(&db_cache, &cb3); insert_into_cache(&db_cache, &cb4); @@ -503,6 +513,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); let (cb2, _) = fake_compact_block( @@ -511,12 +522,13 @@ mod tests { &dfvk, AddressType::DefaultExternal, value2, + 1, ); insert_into_cache(&db_cache, &cb); insert_into_cache(&db_cache, &cb2); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Account balance should reflect both received notes assert_eq!( @@ -551,7 +563,7 @@ mod tests { ); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Account balance should again reflect both received notes assert_eq!( @@ -581,9 +593,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb1); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -596,6 +609,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 1, ); let (cb3, _) = fake_compact_block( sapling_activation_height() + 2, @@ -603,9 +617,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 2, ); insert_into_cache(&db_cache, &cb3); - match scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None) { + match scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None) { Err(Error::Chain(e)) => { assert_matches!( e.cause(), @@ -618,7 +633,7 @@ mod tests { // If we add a block of height SAPLING_ACTIVATION_HEIGHT + 1, we can now scan both insert_into_cache(&db_cache, &cb2); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), Amount::from_u64(150_000).unwrap() @@ -652,11 +667,12 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Account balance should reflect the received note assert_eq!( @@ -672,11 +688,12 @@ mod tests { &dfvk, AddressType::DefaultExternal, value2, + 1, ); insert_into_cache(&db_cache, &cb2); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Account balance should reflect both received notes assert_eq!( @@ -712,11 +729,12 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Account balance should reflect the received note assert_eq!( @@ -737,11 +755,12 @@ mod tests { &dfvk, to2, value2, + 1, ), ); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Account balance should equal the change assert_eq!( diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index cfd326b9c..3f1832411 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -3,6 +3,7 @@ use std::error; use std::fmt; +use shardtree::ShardTreeError; use zcash_client_backend::encoding::{Bech32DecodeError, TransparentCodecError}; use zcash_primitives::{consensus::BlockHeight, zip32::AccountId}; @@ -74,6 +75,9 @@ pub enum SqliteClientError { /// belonging to the wallet #[cfg(feature = "transparent-inputs")] AddressNotRecognized(TransparentAddress), + + /// An error occurred in inserting data into one of the wallet's note commitment trees. + CommitmentTree(ShardTreeError), } impl error::Error for SqliteClientError { @@ -114,6 +118,7 @@ impl fmt::Display for SqliteClientError { SqliteClientError::AccountIdOutOfRange => write!(f, "Wallet account identifiers must be less than 0x7FFFFFFF."), #[cfg(feature = "transparent-inputs")] SqliteClientError::AddressNotRecognized(_) => write!(f, "The address associated with a received txo is not identifiable as belonging to the wallet."), + SqliteClientError::CommitmentTree(err) => write!(f, "An error occurred accessing or updating note commitment tree data: {}.", err), } } } @@ -160,3 +165,9 @@ impl From for SqliteClientError { SqliteClientError::InvalidMemo(e) } } + +impl From> for SqliteClientError { + fn from(e: ShardTreeError) -> Self { + SqliteClientError::CommitmentTree(e) + } +} diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 1835b27a9..e17cbab07 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -34,8 +34,10 @@ use rusqlite::{self, Connection}; use secrecy::{ExposeSecret, SecretVec}; -use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, path::Path}; +use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, ops::Range, path::Path}; +use incrementalmerkletree::Position; +use shardtree::{ShardTree, ShardTreeError}; use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight}, @@ -52,8 +54,10 @@ use zcash_primitives::{ use zcash_client_backend::{ address::{AddressMetadata, UnifiedAddress}, data_api::{ - self, chain::BlockSource, DecryptedTransaction, NullifierQuery, PoolType, PrunedBlock, - Recipient, SentTransaction, WalletRead, WalletWrite, + self, + chain::{BlockSource, CommitmentTreeMeta}, + DecryptedTransaction, NullifierQuery, PoolType, PrunedBlock, Recipient, SentTransaction, + WalletCommitmentTrees, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, }, keys::{UnifiedFullViewingKey, UnifiedSpendingKey}, proto::compact_formats::CompactBlock, @@ -61,7 +65,9 @@ use zcash_client_backend::{ DecryptedOutput, TransferType, }; -use crate::error::SqliteClientError; +use crate::{ + error::SqliteClientError, wallet::sapling::commitment_tree::WalletDbSaplingShardStore, +}; #[cfg(feature = "unstable")] use { @@ -125,15 +131,15 @@ impl WalletDb { }) } - pub fn transactionally(&mut self, f: F) -> Result + pub fn transactionally>(&mut self, f: F) -> Result where - F: FnOnce(&WalletDb, P>) -> Result, + F: FnOnce(&mut WalletDb, P>) -> Result, { - let wdb = WalletDb { + let mut wdb = WalletDb { conn: SqlTransaction(self.conn.transaction()?), params: self.params.clone(), }; - let result = f(&wdb)?; + let result = f(&mut wdb)?; wdb.conn.0.commit()?; Ok(result) } @@ -148,6 +154,20 @@ impl, P: consensus::Parameters> WalletRead for W wallet::block_height_extrema(self.conn.borrow()).map_err(SqliteClientError::from) } + fn fully_scanned_height( + &self, + ) -> Result, Self::Error> { + wallet::fully_scanned_height(self.conn.borrow()) + } + + fn suggest_scan_ranges( + &self, + _batch_size: usize, + _limit: usize, + ) -> Result>, Self::Error> { + todo!() + } + fn get_min_unspent_height(&self) -> Result, Self::Error> { wallet::get_min_unspent_height(self.conn.borrow()).map_err(SqliteClientError::from) } @@ -210,24 +230,9 @@ impl, P: consensus::Parameters> WalletRead for W } } - fn get_commitment_tree( - &self, - block_height: BlockHeight, - ) -> Result, Self::Error> { - wallet::sapling::get_sapling_commitment_tree(self.conn.borrow(), block_height) - } - - #[allow(clippy::type_complexity)] - fn get_witnesses( - &self, - block_height: BlockHeight, - ) -> Result, Self::Error> { - wallet::sapling::get_sapling_witnesses(self.conn.borrow(), block_height) - } - fn get_sapling_nullifiers( &self, - query: data_api::NullifierQuery, + query: NullifierQuery, ) -> Result, Self::Error> { match query { NullifierQuery::Unspent => wallet::sapling::get_sapling_nullifiers(self.conn.borrow()), @@ -386,21 +391,21 @@ impl WalletWrite for WalletDb #[allow(clippy::type_complexity)] fn advance_by_block( &mut self, - block: &PrunedBlock, - updated_witnesses: &[(Self::NoteRef, sapling::IncrementalWitness)], - ) -> Result, Self::Error> { + block: PrunedBlock, + ) -> Result, Self::Error> { self.transactionally(|wdb| { // Insert the block into the database. + let block_height = block.block_height; wallet::insert_block( &wdb.conn.0, - block.block_height, + block_height, block.block_hash, block.block_time, - block.commitment_tree, + block.sapling_commitment_tree_size.map(|s| s.into()), )?; - let mut new_witnesses = vec![]; - for tx in block.transactions { + let mut wallet_note_ids = vec![]; + for tx in &block.transactions { let tx_row = wallet::put_tx_meta(&wdb.conn.0, tx, block.block_height)?; // Mark notes as spent and remove them from the scanning cache @@ -413,32 +418,24 @@ impl WalletWrite for WalletDb wallet::sapling::put_received_note(&wdb.conn.0, output, tx_row)?; // Save witness for note. - new_witnesses.push((received_note_id, output.witness().clone())); + wallet_note_ids.push(received_note_id); } } - // Insert current new_witnesses into the database. - for (received_note_id, witness) in updated_witnesses.iter().chain(new_witnesses.iter()) - { - if let NoteId::ReceivedNoteId(rnid) = *received_note_id { - wallet::sapling::insert_witness( - &wdb.conn.0, - rnid, - witness, - block.block_height, - )?; - } else { - return Err(SqliteClientError::InvalidNoteId); + let mut sapling_commitments = block.sapling_commitments.into_iter(); + wdb.with_sapling_tree_mut::<_, _, SqliteClientError>(move |sapling_tree| { + if let Some(sapling_tree_size) = block.sapling_commitment_tree_size { + let start_position = Position::from(u64::from(sapling_tree_size)) + - u64::try_from(sapling_commitments.len()).unwrap(); + sapling_tree.batch_insert(start_position, &mut sapling_commitments)?; } - } - - // Prune the stored witnesses (we only expect rollbacks of at most PRUNING_HEIGHT blocks). - wallet::prune_witnesses(&wdb.conn.0, block.block_height - PRUNING_HEIGHT)?; + Ok(()) + })?; // Update now-expired transactions that didn't get mined. - wallet::update_expired_notes(&wdb.conn.0, block.block_height)?; + wallet::update_expired_notes(&wdb.conn.0, block_height)?; - Ok(new_witnesses) + Ok(wallet_note_ids) }) } @@ -493,55 +490,37 @@ impl WalletWrite for WalletDb wallet::sapling::put_received_note(&wdb.conn.0, output, tx_ref)?; } } - } - // If any of the utxos spent in the transaction are ours, mark them as spent. - #[cfg(feature = "transparent-inputs")] - for txin in d_tx - .tx - .transparent_bundle() - .iter() - .flat_map(|b| b.vin.iter()) - { - wallet::mark_transparent_utxo_spent(&wdb.conn.0, tx_ref, &txin.prevout)?; - } + // If any of the utxos spent in the transaction are ours, mark them as spent. + #[cfg(feature = "transparent-inputs")] + for txin in d_tx.tx.transparent_bundle().iter().flat_map(|b| b.vin.iter()) { + wallet::mark_transparent_utxo_spent(&wdb.conn.0, tx_ref, &txin.prevout)?; + } - // If we have some transparent outputs: - if !d_tx - .tx - .transparent_bundle() - .iter() - .any(|b| b.vout.is_empty()) - { - let nullifiers = wdb.get_sapling_nullifiers(data_api::NullifierQuery::All)?; - // If the transaction contains shielded spends from our wallet, we will store z->t - // transactions we observe in the same way they would be stored by - // create_spend_to_address. - if let Some((account_id, _)) = nullifiers.iter().find(|(_, nf)| { - d_tx.tx - .sapling_bundle() - .iter() - .flat_map(|b| b.shielded_spends().iter()) - .any(|input| nf == input.nullifier()) - }) { - for (output_index, txout) in d_tx - .tx - .transparent_bundle() - .iter() - .flat_map(|b| b.vout.iter()) - .enumerate() - { - if let Some(address) = txout.recipient_address() { - wallet::put_sent_output( - &wdb.conn.0, - &wdb.params, - *account_id, - tx_ref, - output_index, - &Recipient::Transparent(address), - txout.value, - None, - )?; + // If we have some transparent outputs: + if d_tx.tx.transparent_bundle().iter().any(|b| !b.vout.is_empty()) { + let nullifiers = wdb.get_sapling_nullifiers(NullifierQuery::All)?; + // If the transaction contains shielded spends from our wallet, we will store z->t + // transactions we observe in the same way they would be stored by + // create_spend_to_address. + if let Some((account_id, _)) = nullifiers.iter().find( + |(_, nf)| + d_tx.tx.sapling_bundle().iter().flat_map(|b| b.shielded_spends().iter()) + .any(|input| nf == input.nullifier()) + ) { + for (output_index, txout) in d_tx.tx.transparent_bundle().iter().flat_map(|b| b.vout.iter()).enumerate() { + if let Some(address) = txout.recipient_address() { + wallet::put_sent_output( + &wdb.conn.0, + &wdb.params, + *account_id, + tx_ref, + output_index, + &Recipient::Transparent(address), + txout.value, + None + )?; + } } } } @@ -633,6 +612,59 @@ impl WalletWrite for WalletDb } } +impl WalletCommitmentTrees for WalletDb { + type Error = rusqlite::Error; + type SaplingShardStore<'a> = WalletDbSaplingShardStore<'a, 'a>; + + fn with_sapling_tree_mut(&mut self, mut callback: F) -> Result + where + for<'a> F: FnMut( + &'a mut ShardTree< + Self::SaplingShardStore<'a>, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + SAPLING_SHARD_HEIGHT, + >, + ) -> Result, + E: From>, + { + let tx = self.conn.transaction().map_err(ShardTreeError::Storage)?; + let shard_store = + WalletDbSaplingShardStore::from_connection(&tx).map_err(ShardTreeError::Storage)?; + let result = { + let mut shardtree = ShardTree::new(shard_store, 100); + callback(&mut shardtree)? + }; + tx.commit().map_err(ShardTreeError::Storage)?; + Ok(result) + } +} + +impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb, P> { + type Error = rusqlite::Error; + type SaplingShardStore<'a> = WalletDbSaplingShardStore<'a, 'a>; + + fn with_sapling_tree_mut(&mut self, mut callback: F) -> Result + where + for<'a> F: FnMut( + &'a mut ShardTree< + Self::SaplingShardStore<'a>, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + SAPLING_SHARD_HEIGHT, + >, + ) -> Result, + E: From>, + { + let mut shardtree = ShardTree::new( + WalletDbSaplingShardStore::from_connection(&self.conn.0) + .map_err(ShardTreeError::Storage)?, + 100, + ); + let result = callback(&mut shardtree)?; + + Ok(result) + } +} + /// A handle for the SQLite block source. pub struct BlockDb(Connection); @@ -1024,6 +1056,7 @@ mod tests { dfvk: &DiversifiableFullViewingKey, req: AddressType, value: Amount, + initial_sapling_tree_size: u32, ) -> (CompactBlock, Nullifier) { let to = match req { AddressType::DefaultExternal => dfvk.default_address().1, @@ -1069,6 +1102,8 @@ mod tests { }; cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); + cb.sapling_commitment_tree_size = initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(); (cb, note.nf(&dfvk.fvk().vk.nk, 0)) } @@ -1081,6 +1116,7 @@ mod tests { dfvk: &DiversifiableFullViewingKey, to: PaymentAddress, value: Amount, + initial_sapling_tree_size: u32, ) -> CompactBlock { let mut rng = OsRng; let rseed = generate_random_rseed(&network(), height, &mut rng); @@ -1154,6 +1190,8 @@ mod tests { }; cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); + cb.sapling_commitment_tree_size = initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(); cb } @@ -1267,6 +1305,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(5).unwrap(), + 0, ); let (cb2, _) = fake_compact_block( BlockHeight::from_u32(2), @@ -1274,6 +1313,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(10).unwrap(), + 1, ); // Write the CompactBlocks to the BlockMeta DB's corresponding disk storage. diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index a05597134..406db08c7 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -67,13 +67,13 @@ use rusqlite::{self, named_params, params, OptionalExtension, ToSql}; use std::collections::HashMap; use std::convert::TryFrom; +use std::io::Cursor; use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight, BranchId, NetworkUpgrade, Parameters}, memo::{Memo, MemoBytes}, - merkle_tree::write_commitment_tree, - sapling::CommitmentTree, + merkle_tree::read_commitment_tree, transaction::{components::Amount, Transaction, TxId}, zip32::{ sapling::{DiversifiableFullViewingKey, ExtendedFullViewingKey}, @@ -83,7 +83,7 @@ use zcash_primitives::{ use zcash_client_backend::{ address::{RecipientAddress, UnifiedAddress}, - data_api::{PoolType, Recipient, SentTransactionOutput}, + data_api::{chain::CommitmentTreeMeta, PoolType, Recipient, SentTransactionOutput}, encoding::AddressCodec, keys::UnifiedFullViewingKey, wallet::WalletTx, @@ -536,6 +536,51 @@ pub(crate) fn block_height_extrema( }) } +pub(crate) fn fully_scanned_height( + conn: &rusqlite::Connection, +) -> Result, SqliteClientError> { + let res_opt = conn + .query_row( + "SELECT height, sapling_commitment_tree_size, sapling_tree + FROM blocks + ORDER BY height DESC + LIMIT 1", + [], + |row| { + let max_height: u32 = row.get(0)?; + let sapling_tree_size: Option = row.get(1)?; + let sapling_tree: Vec = row.get(0)?; + Ok(( + BlockHeight::from(max_height), + sapling_tree_size, + sapling_tree, + )) + }, + ) + .optional()?; + + res_opt + .map(|(max_height, sapling_tree_size, sapling_tree)| { + let commitment_tree_meta = + CommitmentTreeMeta::from_parts(if let Some(known_size) = sapling_tree_size { + known_size + } else { + // parse the legacy commitment tree data + read_commitment_tree::< + zcash_primitives::sapling::Node, + _, + { zcash_primitives::sapling::NOTE_COMMITMENT_TREE_DEPTH }, + >(Cursor::new(sapling_tree))? + .size() + .try_into() + .expect("usize values are convertible to u64 on all supported platforms.") + }); + + Ok((max_height, commitment_tree_meta)) + }) + .transpose() +} + /// Returns the block height at which the specified transaction was mined, /// if any. pub(crate) fn get_tx_height( @@ -765,21 +810,24 @@ pub(crate) fn insert_block( block_height: BlockHeight, block_hash: BlockHash, block_time: u32, - commitment_tree: &CommitmentTree, + sapling_commitment_tree_size: Option, ) -> Result<(), SqliteClientError> { - let mut encoded_tree = Vec::new(); - write_commitment_tree(commitment_tree, &mut encoded_tree).unwrap(); - let mut stmt_insert_block = conn.prepare_cached( - "INSERT INTO blocks (height, hash, time, sapling_tree) - VALUES (?, ?, ?, ?)", + "INSERT INTO blocks ( + height, + hash, + time, + sapling_commitment_tree_size, + sapling_tree + ) + VALUES (?, ?, ?, ?, x'00')", )?; stmt_insert_block.execute(params![ u32::from(block_height), &block_hash.0[..], block_time, - encoded_tree + sapling_commitment_tree_size ])?; Ok(()) @@ -951,17 +999,6 @@ pub(crate) fn put_legacy_transparent_utxo( stmt_upsert_legacy_transparent_utxo.query_row(sql_args, |row| row.get::<_, i64>(0).map(UtxoId)) } -/// Removes old incremental witnesses up to the given block height. -pub(crate) fn prune_witnesses( - conn: &rusqlite::Connection, - below_height: BlockHeight, -) -> Result<(), SqliteClientError> { - let mut stmt_prune_witnesses = - conn.prepare_cached("DELETE FROM sapling_witnesses WHERE block < ?")?; - stmt_prune_witnesses.execute([u32::from(below_height)])?; - Ok(()) -} - /// Marks notes that have not been mined in transactions /// as expired, up to the given block height. pub(crate) fn update_expired_notes( diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 7f5a60ccc..bb8834a6a 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -6,6 +6,7 @@ use rusqlite::{self, types::ToSql}; use schemer::{Migrator, MigratorError}; use schemer_rusqlite::RusqliteAdapter; use secrecy::SecretVec; +use shardtree::ShardTreeError; use uuid::Uuid; use zcash_primitives::{ @@ -34,6 +35,9 @@ pub enum WalletMigrationError { /// Wrapper for amount balance violations BalanceError(BalanceError), + + /// Wrapper for commitment tree invariant violations + CommitmentTree(ShardTreeError), } impl From for WalletMigrationError { @@ -48,6 +52,12 @@ impl From for WalletMigrationError { } } +impl From> for WalletMigrationError { + fn from(e: ShardTreeError) -> Self { + WalletMigrationError::CommitmentTree(e) + } +} + impl fmt::Display for WalletMigrationError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match &self { @@ -62,6 +72,7 @@ impl fmt::Display for WalletMigrationError { } WalletMigrationError::DbError(e) => write!(f, "{}", e), WalletMigrationError::BalanceError(e) => write!(f, "Balance error: {:?}", e), + WalletMigrationError::CommitmentTree(e) => write!(f, "Commitment tree error: {:?}", e), } } } @@ -361,8 +372,9 @@ mod tests { height INTEGER PRIMARY KEY, hash BLOB NOT NULL, time INTEGER NOT NULL, - sapling_tree BLOB NOT NULL - )", + sapling_tree BLOB NOT NULL , + sapling_commitment_tree_size INTEGER, + orchard_commitment_tree_size INTEGER)", "CREATE TABLE sapling_received_notes ( id_note INTEGER PRIMARY KEY, tx INTEGER NOT NULL, @@ -375,6 +387,7 @@ mod tests { is_change INTEGER NOT NULL, memo BLOB, spent INTEGER, + commitment_tree_position INTEGER, FOREIGN KEY (tx) REFERENCES transactions(id_tx), FOREIGN KEY (account) REFERENCES accounts(account), FOREIGN KEY (spent) REFERENCES transactions(id_tx), diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 1cc9bcfc5..e51605ccf 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -4,6 +4,7 @@ mod addresses_table; mod initial_setup; mod received_notes_nullable_nf; mod sent_notes_to_internal; +mod shardtree_support; mod ufvk_support; mod utxos_table; mod v_transactions_net; @@ -46,5 +47,6 @@ pub(super) fn all_migrations( Box::new(add_transaction_views::Migration), Box::new(v_transactions_net::Migration), Box::new(received_notes_nullable_nf::Migration), + Box::new(shardtree_support::Migration), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs new file mode 100644 index 000000000..b9e4a6bf0 --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -0,0 +1,56 @@ +//! This migration adds tables to the wallet database that are needed to persist note commitment +//! tree data using the `shardtree` crate, and migrates existing witness data into these data +//! structures. + +use std::collections::HashSet; + +use rusqlite; +use schemer; +use schemer_rusqlite::RusqliteMigration; +use uuid::Uuid; + +use crate::wallet::init::{migrations::received_notes_nullable_nf, WalletMigrationError}; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( + 0x7da6489d, + 0xe835, + 0x4657, + b"\x8b\xe5\xf5\x12\xbc\xce\x6c\xbf", +); + +pub(super) struct Migration; + +impl schemer::Migration for Migration { + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [received_notes_nullable_nf::MIGRATION_ID] + .into_iter() + .collect() + } + + fn description(&self) -> &'static str { + "Add support for receiving storage of note commitment tree data using the `shardtree` crate." + } +} + +impl RusqliteMigration for Migration { + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + // Add commitment tree sizes to block metadata. + transaction.execute_batch( + "ALTER TABLE blocks ADD COLUMN sapling_commitment_tree_size INTEGER; + ALTER TABLE blocks ADD COLUMN orchard_commitment_tree_size INTEGER; + ALTER TABLE sapling_received_notes ADD COLUMN commitment_tree_position INTEGER;", + )?; + + Ok(()) + } + + fn down(&self, _transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + Ok(()) + } +} diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 66734cfdc..511333ff7 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -1,12 +1,13 @@ //! Functions for Sapling support in the wallet. + use group::ff::PrimeField; -use rusqlite::{named_params, params, types::Value, Connection, OptionalExtension, Row}; +use incrementalmerkletree::Position; +use rusqlite::{named_params, params, types::Value, Connection, Row}; use std::rc::Rc; use zcash_primitives::{ consensus::BlockHeight, memo::MemoBytes, - merkle_tree::{read_commitment_tree, read_incremental_witness, write_incremental_witness}, sapling::{self, Diversifier, Note, Nullifier, Rseed}, transaction::components::Amount, zip32::AccountId, @@ -21,6 +22,8 @@ use crate::{error::SqliteClientError, NoteId}; use super::memo_repr; +pub(crate) mod commitment_tree; + /// This trait provides a generalization over shielded output representations. pub(crate) trait ReceivedSaplingOutput { fn index(&self) -> usize; @@ -28,10 +31,11 @@ pub(crate) trait ReceivedSaplingOutput { fn note(&self) -> &Note; fn memo(&self) -> Option<&MemoBytes>; fn is_change(&self) -> bool; - fn nullifier(&self) -> Option<&Nullifier>; + fn nullifier(&self) -> Option<&sapling::Nullifier>; + fn note_commitment_tree_position(&self) -> Option; } -impl ReceivedSaplingOutput for WalletSaplingOutput { +impl ReceivedSaplingOutput for WalletSaplingOutput { fn index(&self) -> usize { self.index() } @@ -47,10 +51,12 @@ impl ReceivedSaplingOutput for WalletSaplingOutput { fn is_change(&self) -> bool { WalletSaplingOutput::is_change(self) } - - fn nullifier(&self) -> Option<&Nullifier> { + fn nullifier(&self) -> Option<&sapling::Nullifier> { Some(self.nf()) } + fn note_commitment_tree_position(&self) -> Option { + Some(WalletSaplingOutput::note_commitment_tree_position(self)) + } } impl ReceivedSaplingOutput for DecryptedOutput { @@ -69,7 +75,10 @@ impl ReceivedSaplingOutput for DecryptedOutput { fn is_change(&self) -> bool { self.transfer_type == TransferType::WalletInternal } - fn nullifier(&self) -> Option<&Nullifier> { + fn nullifier(&self) -> Option<&sapling::Nullifier> { + None + } + fn note_commitment_tree_position(&self) -> Option { None } } @@ -105,17 +114,17 @@ fn to_spendable_note(row: &Row) -> Result, SqliteCli Rseed::BeforeZip212(rcm) }; - let witness = { - let d: Vec<_> = row.get(4)?; - read_incremental_witness(&d[..])? - }; + let note_commitment_tree_position = + Position::from(u64::try_from(row.get::<_, i64>(4)?).map_err(|_| { + SqliteClientError::CorruptedData("Note commitment tree position invalid.".to_string()) + })?); Ok(ReceivedSaplingNote { note_id, diversifier, note_value, rseed, - witness, + note_commitment_tree_position, }) } @@ -126,15 +135,13 @@ pub(crate) fn get_spendable_sapling_notes( exclude: &[NoteId], ) -> Result>, SqliteClientError> { let mut stmt_select_notes = conn.prepare_cached( - "SELECT id_note, diversifier, value, rcm, witness - FROM sapling_received_notes - INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx - INNER JOIN sapling_witnesses ON sapling_witnesses.note = sapling_received_notes.id_note - WHERE account = :account - AND spent IS NULL - AND transactions.block <= :anchor_height - AND sapling_witnesses.block = :anchor_height - AND id_note NOT IN rarray(:exclude)", + "SELECT id_note, diversifier, value, rcm, commitment_tree_position + FROM sapling_received_notes + INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx + WHERE account = :account + AND spent IS NULL + AND transactions.block <= :anchor_height + AND id_note NOT IN rarray(:exclude)", )?; let excluded: Vec = exclude @@ -184,28 +191,22 @@ pub(crate) fn select_spendable_sapling_notes( // // 4) Match the selected notes against the witnesses at the desired height. let mut stmt_select_notes = conn.prepare_cached( - "WITH selected AS ( - WITH eligible AS ( - SELECT id_note, diversifier, value, rcm, - SUM(value) OVER - (PARTITION BY account, spent ORDER BY id_note) AS so_far - FROM sapling_received_notes - INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx - WHERE account = :account - AND spent IS NULL - AND transactions.block <= :anchor_height - AND id_note NOT IN rarray(:exclude) - ) - SELECT * FROM eligible WHERE so_far < :target_value - UNION - SELECT * FROM (SELECT * FROM eligible WHERE so_far >= :target_value LIMIT 1) - ), witnesses AS ( - SELECT note, witness FROM sapling_witnesses - WHERE block = :anchor_height - ) - SELECT selected.id_note, selected.diversifier, selected.value, selected.rcm, witnesses.witness - FROM selected - INNER JOIN witnesses ON selected.id_note = witnesses.note", + "WITH eligible AS ( + SELECT id_note, diversifier, value, rcm, commitment_tree_position, + SUM(value) + OVER (PARTITION BY account, spent ORDER BY id_note) AS so_far + FROM sapling_received_notes + INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx + WHERE account = :account + AND spent IS NULL + AND transactions.block <= :anchor_height + AND id_note NOT IN rarray(:exclude) + ) + SELECT id_note, diversifier, value, rcm, commitment_tree_position + FROM eligible WHERE so_far < :target_value + UNION + SELECT id_note, diversifier, value, rcm, commitment_tree_position + FROM (SELECT * from eligible WHERE so_far >= :target_value LIMIT 1)", )?; let excluded: Vec = exclude @@ -230,73 +231,6 @@ pub(crate) fn select_spendable_sapling_notes( notes.collect::>() } -/// Returns the commitment tree for the block at the specified height, -/// if any. -pub(crate) fn get_sapling_commitment_tree( - conn: &Connection, - block_height: BlockHeight, -) -> Result, SqliteClientError> { - conn.query_row_and_then( - "SELECT sapling_tree FROM blocks WHERE height = ?", - [u32::from(block_height)], - |row| { - let row_data: Vec = row.get(0)?; - read_commitment_tree(&row_data[..]).map_err(|e| { - rusqlite::Error::FromSqlConversionFailure( - row_data.len(), - rusqlite::types::Type::Blob, - Box::new(e), - ) - }) - }, - ) - .optional() - .map_err(SqliteClientError::from) -} - -/// Returns the incremental witnesses for the block at the specified height, -/// if any. -pub(crate) fn get_sapling_witnesses( - conn: &Connection, - block_height: BlockHeight, -) -> Result, SqliteClientError> { - let mut stmt_fetch_witnesses = - conn.prepare_cached("SELECT note, witness FROM sapling_witnesses WHERE block = ?")?; - - let witnesses = stmt_fetch_witnesses - .query_map([u32::from(block_height)], |row| { - let id_note = NoteId::ReceivedNoteId(row.get(0)?); - let witness_data: Vec = row.get(1)?; - Ok(read_incremental_witness(&witness_data[..]).map(|witness| (id_note, witness))) - }) - .map_err(SqliteClientError::from)?; - - // unwrap database error & IO error from IncrementalWitness::read - let res: Vec<_> = witnesses.collect::, _>>()??; - Ok(res) -} - -/// Records the incremental witness for the specified note, -/// as of the given block height. -pub(crate) fn insert_witness( - conn: &Connection, - note_id: i64, - witness: &sapling::IncrementalWitness, - height: BlockHeight, -) -> Result<(), SqliteClientError> { - let mut stmt_insert_witness = conn.prepare_cached( - "INSERT INTO sapling_witnesses (note, block, witness) - VALUES (?, ?, ?)", - )?; - - let mut encoded = Vec::new(); - write_incremental_witness(witness, &mut encoded).unwrap(); - - stmt_insert_witness.execute(params![note_id, u32::from(height), encoded])?; - - Ok(()) -} - /// Retrieves the set of nullifiers for "potentially spendable" Sapling notes that the /// wallet is tracking. /// @@ -320,7 +254,7 @@ pub(crate) fn get_sapling_nullifiers( let nf_bytes: Vec = row.get(2)?; Ok(( AccountId::from(account), - Nullifier::from_slice(&nf_bytes).unwrap(), + sapling::Nullifier::from_slice(&nf_bytes).unwrap(), )) })?; @@ -343,7 +277,7 @@ pub(crate) fn get_all_sapling_nullifiers( let nf_bytes: Vec = row.get(2)?; Ok(( AccountId::from(account), - Nullifier::from_slice(&nf_bytes).unwrap(), + sapling::Nullifier::from_slice(&nf_bytes).unwrap(), )) })?; @@ -359,7 +293,7 @@ pub(crate) fn get_all_sapling_nullifiers( pub(crate) fn mark_sapling_note_spent( conn: &Connection, tx_ref: i64, - nf: &Nullifier, + nf: &sapling::Nullifier, ) -> Result { let mut stmt_mark_sapling_note_spent = conn.prepare_cached("UPDATE sapling_received_notes SET spent = ? WHERE nf = ?")?; @@ -383,9 +317,19 @@ pub(crate) fn put_received_note( ) -> Result { let mut stmt_upsert_received_note = conn.prepare_cached( "INSERT INTO sapling_received_notes - (tx, output_index, account, diversifier, value, rcm, memo, nf, is_change) - VALUES - (:tx, :output_index, :account, :diversifier, :value, :rcm, :memo, :nf, :is_change) + (tx, output_index, account, diversifier, value, rcm, memo, nf, is_change, commitment_tree_position) + VALUES ( + :tx, + :output_index, + :account, + :diversifier, + :value, + :rcm, + :memo, + :nf, + :is_change, + :commitment_tree_position + ) ON CONFLICT (tx, output_index) DO UPDATE SET account = :account, diversifier = :diversifier, @@ -393,7 +337,8 @@ pub(crate) fn put_received_note( rcm = :rcm, nf = IFNULL(:nf, nf), memo = IFNULL(:memo, memo), - is_change = IFNULL(:is_change, is_change) + is_change = IFNULL(:is_change, is_change), + commitment_tree_position = IFNULL(:commitment_tree_position, commitment_tree_position) RETURNING id_note", )?; @@ -410,7 +355,8 @@ pub(crate) fn put_received_note( ":rcm": &rcm.as_ref(), ":nf": output.nullifier().map(|nf| nf.0.as_ref()), ":memo": memo_repr(output.memo()), - ":is_change": output.is_change() + ":is_change": output.is_change(), + ":commitment_tree_position": output.note_commitment_tree_position().map(u64::from), ]; stmt_upsert_received_note @@ -622,9 +568,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance let (_, anchor_height) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); @@ -644,9 +591,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 1, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance does not include the second note let (_, anchor_height2) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); @@ -691,10 +639,11 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + i, ); insert_into_cache(&db_cache, &cb); } - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Second spend still fails assert_matches!( @@ -724,9 +673,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 11, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Second spend should now succeed assert_matches!( @@ -768,9 +718,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -823,10 +774,11 @@ mod tests { &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, + i, ); insert_into_cache(&db_cache, &cb); } - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Second spend still fails assert_matches!( @@ -855,9 +807,10 @@ mod tests { &ExtendedSpendingKey::master(&[42]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, + 42, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Second spend should now succeed create_spend_to_address( @@ -898,9 +851,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -968,10 +922,11 @@ mod tests { &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, + i, ); insert_into_cache(&db_cache, &cb); } - scan_cached_blocks(&network, &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&network, &db_cache, &mut db_data, None, None).unwrap(); // Send the funds again, discarding history. // Neither transaction output is decryptable by the sender. @@ -1001,9 +956,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance let (_, anchor_height) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); @@ -1056,9 +1012,10 @@ mod tests { &dfvk, AddressType::Internal, value, + 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance let (_, anchor_height) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); @@ -1110,6 +1067,7 @@ mod tests { &dfvk, AddressType::Internal, Amount::from_u64(50000).unwrap(), + 0, ); insert_into_cache(&db_cache, &cb); @@ -1121,11 +1079,12 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(1000).unwrap(), + i, ); insert_into_cache(&db_cache, &cb); } - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance let total = Amount::from_u64(60000).unwrap(); @@ -1241,9 +1200,10 @@ mod tests { &dfvk, AddressType::Internal, Amount::from_u64(50000).unwrap(), + 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); assert_matches!( shield_transparent_funds( diff --git a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs new file mode 100644 index 000000000..a02ce4cd8 --- /dev/null +++ b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs @@ -0,0 +1,123 @@ +use incrementalmerkletree::Address; +use rusqlite; +use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore}; + +use zcash_primitives::{consensus::BlockHeight, sapling}; + +pub struct WalletDbSaplingShardStore<'conn, 'a> { + pub(crate) conn: &'a rusqlite::Transaction<'conn>, +} + +impl<'conn, 'a> WalletDbSaplingShardStore<'conn, 'a> { + pub(crate) fn from_connection( + conn: &'a rusqlite::Transaction<'conn>, + ) -> Result { + Ok(WalletDbSaplingShardStore { conn }) + } +} + +impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { + type H = sapling::Node; + type CheckpointId = BlockHeight; + type Error = rusqlite::Error; + + fn get_shard( + &self, + _shard_root: Address, + ) -> Result>, Self::Error> { + // SELECT shard_data FROM sapling_tree WHERE shard_index = shard_root.index + todo!() + } + + fn last_shard(&self) -> Result>, Self::Error> { + // SELECT shard_data FROM sapling_tree ORDER BY shard_index DESC LIMIT 1 + todo!() + } + + fn put_shard(&mut self, _subtree: LocatedPrunableTree) -> Result<(), Self::Error> { + todo!() + } + + fn get_shard_roots(&self) -> Result, Self::Error> { + // SELECT + todo!() + } + + fn truncate(&mut self, _from: Address) -> Result<(), Self::Error> { + todo!() + } + + fn get_cap(&self) -> Result, Self::Error> { + todo!() + } + + fn put_cap(&mut self, _cap: PrunableTree) -> Result<(), Self::Error> { + todo!() + } + + fn min_checkpoint_id(&self) -> Result, Self::Error> { + todo!() + } + + fn max_checkpoint_id(&self) -> Result, Self::Error> { + todo!() + } + + fn add_checkpoint( + &mut self, + _checkpoint_id: Self::CheckpointId, + _checkpoint: Checkpoint, + ) -> Result<(), Self::Error> { + todo!() + } + + fn checkpoint_count(&self) -> Result { + todo!() + } + + fn get_checkpoint_at_depth( + &self, + _checkpoint_depth: usize, + ) -> Result, Self::Error> { + todo!() + } + + fn get_checkpoint( + &self, + _checkpoint_id: &Self::CheckpointId, + ) -> Result, Self::Error> { + todo!() + } + + fn with_checkpoints(&mut self, _limit: usize, _callback: F) -> Result<(), Self::Error> + where + F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, + { + todo!() + } + + fn update_checkpoint_with( + &mut self, + _checkpoint_id: &Self::CheckpointId, + _update: F, + ) -> Result + where + F: Fn(&mut Checkpoint) -> Result<(), Self::Error>, + { + todo!() + } + + fn remove_checkpoint( + &mut self, + _checkpoint_id: &Self::CheckpointId, + ) -> Result<(), Self::Error> { + todo!() + } + + fn truncate_checkpoints( + &mut self, + _checkpoint_id: &Self::CheckpointId, + ) -> Result<(), Self::Error> { + todo!() + } +} diff --git a/zcash_primitives/src/merkle_tree.rs b/zcash_primitives/src/merkle_tree.rs index 176d3b437..6cda449bc 100644 --- a/zcash_primitives/src/merkle_tree.rs +++ b/zcash_primitives/src/merkle_tree.rs @@ -98,7 +98,7 @@ pub fn write_nonempty_frontier_v1( frontier: &NonEmptyFrontier, ) -> io::Result<()> { write_position(&mut writer, frontier.position())?; - if frontier.position().is_odd() { + if frontier.position().is_right_child() { // The v1 serialization wrote the sibling of a right-hand leaf as an optional value, rather // than as part of the ommers vector. frontier