From 374ed8cf9447f1e9e0dc67f453f0a71878271d26 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 6 Mar 2024 18:25:40 -0700 Subject: [PATCH 1/2] zcash_client_sqlite: Add backend impl for the Orchard note commitment tree --- zcash_client_backend/CHANGELOG.md | 3 + zcash_client_backend/src/data_api.rs | 16 +- zcash_client_sqlite/src/lib.rs | 57 ++++- zcash_client_sqlite/src/wallet.rs | 28 +++ zcash_client_sqlite/src/wallet/init.rs | 74 ++++++- .../src/wallet/init/migrations.rs | 34 +-- .../init/migrations/orchard_shardtree.rs | 196 ++++++++++++++++++ zcash_client_sqlite/src/wallet/orchard.rs | 2 +- zcash_client_sqlite/src/wallet/scanning.rs | 60 +++++- 9 files changed, 431 insertions(+), 39 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/orchard_shardtree.rs diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 0b4cb668d..86cbe6337 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -22,6 +22,7 @@ and this library adheres to Rust's notion of - `SentTransaction::new` - `ORCHARD_SHARD_HEIGHT` - `BlockMetadata::orchard_tree_size` + - `WalletSummary::next_orchard_subtree_index` - `chain::ScanSummary::{spent_orchard_note_count, received_orchard_note_count}` - `zcash_client_backend::fees`: - `orchard` @@ -64,6 +65,8 @@ and this library adheres to Rust's notion of - `fn put_orchard_subtree_roots` - Added method `WalletRead::validate_seed` - Removed `Error::AccountNotFound` variant. + - `WalletSummary::new` now takes an additional `next_orchard_subtree_index` + argument when the `orchard` feature flag is enabled. - `zcash_client_backend::decrypt`: - Fields of `DecryptedOutput` are now private. Use `DecryptedOutput::new` and the newly provided accessors instead. diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 9c32c6480..b16bd7b96 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -353,6 +353,8 @@ pub struct WalletSummary { fully_scanned_height: BlockHeight, scan_progress: Option>, next_sapling_subtree_index: u64, + #[cfg(feature = "orchard")] + next_orchard_subtree_index: u64, } impl WalletSummary { @@ -362,14 +364,17 @@ impl WalletSummary { chain_tip_height: BlockHeight, fully_scanned_height: BlockHeight, scan_progress: Option>, - next_sapling_subtree_idx: u64, + next_sapling_subtree_index: u64, + #[cfg(feature = "orchard")] next_orchard_subtree_index: u64, ) -> Self { Self { account_balances, chain_tip_height, fully_scanned_height, scan_progress, - next_sapling_subtree_index: next_sapling_subtree_idx, + next_sapling_subtree_index, + #[cfg(feature = "orchard")] + next_orchard_subtree_index, } } @@ -405,6 +410,13 @@ impl WalletSummary { self.next_sapling_subtree_index } + /// Returns the Orchard subtree index that should start the next range of subtree + /// roots passed to [`WalletCommitmentTrees::put_orchard_subtree_roots`]. + #[cfg(feature = "orchard")] + pub fn next_orchard_subtree_index(&self) -> u64 { + self.next_orchard_subtree_index + } + /// Returns whether or not wallet scanning is complete. pub fn is_synced(&self) -> bool { self.chain_tip_height == self.fully_scanned_height diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 8c1dae2aa..6b3361769 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -956,7 +956,7 @@ impl WalletCommitmentTrees for WalletDb; #[cfg(feature = "orchard")] - fn with_orchard_tree_mut(&mut self, _callback: F) -> Result + fn with_orchard_tree_mut(&mut self, mut callback: F) -> Result where for<'a> F: FnMut( &'a mut ShardTree< @@ -967,16 +967,41 @@ impl WalletCommitmentTrees for WalletDb Result, E: From>, { - todo!() + let tx = self + .conn + .transaction() + .map_err(|e| ShardTreeError::Storage(commitment_tree::Error::Query(e)))?; + let shard_store = SqliteShardStore::from_connection(&tx, ORCHARD_TABLES_PREFIX) + .map_err(|e| ShardTreeError::Storage(commitment_tree::Error::Query(e)))?; + let result = { + let mut shardtree = ShardTree::new(shard_store, PRUNING_DEPTH.try_into().unwrap()); + callback(&mut shardtree)? + }; + + tx.commit() + .map_err(|e| ShardTreeError::Storage(commitment_tree::Error::Query(e)))?; + Ok(result) } #[cfg(feature = "orchard")] fn put_orchard_subtree_roots( &mut self, - _start_index: u64, - _roots: &[CommitmentTreeRoot], + start_index: u64, + roots: &[CommitmentTreeRoot], ) -> Result<(), ShardTreeError> { - todo!() + let tx = self + .conn + .transaction() + .map_err(|e| ShardTreeError::Storage(commitment_tree::Error::Query(e)))?; + put_shard_roots::<_, { ORCHARD_SHARD_HEIGHT * 2 }, ORCHARD_SHARD_HEIGHT>( + &tx, + ORCHARD_TABLES_PREFIX, + start_index, + roots, + )?; + tx.commit() + .map_err(|e| ShardTreeError::Storage(commitment_tree::Error::Query(e)))?; + Ok(()) } } @@ -1027,7 +1052,7 @@ impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb; #[cfg(feature = "orchard")] - fn with_orchard_tree_mut(&mut self, _callback: F) -> Result + fn with_orchard_tree_mut(&mut self, mut callback: F) -> Result where for<'a> F: FnMut( &'a mut ShardTree< @@ -1038,16 +1063,28 @@ impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb Result, E: From>, { - todo!() + let mut shardtree = ShardTree::new( + SqliteShardStore::from_connection(self.conn.0, ORCHARD_TABLES_PREFIX) + .map_err(|e| ShardTreeError::Storage(commitment_tree::Error::Query(e)))?, + PRUNING_DEPTH.try_into().unwrap(), + ); + let result = callback(&mut shardtree)?; + + Ok(result) } #[cfg(feature = "orchard")] fn put_orchard_subtree_roots( &mut self, - _start_index: u64, - _roots: &[CommitmentTreeRoot], + start_index: u64, + roots: &[CommitmentTreeRoot], ) -> Result<(), ShardTreeError> { - todo!() + put_shard_roots::<_, { orchard::NOTE_COMMITMENT_TREE_DEPTH as u8 }, ORCHARD_SHARD_HEIGHT>( + self.conn.0, + ORCHARD_TABLES_PREFIX, + start_index, + roots, + ) } } diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index ff0c7cf6a..c29eac1b1 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -110,6 +110,9 @@ use crate::{ use self::scanning::{parse_priority_code, priority_code, replace_queue_entries}; +#[cfg(feature = "orchard")] +use {crate::ORCHARD_TABLES_PREFIX, zcash_client_backend::data_api::ORCHARD_SHARD_HEIGHT}; + #[cfg(feature = "transparent-inputs")] use { crate::UtxoId, @@ -948,6 +951,9 @@ pub(crate) fn get_wallet_summary( drop(transparent_trace); } + // The approach used here for Sapling and Orchard subtree indexing was a quick hack + // that has not yet been replaced. TODO: Make less hacky. + // https://github.com/zcash/librustzcash/issues/1249 let next_sapling_subtree_index = { let shard_store = SqliteShardStore::<_, ::sapling::Node, SAPLING_SHARD_HEIGHT>::from_connection( @@ -967,12 +973,34 @@ pub(crate) fn get_wallet_summary( .unwrap_or(0) }; + #[cfg(feature = "orchard")] + let next_orchard_subtree_index = { + let shard_store = SqliteShardStore::< + _, + ::orchard::tree::MerkleHashOrchard, + ORCHARD_SHARD_HEIGHT, + >::from_connection(tx, ORCHARD_TABLES_PREFIX)?; + + // The last shard will be incomplete, and we want the next range to overlap with + // the last complete shard, so return the index of the second-to-last shard root. + shard_store + .get_shard_roots() + .map_err(ShardTreeError::Storage)? + .iter() + .rev() + .nth(1) + .map(|addr| addr.index()) + .unwrap_or(0) + }; + let summary = WalletSummary::new( account_balances, chain_tip_height, fully_scanned_height, sapling_scan_progress, next_sapling_subtree_index, + #[cfg(feature = "orchard")] + next_orchard_subtree_index, ); Ok(Some(summary)) diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index bb59d303d..77cf2028d 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -11,9 +11,8 @@ use uuid::Uuid; use zcash_client_backend::keys::AddressGenerationError; use zcash_primitives::{consensus, transaction::components::amount::BalanceError}; -use crate::WalletDb; - use super::commitment_tree; +use crate::WalletDb; mod migrations; @@ -263,6 +262,31 @@ mod tests { ON UPDATE RESTRICT, CONSTRAINT nf_uniq UNIQUE (spend_pool, nf) )", + "CREATE TABLE orchard_tree_cap ( + -- cap_id exists only to be able to take advantage of `ON CONFLICT` + -- upsert functionality; the table will only ever contain one row + cap_id INTEGER PRIMARY KEY, + cap_data BLOB NOT NULL + )", + "CREATE TABLE orchard_tree_checkpoint_marks_removed ( + checkpoint_id INTEGER NOT NULL, + mark_removed_position INTEGER NOT NULL, + FOREIGN KEY (checkpoint_id) REFERENCES orchard_tree_checkpoints(checkpoint_id) + ON DELETE CASCADE, + CONSTRAINT spend_position_unique UNIQUE (checkpoint_id, mark_removed_position) + )", + "CREATE TABLE orchard_tree_checkpoints ( + checkpoint_id INTEGER PRIMARY KEY, + position INTEGER + )", + "CREATE TABLE orchard_tree_shards ( + shard_index INTEGER PRIMARY KEY, + subtree_end_height INTEGER, + root_hash BLOB, + shard_data BLOB, + contains_marked INTEGER, + CONSTRAINT root_unique UNIQUE (root_hash) + )", r#"CREATE TABLE "sapling_received_notes" ( id INTEGER PRIMARY KEY, tx INTEGER NOT NULL, @@ -390,6 +414,52 @@ mod tests { } let expected_views = vec![ + // v_orchard_shard_scan_ranges + format!( + "CREATE VIEW v_orchard_shard_scan_ranges AS + SELECT + shard.shard_index, + shard.shard_index << 16 AS start_position, + (shard.shard_index + 1) << 16 AS end_position_exclusive, + IFNULL(prev_shard.subtree_end_height, {}) AS subtree_start_height, + shard.subtree_end_height, + shard.contains_marked, + scan_queue.block_range_start, + scan_queue.block_range_end, + scan_queue.priority + FROM orchard_tree_shards shard + LEFT OUTER JOIN orchard_tree_shards prev_shard + ON shard.shard_index = prev_shard.shard_index + 1 + -- Join with scan ranges that overlap with the subtree's involved blocks. + INNER JOIN scan_queue ON ( + subtree_start_height < scan_queue.block_range_end AND + ( + scan_queue.block_range_start <= shard.subtree_end_height OR + shard.subtree_end_height IS NULL + ) + )", + u32::from(st.network().activation_height(NetworkUpgrade::Nu5).unwrap()), + ), + //v_orchard_shard_unscanned_ranges + format!( + "CREATE VIEW v_orchard_shard_unscanned_ranges AS + WITH wallet_birthday AS (SELECT MIN(birthday_height) AS height FROM accounts) + SELECT + shard_index, + start_position, + end_position_exclusive, + subtree_start_height, + subtree_end_height, + contains_marked, + block_range_start, + block_range_end, + priority + FROM v_orchard_shard_scan_ranges + INNER JOIN wallet_birthday + WHERE priority > {} + AND block_range_end > wallet_birthday.height", + priority_code(&ScanPriority::Scanned), + ), // v_sapling_shard_scan_ranges format!( "CREATE VIEW v_sapling_shard_scan_ranges AS diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index b9ad7561f..553306571 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -5,6 +5,7 @@ mod addresses_table; mod full_account_ids; mod initial_setup; mod nullifier_map; +mod orchard_shardtree; mod received_notes_nullable_nf; mod receiving_key_scopes; mod sapling_memo_consistency; @@ -24,7 +25,7 @@ use std::rc::Rc; use schemer_rusqlite::RusqliteMigration; use secrecy::SecretVec; -use zcash_primitives::consensus; +use zcash_protocol::consensus; use super::WalletMigrationError; @@ -45,20 +46,20 @@ pub(super) fn all_migrations( // | // v_transactions_net // | - // received_notes_nullable_nf - // / | \ - // / | \ - // shardtree_support sapling_memo_consistency nullifier_map - // / \ \ - // add_account_birthdays receiving_key_scopes v_transactions_transparent_history - // | \ | | - // v_sapling_shard_unscanned_ranges \ | v_tx_outputs_use_legacy_false - // | \ | | - // wallet_summaries \ | v_transactions_shielding_balance - // \ | | - // \ | v_transactions_note_uniqueness - // \ | / - // full_account_ids + // received_notes_nullable_nf------ + // / | \ + // / | \ + // --------------- shardtree_support sapling_memo_consistency nullifier_map + // / / \ \ + // orchard_shardtree add_account_birthdays receiving_key_scopes v_transactions_transparent_history + // | \ | | + // v_sapling_shard_unscanned_ranges \ | v_tx_outputs_use_legacy_false + // | \ | | + // wallet_summaries \ | v_transactions_shielding_balance + // \ | | + // \ | v_transactions_note_uniqueness + // \ | / + // full_account_ids vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -101,5 +102,8 @@ pub(super) fn all_migrations( seed, params: params.clone(), }), + Box::new(orchard_shardtree::Migration { + params: params.clone(), + }), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/orchard_shardtree.rs b/zcash_client_sqlite/src/wallet/init/migrations/orchard_shardtree.rs new file mode 100644 index 000000000..68b57b90d --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/orchard_shardtree.rs @@ -0,0 +1,196 @@ +//! This migration adds tables to the wallet database that are needed to persist Orchard note +//! commitment tree data using the `shardtree` crate. + +use std::collections::HashSet; + +use rusqlite::{named_params, OptionalExtension}; +use schemer_rusqlite::RusqliteMigration; +use tracing::debug; +use uuid::Uuid; +use zcash_client_backend::data_api::scanning::ScanPriority; +use zcash_protocol::consensus::{self, BlockHeight, NetworkUpgrade}; + +use super::shardtree_support; +use crate::wallet::{init::WalletMigrationError, scan_queue_extrema, scanning::priority_code}; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x3a6487f7_e068_42bb_9d12_6bb8dbe6da00); + +pub(super) struct Migration

{ + pub(super) params: P, +} + +impl

schemer::Migration for Migration

{ + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [shardtree_support::MIGRATION_ID].into_iter().collect() + } + + fn description(&self) -> &'static str { + "Add support for storage of Orchard note commitment tree data using the `shardtree` crate." + } +} + +impl RusqliteMigration for Migration

{ + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + // Add shard persistence + debug!("Creating tables for Orchard shard persistence"); + transaction.execute_batch( + "CREATE TABLE orchard_tree_shards ( + shard_index INTEGER PRIMARY KEY, + subtree_end_height INTEGER, + root_hash BLOB, + shard_data BLOB, + contains_marked INTEGER, + CONSTRAINT root_unique UNIQUE (root_hash) + ); + CREATE TABLE orchard_tree_cap ( + -- cap_id exists only to be able to take advantage of `ON CONFLICT` + -- upsert functionality; the table will only ever contain one row + cap_id INTEGER PRIMARY KEY, + cap_data BLOB NOT NULL + );", + )?; + + // Add checkpoint persistence + debug!("Creating tables for checkpoint persistence"); + transaction.execute_batch( + "CREATE TABLE orchard_tree_checkpoints ( + checkpoint_id INTEGER PRIMARY KEY, + position INTEGER + ); + CREATE TABLE orchard_tree_checkpoint_marks_removed ( + checkpoint_id INTEGER NOT NULL, + mark_removed_position INTEGER NOT NULL, + FOREIGN KEY (checkpoint_id) REFERENCES orchard_tree_checkpoints(checkpoint_id) + ON DELETE CASCADE, + CONSTRAINT spend_position_unique UNIQUE (checkpoint_id, mark_removed_position) + );", + )?; + + transaction.execute_batch(&format!( + "CREATE VIEW v_orchard_shard_scan_ranges AS + SELECT + shard.shard_index, + shard.shard_index << {} AS start_position, + (shard.shard_index + 1) << {} AS end_position_exclusive, + IFNULL(prev_shard.subtree_end_height, {}) AS subtree_start_height, + shard.subtree_end_height, + shard.contains_marked, + scan_queue.block_range_start, + scan_queue.block_range_end, + scan_queue.priority + FROM orchard_tree_shards shard + LEFT OUTER JOIN orchard_tree_shards prev_shard + ON shard.shard_index = prev_shard.shard_index + 1 + -- Join with scan ranges that overlap with the subtree's involved blocks. + INNER JOIN scan_queue ON ( + subtree_start_height < scan_queue.block_range_end AND + ( + scan_queue.block_range_start <= shard.subtree_end_height OR + shard.subtree_end_height IS NULL + ) + )", + 16, // ORCHARD_SHARD_HEIGHT is only available when `feature = "orchard"` is enabled. + 16, // ORCHARD_SHARD_HEIGHT is only available when `feature = "orchard"` is enabled. + u32::from(self.params.activation_height(NetworkUpgrade::Nu5).unwrap()), + ))?; + + transaction.execute_batch(&format!( + "CREATE VIEW v_orchard_shard_unscanned_ranges AS + WITH wallet_birthday AS (SELECT MIN(birthday_height) AS height FROM accounts) + SELECT + shard_index, + start_position, + end_position_exclusive, + subtree_start_height, + subtree_end_height, + contains_marked, + block_range_start, + block_range_end, + priority + FROM v_orchard_shard_scan_ranges + INNER JOIN wallet_birthday + WHERE priority > {} + AND block_range_end > wallet_birthday.height;", + priority_code(&ScanPriority::Scanned), + ))?; + + // Treat the current best-known chain tip height as the height to use for Orchard + // initialization, bounded below by NU5 activation. + if let Some(orchard_init_height) = scan_queue_extrema(transaction)?.and_then(|r| { + self.params + .activation_height(NetworkUpgrade::Nu5) + .map(|orchard_activation| std::cmp::max(orchard_activation, *r.end())) + }) { + // If a scan range exists that contains the Orchard init height, split it in two at the + // init height. + if let Some((start, end, range_priority)) = transaction + .query_row_and_then( + "SELECT block_range_start, block_range_end, priority + FROM scan_queue + WHERE block_range_start <= :orchard_init_height + AND block_range_end > :orchard_init_height", + named_params![":orchard_init_height": u32::from(orchard_init_height)], + |row| { + let start = BlockHeight::from(row.get::<_, u32>(0)?); + let end = BlockHeight::from(row.get::<_, u32>(1)?); + let range_priority: i64 = row.get(2)?; + Ok((start, end, range_priority)) + }, + ) + .optional()? + { + transaction.execute( + "DELETE from scan_queue WHERE block_range_start = :start", + named_params![":start": u32::from(start)], + )?; + if start < orchard_init_height { + // Rewrite the start of the scan range to be exactly what it was prior to the + // change. + transaction.execute( + "INSERT INTO scan_queue (block_range_start, block_range_end, priority) + VALUES (:block_range_start, :block_range_end, :priority)", + named_params![ + ":block_range_start": u32::from(start), + ":block_range_end": u32::from(orchard_init_height), + ":priority": range_priority, + ], + )?; + } + // Rewrite the remainder of the range to have at least priority `Historic` + transaction.execute( + "INSERT INTO scan_queue (block_range_start, block_range_end, priority) + VALUES (:block_range_start, :block_range_end, :priority)", + named_params![ + ":block_range_start": u32::from(orchard_init_height), + ":block_range_end": u32::from(end), + ":priority": + std::cmp::max(range_priority, priority_code(&ScanPriority::Historic)), + ], + )?; + // Rewrite any scanned ranges above the end of the first Orchard + // range to have at least priority `Historic` + transaction.execute( + "UPDATE scan_queue SET priority = :historic + WHERE :block_range_start >= :orchard_initial_range_end + AND priority < :historic", + named_params![ + ":historic": priority_code(&ScanPriority::Historic), + ":orchard_initial_range_end": u32::from(end), + ], + )?; + } + } + + Ok(()) + } + + fn down(&self, _transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + Err(WalletMigrationError::CannotRevert(MIGRATION_ID)) + } +} diff --git a/zcash_client_sqlite/src/wallet/orchard.rs b/zcash_client_sqlite/src/wallet/orchard.rs index 2de55071d..fac22c8e2 100644 --- a/zcash_client_sqlite/src/wallet/orchard.rs +++ b/zcash_client_sqlite/src/wallet/orchard.rs @@ -99,7 +99,7 @@ pub(crate) mod tests { } fn next_subtree_index(s: &WalletSummary) -> u64 { - todo!() + s.next_orchard_subtree_index() } fn select_spendable_notes( diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index e23dfc46b..73de6cae6 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -1,3 +1,4 @@ +use incrementalmerkletree::{Address, Position}; use rusqlite::{self, named_params, types::Value, OptionalExtension}; use shardtree::error::ShardTreeError; use std::cmp::{max, min}; @@ -6,14 +7,14 @@ use std::ops::Range; use std::rc::Rc; use tracing::{debug, trace}; -use incrementalmerkletree::{Address, Position}; -use zcash_primitives::consensus::{self, BlockHeight, NetworkUpgrade}; - -use zcash_client_backend::data_api::{ - scanning::{spanning_tree::SpanningTree, ScanPriority, ScanRange}, - SAPLING_SHARD_HEIGHT, +use zcash_client_backend::{ + data_api::{ + scanning::{spanning_tree::SpanningTree, ScanPriority, ScanRange}, + SAPLING_SHARD_HEIGHT, + }, + ShieldedProtocol, }; -use zcash_protocol::{PoolType, ShieldedProtocol}; +use zcash_primitives::consensus::{self, BlockHeight, NetworkUpgrade}; use crate::{ error::SqliteClientError, @@ -23,6 +24,12 @@ use crate::{ use super::wallet_birthday; +#[cfg(feature = "orchard")] +use {crate::ORCHARD_TABLES_PREFIX, zcash_client_backend::data_api::ORCHARD_SHARD_HEIGHT}; + +#[cfg(not(feature = "orchard"))] +use zcash_client_backend::PoolType; + pub(crate) fn priority_code(priority: &ScanPriority) -> i64 { use ScanPriority::*; match priority { @@ -301,6 +308,7 @@ pub(crate) fn scan_complete( wallet_note_positions: &[(ShieldedProtocol, Position)], ) -> Result<(), SqliteClientError> { // Read the wallet birthday (if known). + // TODO: use per-pool birthdays? let wallet_birthday = wallet_birthday(conn)?; // Determine the range of block heights for which we will be updating the scan queue. @@ -310,6 +318,8 @@ pub(crate) fn scan_complete( // the note commitment tree subtrees containing the positions of the discovered notes. // We will query by subtree index to find these bounds. let mut required_sapling_subtrees = BTreeSet::new(); + #[cfg(feature = "orchard")] + let mut required_orchard_subtrees = BTreeSet::new(); for (protocol, position) in wallet_note_positions { match protocol { ShieldedProtocol::Sapling => { @@ -318,6 +328,12 @@ pub(crate) fn scan_complete( ); } ShieldedProtocol::Orchard => { + #[cfg(feature = "orchard")] + required_orchard_subtrees.insert( + Address::above_position(ORCHARD_SHARD_HEIGHT.into(), *position).index(), + ); + + #[cfg(not(feature = "orchard"))] return Err(SqliteClientError::UnsupportedPoolType(PoolType::Shielded( *protocol, ))); @@ -325,14 +341,28 @@ pub(crate) fn scan_complete( } } - extend_range( + let extended_range = extend_range( conn, &range, required_sapling_subtrees, SAPLING_TABLES_PREFIX, params.activation_height(NetworkUpgrade::Sapling), wallet_birthday, + )?; + + #[cfg(feature = "orchard")] + let extended_range = extend_range( + conn, + extended_range.as_ref().unwrap_or(&range), + required_orchard_subtrees, + ORCHARD_TABLES_PREFIX, + params.activation_height(NetworkUpgrade::Nu5), + wallet_birthday, )? + .or(extended_range); + + #[allow(clippy::let_and_return)] + extended_range }; let query_range = extended_range.clone().unwrap_or_else(|| range.clone()); @@ -415,9 +445,21 @@ pub(crate) fn update_chain_tip( // `ScanRange` uses an exclusive upper bound. let chain_end = new_tip + 1; - // Read the maximum height from the shards table. + // Read the maximum height from each of the the shards tables. The minimum of the two + // gives the start of a height range that covers the last incomplete shard of both the + // Sapling and Orchard pools. let sapling_shard_tip = tip_shard_end_height(conn, SAPLING_TABLES_PREFIX)?; + #[cfg(feature = "orchard")] + let orchard_shard_tip = tip_shard_end_height(conn, ORCHARD_TABLES_PREFIX)?; + #[cfg(feature = "orchard")] + let min_shard_tip = match (sapling_shard_tip, orchard_shard_tip) { + (None, None) => None, + (None, Some(o)) => Some(o), + (Some(s), None) => Some(s), + (Some(s), Some(o)) => Some(std::cmp::min(s, o)), + }; + #[cfg(not(feature = "orchard"))] let min_shard_tip = sapling_shard_tip; // Create a scanning range for the fragment of the last shard leading up to new tip. From ae9dd25525432421c5927b3fced8c73851117f61 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 7 Mar 2024 11:47:00 -0700 Subject: [PATCH 2/2] zcash_client_sqlite: Add `orchard_received_notes` table and update related views. --- zcash_client_sqlite/CHANGELOG.md | 1 + zcash_client_sqlite/src/testing.rs | 23 +- zcash_client_sqlite/src/wallet/init.rs | 376 +++++++++++------- .../src/wallet/init/migrations.rs | 4 + .../init/migrations/orchard_received_notes.rs | 273 +++++++++++++ .../init/migrations/orchard_shardtree.rs | 20 + 6 files changed, 530 insertions(+), 167 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/orchard_received_notes.rs diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index bb97cb4b8..1490aeb22 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -35,6 +35,7 @@ and this library adheres to Rust's notion of - `init::WalletMigrationError` has added variants: - `WalletMigrationError::AddressGeneration` - `WalletMigrationError::CannotRevert` +- The `v_transactions` and `v_tx_outputs` views now include Orchard notes. ## [0.9.1] - 2024-03-09 diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index 1fd22e1c5..8682e5449 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -962,7 +962,7 @@ impl TestFvk for orchard::keys::FullViewingKey { fn add_spend( &self, ctx: &mut CompactTx, - nf: Self::Nullifier, + revealed_spent_note_nullifier: Self::Nullifier, rng: &mut R, ) { // Generate a dummy recipient. @@ -977,7 +977,7 @@ impl TestFvk for orchard::keys::FullViewingKey { }; let (cact, _) = compact_orchard_action( - nf, + revealed_spent_note_nullifier, recipient, NonNegativeAmount::ZERO, self.orchard_ovk(zip32::Scope::Internal), @@ -997,7 +997,7 @@ impl TestFvk for orchard::keys::FullViewingKey { mut rng: &mut R, ) -> Self::Nullifier { // Generate a dummy nullifier - let nullifier = + let revealed_spent_note_nullifier = orchard::note::Nullifier::from_bytes(&pallas::Base::random(&mut rng).to_repr()) .unwrap(); @@ -1008,7 +1008,7 @@ impl TestFvk for orchard::keys::FullViewingKey { }; let (cact, note) = compact_orchard_action( - nullifier, + revealed_spent_note_nullifier, self.address_at(j, scope), value, self.orchard_ovk(scope), @@ -1025,7 +1025,7 @@ impl TestFvk for orchard::keys::FullViewingKey { ctx: &mut CompactTx, _: &P, _: BlockHeight, - nf: Self::Nullifier, + revealed_spent_note_nullifier: Self::Nullifier, req: AddressType, value: NonNegativeAmount, _: u32, @@ -1038,7 +1038,7 @@ impl TestFvk for orchard::keys::FullViewingKey { }; let (cact, note) = compact_orchard_action( - nf, + revealed_spent_note_nullifier, self.address_at(j, scope), value, self.orchard_ovk(scope), @@ -1046,6 +1046,7 @@ impl TestFvk for orchard::keys::FullViewingKey { ); ctx.actions.push(cact); + // Return the nullifier of the newly created output note note.nullifier(self) } } @@ -1100,8 +1101,6 @@ fn compact_orchard_action( ovk: Option, rng: &mut R, ) -> (CompactOrchardAction, orchard::Note) { - let nf = nullifier.to_bytes().to_vec(); - let rseed = { loop { let mut bytes = [0; 32]; @@ -1120,16 +1119,14 @@ fn compact_orchard_action( ) .unwrap(); let encryptor = OrchardNoteEncryption::new(ovk, note, *MemoBytes::empty().as_array()); - let cmx = orchard::note::ExtractedNoteCommitment::from(note.commitment()) - .to_bytes() - .to_vec(); + let cmx = orchard::note::ExtractedNoteCommitment::from(note.commitment()); let ephemeral_key = OrchardDomain::epk_bytes(encryptor.epk()).0.to_vec(); let enc_ciphertext = encryptor.encrypt_note_plaintext(); ( CompactOrchardAction { - nullifier: nf, - cmx, + nullifier: nullifier.to_bytes().to_vec(), + cmx: cmx.to_bytes().to_vec(), ephemeral_key, ciphertext: enc_ciphertext.as_ref()[..52].to_vec(), }, diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 77cf2028d..3483a93dd 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -262,6 +262,26 @@ mod tests { ON UPDATE RESTRICT, CONSTRAINT nf_uniq UNIQUE (spend_pool, nf) )", + "CREATE TABLE orchard_received_notes ( + id INTEGER PRIMARY KEY, + tx INTEGER NOT NULL, + action_index INTEGER NOT NULL, + account_id INTEGER NOT NULL, + diversifier BLOB NOT NULL, + value INTEGER NOT NULL, + rho BLOB NOT NULL, + rseed BLOB NOT NULL, + nf BLOB UNIQUE, + is_change INTEGER NOT NULL, + memo BLOB, + spent INTEGER, + commitment_tree_position INTEGER, + recipient_key_scope INTEGER, + FOREIGN KEY (tx) REFERENCES transactions(id_tx), + FOREIGN KEY (account_id) REFERENCES accounts(id), + FOREIGN KEY (spent) REFERENCES transactions(id_tx), + CONSTRAINT tx_output UNIQUE (tx, action_index) + )", "CREATE TABLE orchard_tree_cap ( -- cap_id exists only to be able to take advantage of `ON CONFLICT` -- upsert functionality; the table will only ever contain one row @@ -460,6 +480,57 @@ mod tests { AND block_range_end > wallet_birthday.height", priority_code(&ScanPriority::Scanned), ), + // v_orchard_shards_scan_state + "CREATE VIEW v_orchard_shards_scan_state AS + SELECT + shard_index, + start_position, + end_position_exclusive, + subtree_start_height, + subtree_end_height, + contains_marked, + MAX(priority) AS max_priority + FROM v_orchard_shard_scan_ranges + GROUP BY + shard_index, + start_position, + end_position_exclusive, + subtree_start_height, + subtree_end_height, + contains_marked".to_owned(), + // v_received_notes + "CREATE VIEW v_received_notes AS + SELECT + id, + tx, + 2 AS pool, + sapling_received_notes.output_index AS output_index, + account_id, + value, + is_change, + memo, + spent, + sent_notes.id AS sent_note_id + FROM sapling_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, 2, sapling_received_notes.output_index) + UNION + SELECT + id, + tx, + 3 AS pool, + orchard_received_notes.action_index AS output_index, + account_id, + value, + is_change, + memo, + spent, + sent_notes.id AS sent_note_id + FROM orchard_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (orchard_received_notes.tx, 3, orchard_received_notes.action_index)".to_owned(), // v_sapling_shard_scan_ranges format!( "CREATE VIEW v_sapling_shard_scan_ranges AS @@ -526,162 +597,159 @@ mod tests { contains_marked".to_owned(), // v_transactions "CREATE VIEW v_transactions AS - WITH - notes AS ( - SELECT sapling_received_notes.id AS id, - sapling_received_notes.account_id AS account_id, - transactions.block AS block, - transactions.txid AS txid, - 2 AS pool, - sapling_received_notes.value AS value, - CASE - WHEN sapling_received_notes.is_change THEN 1 - ELSE 0 - END AS is_change, - CASE - WHEN sapling_received_notes.is_change THEN 0 - ELSE 1 - END AS received_count, - CASE - WHEN (sapling_received_notes.memo IS NULL OR sapling_received_notes.memo = X'F6') - THEN 0 - ELSE 1 - END AS memo_present - FROM sapling_received_notes - JOIN transactions - ON transactions.id_tx = sapling_received_notes.tx - UNION - SELECT utxos.id AS id, - utxos.received_by_account_id AS account_id, - utxos.height AS block, - utxos.prevout_txid AS txid, - 0 AS pool, - utxos.value_zat AS value, - 0 AS is_change, - 1 AS received_count, - 0 AS memo_present - FROM utxos - UNION - SELECT sapling_received_notes.id AS id, - sapling_received_notes.account_id AS account_id, - transactions.block AS block, - transactions.txid AS txid, - 2 AS pool, - -sapling_received_notes.value AS value, - 0 AS is_change, - 0 AS received_count, - 0 AS memo_present - FROM sapling_received_notes - JOIN transactions - ON transactions.id_tx = sapling_received_notes.spent - UNION - SELECT utxos.id AS id, - utxos.received_by_account_id AS account_id, - transactions.block AS block, - transactions.txid AS txid, - 0 AS pool, - -utxos.value_zat AS value, - 0 AS is_change, - 0 AS received_count, - 0 AS memo_present - FROM utxos - JOIN transactions - ON transactions.id_tx = utxos.spent_in_tx - ), - sent_note_counts AS ( - SELECT sent_notes.from_account_id AS account_id, - transactions.txid AS txid, - COUNT(DISTINCT sent_notes.id) as sent_notes, - SUM( - CASE - WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6' OR sapling_received_notes.tx IS NOT NULL) - THEN 0 - ELSE 1 - END - ) AS memo_count - FROM sent_notes - JOIN transactions - ON transactions.id_tx = sent_notes.tx - LEFT JOIN sapling_received_notes - ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = - (sapling_received_notes.tx, 2, sapling_received_notes.output_index) - WHERE COALESCE(sapling_received_notes.is_change, 0) = 0 - GROUP BY account_id, txid - ), - blocks_max_height AS ( - SELECT MAX(blocks.height) as max_height FROM blocks - ) - SELECT notes.account_id AS account_id, - notes.block AS mined_height, - notes.txid AS txid, - transactions.tx_index AS tx_index, - transactions.expiry_height AS expiry_height, - transactions.raw AS raw, - SUM(notes.value) AS account_balance_delta, - transactions.fee AS fee_paid, - SUM(notes.is_change) > 0 AS has_change, - MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, - SUM(notes.received_count) AS received_note_count, - SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, - blocks.time AS block_time, - ( - blocks.height IS NULL - AND transactions.expiry_height BETWEEN 1 AND blocks_max_height.max_height - ) AS expired_unmined - FROM notes - LEFT JOIN transactions - ON notes.txid = transactions.txid - JOIN blocks_max_height - LEFT JOIN blocks ON blocks.height = notes.block - LEFT JOIN sent_note_counts - ON sent_note_counts.account_id = notes.account_id - AND sent_note_counts.txid = notes.txid - GROUP BY notes.account_id, notes.txid".to_owned(), + WITH + notes AS ( + SELECT v_received_notes.id AS id, + v_received_notes.account_id AS account_id, + transactions.block AS block, + transactions.txid AS txid, + v_received_notes.pool AS pool, + v_received_notes.value AS value, + CASE + WHEN v_received_notes.is_change THEN 1 + ELSE 0 + END AS is_change, + CASE + WHEN v_received_notes.is_change THEN 0 + ELSE 1 + END AS received_count, + CASE + WHEN (v_received_notes.memo IS NULL OR v_received_notes.memo = X'F6') + THEN 0 + ELSE 1 + END AS memo_present + FROM v_received_notes + JOIN transactions + ON transactions.id_tx = v_received_notes.tx + UNION + SELECT utxos.id AS id, + utxos.received_by_account_id AS account_id, + utxos.height AS block, + utxos.prevout_txid AS txid, + 0 AS pool, + utxos.value_zat AS value, + 0 AS is_change, + 1 AS received_count, + 0 AS memo_present + FROM utxos + UNION + SELECT v_received_notes.id AS id, + v_received_notes.account_id AS account_id, + transactions.block AS block, + transactions.txid AS txid, + v_received_notes.pool AS pool, + -v_received_notes.value AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM v_received_notes + JOIN transactions + ON transactions.id_tx = v_received_notes.spent + UNION + SELECT utxos.id AS id, + utxos.received_by_account_id AS account_id, + transactions.block AS block, + transactions.txid AS txid, + 0 AS pool, + -utxos.value_zat AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM utxos + JOIN transactions + ON transactions.id_tx = utxos.spent_in_tx + ), + sent_note_counts AS ( + SELECT sent_notes.from_account_id AS account_id, + transactions.txid AS txid, + COUNT(DISTINCT sent_notes.id) as sent_notes, + SUM( + CASE + WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6' OR v_received_notes.tx IS NOT NULL) + THEN 0 + ELSE 1 + END + ) AS memo_count + FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx + LEFT JOIN v_received_notes + ON sent_notes.id = v_received_notes.sent_note_id + WHERE COALESCE(v_received_notes.is_change, 0) = 0 + GROUP BY account_id, txid + ), + blocks_max_height AS ( + SELECT MAX(blocks.height) as max_height FROM blocks + ) + SELECT notes.account_id AS account_id, + notes.block AS mined_height, + notes.txid AS txid, + transactions.tx_index AS tx_index, + transactions.expiry_height AS expiry_height, + transactions.raw AS raw, + SUM(notes.value) AS account_balance_delta, + transactions.fee AS fee_paid, + SUM(notes.is_change) > 0 AS has_change, + MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, + SUM(notes.received_count) AS received_note_count, + SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, + blocks.time AS block_time, + ( + blocks.height IS NULL + AND transactions.expiry_height BETWEEN 1 AND blocks_max_height.max_height + ) AS expired_unmined + FROM notes + LEFT JOIN transactions + ON notes.txid = transactions.txid + JOIN blocks_max_height + LEFT JOIN blocks ON blocks.height = notes.block + LEFT JOIN sent_note_counts + ON sent_note_counts.account_id = notes.account_id + AND sent_note_counts.txid = notes.txid + GROUP BY notes.account_id, notes.txid".to_owned(), // v_tx_outputs "CREATE VIEW v_tx_outputs AS - SELECT transactions.txid AS txid, - 2 AS output_pool, - sapling_received_notes.output_index AS output_index, - sent_notes.from_account_id AS from_account_id, - sapling_received_notes.account_id AS to_account_id, - NULL AS to_address, - sapling_received_notes.value AS value, - sapling_received_notes.is_change AS is_change, - sapling_received_notes.memo AS memo - FROM sapling_received_notes - JOIN transactions - ON transactions.id_tx = sapling_received_notes.tx - LEFT JOIN sent_notes - ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = - (sapling_received_notes.tx, 2, sent_notes.output_index) - UNION - SELECT utxos.prevout_txid AS txid, - 0 AS output_pool, - utxos.prevout_idx AS output_index, - NULL AS from_account_id, - utxos.received_by_account_id AS to_account_id, - utxos.address AS to_address, - utxos.value_zat AS value, - 0 AS is_change, - NULL AS memo - FROM utxos - UNION - SELECT transactions.txid AS txid, - sent_notes.output_pool AS output_pool, - sent_notes.output_index AS output_index, - sent_notes.from_account_id AS from_account_id, - sapling_received_notes.account_id AS to_account_id, - sent_notes.to_address AS to_address, - sent_notes.value AS value, - 0 AS is_change, - sent_notes.memo AS memo - FROM sent_notes - JOIN transactions - ON transactions.id_tx = sent_notes.tx - LEFT JOIN sapling_received_notes - ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = - (sapling_received_notes.tx, 2, sapling_received_notes.output_index) - WHERE COALESCE(sapling_received_notes.is_change, 0) = 0".to_owned(), + SELECT transactions.txid AS txid, + v_received_notes.pool AS output_pool, + v_received_notes.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + v_received_notes.account_id AS to_account_id, + NULL AS to_address, + v_received_notes.value AS value, + v_received_notes.is_change AS is_change, + v_received_notes.memo AS memo + FROM v_received_notes + JOIN transactions + ON transactions.id_tx = v_received_notes.tx + LEFT JOIN sent_notes + ON sent_notes.id = v_received_notes.sent_note_id + UNION + SELECT utxos.prevout_txid AS txid, + 0 AS output_pool, + utxos.prevout_idx AS output_index, + NULL AS from_account_id, + utxos.received_by_account_id AS to_account_id, + utxos.address AS to_address, + utxos.value_zat AS value, + 0 AS is_change, + NULL AS memo + FROM utxos + UNION + SELECT transactions.txid AS txid, + sent_notes.output_pool AS output_pool, + sent_notes.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + v_received_notes.account_id AS to_account_id, + sent_notes.to_address AS to_address, + sent_notes.value AS value, + 0 AS is_change, + sent_notes.memo AS memo + FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx + LEFT JOIN v_received_notes + ON sent_notes.id = v_received_notes.sent_note_id + WHERE COALESCE(v_received_notes.is_change, 0) = 0".to_owned(), ]; let mut views_query = st diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 553306571..044065082 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -5,6 +5,7 @@ mod addresses_table; mod full_account_ids; mod initial_setup; mod nullifier_map; +mod orchard_received_notes; mod orchard_shardtree; mod received_notes_nullable_nf; mod receiving_key_scopes; @@ -60,6 +61,8 @@ pub(super) fn all_migrations( // \ | v_transactions_note_uniqueness // \ | / // full_account_ids + // | + // orchard_received_notes vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -105,5 +108,6 @@ pub(super) fn all_migrations( Box::new(orchard_shardtree::Migration { params: params.clone(), }), + Box::new(orchard_received_notes::Migration), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/orchard_received_notes.rs b/zcash_client_sqlite/src/wallet/init/migrations/orchard_received_notes.rs new file mode 100644 index 000000000..014f80dc4 --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/orchard_received_notes.rs @@ -0,0 +1,273 @@ +//! This migration adds tables to the wallet database that are needed to persist Orchard received +//! notes. + +use std::collections::HashSet; + +use schemer_rusqlite::RusqliteMigration; +use uuid::Uuid; +use zcash_client_backend::{PoolType, ShieldedProtocol}; + +use super::full_account_ids; +use crate::wallet::{init::WalletMigrationError, pool_code}; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x51d7a273_aa19_4109_9325_80e4a5545048); + +pub(super) struct Migration; + +impl schemer::Migration for Migration { + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [full_account_ids::MIGRATION_ID].into_iter().collect() + } + + fn description(&self) -> &'static str { + "Add support for storage of Orchard received notes." + } +} + +impl RusqliteMigration for Migration { + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction<'_>) -> Result<(), Self::Error> { + transaction.execute_batch( + "CREATE TABLE orchard_received_notes ( + id INTEGER PRIMARY KEY, + tx INTEGER NOT NULL, + action_index INTEGER NOT NULL, + account_id INTEGER NOT NULL, + diversifier BLOB NOT NULL, + value INTEGER NOT NULL, + rho BLOB NOT NULL, + rseed BLOB NOT NULL, + nf BLOB UNIQUE, + is_change INTEGER NOT NULL, + memo BLOB, + spent INTEGER, + commitment_tree_position INTEGER, + recipient_key_scope INTEGER, + FOREIGN KEY (tx) REFERENCES transactions(id_tx), + FOREIGN KEY (account_id) REFERENCES accounts(id), + FOREIGN KEY (spent) REFERENCES transactions(id_tx), + CONSTRAINT tx_output UNIQUE (tx, action_index) + ); + CREATE INDEX orchard_received_notes_account ON orchard_received_notes ( + account_id ASC + ); + CREATE INDEX orchard_received_notes_tx ON orchard_received_notes ( + tx ASC + ); + CREATE INDEX orchard_received_notes_spent ON orchard_received_notes ( + spent ASC + );", + )?; + + transaction.execute_batch({ + let sapling_pool_code = pool_code(PoolType::Shielded(ShieldedProtocol::Sapling)); + let orchard_pool_code = pool_code(PoolType::Shielded(ShieldedProtocol::Orchard)); + &format!( + "CREATE VIEW v_received_notes AS + SELECT + id, + tx, + {sapling_pool_code} AS pool, + sapling_received_notes.output_index AS output_index, + account_id, + value, + is_change, + memo, + spent, + sent_notes.id AS sent_note_id + FROM sapling_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, {sapling_pool_code}, sapling_received_notes.output_index) + UNION + SELECT + id, + tx, + {orchard_pool_code} AS pool, + orchard_received_notes.action_index AS output_index, + account_id, + value, + is_change, + memo, + spent, + sent_notes.id AS sent_note_id + FROM orchard_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (orchard_received_notes.tx, {orchard_pool_code}, orchard_received_notes.action_index);" + ) + })?; + + transaction.execute_batch({ + let transparent_pool_code = pool_code(PoolType::Transparent); + &format!( + "DROP VIEW v_transactions; + CREATE VIEW v_transactions AS + WITH + notes AS ( + SELECT v_received_notes.id AS id, + v_received_notes.account_id AS account_id, + transactions.block AS block, + transactions.txid AS txid, + v_received_notes.pool AS pool, + v_received_notes.value AS value, + CASE + WHEN v_received_notes.is_change THEN 1 + ELSE 0 + END AS is_change, + CASE + WHEN v_received_notes.is_change THEN 0 + ELSE 1 + END AS received_count, + CASE + WHEN (v_received_notes.memo IS NULL OR v_received_notes.memo = X'F6') + THEN 0 + ELSE 1 + END AS memo_present + FROM v_received_notes + JOIN transactions + ON transactions.id_tx = v_received_notes.tx + UNION + SELECT utxos.id AS id, + utxos.received_by_account_id AS account_id, + utxos.height AS block, + utxos.prevout_txid AS txid, + {transparent_pool_code} AS pool, + utxos.value_zat AS value, + 0 AS is_change, + 1 AS received_count, + 0 AS memo_present + FROM utxos + UNION + SELECT v_received_notes.id AS id, + v_received_notes.account_id AS account_id, + transactions.block AS block, + transactions.txid AS txid, + v_received_notes.pool AS pool, + -v_received_notes.value AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM v_received_notes + JOIN transactions + ON transactions.id_tx = v_received_notes.spent + UNION + SELECT utxos.id AS id, + utxos.received_by_account_id AS account_id, + transactions.block AS block, + transactions.txid AS txid, + {transparent_pool_code} AS pool, + -utxos.value_zat AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM utxos + JOIN transactions + ON transactions.id_tx = utxos.spent_in_tx + ), + sent_note_counts AS ( + SELECT sent_notes.from_account_id AS account_id, + transactions.txid AS txid, + COUNT(DISTINCT sent_notes.id) as sent_notes, + SUM( + CASE + WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6' OR v_received_notes.tx IS NOT NULL) + THEN 0 + ELSE 1 + END + ) AS memo_count + FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx + LEFT JOIN v_received_notes + ON sent_notes.id = v_received_notes.sent_note_id + WHERE COALESCE(v_received_notes.is_change, 0) = 0 + GROUP BY account_id, txid + ), + blocks_max_height AS ( + SELECT MAX(blocks.height) as max_height FROM blocks + ) + SELECT notes.account_id AS account_id, + notes.block AS mined_height, + notes.txid AS txid, + transactions.tx_index AS tx_index, + transactions.expiry_height AS expiry_height, + transactions.raw AS raw, + SUM(notes.value) AS account_balance_delta, + transactions.fee AS fee_paid, + SUM(notes.is_change) > 0 AS has_change, + MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, + SUM(notes.received_count) AS received_note_count, + SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, + blocks.time AS block_time, + ( + blocks.height IS NULL + AND transactions.expiry_height BETWEEN 1 AND blocks_max_height.max_height + ) AS expired_unmined + FROM notes + LEFT JOIN transactions + ON notes.txid = transactions.txid + JOIN blocks_max_height + LEFT JOIN blocks ON blocks.height = notes.block + LEFT JOIN sent_note_counts + ON sent_note_counts.account_id = notes.account_id + AND sent_note_counts.txid = notes.txid + GROUP BY notes.account_id, notes.txid; + + DROP VIEW v_tx_outputs; + CREATE VIEW v_tx_outputs AS + SELECT transactions.txid AS txid, + v_received_notes.pool AS output_pool, + v_received_notes.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + v_received_notes.account_id AS to_account_id, + NULL AS to_address, + v_received_notes.value AS value, + v_received_notes.is_change AS is_change, + v_received_notes.memo AS memo + FROM v_received_notes + JOIN transactions + ON transactions.id_tx = v_received_notes.tx + LEFT JOIN sent_notes + ON sent_notes.id = v_received_notes.sent_note_id + UNION + SELECT utxos.prevout_txid AS txid, + {transparent_pool_code} AS output_pool, + utxos.prevout_idx AS output_index, + NULL AS from_account_id, + utxos.received_by_account_id AS to_account_id, + utxos.address AS to_address, + utxos.value_zat AS value, + 0 AS is_change, + NULL AS memo + FROM utxos + UNION + SELECT transactions.txid AS txid, + sent_notes.output_pool AS output_pool, + sent_notes.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + v_received_notes.account_id AS to_account_id, + sent_notes.to_address AS to_address, + sent_notes.value AS value, + 0 AS is_change, + sent_notes.memo AS memo + FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx + LEFT JOIN v_received_notes + ON sent_notes.id = v_received_notes.sent_note_id + WHERE COALESCE(v_received_notes.is_change, 0) = 0;") + })?; + + Ok(()) + } + + fn down(&self, _transaction: &rusqlite::Transaction<'_>) -> Result<(), Self::Error> { + Err(WalletMigrationError::CannotRevert(MIGRATION_ID)) + } +} diff --git a/zcash_client_sqlite/src/wallet/init/migrations/orchard_shardtree.rs b/zcash_client_sqlite/src/wallet/init/migrations/orchard_shardtree.rs index 68b57b90d..10e2796b8 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/orchard_shardtree.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/orchard_shardtree.rs @@ -120,6 +120,26 @@ impl RusqliteMigration for Migration

{ priority_code(&ScanPriority::Scanned), ))?; + transaction.execute_batch( + "CREATE VIEW v_orchard_shards_scan_state AS + SELECT + shard_index, + start_position, + end_position_exclusive, + subtree_start_height, + subtree_end_height, + contains_marked, + MAX(priority) AS max_priority + FROM v_orchard_shard_scan_ranges + GROUP BY + shard_index, + start_position, + end_position_exclusive, + subtree_start_height, + subtree_end_height, + contains_marked;", + )?; + // Treat the current best-known chain tip height as the height to use for Orchard // initialization, bounded below by NU5 activation. if let Some(orchard_init_height) = scan_queue_extrema(transaction)?.and_then(|r| {