zec
/
librustzcash
mirror of https://github.com/zcash/librustzcash.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #1307 from nuttycom/fix/update_chain_tip_stable_max_scanned 

zcash_client_sqlite: Use account birthday subtree sizes for progress
This commit is contained in:
Kris Nuttycombe 2024-03-25 07:56:34 -06:00 committed by GitHub
parent 33edf74126 e22cf68a4a
commit 22f90bcd8e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 347 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
zcash_client_sqlite/src/testing.rs
View File

@ -235,7 +235,7 @@ impl<Cache> TestBuilder<Cache> {
let mut cached_blocks = BTreeMap::new(); let mut cached_blocks = BTreeMap::new();
if let Some(initial_state) = self.initial_chain_state { if let Some(initial_state) = &self.initial_chain_state {
db_data db_data
.put_sapling_subtree_roots(0, &initial_state.prior_sapling_roots) .put_sapling_subtree_roots(0, &initial_state.prior_sapling_roots)
.unwrap(); .unwrap();
@ -302,7 +302,9 @@ impl<Cache> TestBuilder<Cache> {
TestState { TestState {
cache: self.cache, cache: self.cache,
cached_blocks, cached_blocks,
latest_block_height: None, latest_block_height: self
.initial_chain_state
.map(|s| s.chain_state.block_height()),
_data_file: data_file, _data_file: data_file,
db_data, db_data,
test_account, test_account,
@ -474,6 +476,7 @@ where
value, value,
prior_cached_block.sapling_end_size, prior_cached_block.sapling_end_size,
prior_cached_block.orchard_end_size, prior_cached_block.orchard_end_size,
false,
); );
(height, res, nf) (height, res, nf)
@ -529,6 +532,7 @@ where
value: NonNegativeAmount, value: NonNegativeAmount,
initial_sapling_tree_size: u32, initial_sapling_tree_size: u32,
initial_orchard_tree_size: u32, initial_orchard_tree_size: u32,
allow_broken_hash_chain: bool,
) -> (Cache::InsertResult, Fvk::Nullifier) { ) -> (Cache::InsertResult, Fvk::Nullifier) {
let mut prior_cached_block = self let mut prior_cached_block = self
.latest_cached_block_below_height(height) .latest_cached_block_below_height(height)
@ -542,7 +546,9 @@ where
// we need to generate a new prior cached block that the block to be generated can // we need to generate a new prior cached block that the block to be generated can
// successfully chain from, with the provided tree sizes. // successfully chain from, with the provided tree sizes.
if prior_cached_block.chain_state.block_height() == height - 1 { if prior_cached_block.chain_state.block_height() == height - 1 {
if !allow_broken_hash_chain {
assert_eq!(prev_hash, prior_cached_block.chain_state.block_hash()); assert_eq!(prev_hash, prior_cached_block.chain_state.block_hash());
}
} else { } else {
let final_sapling_tree = let final_sapling_tree =
(prior_cached_block.sapling_end_size..initial_sapling_tree_size).fold( (prior_cached_block.sapling_end_size..initial_sapling_tree_size).fold(
@ -774,6 +780,11 @@ impl<Cache> TestState<Cache> {
&mut self.db_data &mut self.db_data
} }
/// Exposes the test framework's source of randomness.
pub(crate) fn rng_mut(&mut self) -> &mut ChaChaRng {
&mut self.rng
}
/// Exposes the network in use. /// Exposes the network in use.
pub(crate) fn network(&self) -> LocalNetwork { pub(crate) fn network(&self) -> LocalNetwork {
self.db_data.params self.db_data.params

121
zcash_client_sqlite/src/testing/pool.rs
View File

@ -2,16 +2,19 @@
//! //!
//! Generalised for sharing across the Sapling and Orchard implementations. //! Generalised for sharing across the Sapling and Orchard implementations.
use std::{convert::Infallible, num::NonZeroU32}; use std::{
convert::Infallible,
num::{NonZeroU32, NonZeroU8},
};
use incrementalmerkletree::Level; use incrementalmerkletree::{frontier::Frontier, Level};
use rand_core::RngCore; use rand_core::RngCore;
use rusqlite::params; use rusqlite::params;
use secrecy::Secret; use secrecy::Secret;
use shardtree::error::ShardTreeError; use shardtree::error::ShardTreeError;
use zcash_primitives::{ use zcash_primitives::{
block::BlockHash, block::BlockHash,
consensus::BranchId, consensus::{BranchId, NetworkUpgrade, Parameters},
legacy::TransparentAddress, legacy::TransparentAddress,
memo::{Memo, MemoBytes}, memo::{Memo, MemoBytes},
transaction::{ transaction::{
@ -28,7 +31,7 @@ use zcash_client_backend::{
address::Address, address::Address,
data_api::{ data_api::{
self, self,
chain::{self, CommitmentTreeRoot, ScanSummary}, chain::{self, ChainState, CommitmentTreeRoot, ScanSummary},
error::Error, error::Error,
wallet::input_selection::{GreedyInputSelector, GreedyInputSelectorError}, wallet::input_selection::{GreedyInputSelector, GreedyInputSelectorError},
AccountBirthday, DecryptedTransaction, Ratio, WalletRead, WalletSummary, WalletWrite, AccountBirthday, DecryptedTransaction, Ratio, WalletRead, WalletSummary, WalletWrite,
@ -46,11 +49,8 @@ use zcash_protocol::consensus::BlockHeight;
use super::TestFvk; use super::TestFvk;
use crate::{ use crate::{
error::SqliteClientError, error::SqliteClientError,
testing::{input_selector, AddressType, BlockCache, TestBuilder, TestState}, testing::{input_selector, AddressType, BlockCache, InitialChainState, TestBuilder, TestState},
wallet::{ wallet::{block_max_scanned, commitment_tree, parse_scope, truncate_to_height},
block_max_scanned, commitment_tree, parse_scope,
scanning::tests::test_with_nu5_birthday_offset, truncate_to_height,
},
AccountId, NoteId, ReceivedNoteId, AccountId, NoteId, ReceivedNoteId,
}; };
@ -1260,66 +1260,75 @@ pub(crate) fn shield_transparent<T: ShieldedPoolTester>() {
// FIXME: This requires fixes to the test framework. // FIXME: This requires fixes to the test framework.
#[allow(dead_code)] #[allow(dead_code)]
pub(crate) fn birthday_in_anchor_shard<T: ShieldedPoolTester>() { pub(crate) fn birthday_in_anchor_shard<T: ShieldedPoolTester>() {
// Use a non-zero birthday offset because Sapling and NU5 are activated at the same height.
let (mut st, dfvk, birthday, _) = test_with_nu5_birthday_offset::<T>(76, BlockHash([0; 32]));
// Set up the following situation: // Set up the following situation:
// //
// |<------ 500 ------->|<--- 10 --->|<--- 10 --->| // |<------ 500 ------->|<--- 10 --->|<--- 10 --->|
// last_shard_start wallet_birthday received_tx anchor_height // last_shard_start wallet_birthday received_tx anchor_height
// //
// Set up some shard root history before the wallet birthday. // We set the Sapling and Orchard frontiers at the birthday block initial state to 1234
let prev_shard_start = birthday.height() - 500; // notes beyond the end of the first shard.
T::put_subtree_roots( let frontier_tree_size: u32 = (0x1 << 16) + 1234;
&mut st, let mut st = TestBuilder::new()
0, .with_block_cache()
&[CommitmentTreeRoot::from_parts( .with_initial_chain_state(|rng, network| {
prev_shard_start, let birthday_height = network.activation_height(NetworkUpgrade::Nu5).unwrap() + 1000;
// fake a hash, the value doesn't matter
T::empty_tree_leaf(),
)],
)
.unwrap();
let received_tx_height = birthday.height() + 10; // Construct a fake chain state for the end of the block with the given
// birthday_offset from the Nu5 birthday.
let (prior_sapling_roots, sapling_initial_tree) =
Frontier::random_with_prior_subtree_roots(
rng,
frontier_tree_size.into(),
NonZeroU8::new(16).unwrap(),
);
// There will only be one prior root
let prior_sapling_roots = prior_sapling_roots
.into_iter()
.map(|root| CommitmentTreeRoot::from_parts(birthday_height - 500, root))
.collect::<Vec<_>>();
let initial_sapling_tree_size = birthday
.sapling_frontier()
.value()
.map(|f| u64::from(f.position() + 1))
.unwrap_or(0)
.try_into()
.unwrap();
#[cfg(feature = "orchard")] #[cfg(feature = "orchard")]
let initial_orchard_tree_size = birthday let (prior_orchard_roots, orchard_initial_tree) =
.orchard_frontier() Frontier::random_with_prior_subtree_roots(
.value() rng,
.map(|f| u64::from(f.position() + 1)) frontier_tree_size.into(),
.unwrap_or(0) NonZeroU8::new(16).unwrap(),
.try_into() );
.unwrap(); // There will only be one prior root
#[cfg(not(feature = "orchard"))] #[cfg(feature = "orchard")]
let initial_orchard_tree_size = 0; let prior_orchard_roots = prior_orchard_roots
.into_iter()
.map(|root| CommitmentTreeRoot::from_parts(birthday_height - 500, root))
.collect::<Vec<_>>();
InitialChainState {
chain_state: ChainState::new(
birthday_height - 1,
BlockHash([5; 32]),
sapling_initial_tree,
#[cfg(feature = "orchard")]
orchard_initial_tree,
),
prior_sapling_roots,
#[cfg(feature = "orchard")]
prior_orchard_roots,
}
})
.with_account_having_current_birthday()
.build();
// Generate 9 blocks that have no value for us, starting at the birthday height. // Generate 9 blocks that have no value for us, starting at the birthday height.
let not_our_key = T::sk_to_fvk(&T::sk(&[0xf5; 32]));
let not_our_value = NonNegativeAmount::const_from_u64(10000); let not_our_value = NonNegativeAmount::const_from_u64(10000);
st.generate_block_at( let not_our_key = T::random_fvk(st.rng_mut());
birthday.height(), let (initial_height, _, _) =
BlockHash([0; 32]), st.generate_next_block(&not_our_key, AddressType::DefaultExternal, not_our_value);
&not_our_key,
AddressType::DefaultExternal,
not_our_value,
initial_sapling_tree_size,
initial_orchard_tree_size,
);
for _ in 1..9 { for _ in 1..9 {
st.generate_next_block(&not_our_key, AddressType::DefaultExternal, not_our_value); st.generate_next_block(&not_our_key, AddressType::DefaultExternal, not_our_value);
} }
// Now, generate a block that belongs to our wallet // Now, generate a block that belongs to our wallet
st.generate_next_block( let (received_tx_height, _, _) = st.generate_next_block(
&dfvk, &T::test_account_fvk(&st),
AddressType::DefaultExternal, AddressType::DefaultExternal,
NonNegativeAmount::const_from_u64(500000), NonNegativeAmount::const_from_u64(500000),
); );
@ -1331,7 +1340,7 @@ pub(crate) fn birthday_in_anchor_shard<T: ShieldedPoolTester>() {
// Scan a block range that includes our received note, but skips some blocks we need to // Scan a block range that includes our received note, but skips some blocks we need to
// make it spendable. // make it spendable.
st.scan_cached_blocks(birthday.height() + 5, 20); st.scan_cached_blocks(initial_height + 5, 20);
// Verify that the received note is not considered spendable // Verify that the received note is not considered spendable
let account = st.test_account().unwrap(); let account = st.test_account().unwrap();
@ -1348,7 +1357,7 @@ pub(crate) fn birthday_in_anchor_shard<T: ShieldedPoolTester>() {
assert_eq!(spendable.len(), 0); assert_eq!(spendable.len(), 0);
// Scan the blocks we skipped // Scan the blocks we skipped
st.scan_cached_blocks(birthday.height(), 5); st.scan_cached_blocks(initial_height, 5);
// Verify that the received note is now considered spendable // Verify that the received note is now considered spendable
let spendable = T::select_spendable_notes( let spendable = T::select_spendable_notes(
@ -1392,6 +1401,7 @@ pub(crate) fn checkpoint_gaps<T: ShieldedPoolTester>() {
not_our_value, not_our_value,
st.latest_cached_block().unwrap().sapling_end_size, st.latest_cached_block().unwrap().sapling_end_size,
st.latest_cached_block().unwrap().orchard_end_size, st.latest_cached_block().unwrap().orchard_end_size,
false,
); );
// Scan the block // Scan the block
@ -1888,6 +1898,7 @@ pub(crate) fn invalid_chain_cache_disconnected<T: ShieldedPoolTester>() {
NonNegativeAmount::const_from_u64(8), NonNegativeAmount::const_from_u64(8),
2, 2,
2, 2,
true,
); );
st.generate_next_block( st.generate_next_block(
&dfvk, &dfvk,

110
zcash_client_sqlite/src/wallet.rs
View File

@ -374,19 +374,27 @@ pub(crate) fn add_account<P: consensus::Parameters>(
#[cfg(not(feature = "transparent-inputs"))] #[cfg(not(feature = "transparent-inputs"))]
let transparent_item: Option<Vec<u8>> = None; let transparent_item: Option<Vec<u8>> = None;
let birthday_sapling_tree_size = Some(birthday.sapling_frontier().tree_size());
#[cfg(feature = "orchard")]
let birthday_orchard_tree_size = Some(birthday.orchard_frontier().tree_size());
#[cfg(not(feature = "orchard"))]
let birthday_orchard_tree_size: Option<u64> = None;
let account_id: AccountId = conn.query_row( let account_id: AccountId = conn.query_row(
r#" r#"
INSERT INTO accounts ( INSERT INTO accounts (
account_kind, hd_seed_fingerprint, hd_account_index, account_kind, hd_seed_fingerprint, hd_account_index,
ufvk, uivk, ufvk, uivk,
orchard_fvk_item_cache, sapling_fvk_item_cache, p2pkh_fvk_item_cache, orchard_fvk_item_cache, sapling_fvk_item_cache, p2pkh_fvk_item_cache,
birthday_height, recover_until_height birthday_height, birthday_sapling_tree_size, birthday_orchard_tree_size,
recover_until_height
) )
VALUES ( VALUES (
:account_kind, :hd_seed_fingerprint, :hd_account_index, :account_kind, :hd_seed_fingerprint, :hd_account_index,
:ufvk, :uivk, :ufvk, :uivk,
:orchard_fvk_item_cache, :sapling_fvk_item_cache, :p2pkh_fvk_item_cache, :orchard_fvk_item_cache, :sapling_fvk_item_cache, :p2pkh_fvk_item_cache,
:birthday_height, :recover_until_height :birthday_height, :birthday_sapling_tree_size, :birthday_orchard_tree_size,
:recover_until_height
) )
RETURNING id; RETURNING id;
"#, "#,
@ -400,6 +408,8 @@ pub(crate) fn add_account<P: consensus::Parameters>(
":sapling_fvk_item_cache": sapling_item, ":sapling_fvk_item_cache": sapling_item,
":p2pkh_fvk_item_cache": transparent_item, ":p2pkh_fvk_item_cache": transparent_item,
":birthday_height": u32::from(birthday.height()), ":birthday_height": u32::from(birthday.height()),
":birthday_sapling_tree_size": birthday_sapling_tree_size,
":birthday_orchard_tree_size": birthday_orchard_tree_size,
":recover_until_height": birthday.recover_until().map(u32::from) ":recover_until_height": birthday.recover_until().map(u32::from)
], ],
|row| Ok(AccountId(row.get(0)?)), |row| Ok(AccountId(row.get(0)?)),
@ -884,22 +894,39 @@ impl ScanProgress for SubtreeScanProgress {
) )
.map_err(SqliteClientError::from) .map_err(SqliteClientError::from)
} else { } else {
let start_height = birthday_height; // Get the starting note commitment tree size from the wallet birthday, or failing that
// Compute the starting number of notes directly from the blocks table // from the blocks table.
let start_size = conn.query_row( let start_size = conn
"SELECT MAX(sapling_commitment_tree_size) .query_row(
"SELECT birthday_sapling_tree_size
FROM accounts
WHERE birthday_height = :birthday_height",
named_params![":birthday_height": u32::from(birthday_height)],
|row| row.get::<_, Option<u64>>(0),
)
.optional()?
.flatten()
.map(Ok)
.or_else(|| {
conn.query_row(
"SELECT MAX(sapling_commitment_tree_size - sapling_output_count)
FROM blocks FROM blocks
WHERE height <= :start_height", WHERE height <= :start_height",
named_params![":start_height": u32::from(start_height)], named_params![":start_height": u32::from(birthday_height)],
|row| row.get::<_, Option<u64>>(0), |row| row.get::<_, Option<u64>>(0),
)?; )
.optional()
.map(|opt| opt.flatten())
.transpose()
})
.transpose()?;
// Compute the total blocks scanned so far above the starting height // Compute the total blocks scanned so far above the starting height
let scanned_count = conn.query_row( let scanned_count = conn.query_row(
"SELECT SUM(sapling_output_count) "SELECT SUM(sapling_output_count)
FROM blocks FROM blocks
WHERE height > :start_height", WHERE height > :start_height",
named_params![":start_height": u32::from(start_height)], named_params![":start_height": u32::from(birthday_height)],
|row| row.get::<_, Option<u64>>(0), |row| row.get::<_, Option<u64>>(0),
)?; )?;
@ -915,22 +942,22 @@ impl ScanProgress for SubtreeScanProgress {
FROM sapling_tree_shards FROM sapling_tree_shards
WHERE subtree_end_height > :start_height WHERE subtree_end_height > :start_height
OR subtree_end_height IS NULL", OR subtree_end_height IS NULL",
named_params![":start_height": u32::from(start_height)], named_params![":start_height": u32::from(birthday_height)],
|row| { |row| {
let min_tree_size = row let min_tree_size = row
.get::<_, Option<u64>>(0)? .get::<_, Option<u64>>(0)?
.map(|min| min << SAPLING_SHARD_HEIGHT); .map(|min_idx| min_idx << SAPLING_SHARD_HEIGHT);
let max_idx = row.get::<_, Option<u64>>(1)?; let max_tree_size = row
Ok(start_size .get::<_, Option<u64>>(1)?
.or(min_tree_size) .map(|max_idx| (max_idx + 1) << SAPLING_SHARD_HEIGHT);
.zip(max_idx) Ok(start_size.or(min_tree_size).zip(max_tree_size).map(
.map(|(min_tree_size, max)| { |(min_tree_size, max_tree_size)| {
let max_tree_size = (max + 1) << SAPLING_SHARD_HEIGHT;
Ratio::new( Ratio::new(
scanned_count.unwrap_or(0), scanned_count.unwrap_or(0),
max_tree_size - min_tree_size, max_tree_size - min_tree_size,
) )
})) },
))
}, },
) )
.optional()? .optional()?
@ -961,22 +988,38 @@ impl ScanProgress for SubtreeScanProgress {
) )
.map_err(SqliteClientError::from) .map_err(SqliteClientError::from)
} else { } else {
let start_height = birthday_height;
// Compute the starting number of notes directly from the blocks table // Compute the starting number of notes directly from the blocks table
let start_size = conn.query_row( let start_size = conn
"SELECT MAX(orchard_commitment_tree_size) .query_row(
"SELECT birthday_orchard_tree_size
FROM accounts
WHERE birthday_height = :birthday_height",
named_params![":birthday_height": u32::from(birthday_height)],
|row| row.get::<_, Option<u64>>(0),
)
.optional()?
.flatten()
.map(Ok)
.or_else(|| {
conn.query_row(
"SELECT MAX(orchard_commitment_tree_size - orchard_action_count)
FROM blocks FROM blocks
WHERE height <= :start_height", WHERE height <= :start_height",
named_params![":start_height": u32::from(start_height)], named_params![":start_height": u32::from(birthday_height)],
|row| row.get::<_, Option<u64>>(0), |row| row.get::<_, Option<u64>>(0),
)?; )
.optional()
.map(|opt| opt.flatten())
.transpose()
})
.transpose()?;
// Compute the total blocks scanned so far above the starting height // Compute the total blocks scanned so far above the starting height
let scanned_count = conn.query_row( let scanned_count = conn.query_row(
"SELECT SUM(orchard_action_count) "SELECT SUM(orchard_action_count)
FROM blocks FROM blocks
WHERE height > :start_height", WHERE height > :start_height",
named_params![":start_height": u32::from(start_height)], named_params![":start_height": u32::from(birthday_height)],
|row| row.get::<_, Option<u64>>(0), |row| row.get::<_, Option<u64>>(0),
)?; )?;
@ -992,22 +1035,22 @@ impl ScanProgress for SubtreeScanProgress {
FROM orchard_tree_shards FROM orchard_tree_shards
WHERE subtree_end_height > :start_height WHERE subtree_end_height > :start_height
OR subtree_end_height IS NULL", OR subtree_end_height IS NULL",
named_params![":start_height": u32::from(start_height)], named_params![":start_height": u32::from(birthday_height)],
|row| { |row| {
let min_tree_size = row let min_tree_size = row
.get::<_, Option<u64>>(0)? .get::<_, Option<u64>>(0)?
.map(|min| min << ORCHARD_SHARD_HEIGHT); .map(|min_idx| min_idx << ORCHARD_SHARD_HEIGHT);
let max_idx = row.get::<_, Option<u64>>(1)?; let max_tree_size = row
Ok(start_size .get::<_, Option<u64>>(1)?
.or(min_tree_size) .map(|max_idx| (max_idx + 1) << ORCHARD_SHARD_HEIGHT);
.zip(max_idx) Ok(start_size.or(min_tree_size).zip(max_tree_size).map(
.map(|(min_tree_size, max)| { |(min_tree_size, max_tree_size)| {
let max_tree_size = (max + 1) << ORCHARD_SHARD_HEIGHT;
Ratio::new( Ratio::new(
scanned_count.unwrap_or(0), scanned_count.unwrap_or(0),
max_tree_size - min_tree_size, max_tree_size - min_tree_size,
) )
})) },
))
}, },
) )
.optional()? .optional()?
@ -3060,6 +3103,7 @@ mod tests {
not_our_value, not_our_value,
0, 0,
0, 0,
false,
); );
let (mid_height, _, _) = let (mid_height, _, _) =
st.generate_next_block(&not_our_key, AddressType::DefaultExternal, not_our_value); st.generate_next_block(&not_our_key, AddressType::DefaultExternal, not_our_value);

17
zcash_client_sqlite/src/wallet/init.rs
View File

@ -370,8 +370,23 @@ mod tests {
sapling_fvk_item_cache BLOB, sapling_fvk_item_cache BLOB,
p2pkh_fvk_item_cache BLOB, p2pkh_fvk_item_cache BLOB,
birthday_height INTEGER NOT NULL, birthday_height INTEGER NOT NULL,
birthday_sapling_tree_size INTEGER,
birthday_orchard_tree_size INTEGER,
recover_until_height INTEGER, recover_until_height INTEGER,
CHECK ( (account_kind = 0 AND hd_seed_fingerprint IS NOT NULL AND hd_account_index IS NOT NULL AND ufvk IS NOT NULL) OR (account_kind = 1 AND hd_seed_fingerprint IS NULL AND hd_account_index IS NULL) ) CHECK (
(
account_kind = 0
AND hd_seed_fingerprint IS NOT NULL
AND hd_account_index IS NOT NULL
AND ufvk IS NOT NULL
)
OR
(
account_kind = 1
AND hd_seed_fingerprint IS NULL
AND hd_account_index IS NULL
)
)
)"#, )"#,
r#"CREATE TABLE "addresses" ( r#"CREATE TABLE "addresses" (
account_id INTEGER NOT NULL, account_id INTEGER NOT NULL,

8
zcash_client_sqlite/src/wallet/init/migrations.rs
View File

@ -56,10 +56,10 @@ pub(super) fn all_migrations<P: consensus::Parameters + 'static>(
// v_sapling_shard_unscanned_ranges \ | v_tx_outputs_use_legacy_false // v_sapling_shard_unscanned_ranges \ | v_tx_outputs_use_legacy_false
// | \ | | // | \ | |
// wallet_summaries \ | v_transactions_shielding_balance // wallet_summaries \ | v_transactions_shielding_balance
// \ | | // \ \ | |
// \ | v_transactions_note_uniqueness // \ \ | v_transactions_note_uniqueness
// \ | / // \ \ | /
// full_account_ids // -------------------- full_account_ids
// | // |
// orchard_received_notes // orchard_received_notes
vec![ vec![

57
zcash_client_sqlite/src/wallet/init/migrations/full_account_ids.rs
View File

@ -1,7 +1,7 @@
use std::{collections::HashSet, rc::Rc}; use std::{collections::HashSet, rc::Rc};
use crate::wallet::{account_kind_code, init::WalletMigrationError}; use crate::wallet::{account_kind_code, init::WalletMigrationError};
use rusqlite::{named_params, Transaction}; use rusqlite::{named_params, OptionalExtension, Transaction};
use schemer_rusqlite::RusqliteMigration; use schemer_rusqlite::RusqliteMigration;
use secrecy::{ExposeSecret, SecretVec}; use secrecy::{ExposeSecret, SecretVec};
use uuid::Uuid; use uuid::Uuid;
@ -10,7 +10,9 @@ use zcash_keys::keys::UnifiedFullViewingKey;
use zcash_primitives::consensus; use zcash_primitives::consensus;
use zip32::fingerprint::SeedFingerprint; use zip32::fingerprint::SeedFingerprint;
use super::{add_account_birthdays, receiving_key_scopes, v_transactions_note_uniqueness}; use super::{
add_account_birthdays, receiving_key_scopes, v_transactions_note_uniqueness, wallet_summaries,
};
/// The migration that switched from presumed seed-derived account IDs to supporting /// The migration that switched from presumed seed-derived account IDs to supporting
/// HD accounts and all sorts of imported keys. /// HD accounts and all sorts of imported keys.
@ -31,6 +33,7 @@ impl<P: consensus::Parameters> schemer::Migration for Migration<P> {
receiving_key_scopes::MIGRATION_ID, receiving_key_scopes::MIGRATION_ID,
add_account_birthdays::MIGRATION_ID, add_account_birthdays::MIGRATION_ID,
v_transactions_note_uniqueness::MIGRATION_ID, v_transactions_note_uniqueness::MIGRATION_ID,
wallet_summaries::MIGRATION_ID,
] ]
.into_iter() .into_iter()
.collect() .collect()
@ -50,8 +53,8 @@ impl<P: consensus::Parameters> RusqliteMigration for Migration<P> {
account_index: zip32::AccountId::ZERO, account_index: zip32::AccountId::ZERO,
}); });
let account_kind_imported = account_kind_code(AccountSource::Imported); let account_kind_imported = account_kind_code(AccountSource::Imported);
transaction.execute_batch( transaction.execute_batch(&format!(
&format!(r#" r#"
PRAGMA foreign_keys = OFF; PRAGMA foreign_keys = OFF;
PRAGMA legacy_alter_table = ON; PRAGMA legacy_alter_table = ON;
@ -66,18 +69,29 @@ impl<P: consensus::Parameters> RusqliteMigration for Migration<P> {
sapling_fvk_item_cache BLOB, sapling_fvk_item_cache BLOB,
p2pkh_fvk_item_cache BLOB, p2pkh_fvk_item_cache BLOB,
birthday_height INTEGER NOT NULL, birthday_height INTEGER NOT NULL,
birthday_sapling_tree_size INTEGER,
birthday_orchard_tree_size INTEGER,
recover_until_height INTEGER, recover_until_height INTEGER,
CHECK ( CHECK (
(account_kind = {account_kind_derived} AND hd_seed_fingerprint IS NOT NULL AND hd_account_index IS NOT NULL AND ufvk IS NOT NULL) (
account_kind = {account_kind_derived}
AND hd_seed_fingerprint IS NOT NULL
AND hd_account_index IS NOT NULL
AND ufvk IS NOT NULL
)
OR OR
(account_kind = {account_kind_imported} AND hd_seed_fingerprint IS NULL AND hd_account_index IS NULL) (
account_kind = {account_kind_imported}
AND hd_seed_fingerprint IS NULL
AND hd_account_index IS NULL
)
) )
); );
CREATE UNIQUE INDEX hd_account ON accounts_new (hd_seed_fingerprint, hd_account_index); CREATE UNIQUE INDEX hd_account ON accounts_new (hd_seed_fingerprint, hd_account_index);
CREATE UNIQUE INDEX accounts_uivk ON accounts_new (uivk); CREATE UNIQUE INDEX accounts_uivk ON accounts_new (uivk);
CREATE UNIQUE INDEX accounts_ufvk ON accounts_new (ufvk); CREATE UNIQUE INDEX accounts_ufvk ON accounts_new (ufvk);
"#), "#
)?; ))?;
// We require the seed *if* there are existing accounts in the table. // We require the seed *if* there are existing accounts in the table.
if transaction.query_row("SELECT COUNT(*) FROM accounts", [], |row| { if transaction.query_row("SELECT COUNT(*) FROM accounts", [], |row| {
@ -158,19 +172,40 @@ impl<P: consensus::Parameters> RusqliteMigration for Migration<P> {
#[cfg(not(feature = "transparent-inputs"))] #[cfg(not(feature = "transparent-inputs"))]
let transparent_item: Option<Vec<u8>> = None; let transparent_item: Option<Vec<u8>> = None;
// Get the tree sizes for the birthday height from the blocks table, if
// available.
let (birthday_sapling_tree_size, birthday_orchard_tree_size) = transaction
.query_row(
"SELECT sapling_commitment_tree_size - sapling_output_count,
orchard_commitment_tree_size - orchard_action_count
FROM blocks
WHERE height = :birthday_height",
named_params![":birthday_height": birthday_height],
|row| {
Ok(row
.get::<_, Option<u32>>(0)?
.zip(row.get::<_, Option<u32>>(1)?))
},
)
.optional()?
.flatten()
.map_or((None, None), |(s, o)| (Some(s), Some(o)));
transaction.execute( transaction.execute(
r#" r#"
INSERT INTO accounts_new ( INSERT INTO accounts_new (
id, account_kind, hd_seed_fingerprint, hd_account_index, id, account_kind, hd_seed_fingerprint, hd_account_index,
ufvk, uivk, ufvk, uivk,
orchard_fvk_item_cache, sapling_fvk_item_cache, p2pkh_fvk_item_cache, orchard_fvk_item_cache, sapling_fvk_item_cache, p2pkh_fvk_item_cache,
birthday_height, recover_until_height birthday_height, birthday_sapling_tree_size, birthday_orchard_tree_size,
recover_until_height
) )
VALUES ( VALUES (
:account_id, :account_kind, :seed_id, :account_index, :account_id, :account_kind, :seed_id, :account_index,
:ufvk, :uivk, :ufvk, :uivk,
:orchard_fvk_item_cache, :sapling_fvk_item_cache, :p2pkh_fvk_item_cache, :orchard_fvk_item_cache, :sapling_fvk_item_cache, :p2pkh_fvk_item_cache,
:birthday_height, :recover_until_height :birthday_height, :birthday_sapling_tree_size, :birthday_orchard_tree_size,
:recover_until_height
); );
"#, "#,
named_params![ named_params![
@ -184,6 +219,8 @@ impl<P: consensus::Parameters> RusqliteMigration for Migration<P> {
":sapling_fvk_item_cache": ufvk_parsed.sapling().map(|k| k.to_bytes()), ":sapling_fvk_item_cache": ufvk_parsed.sapling().map(|k| k.to_bytes()),
":p2pkh_fvk_item_cache": transparent_item, ":p2pkh_fvk_item_cache": transparent_item,
":birthday_height": birthday_height, ":birthday_height": birthday_height,
":birthday_sapling_tree_size": birthday_sapling_tree_size,
":birthday_orchard_tree_size": birthday_orchard_tree_size,
":recover_until_height": recover_until_height, ":recover_until_height": recover_until_height,
], ],
)?; )?;

183
zcash_client_sqlite/src/wallet/scanning.rs
View File

@ -594,7 +594,6 @@ pub(crate) mod tests {
consensus::{BlockHeight, NetworkUpgrade, Parameters}, consensus::{BlockHeight, NetworkUpgrade, Parameters},
transaction::components::amount::NonNegativeAmount, transaction::components::amount::NonNegativeAmount,
}; };
use zcash_protocol::ShieldedProtocol;
use crate::{ use crate::{
error::SqliteClientError, error::SqliteClientError,
@ -610,10 +609,7 @@ pub(crate) mod tests {
}; };
#[cfg(feature = "orchard")] #[cfg(feature = "orchard")]
use { use {crate::wallet::orchard::tests::OrchardPoolTester, orchard::tree::MerkleHashOrchard};
crate::wallet::orchard::tests::OrchardPoolTester, orchard::tree::MerkleHashOrchard,
zcash_client_backend::data_api::ORCHARD_SHARD_HEIGHT,
};
#[test] #[test]
fn sapling_scan_complete() { fn sapling_scan_complete() {
@ -703,6 +699,7 @@ pub(crate) mod tests {
value, value,
initial_sapling_tree_size, initial_sapling_tree_size,
initial_orchard_tree_size, initial_orchard_tree_size,
false,
); );
for _ in 1..=10 { for _ in 1..=10 {
@ -1111,6 +1108,7 @@ pub(crate) mod tests {
NonNegativeAmount::const_from_u64(10000), NonNegativeAmount::const_from_u64(10000),
frontier_tree_size + 10, frontier_tree_size + 10,
frontier_tree_size + 10, frontier_tree_size + 10,
false,
); );
st.scan_cached_blocks(max_scanned, 1); st.scan_cached_blocks(max_scanned, 1);
@ -1188,9 +1186,7 @@ pub(crate) mod tests {
assert_eq!(actual, expected); assert_eq!(actual, expected);
} }
// FIXME: This requires fixes to the test framework.
#[test] #[test]
#[cfg(feature = "orchard")]
fn sapling_update_chain_tip_stable_max_scanned() { fn sapling_update_chain_tip_stable_max_scanned() {
update_chain_tip_stable_max_scanned::<SaplingPoolTester>(); update_chain_tip_stable_max_scanned::<SaplingPoolTester>();
} }
@ -1201,36 +1197,74 @@ pub(crate) mod tests {
update_chain_tip_stable_max_scanned::<OrchardPoolTester>(); update_chain_tip_stable_max_scanned::<OrchardPoolTester>();
} }
// FIXME: This requires fixes to the test framework.
#[allow(dead_code)]
fn update_chain_tip_stable_max_scanned<T: ShieldedPoolTester>() { fn update_chain_tip_stable_max_scanned<T: ShieldedPoolTester>() {
use ScanPriority::*; use ScanPriority::*;
// Use a non-zero birthday offset because Sapling and NU5 are activated at the same height.
let (mut st, dfvk, birthday, sap_active) =
test_with_nu5_birthday_offset::<T>(76, BlockHash([0; 32]));
// Set up the following situation: // Set up the following situation:
// //
// prior_tip new_tip // prior_tip new_tip
// |<--- 500 --->|<- 20 ->|<-- 50 -->|<- 20 ->| // |<--- 500 --->|<- 20 ->|<-- 50 -->|<- 20 ->|
// wallet_birthday max_scanned last_shard_start // wallet_birthday max_scanned last_shard_start
// //
let max_scanned = birthday.height() + 500; let birthday_offset = 76;
let prior_tip = max_scanned + 20; let birthday_prior_block_hash = BlockHash([0; 32]);
// We set the Sapling and Orchard frontiers at the birthday block initial state to 1234
// notes beyond the end of the first shard.
let frontier_tree_size: u32 = (0x1 << 16) + 1234;
let mut st = TestBuilder::new()
.with_block_cache()
.with_initial_chain_state(|rng, network| {
let birthday_height =
network.activation_height(NetworkUpgrade::Nu5).unwrap() + birthday_offset;
// Set up some shard root history before the wallet birthday. // Construct a fake chain state for the end of the block with the given
let second_to_last_shard_start = birthday.height() - 1000; // birthday_offset from the Nu5 birthday.
T::put_subtree_roots( let (prior_sapling_roots, sapling_initial_tree) =
&mut st, Frontier::random_with_prior_subtree_roots(
0, rng,
&[CommitmentTreeRoot::from_parts( frontier_tree_size.into(),
second_to_last_shard_start, NonZeroU8::new(16).unwrap(),
// fake a hash, the value doesn't matter );
T::empty_tree_leaf(), // There will only be one prior root
)], let prior_sapling_roots = prior_sapling_roots
) .into_iter()
.unwrap(); .map(|root| CommitmentTreeRoot::from_parts(birthday_height - 10, root))
.collect::<Vec<_>>();
#[cfg(feature = "orchard")]
let (prior_orchard_roots, orchard_initial_tree) =
Frontier::random_with_prior_subtree_roots(
rng,
frontier_tree_size.into(),
NonZeroU8::new(16).unwrap(),
);
// There will only be one prior root
#[cfg(feature = "orchard")]
let prior_orchard_roots = prior_orchard_roots
.into_iter()
.map(|root| CommitmentTreeRoot::from_parts(birthday_height - 10, root))
.collect::<Vec<_>>();
InitialChainState {
chain_state: ChainState::new(
birthday_height - 1,
birthday_prior_block_hash,
sapling_initial_tree,
#[cfg(feature = "orchard")]
orchard_initial_tree,
),
prior_sapling_roots,
#[cfg(feature = "orchard")]
prior_orchard_roots,
}
})
.with_account_having_current_birthday()
.build();
let account = st.test_account().cloned().unwrap();
let dfvk = T::test_account_fvk(&st);
let birthday = account.birthday();
let sap_active = st.sapling_activation_height();
// We have scan ranges and a subtree, but have scanned no blocks. // We have scan ranges and a subtree, but have scanned no blocks.
let summary = st.get_wallet_summary(1); let summary = st.get_wallet_summary(1);
@ -1238,67 +1272,56 @@ pub(crate) mod tests {
// Set up prior chain state. This simulates us having imported a wallet // Set up prior chain state. This simulates us having imported a wallet
// with a birthday 520 blocks below the chain tip. // with a birthday 520 blocks below the chain tip.
let max_scanned = birthday.height() + 500;
let prior_tip = max_scanned + 20;
st.wallet_mut().update_chain_tip(prior_tip).unwrap(); st.wallet_mut().update_chain_tip(prior_tip).unwrap();
// Verify that the suggested scan ranges match what is expected. // Verify that the suggested scan ranges match what is expected.
let expected = vec![ let expected = vec![
scan_range(birthday.height().into()..(prior_tip + 1).into(), ChainTip), scan_range(birthday.height().into()..(prior_tip + 1).into(), ChainTip),
scan_range(sap_active..birthday.height().into(), Ignored), scan_range(sap_active.into()..birthday.height().into(), Ignored),
]; ];
let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap();
assert_eq!(actual, expected); assert_eq!(actual, expected);
// Now, scan the max scanned block. // Simulate that in the blocks between the wallet birthday and the max_scanned height,
let initial_sapling_tree_size = birthday // there are 10 Sapling notes and 10 Orchard notes created on the chain.
.sapling_frontier()
.value()
.map(|f| u64::from(f.position() + 1))
.unwrap_or(0)
.try_into()
.unwrap();
#[cfg(feature = "orchard")]
let initial_orchard_tree_size = birthday
.orchard_frontier()
.value()
.map(|f| u64::from(f.position() + 1))
.unwrap_or(0)
.try_into()
.unwrap();
#[cfg(not(feature = "orchard"))]
let initial_orchard_tree_size = 0;
st.generate_block_at( st.generate_block_at(
max_scanned, max_scanned,
BlockHash([0u8; 32]), BlockHash([1; 32]),
&dfvk, &dfvk,
AddressType::DefaultExternal, AddressType::DefaultExternal,
NonNegativeAmount::const_from_u64(10000), NonNegativeAmount::const_from_u64(10000),
initial_sapling_tree_size, frontier_tree_size + 10,
initial_orchard_tree_size, frontier_tree_size + 10,
false,
); );
st.scan_cached_blocks(max_scanned, 1); st.scan_cached_blocks(max_scanned, 1);
// We have scanned a block, so we now have a starting tree position, 500 blocks above the // We have scanned a block, so we now have a starting tree position, 500 blocks above the
// wallet birthday but before the end of the shard. // wallet birthday but before the end of the shard.
let summary = st.get_wallet_summary(1); let summary = st.get_wallet_summary(1);
assert_eq!(summary.as_ref().map(|s| T::next_subtree_index(s)), Some(0),); assert_eq!(summary.as_ref().map(|s| T::next_subtree_index(s)), Some(0));
// Progress denominator depends on which pools are enabled (which changes the // Progress denominator depends on which pools are enabled (which changes the
// initial tree states in `test_with_nu5_birthday_offset`). // initial tree states). Here we compute the denominator based upon the fact that
let expected_denom = 1 << SAPLING_SHARD_HEIGHT; // the trees are the same size at present.
let expected_denom = (1 << SAPLING_SHARD_HEIGHT) * 2 - frontier_tree_size;
#[cfg(feature = "orchard")] #[cfg(feature = "orchard")]
let expected_denom = expected_denom + (1 << ORCHARD_SHARD_HEIGHT); let expected_denom = expected_denom * 2;
assert_eq!( assert_eq!(
summary.and_then(|s| s.scan_progress()), summary.and_then(|s| s.scan_progress()),
Some(Ratio::new(1, expected_denom)) Some(Ratio::new(1, u64::from(expected_denom)))
); );
// Now simulate shutting down, and then restarting 70 blocks later, after a shard // Now simulate shutting down, and then restarting 70 blocks later, after a shard
// has been completed. // has been completed in one pool. This shard will have index 2, as our birthday
// was in shard 1.
let last_shard_start = prior_tip + 50; let last_shard_start = prior_tip + 50;
T::put_subtree_roots( T::put_subtree_roots(
&mut st, &mut st,
0, 2,
&[CommitmentTreeRoot::from_parts( &[CommitmentTreeRoot::from_parts(
last_shard_start, last_shard_start,
// fake a hash, the value doesn't matter // fake a hash, the value doesn't matter
@ -1307,6 +1330,36 @@ pub(crate) mod tests {
) )
.unwrap(); .unwrap();
{
let mut shard_stmt = st
.wallet_mut()
.conn
.prepare("SELECT shard_index, subtree_end_height FROM sapling_tree_shards")
.unwrap();
(shard_stmt
.query_and_then::<_, rusqlite::Error, _, _>([], |row| {
Ok((row.get::<_, u32>(0)?, row.get::<_, Option<u32>>(1)?))
})
.unwrap()
.collect::<Result<Vec<_>, _>>())
.unwrap();
}
{
let mut shard_stmt = st
.wallet_mut()
.conn
.prepare("SELECT shard_index, subtree_end_height FROM orchard_tree_shards")
.unwrap();
(shard_stmt
.query_and_then::<_, rusqlite::Error, _, _>([], |row| {
Ok((row.get::<_, u32>(0)?, row.get::<_, Option<u32>>(1)?))
})
.unwrap()
.collect::<Result<Vec<_>, _>>())
.unwrap();
}
let new_tip = last_shard_start + 20; let new_tip = last_shard_start + 20;
st.wallet_mut().update_chain_tip(new_tip).unwrap(); st.wallet_mut().update_chain_tip(new_tip).unwrap();
let chain_end = u32::from(new_tip + 1); let chain_end = u32::from(new_tip + 1);
@ -1320,26 +1373,20 @@ pub(crate) mod tests {
// The max scanned block itself is left as-is. // The max scanned block itself is left as-is.
scan_range(max_scanned.into()..(max_scanned + 1).into(), Scanned), scan_range(max_scanned.into()..(max_scanned + 1).into(), Scanned),
// The range below the second-to-last shard is ignored. // The range below the second-to-last shard is ignored.
scan_range(sap_active..birthday.height().into(), Ignored), scan_range(sap_active.into()..birthday.height().into(), Ignored),
]; ];
let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap();
assert_eq!(actual, expected); assert_eq!(actual, expected);
// We've crossed a subtree boundary, and so still only have one scanned note but have two // We've crossed a subtree boundary, but only in one pool. We still only have one scanned
// shards worth of notes to scan. // note but in the pool where we crossed the subtree boundary we have two shards worth of
let expected_denom = expected_denom // notes to scan.
+ match T::SHIELDED_PROTOCOL { let expected_denom = expected_denom + (1 << 16);
ShieldedProtocol::Sapling => 1 << SAPLING_SHARD_HEIGHT,
#[cfg(feature = "orchard")]
ShieldedProtocol::Orchard => 1 << ORCHARD_SHARD_HEIGHT,
#[cfg(not(feature = "orchard"))]
ShieldedProtocol::Orchard => unreachable!(),
};
let summary = st.get_wallet_summary(1); let summary = st.get_wallet_summary(1);
assert_eq!( assert_eq!(
summary.and_then(|s| s.scan_progress()), summary.and_then(|s| s.scan_progress()),
Some(Ratio::new(1, expected_denom)) Some(Ratio::new(1, u64::from(expected_denom)))
); );
} }