zec
/
zcash-devtool
mirror of https://github.com/zcash/zcash-devtool.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

sync: Refresh UTXOs at the start of each scanning cycle 

The new `transparent-inputs` feature flag is default-enabled because
most testing of the Rust crates that we want to do will be with that
flag enabled, and anyone using this tool to investigate existing wallet
databases will also need it enabled. But the flag can be disabled for
testing shielded-only workflows.

Closes Electric-Coin-Company/zec-sqlite-cli#21.
This commit is contained in:
Jack Grigg 2024-05-20 19:29:55 +00:00
parent 07d1b58103
commit 97fc031013
4 changed files with 199 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
Cargo.lock generated
View File

@ -924,6 +924,19 @@ dependencies = [
"hashbrown 0.14.3",
]
[[package]]
name = "hdwallet"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a03ba7d4c9ea41552cd4351965ff96883e629693ae85005c501bb4b9e1c48a7"
dependencies = [
"lazy_static",
"rand_core",
"ring 0.16.20",
"secp256k1",
"thiserror",
]
[[package]]
name = "heck"
version = "0.4.1"
@ -1824,6 +1837,21 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56"
[[package]]
name = "ring"
version = "0.16.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"
dependencies = [
"cc",
"libc",
"once_cell",
"spin 0.5.2",
"untrusted 0.7.1",
"web-sys",
"winapi",
]
[[package]]
name = "ring"
version = "0.17.8"
@ -1835,10 +1863,19 @@ dependencies = [
"getrandom",
"libc",
"spin 0.9.8",
"untrusted",
"untrusted 0.9.0",
"windows-sys 0.52.0",
]
[[package]]
name = "ripemd"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd124222d17ad93a644ed9d011a40f4fb64aa54275c08cc216524a9ea82fb09f"
dependencies = [
"digest",
]
[[package]]
name = "roaring"
version = "0.10.4"
@ -1899,7 +1936,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4"
dependencies = [
"log",
"ring",
"ring 0.17.8",
"rustls-webpki",
"sct",
]
@ -1919,8 +1956,8 @@ version = "0.101.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
dependencies = [
"ring",
"untrusted",
"ring 0.17.8",
"untrusted 0.9.0",
]
[[package]]
@ -2002,8 +2039,26 @@ version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414"
dependencies = [
"ring",
"untrusted",
"ring 0.17.8",
"untrusted 0.9.0",
]
[[package]]
name = "secp256k1"
version = "0.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4124a35fe33ae14259c490fd70fa199a32b9ce9502f2ee6bc4f81ec06fa65894"
dependencies = [
"secp256k1-sys",
]
[[package]]
name = "secp256k1-sys"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70a129b9e9efbfb223753b9163c4ab3b13cff7fd9c7f010fbac25ab4099fa07e"
dependencies = [
"cc",
]
[[package]]
@ -2726,6 +2781,12 @@ dependencies = [
"subtle",
]
[[package]]
name = "untrusted"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
[[package]]
name = "untrusted"
version = "0.9.0"
@ -2825,6 +2886,16 @@ version = "0.2.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
[[package]]
name = "web-sys"
version = "0.3.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webpki-roots"
version = "0.25.4"
@ -3053,6 +3124,7 @@ dependencies = [
"crossbeam-channel",
"document-features",
"group",
"hdwallet",
"hex",
"incrementalmerkletree",
"memuse",
@ -3091,6 +3163,7 @@ dependencies = [
"byteorder",
"document-features",
"group",
"hdwallet",
"incrementalmerkletree",
"jubjub",
"maybe-rayon",
@ -3137,6 +3210,7 @@ dependencies = [
"byteorder",
"document-features",
"group",
"hdwallet",
"memuse",
"nonempty",
"orchard",
@ -3179,6 +3253,7 @@ dependencies = [
"ff",
"fpe",
"group",
"hdwallet",
"hex",
"incrementalmerkletree",
"jubjub",
@ -3188,7 +3263,9 @@ dependencies = [
"rand",
"rand_core",
"redjubjub",
"ripemd",
"sapling-crypto",
"secp256k1",
"sha2",
"subtle",
"tracing",

5
Cargo.toml
View File

@ -39,7 +39,10 @@ tokio-util = { version = "0.7", optional = true }
tui-logger = { version = "0.11", optional = true, features = ["tracing-support"] }
[features]
default = []
default = ["transparent-inputs"]
transparent-inputs = [
"zcash_client_sqlite/transparent-inputs",
]
tui = [
"dep:crossterm",
"dep:ratatui",

2
src/commands/balance.rs
View File

@ -54,6 +54,8 @@ impl Command {
" Orchard Spendable: {}",
format_zec(balance.orchard_balance().spendable_value())
);
#[cfg(feature = "transparent-inputs")]
println!(" Unshielded: {}", format_zec(balance.unshielded()));
} else {
println!("Insufficient information to build a wallet summary.");
}

117
src/commands/sync.rs
View File

@ -31,8 +31,22 @@ use crate::{
ShutdownListener,
};
#[cfg(feature = "transparent-inputs")]
use {
zcash_client_backend::{encoding::AddressCodec, wallet::WalletTransparentOutput},
zcash_client_sqlite::AccountId,
zcash_primitives::{
legacy::Script,
transaction::components::transparent::{OutPoint, TxOut},
},
zcash_protocol::value::Zatoshis,
};
#[cfg(any(feature = "transparent-inputs", feature = "tui"))]
use zcash_protocol::consensus::NetworkUpgrade;
#[cfg(feature = "tui")]
use {crate::tui::Tui, zcash_protocol::consensus::NetworkUpgrade};
use crate::tui::Tui;
#[cfg(feature = "tui")]
mod defrag;
@ -68,14 +82,14 @@ impl Command {
let mut db_data = WalletDb::for_path(db_data, params)?;
let mut client = connect_to_lightwalletd(self.server.pick(params)?).await?;
#[cfg(any(feature = "transparent-inputs", feature = "tui"))]
let wallet_birthday = db_data
.get_wallet_birthday()?
.unwrap_or_else(|| params.activation_height(NetworkUpgrade::Sapling).unwrap());
#[cfg(feature = "tui")]
let tui_handle = if self.defrag {
let mut app = defrag::App::new(
shutdown.tui_quit_signal(),
db_data
.get_wallet_birthday()?
.unwrap_or_else(|| params.activation_height(NetworkUpgrade::Sapling).unwrap()),
);
let mut app = defrag::App::new(shutdown.tui_quit_signal(), wallet_birthday);
let handle = app.handle();
tokio::spawn(async move {
if let Err(e) = app.run(tui).await {
@ -98,12 +112,29 @@ impl Command {
fsblockdb_root: &Path,
db_cache: &mut FsBlockDb,
db_data: &mut WalletDb<rusqlite::Connection, P>,
#[cfg(feature = "transparent-inputs")] wallet_birthday: BlockHeight,
#[cfg(feature = "tui")] tui_handle: Option<&defrag::AppHandle>,
) -> Result<bool, anyhow::Error> {
// 3) Download chain tip metadata from lightwalletd
// 4) Notify the wallet of the updated chain tip.
let _chain_tip = update_chain_tip(client, db_data).await?;
// Refresh UTXOs for the accounts in the wallet. We do this before we perform
// any shielded scanning, to ensure that we discover any UTXOs between the old
// fully-scanned height and the current chain tip.
#[cfg(feature = "transparent-inputs")]
for account_id in db_data.get_account_ids()? {
let start_height = db_data
.block_fully_scanned()?
.map(|meta| meta.block_height())
.unwrap_or(wallet_birthday);
info!(
"Refreshing UTXOs for {:?} from height {}",
account_id, start_height,
);
refresh_utxos(params, client, db_data, account_id, start_height).await?;
}
// 5) Get the suggested scan ranges from the wallet database
info!("Fetching scan ranges");
let mut scan_ranges = db_data.suggest_scan_ranges()?;
@ -293,6 +324,8 @@ impl Command {
fsblockdb_root,
&mut db_cache,
&mut db_data,
#[cfg(feature = "transparent-inputs")]
wallet_birthday,
#[cfg(feature = "tui")]
tui_handle.as_ref(),
)
@ -565,3 +598,73 @@ fn scan_blocks<P: Parameters + Send + 'static>(
Err(e) => Err(anyhow!("{:?}", e)),
}
}
/// Refreshes the given account's view of UTXOs that exist starting at the given height.
///
/// ## Note about UTXO tracking
///
/// (Extracted from [a comment in the Android SDK].)
///
/// We no longer clear UTXOs here, as `WalletDb::put_received_transparent_utxo` now uses
/// an upsert instead of an insert. This means that now-spent UTXOs would previously have
/// been deleted, but now are left in the database (like shielded notes).
///
/// Due to the fact that the `lightwalletd` query only returns _current_ UTXOs, we don't
/// learn about recently-spent UTXOs here, so the transparent balance does not get updated
/// here.
///
/// Instead, when a received shielded note is "enhanced" by downloading the full
/// transaction, we mark any UTXOs spent in that transaction as spent in the database.
/// This relies on two current properties:
/// - UTXOs are only ever spent in shielding transactions.
/// - At least one shielded note from each shielding transaction is always enhanced.
///
/// However, for greater reliability, we may want to alter the Data Access API to support
/// "inferring spentness" from what is _not_ returned as a UTXO, or alternatively fetch
/// TXOs from `lightwalletd` instead of just UTXOs.
///
/// [a comment in the Android SDK]: https://github.com/Electric-Coin-Company/zcash-android-wallet-sdk/blob/855204fc8ae4057fdac939f98df4aa38c8e662f1/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt#L979-L991
#[cfg(feature = "transparent-inputs")]
async fn refresh_utxos<P: Parameters>(
params: &P,
client: &mut CompactTxStreamerClient<Channel>,
db_data: &mut WalletDb<rusqlite::Connection, P>,
account_id: AccountId,
start_height: BlockHeight,
) -> Result<(), anyhow::Error> {
let request = service::GetAddressUtxosArg {
addresses: db_data
.get_transparent_receivers(account_id)?
.into_keys()
.map(|addr| addr.encode(params))
.collect(),
start_height: start_height.into(),
max_entries: 0,
};
client
.get_address_utxos_stream(request)
.await?
.into_inner()
.map_err(anyhow::Error::from)
.and_then(|reply| async move {
WalletTransparentOutput::from_parts(
OutPoint::new(reply.txid[..].try_into()?, reply.index.try_into()?),
TxOut {
value: Zatoshis::from_nonnegative_i64(reply.value_zat)?,
script_pubkey: Script(reply.script),
},
BlockHeight::from(u32::try_from(reply.height)?),
)
.ok_or(anyhow!(
"Received UTXO that doesn't correspond to a valid P2PKH or P2SH address"
))
})
.try_for_each(|output| {
let res = db_data.put_received_transparent_utxo(&output).map(|_| ());
async move { res.map_err(anyhow::Error::from) }
})
.await?;
Ok(())
}