zec
/
librustzcash
mirror of https://github.com/zcash/librustzcash.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #657 from nuttycom/wallet/upsert_utxos 

Use upsert functionality for transparent UTXOs, rather than delete/repopulate.
This commit is contained in:
str4d 2022-10-11 04:17:49 +01:00 committed by GitHub
parent 709577cec7 6df6fec860
commit 1dc3cfe724
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 319 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
zcash_client_backend/src/wallet.rs
View File

@ -31,6 +31,7 @@ pub struct WalletTx<N> {
#[cfg(feature = "transparent-inputs")]
pub struct WalletTransparentOutput {
pub received_by_account: AccountId,
pub outpoint: OutPoint,
pub txout: TxOut,
pub height: BlockHeight,

5
zcash_client_sqlite/CHANGELOG.md
View File

@ -17,7 +17,7 @@ and this library adheres to Rust's notion of
- `SqliteClientError::RequestedRewindInvalid`, to report when requested
rewinds exceed supported bounds.
- `SqliteClientError::DiversifierIndexOutOfRange`, to report when the space
of available diversifier indices has been exhausted.
of available diversifier indices has been exhausted.
- `SqliteClientError::AccountIdDiscontinuity`, to report when a user attempts
to initialize the accounts table with a noncontiguous set of account identifiers.
- `SqliteClientError::AccountIdOutOfRange`, to report when the maximum account
@ -93,6 +93,8 @@ and this library adheres to Rust's notion of
- `get_extended_full_viewing_keys` (use
`zcash_client_backend::data_api::WalletRead::get_unified_full_viewing_keys`
instead).
- `delete_utxos_above` (use
`zcash_client_backend::data_api::WalletWrite::rewind_to_height` instead)
- `zcash_client_sqlite::with_blocks` (use
`zcash_client_backend::data_api::BlockSource::with_blocks` instead)
@ -125,7 +127,6 @@ and this library adheres to Rust's notion of
- `put_tx_meta`
- `put_tx_data`
- `mark_sapling_note_spent`
- `delete_utxos_above`
- `put_receiverd_note`
- `insert_witness`
- `prune_witnesses`

4
zcash_client_sqlite/src/lib.rs
View File

@ -111,7 +111,7 @@ impl fmt::Display for NoteId {
/// A newtype wrapper for sqlite primary key values for the utxos
/// table.
#[derive(Debug, Copy, Clone)]
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub struct UtxoId(pub i64);
/// A wrapper for the SQLite connection to the wallet database.
@ -252,7 +252,7 @@ impl<P: consensus::Parameters> WalletReadTransparent for WalletDb<P> {
&self,
account: AccountId,
) -> Result<HashSet<TransparentAddress>, Self::Error> {
wallet::get_transparent_receivers(self, account)
wallet::get_transparent_receivers(&self.params, &self.conn, account)
}
fn get_unspent_transparent_outputs(

100
zcash_client_sqlite/src/prepared.rs
View File

@ -24,10 +24,10 @@ use crate::{error::SqliteClientError, wallet::PoolType, NoteId, WalletDb};
#[cfg(feature = "transparent-inputs")]
use {
crate::UtxoId,
rusqlite::{named_params, OptionalExtension},
zcash_client_backend::{encoding::AddressCodec, wallet::WalletTransparentOutput},
zcash_primitives::{
legacy::TransparentAddress, transaction::components::transparent::OutPoint,
},
zcash_primitives::transaction::components::transparent::OutPoint,
};
/// The primary type used to implement [`WalletWrite`] for the SQLite database.
@ -55,7 +55,7 @@ pub struct DataConnStmtCache<'a, P> {
#[cfg(feature = "transparent-inputs")]
stmt_insert_received_transparent_utxo: Statement<'a>,
#[cfg(feature = "transparent-inputs")]
stmt_delete_utxos: Statement<'a>,
stmt_update_received_transparent_utxo: Statement<'a>,
stmt_insert_received_note: Statement<'a>,
stmt_update_received_note: Statement<'a>,
stmt_select_received_note: Statement<'a>,
@ -112,12 +112,27 @@ impl<'a, P> DataConnStmtCache<'a, P> {
)?,
#[cfg(feature = "transparent-inputs")]
stmt_insert_received_transparent_utxo: wallet_db.conn.prepare(
"INSERT INTO utxos (address, prevout_txid, prevout_idx, script, value_zat, height)
VALUES (:address, :prevout_txid, :prevout_idx, :script, :value_zat, :height)"
"INSERT INTO utxos (
received_by_account, address,
prevout_txid, prevout_idx, script,
value_zat, height)
VALUES (
:received_by_account, :address,
:prevout_txid, :prevout_idx, :script,
:value_zat, :height)
RETURNING id_utxo"
)?,
#[cfg(feature = "transparent-inputs")]
stmt_delete_utxos: wallet_db.conn.prepare(
"DELETE FROM utxos WHERE address = :address AND height > :above_height"
stmt_update_received_transparent_utxo: wallet_db.conn.prepare(
"UPDATE utxos
SET received_by_account = :received_by_account,
height = :height,
address = :address,
script = :script,
value_zat = :value_zat
WHERE prevout_txid = :prevout_txid
AND prevout_idx = :prevout_idx
RETURNING id_utxo"
)?,
stmt_insert_received_note: wallet_db.conn.prepare(
"INSERT INTO received_notes (tx, output_index, account, diversifier, value, rcm, memo, nf, is_change)
@ -339,40 +354,53 @@ impl<'a, P: consensus::Parameters> DataConnStmtCache<'a, P> {
pub(crate) fn stmt_insert_received_transparent_utxo(
&mut self,
output: &WalletTransparentOutput,
) -> Result<i64, SqliteClientError> {
let sql_args: &[(&str, &dyn ToSql)] = &[
(":address", &output.address().encode(&self.wallet_db.params)),
(":prevout_txid", &output.outpoint.hash().to_vec()),
(":prevout_idx", &output.outpoint.n()),
(":script", &output.txout.script_pubkey.0),
(":value_zat", &i64::from(output.txout.value)),
(":height", &u32::from(output.height)),
];
) -> Result<UtxoId, SqliteClientError> {
self.stmt_insert_received_transparent_utxo
.execute(sql_args)?;
Ok(self.wallet_db.conn.last_insert_rowid())
.query_row(
named_params![
":received_by_account": &u32::from(output.received_by_account),
":address": &output.address().encode(&self.wallet_db.params),
":prevout_txid": &output.outpoint.hash().to_vec(),
":prevout_idx": &output.outpoint.n(),
":script": &output.txout.script_pubkey.0,
":value_zat": &i64::from(output.txout.value),
":height": &u32::from(output.height),
],
|row| {
let id = row.get(0)?;
Ok(UtxoId(id))
},
)
.map_err(SqliteClientError::from)
}
/// Removes all records of UTXOs that were recorded as having been received at block
/// heights greater than the given height.
/// Adds the given received UTXO to the datastore.
///
/// Returns the number of UTXOs that were removed.
/// Returns the database row for the newly-inserted UTXO, or an error if the UTXO
/// exists.
#[cfg(feature = "transparent-inputs")]
pub(crate) fn stmt_delete_utxos(
pub(crate) fn stmt_update_received_transparent_utxo(
&mut self,
taddr: &TransparentAddress,
height: BlockHeight,
) -> Result<usize, SqliteClientError> {
let sql_args: &[(&str, &dyn ToSql)] = &[
(":address", &taddr.encode(&self.wallet_db.params)),
(":above_height", &u32::from(height)),
];
let rows = self.stmt_delete_utxos.execute(sql_args)?;
Ok(rows)
output: &WalletTransparentOutput,
) -> Result<Option<UtxoId>, SqliteClientError> {
self.stmt_update_received_transparent_utxo
.query_row(
named_params![
":prevout_txid": &output.outpoint.hash().to_vec(),
":prevout_idx": &output.outpoint.n(),
":received_by_account": &u32::from(output.received_by_account),
":address": &output.address().encode(&self.wallet_db.params),
":script": &output.txout.script_pubkey.0,
":value_zat": &i64::from(output.txout.value),
":height": &u32::from(output.height),
],
|row| {
let id = row.get(0)?;
Ok(UtxoId(id))
},
)
.optional()
.map_err(SqliteClientError::from)
}
/// Adds the given address and diversifier index to the addresses table.

124
zcash_client_sqlite/src/wallet.rs
View File

@ -12,9 +12,6 @@ use rusqlite::{named_params, OptionalExtension, ToSql};
use std::collections::HashMap;
use std::convert::TryFrom;
#[cfg(feature = "transparent-inputs")]
use std::collections::HashSet;
use zcash_primitives::{
block::BlockHash,
consensus::{self, BlockHeight, BranchId, NetworkUpgrade, Parameters},
@ -44,7 +41,8 @@ use zcash_primitives::legacy::TransparentAddress;
#[cfg(feature = "transparent-inputs")]
use {
crate::UtxoId,
rusqlite::params,
rusqlite::{params, Connection},
std::collections::HashSet,
zcash_client_backend::{encoding::AddressCodec, wallet::WalletTransparentOutput},
zcash_primitives::{
legacy::{keys::IncomingViewingKey, Script},
@ -279,20 +277,19 @@ pub(crate) fn get_current_address<P: consensus::Parameters>(
#[cfg(feature = "transparent-inputs")]
pub(crate) fn get_transparent_receivers<P: consensus::Parameters>(
wdb: &WalletDb<P>,
params: &P,
conn: &Connection,
account: AccountId,
) -> Result<HashSet<TransparentAddress>, SqliteClientError> {
let mut ret = HashSet::new();
// Get all UAs derived
let mut ua_query = wdb
.conn
.prepare("SELECT address FROM addresses WHERE account = :account")?;
let mut ua_query = conn.prepare("SELECT address FROM addresses WHERE account = :account")?;
let mut rows = ua_query.query(named_params![":account": &u32::from(account)])?;
while let Some(row) = rows.next()? {
let ua_str: String = row.get(0)?;
let ua = RecipientAddress::decode(&wdb.params, &ua_str)
let ua = RecipientAddress::decode(params, &ua_str)
.ok_or_else(|| {
SqliteClientError::CorruptedData("Not a valid Zcash recipient address".to_owned())
})
@ -309,12 +306,12 @@ pub(crate) fn get_transparent_receivers<P: consensus::Parameters>(
}
// Get the UFVK for the account.
let ufvk_str: String = wdb.conn.query_row(
let ufvk_str: String = conn.query_row(
"SELECT ufvk FROM accounts WHERE account = :account",
[u32::from(account)],
|row| row.get(0),
)?;
let ufvk = UnifiedFullViewingKey::decode(&wdb.params, &ufvk_str)
let ufvk = UnifiedFullViewingKey::decode(params, &ufvk_str)
.map_err(SqliteClientError::CorruptedData)?;
// Derive the default transparent address (if it wasn't already part of a derived UA).
@ -923,7 +920,9 @@ pub(crate) fn get_unspent_transparent_outputs<P: consensus::Parameters>(
max_height: BlockHeight,
) -> Result<Vec<WalletTransparentOutput>, SqliteClientError> {
let mut stmt_blocks = wdb.conn.prepare(
"SELECT u.prevout_txid, u.prevout_idx, u.script, u.value_zat, u.height, tx.block as block
"SELECT u.received_by_account,
u.prevout_txid, u.prevout_idx, u.script,
u.value_zat, u.height, tx.block as block
FROM utxos u
LEFT OUTER JOIN transactions tx
ON tx.id_tx = u.spent_in_tx
@ -935,16 +934,19 @@ pub(crate) fn get_unspent_transparent_outputs<P: consensus::Parameters>(
let addr_str = address.encode(&wdb.params);
let rows = stmt_blocks.query_map(params![addr_str, u32::from(max_height)], |row| {
let id: Vec<u8> = row.get(0)?;
let received_by_account: u32 = row.get(0)?;
let txid: Vec<u8> = row.get(1)?;
let mut txid_bytes = [0u8; 32];
txid_bytes.copy_from_slice(&id);
let index: u32 = row.get(1)?;
let script_pubkey = Script(row.get(2)?);
let value = Amount::from_i64(row.get(3)?).unwrap();
let height: u32 = row.get(4)?;
txid_bytes.copy_from_slice(&txid);
let index: u32 = row.get(2)?;
let script_pubkey = Script(row.get(3)?);
let value = Amount::from_i64(row.get(4)?).unwrap();
let height: u32 = row.get(5)?;
Ok(WalletTransparentOutput {
received_by_account: AccountId::from(received_by_account),
outpoint: OutPoint::new(txid_bytes, index),
txout: TxOut {
value,
@ -1054,23 +1056,11 @@ pub(crate) fn put_received_transparent_utxo<'a, P: consensus::Parameters>(
stmts: &mut DataConnStmtCache<'a, P>,
output: &WalletTransparentOutput,
) -> Result<UtxoId, SqliteClientError> {
stmts
.stmt_insert_received_transparent_utxo(output)
.map(UtxoId)
}
/// Removes all records of UTXOs that were recorded as having been received
/// at block heights greater than the given height.
#[cfg(feature = "transparent-inputs")]
#[deprecated(
note = "This method will be removed in a future update. Use zcash_client_backend::data_api::WalletWrite::rewind_to_height instead."
)]
pub fn delete_utxos_above<'a, P: consensus::Parameters>(
stmts: &mut DataConnStmtCache<'a, P>,
taddr: &TransparentAddress,
height: BlockHeight,
) -> Result<usize, SqliteClientError> {
stmts.stmt_delete_utxos(taddr, height)
let update_result = stmts.stmt_update_received_transparent_utxo(output)?;
match update_result {
None => stmts.stmt_insert_received_transparent_utxo(output),
Some(id) => Ok(id),
}
}
/// Records the specified shielded output as having been received.
@ -1301,6 +1291,15 @@ mod tests {
use super::{get_address, get_balance};
#[cfg(feature = "transparent-inputs")]
use {
zcash_client_backend::{data_api::WalletWrite, wallet::WalletTransparentOutput},
zcash_primitives::{
consensus::BlockHeight,
transaction::components::{OutPoint, TxOut},
},
};
#[test]
fn empty_database_has_no_balance() {
let data_file = NamedTempFile::new().unwrap();
@ -1326,4 +1325,59 @@ mod tests {
Amount::zero()
);
}
#[test]
#[cfg(feature = "transparent-inputs")]
fn put_received_transparent_utxo() {
let data_file = NamedTempFile::new().unwrap();
let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap();
init_wallet_db(&mut db_data, None).unwrap();
// Add an account to the wallet
let mut ops = db_data.get_update_ops().unwrap();
let seed = Secret::new([0u8; 32].to_vec());
let (account_id, usk) = ops.create_account(&seed).unwrap();
let (uaddr, _) = usk.to_unified_full_viewing_key().default_address();
let taddr = uaddr.transparent().unwrap();
let mut utxo = WalletTransparentOutput {
received_by_account: account_id,
outpoint: OutPoint::new([1u8; 32], 1),
txout: TxOut {
value: Amount::from_u64(100000).unwrap(),
script_pubkey: taddr.script(),
},
height: BlockHeight::from_u32(12345),
};
let res0 = super::put_received_transparent_utxo(&mut ops, &utxo);
assert!(matches!(res0, Ok(_)));
// Change something about the UTXO and upsert; we should get back
// the same utxoid
utxo.height = BlockHeight::from_u32(34567);
let res1 = super::put_received_transparent_utxo(&mut ops, &utxo);
assert!(matches!(res1, Ok(id) if id == res0.unwrap()));
assert!(matches!(
super::get_unspent_transparent_outputs(
&db_data,
taddr,
BlockHeight::from_u32(12345)
),
Ok(utxos) if utxos.is_empty()
));
assert!(matches!(
super::get_unspent_transparent_outputs(
&db_data,
taddr,
BlockHeight::from_u32(34567)
),
Ok(utxos) if {
utxos.len() == 1 &&
utxos.iter().any(|rutxo| rutxo.height == utxo.height)
}
));
}
}

4
zcash_client_sqlite/src/wallet/init.rs
View File

@ -393,8 +393,9 @@ mod tests {
fee INTEGER,
FOREIGN KEY (block) REFERENCES blocks(height)
)",
"CREATE TABLE utxos (
"CREATE TABLE \"utxos\" (
id_utxo INTEGER PRIMARY KEY,
received_by_account INTEGER NOT NULL,
address TEXT NOT NULL,
prevout_txid BLOB NOT NULL,
prevout_idx INTEGER NOT NULL,
@ -402,6 +403,7 @@ mod tests {
value_zat INTEGER NOT NULL,
height INTEGER NOT NULL,
spent_in_tx INTEGER,
FOREIGN KEY (received_by_account) REFERENCES accounts(account),
FOREIGN KEY (spent_in_tx) REFERENCES transactions(id_tx),
CONSTRAINT tx_outpoint UNIQUE (prevout_txid, prevout_idx)
)",

4
zcash_client_sqlite/src/wallet/init/migrations.rs
View File

@ -1,4 +1,5 @@
mod add_transaction_views;
mod add_utxo_account;
mod addresses_table;
mod initial_setup;
mod ufvk_support;
@ -27,5 +28,8 @@ pub(super) fn all_migrations<P: consensus::Parameters + 'static>(
Box::new(add_transaction_views::Migration {
params: params.clone(),
}),
Box::new(add_utxo_account::Migration {
_params: params.clone(),
}),
]
}

35
zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs
View File

@ -247,13 +247,22 @@ impl<P: consensus::Parameters> RusqliteMigration for Migration<P> {
#[cfg(test)]
mod tests {
use rusqlite::{self, params};
use tempfile::NamedTempFile;
use zcash_client_backend::keys::UnifiedSpendingKey;
use zcash_primitives::zip32::AccountId;
use crate::{
tests,
wallet::init::{init_wallet_db, init_wallet_db_internal, migrations::addresses_table},
WalletDb,
};
#[cfg(feature = "transparent-inputs")]
use {
crate::wallet::init::migrations::ufvk_support,
rusqlite::params,
zcash_client_backend::{encoding::AddressCodec, keys::UnifiedSpendingKey},
zcash_client_backend::encoding::AddressCodec,
zcash_primitives::{
consensus::{BlockHeight, BranchId},
legacy::{keys::IncomingViewingKey, Script},
@ -264,25 +273,29 @@ mod tests {
},
TransactionData, TxVersion,
},
zip32::AccountId,
},
};
use crate::{
tests,
wallet::init::{init_wallet_db, init_wallet_db_internal, migrations::addresses_table},
WalletDb,
};
#[test]
fn transaction_views() {
let data_file = NamedTempFile::new().unwrap();
let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap();
init_wallet_db_internal(&mut db_data, None, Some(addresses_table::MIGRATION_ID)).unwrap();
let usk =
UnifiedSpendingKey::from_seed(&tests::network(), &[0u8; 32][..], AccountId::from(0))
.unwrap();
let ufvk = usk.to_unified_full_viewing_key();
db_data
.conn
.execute(
"INSERT INTO accounts (account, ufvk) VALUES (0, ?)",
params![ufvk.encode(&tests::network())],
)
.unwrap();
db_data.conn.execute_batch(
"INSERT INTO accounts (account, ufvk) VALUES (0, '');
INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, '');
"INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, '');
INSERT INTO transactions (block, id_tx, txid) VALUES (0, 0, '');
INSERT INTO sent_notes (tx, output_pool, output_index, from_account, address, value)

129
zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs Normal file
View File

@ -0,0 +1,129 @@
//! A migration that adds an identifier for the account that received a UTXO to the utxos table
use std::collections::HashSet;
use rusqlite;
use schemer;
use schemer_rusqlite::RusqliteMigration;
use uuid::Uuid;
use zcash_primitives::consensus;
use super::{addresses_table, utxos_table};
use crate::wallet::init::WalletMigrationError;
#[cfg(feature = "transparent-inputs")]
use {
crate::{error::SqliteClientError, wallet::get_transparent_receivers},
rusqlite::named_params,
zcash_client_backend::encoding::AddressCodec,
zcash_primitives::zip32::AccountId,
};
/// This migration adds an account identifier column to the UTXOs table.
///
/// 761884d6-30d8-44ef-b204-0b82551c4ca1
pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields(
0x761884d6,
0x30d8,
0x44ef,
b"\xb2\x04\x0b\x82\x55\x1c\x4c\xa1",
);
pub(super) struct Migration<P> {
pub(super) _params: P,
}
impl<P> schemer::Migration for Migration<P> {
fn id(&self) -> Uuid {
MIGRATION_ID
}
fn dependencies(&self) -> HashSet<Uuid> {
[utxos_table::MIGRATION_ID, addresses_table::MIGRATION_ID]
.into_iter()
.collect()
}
fn description(&self) -> &'static str {
"Adds an identifier for the account that received a UTXO to the utxos table"
}
}
impl<P: consensus::Parameters> RusqliteMigration for Migration<P> {
type Error = WalletMigrationError;
fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> {
transaction.execute_batch("ALTER TABLE utxos ADD COLUMN received_by_account INTEGER;")?;
#[cfg(feature = "transparent-inputs")]
{
let mut stmt_update_utxo_account = transaction.prepare(
"UPDATE utxos SET received_by_account = :account WHERE address = :address",
)?;
let mut stmt_fetch_accounts = transaction.prepare("SELECT account FROM accounts")?;
let mut rows = stmt_fetch_accounts.query([])?;
while let Some(row) = rows.next()? {
let account: u32 = row.get(0)?;
let taddrs =
get_transparent_receivers(&self._params, transaction, AccountId::from(account))
.map_err(|e| match e {
SqliteClientError::DbError(e) => WalletMigrationError::DbError(e),
SqliteClientError::CorruptedData(s) => {
WalletMigrationError::CorruptedData(s)
}
other => WalletMigrationError::CorruptedData(format!(
"Unexpected error in migration: {}",
other
)),
})?;
for taddr in taddrs {
stmt_update_utxo_account.execute(named_params![
":account": &account,
":address": &taddr.encode(&self._params),
])?;
}
}
}
transaction.execute_batch(
"CREATE TABLE utxos_new (
id_utxo INTEGER PRIMARY KEY,
received_by_account INTEGER NOT NULL,
address TEXT NOT NULL,
prevout_txid BLOB NOT NULL,
prevout_idx INTEGER NOT NULL,
script BLOB NOT NULL,
value_zat INTEGER NOT NULL,
height INTEGER NOT NULL,
spent_in_tx INTEGER,
FOREIGN KEY (received_by_account) REFERENCES accounts(account),
FOREIGN KEY (spent_in_tx) REFERENCES transactions(id_tx),
CONSTRAINT tx_outpoint UNIQUE (prevout_txid, prevout_idx)
);
INSERT INTO utxos_new (
id_utxo, received_by_account, address,
prevout_txid, prevout_idx, script, value_zat,
height, spent_in_tx)
SELECT
id_utxo, received_by_account, address,
prevout_txid, prevout_idx, script, value_zat,
height, spent_in_tx
FROM utxos;",
)?;
transaction.execute_batch(
"DROP TABLE utxos;
ALTER TABLE utxos_new RENAME TO utxos;",
)?;
Ok(())
}
fn down(&self, _transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> {
// TODO: something better than just panic?
panic!("Cannot revert this migration.");
}
}