zecfoundation
/
zcashd
mirror of https://github.com/ZcashFoundation/zcashd.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #5779 from str4d/persist-orchard-note-positions 

wallet: Persist Orchard note positions with the witness tree
This commit is contained in:
str4d 2022-03-31 19:33:08 +01:00 committed by GitHub
parent a394770ab8 fbd2912885
commit bd2bc8834f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 256 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
qa/pull-tester/rpc-tests.py
View File

@ -75,6 +75,7 @@ BASE_SCRIPTS= [
'wallet_doublespend.py',
'wallet_import_export.py',
'wallet_isfromme.py',
'wallet_orchard_change.py',
'wallet_orchard_persistence.py',
'wallet_nullifiers.py',
'wallet_sapling.py',

123
qa/rpc-tests/wallet_orchard_change.py Executable file
View File

@ -0,0 +1,123 @@
#!/usr/bin/env python3
# Copyright (c) 2022 The Zcash developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or https://www.opensource.org/licenses/mit-license.php .
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import (
NU5_BRANCH_ID,
assert_equal,
get_coinbase_address,
nuparams,
start_nodes,
stop_nodes,
wait_and_assert_operationid_status,
wait_bitcoinds,
)
# Test wallet behaviour with the Orchard protocol
class WalletOrchardChangeTest(BitcoinTestFramework):
def __init__(self):
super().__init__()
self.num_nodes = 4
def setup_nodes(self):
return start_nodes(self.num_nodes, self.options.tmpdir, [[
nuparams(NU5_BRANCH_ID, 205),
'-regtestwalletsetbestchaineveryblock',
]] * self.num_nodes)
def check_has_output(self, node, tx, expected):
tx_outputs = self.nodes[0].z_viewtransaction(tx)['outputs']
tx_outputs = [{x: y for (x, y) in output.items() if x in expected} for output in tx_outputs]
found = False
for output in tx_outputs:
if output == expected:
found = True
break
assert found, 'Node %s is missing output %s in tx %s (actual: %s)' % (node, expected, tx, tx_outputs)
def run_test(self):
# Sanity-check the test harness.
assert_equal(self.nodes[0].getblockcount(), 200)
# Create an account with funds in the Sapling receiver.
acct0 = self.nodes[0].z_getnewaccount()['account']
ua0 = self.nodes[0].z_getaddressforaccount(acct0, ['sapling', 'orchard'])['address']
recipients = [{"address": ua0, "amount": 10}]
myopid = self.nodes[0].z_sendmany(get_coinbase_address(self.nodes[0]), recipients, 1, 0, 'AllowRevealedSenders')
wait_and_assert_operationid_status(self.nodes[0], myopid)
# Mine the tx & activate NU5.
self.sync_all()
self.nodes[0].generate(5)
self.sync_all()
assert_equal(
{'pools': {'sapling': {'valueZat': 10_0000_0000}}, 'minimum_confirmations': 1},
self.nodes[0].z_getbalanceforaccount(acct0),
)
# We want to generate an Orchard change note on node 0. We do this by
# sending funds to an Orchard-only UA on node 1.
acct1 = self.nodes[1].z_getnewaccount()['account']
ua1 = self.nodes[1].z_getaddressforaccount(acct1, ['orchard'])['address']
recipients = [{"address": ua1, "amount": 1}]
myopid = self.nodes[0].z_sendmany(ua0, recipients, 1, 0)
source_tx = wait_and_assert_operationid_status(self.nodes[0], myopid)
self.sync_all()
self.nodes[0].generate(1)
self.sync_all()
# The nodes have the expected split of funds.
assert_equal(
{'pools': {'orchard': {'valueZat': 9_0000_0000}}, 'minimum_confirmations': 1},
self.nodes[0].z_getbalanceforaccount(acct0),
)
assert_equal(
{'pools': {'orchard': {'valueZat': 1_0000_0000}}, 'minimum_confirmations': 1},
self.nodes[1].z_getbalanceforaccount(acct1),
)
# The Orchard note for node 0 is a change note.
self.check_has_output(0, source_tx, {
'pool': 'orchard',
'outgoing': False,
'walletInternal': True,
'value': 9,
'valueZat': 9_0000_0000,
})
# Shut down the nodes, and restart so that we can check wallet load
stop_nodes(self.nodes)
wait_bitcoinds()
self.setup_network()
# The nodes have unaltered balances.
assert_equal(
{'pools': {'orchard': {'valueZat': 9_0000_0000}}, 'minimum_confirmations': 1},
self.nodes[0].z_getbalanceforaccount(acct0),
)
assert_equal(
{'pools': {'orchard': {'valueZat': 1_0000_0000}}, 'minimum_confirmations': 1},
self.nodes[1].z_getbalanceforaccount(acct1),
)
# Send another Orchard transaction from node 0 to node 1.
myopid = self.nodes[0].z_sendmany(ua0, recipients, 1, 0)
wait_and_assert_operationid_status(self.nodes[0], myopid)
self.sync_all()
self.nodes[0].generate(1)
self.sync_all()
assert_equal(
{'pools': {'orchard': {'valueZat': 8_0000_0000}}, 'minimum_confirmations': 1},
self.nodes[0].z_getbalanceforaccount(acct0),
)
if __name__ == '__main__':
WalletOrchardChangeTest().main()

4
src/rust/include/rust/orchard/wallet.h
View File

@ -132,13 +132,9 @@ bool orchard_wallet_add_notes_from_bundle(
*
* The provided bundle must be a component of the transaction from which
* `txid` was derived.
*
* The value the `blockHeight` pointer points to be set to the height at which the
* transaction was mined, or `nullptr` if the transaction is not in the main chain.
*/
bool orchard_wallet_load_bundle(
OrchardWalletPtr* wallet,
const uint32_t* blockHeight,
const unsigned char txid[32],
const OrchardBundlePtr* bundle,
const RawOrchardActionIVK* actionIvks,

193
src/rust/src/wallet.rs
View File

@ -8,9 +8,10 @@ use std::ptr;
use std::slice;
use tracing::error;
use zcash_encoding::Optional;
use zcash_encoding::{Optional, Vector};
use zcash_primitives::{
consensus::BlockHeight,
merkle_tree::incremental::{read_position, write_position},
transaction::{components::Amount, TxId},
};
@ -58,22 +59,26 @@ pub struct InPoint {
pub struct DecryptedNote {
note: Note,
memo: [u8; 512],
/// The position of the note's commitment within the global Merkle tree.
position: Option<Position>,
}
/// A data structure tracking the note data that was decrypted from a single transaction,
/// along with the height at which that transaction was discovered in the chain.
/// A data structure tracking the note data that was decrypted from a single transaction.
#[derive(Debug, Clone)]
pub struct TxNotes {
/// The block height containing the transaction from which these
/// notes were decrypted.
tx_height: Option<BlockHeight>,
/// A map from the index of the Orchard action from which this note
/// was decrypted to the decrypted note value.
decrypted_notes: BTreeMap<usize, DecryptedNote>,
}
/// A data structure holding chain position information for a single transaction.
#[derive(Clone, Debug)]
struct NotePositions {
/// The height of the block containing the transaction.
tx_height: BlockHeight,
/// A map from the index of an Orchard action tracked by this wallet, to the position
/// of the output note's commitment within the global Merkle tree.
note_positions: BTreeMap<usize, Position>,
}
struct KeyStore {
payment_addresses: BTreeMap<OrderedAddress, IncomingViewingKey>,
viewing_keys: BTreeMap<IncomingViewingKey, FullViewingKey>,
@ -136,6 +141,9 @@ pub struct Wallet {
/// The in-memory index from txid to notes from the associated transaction that have
/// been decrypted with the IVKs known to this wallet.
wallet_received_notes: BTreeMap<TxId, TxNotes>,
/// The in-memory index from txid to note positions from the associated transaction.
/// This map should always have a subset of the keys in `wallet_received_notes`.
wallet_note_positions: BTreeMap<TxId, NotePositions>,
/// The in-memory index from nullifier to the outpoint of the note from which that
/// nullifier was derived.
nullifiers: BTreeMap<Nullifier, OutPoint>,
@ -209,6 +217,7 @@ impl Wallet {
Wallet {
key_store: KeyStore::empty(),
wallet_received_notes: BTreeMap::new(),
wallet_note_positions: BTreeMap::new(),
nullifiers: BTreeMap::new(),
witness_tree: BridgeTree::new(MAX_CHECKPOINTS),
last_checkpoint: None,
@ -224,9 +233,7 @@ impl Wallet {
/// in place with the expectation that they will be overwritten and/or updated in
/// the rescan process.
pub fn reset(&mut self) {
for tx_notes in self.wallet_received_notes.values_mut() {
tx_notes.tx_height = None;
}
self.wallet_note_positions.clear();
self.witness_tree = BridgeTree::new(MAX_CHECKPOINTS);
self.last_checkpoint = None;
self.last_observed = None;
@ -296,12 +303,10 @@ impl Wallet {
// retain notes that correspond to transactions that are not "un-mined" after
// the rewind
let to_retain: BTreeSet<_> = self
.wallet_received_notes
.wallet_note_positions
.iter()
.filter_map(|(txid, n)| {
if n.tx_height
.map_or(true, |h| h <= last_observed.block_height)
{
if n.tx_height <= last_observed.block_height {
Some(*txid)
} else {
None
@ -312,18 +317,10 @@ impl Wallet {
self.mined_notes.retain(|_, v| to_retain.contains(&v.txid));
// nullifier and received note data are retained, because these values are stable
// once we've observed a note for the first time. The block height at which we
// observed the note is set to `None` as the transaction will no longer have been
// observed as having been mined.
for (txid, n) in self.wallet_received_notes.iter_mut() {
// Erase block height and commitment tree information for any received
// notes that have been un-mined by the rewind.
if !to_retain.contains(txid) {
n.tx_height = None;
for dnote in n.decrypted_notes.values_mut() {
dnote.position = None;
}
}
}
// observed the note is removed along with the note positions, because the
// transaction will no longer have been observed as having been mined.
self.wallet_note_positions
.retain(|txid, _| to_retain.contains(txid));
self.last_observed = Some(last_observed);
self.last_checkpoint = if checkpoint_count > blocks_to_rewind as usize {
Some(checkpoint_height - blocks_to_rewind)
@ -360,15 +357,7 @@ impl Wallet {
.collect::<Vec<_>>();
for (action_idx, ivk, note, recipient, memo) in bundle.decrypt_outputs_with_keys(&keys) {
assert!(self.add_decrypted_note(
None,
txid,
action_idx,
ivk.clone(),
note,
recipient,
memo
));
assert!(self.add_decrypted_note(txid, action_idx, ivk.clone(), note, recipient, memo));
involvement.receive_action_metadata.insert(action_idx, ivk);
}
@ -378,8 +367,6 @@ impl Wallet {
/// Restore note and potential spend data from a bundle using the provided
/// metadata.
///
/// - `tx_height`: if the transaction containing the bundle has been mined,
/// this should contain the block height it was mined at
/// - `txid`: The ID for the transaction from which the provided bundle was
/// extracted.
/// - `bundle`: the bundle to decrypt notes from
@ -391,7 +378,6 @@ impl Wallet {
/// will return an error.
pub fn load_bundle(
&mut self,
tx_height: Option<BlockHeight>,
txid: &TxId,
bundle: &Bundle<Authorized, Amount>,
hints: BTreeMap<usize, &IncomingViewingKey>,
@ -414,21 +400,14 @@ impl Wallet {
for (action_idx, ivk) in hints.into_iter() {
if let Some((note, recipient, memo)) = bundle.decrypt_output_with_key(action_idx, ivk) {
if !self.add_decrypted_note(
tx_height,
txid,
action_idx,
ivk.clone(),
note,
recipient,
memo,
) {
if !self.add_decrypted_note(txid, action_idx, ivk.clone(), note, recipient, memo) {
return Err(BundleLoadError::FvkNotFound(ivk.clone()));
}
} else {
return Err(BundleLoadError::ActionDecryptionFailed(action_idx));
}
}
Ok(())
}
@ -436,7 +415,6 @@ impl Wallet {
#[allow(clippy::too_many_arguments)]
fn add_decrypted_note(
&mut self,
tx_height: Option<BlockHeight>,
txid: &TxId,
action_idx: usize,
ivk: IncomingViewingKey,
@ -458,15 +436,10 @@ impl Wallet {
self.nullifiers.insert(nf, outpoint);
// add the decrypted note data to the wallet
let note_data = DecryptedNote {
note,
memo,
position: None,
};
let note_data = DecryptedNote { note, memo };
self.wallet_received_notes
.entry(*txid)
.or_insert_with(|| TxNotes {
tx_height,
decrypted_notes: BTreeMap::new(),
})
.decrypted_notes
@ -560,11 +533,17 @@ impl Wallet {
});
// update the block height recorded for the transaction
if let Some(tx_notes) = self.wallet_received_notes.get_mut(txid) {
tx_notes.tx_height = Some(block_height);
let my_notes_for_tx = self.wallet_received_notes.get(txid);
if my_notes_for_tx.is_some() {
self.wallet_note_positions.insert(
*txid,
NotePositions {
tx_height: block_height,
note_positions: BTreeMap::default(),
},
);
}
let mut my_notes_for_tx = self.wallet_received_notes.get_mut(txid);
for (action_idx, action) in bundle.actions().iter().enumerate() {
// append the note commitment for each action to the witness tree
if !self
@ -575,13 +554,18 @@ impl Wallet {
}
// for notes that are ours, witness the current state of the tree
if let Some(dnote) = my_notes_for_tx
.as_mut()
.and_then(|n| n.decrypted_notes.get_mut(&action_idx))
if my_notes_for_tx
.as_ref()
.and_then(|n| n.decrypted_notes.get(&action_idx))
.is_some()
{
let (pos, cmx) = self.witness_tree.witness().expect("tree is not empty");
assert_eq!(cmx, MerkleHashOrchard::from_cmx(action.cmx()));
dnote.position = Some(pos);
self.wallet_note_positions
.get_mut(txid)
.expect("We created this above")
.note_positions
.insert(action_idx, pos);
}
// For nullifiers that are ours that we detect as spent by this action,
@ -663,12 +647,16 @@ impl Wallet {
self.key_store
.ivk_for_address(&dnote.note.recipient())
.and_then(|ivk| self.key_store.viewing_keys.get(ivk))
.zip(dnote.position)
.zip(
self.wallet_note_positions
.get(&outpoint.txid)
.and_then(|tx_notes| tx_notes.note_positions.get(&outpoint.action_idx)),
)
.map(|(fvk, position)| {
let path = self
.witness_tree
.authentication_path(
position,
*position,
&MerkleHashOrchard::from_cmx(&dnote.note.commitment().into()),
)
.expect("wallet always has paths to positioned notes");
@ -676,7 +664,7 @@ impl Wallet {
fvk.clone(),
dnote.note,
MerklePath::from_parts(
u64::from(position).try_into().unwrap(),
u64::from(*position).try_into().unwrap(),
path.try_into().unwrap(),
),
)
@ -810,7 +798,6 @@ pub extern "C" fn orchard_wallet_add_notes_from_bundle(
#[no_mangle]
pub extern "C" fn orchard_wallet_load_bundle(
wallet: *mut Wallet,
block_height: *const u32,
txid: *const [c_uchar; 32],
bundle: *const Bundle<Authorized, Amount>,
hints: *const FFIActionIvk,
@ -819,7 +806,6 @@ pub extern "C" fn orchard_wallet_load_bundle(
potential_spend_idxs_len: usize,
) -> bool {
let wallet = unsafe { wallet.as_mut() }.expect("Wallet pointer may not be null");
let block_height = unsafe { block_height.as_ref() }.map(|h| BlockHeight::from(*h));
let txid = TxId::from_bytes(*unsafe { txid.as_ref() }.expect("txid may not be null."));
let bundle = unsafe { bundle.as_ref() }.expect("bundle pointer may not be null.");
@ -835,7 +821,7 @@ pub extern "C" fn orchard_wallet_load_bundle(
);
}
match wallet.load_bundle(block_height, &txid, bundle, hints, potential_spend_idxs) {
match wallet.load_bundle(&txid, bundle, hints, potential_spend_idxs) {
Ok(_) => true,
Err(e) => {
error!("Failed to restore decrypted notes to wallet: {:?}", e);
@ -1172,6 +1158,8 @@ pub extern "C" fn orchard_wallet_gc_note_commitment_tree(wallet: *mut Wallet) {
wallet.witness_tree.garbage_collect();
}
const NOTE_STATE_V1: u8 = 1;
#[no_mangle]
pub extern "C" fn orchard_wallet_write_note_commitment_tree(
wallet: *const Wallet,
@ -1181,14 +1169,37 @@ pub extern "C" fn orchard_wallet_write_note_commitment_tree(
let wallet = unsafe { wallet.as_ref() }.expect("Wallet pointer may not be null.");
let mut writer = CppStreamWriter::from_raw_parts(stream, write_cb.unwrap());
let mut write_all = move || -> io::Result<()> {
let write_v1 = move |mut writer: CppStreamWriter| -> io::Result<()> {
Optional::write(&mut writer, wallet.last_checkpoint, |w, h| {
w.write_u32::<LittleEndian>(h.into())
})?;
write_tree(&mut writer, &wallet.witness_tree)
write_tree(&mut writer, &wallet.witness_tree)?;
// Write note positions.
Vector::write_sized(
&mut writer,
wallet.wallet_note_positions.iter(),
|mut w, (txid, tx_notes)| {
txid.write(&mut w)?;
w.write_u32::<LittleEndian>(tx_notes.tx_height.into())?;
Vector::write_sized(
w,
tx_notes.note_positions.iter(),
|w, (action_idx, position)| {
w.write_u32::<LittleEndian>(*action_idx as u32)?;
write_position(w, *position)
},
)
},
)?;
Ok(())
};
match write_all() {
match writer
.write_u8(NOTE_STATE_V1)
.and_then(|()| write_v1(writer))
{
Ok(()) => true,
Err(e) => {
error!("Failure in writing Orchard note commitment tree: {}", e);
@ -1206,25 +1217,57 @@ pub extern "C" fn orchard_wallet_load_note_commitment_tree(
let wallet = unsafe { wallet.as_mut() }.expect("Wallet pointer may not be null.");
let mut reader = CppStreamReader::from_raw_parts(stream, read_cb.unwrap());
let mut read_all = move || -> io::Result<()> {
let mut read_v1 = move |mut reader: CppStreamReader| -> io::Result<()> {
let last_checkpoint = Optional::read(&mut reader, |r| {
r.read_u32::<LittleEndian>().map(BlockHeight::from)
})?;
let witness_tree = read_tree(&mut reader)?;
// Read note positions.
wallet.wallet_note_positions = Vector::read_collected(&mut reader, |mut r| {
Ok((
TxId::read(&mut r)?,
NotePositions {
tx_height: r.read_u32::<LittleEndian>().map(BlockHeight::from)?,
note_positions: Vector::read_collected(r, |r| {
Ok((
r.read_u32::<LittleEndian>().map(|idx| idx as usize)?,
read_position(r)?,
))
})?,
},
))
})?;
wallet.last_checkpoint = last_checkpoint;
wallet.witness_tree = witness_tree;
Ok(())
};
match read_all() {
Ok(_) => true,
match reader.read_u8() {
Err(e) => {
error!(
"Failed to read Orchard note commitment or last checkpoint height: {}",
"Failed to read Orchard note position serialization flag: {}",
e
);
false
}
Ok(NOTE_STATE_V1) => match read_v1(reader) {
Ok(_) => true,
Err(e) => {
error!(
"Failed to read Orchard note commitment or last checkpoint height: {}",
e
);
false
}
},
Ok(flag) => {
error!(
"Unrecognized Orchard note position serialization version: {}",
flag
);
false
}
}
}

11
src/wallet/orchard.h
View File

@ -288,7 +288,6 @@ public:
* notes to the wallet.
*/
bool LoadWalletTx(
const std::optional<int> nBlockHeight,
const CTransaction& tx,
const OrchardWalletTxMeta& txMeta
) {
@ -296,10 +295,8 @@ public:
for (const auto& [action_idx, ivk] : txMeta.mapOrchardActionData) {
rawHints.push_back({ action_idx, ivk.inner.get() });
}
uint32_t blockHeight = nBlockHeight.has_value() ? (uint32_t) nBlockHeight.value() : 0;
return orchard_wallet_load_bundle(
inner.get(),
nBlockHeight.has_value() ? &blockHeight : nullptr,
tx.GetHash().begin(),
tx.GetOrchardBundle().inner.get(),
rawHints.data(),
@ -485,6 +482,10 @@ public:
template<typename Stream>
void Serialize(Stream& s) const {
int nVersion = s.GetVersion();
if (!(s.GetType() & SER_GETHASH)) {
::Serialize(s, nVersion);
}
RustStream rs(s);
if (!orchard_wallet_write_note_commitment_tree(
wallet.inner.get(),
@ -503,6 +504,10 @@ public:
template<typename Stream>
void Unserialize(Stream& s) {
int nVersion = s.GetVersion();
if (!(s.GetType() & SER_GETHASH)) {
::Unserialize(s, nVersion);
}
RustStream rs(s);
if (!orchard_wallet_load_note_commitment_tree(
wallet.inner.get(),

15
src/wallet/wallet.cpp
View File

@ -1040,7 +1040,6 @@ void CWallet::LoadRecipientMapping(const uint256& txid, const RecipientMapping&
bool CWallet::LoadCaches()
{
AssertLockHeld(cs_wallet);
AssertLockHeld(cs_main);
auto seed = GetMnemonicSeed();
@ -1160,12 +1159,7 @@ bool CWallet::LoadCaches()
// Restore decrypted Orchard notes.
for (const auto& [_, walletTx] : mapWallet) {
if (!walletTx.orchardTxMeta.empty()) {
const CBlockIndex* pTxIndex;
std::optional<int> blockHeight;
if (walletTx.GetDepthInMainChain(pTxIndex) > 0) {
blockHeight = pTxIndex->nHeight;
}
if (!orchardWallet.LoadWalletTx(blockHeight, walletTx, walletTx.orchardTxMeta)) {
if (!orchardWallet.LoadWalletTx(walletTx, walletTx.orchardTxMeta)) {
LogPrintf("%s: Error: Failed to decrypt previously decrypted notes for txid %s.\n",
__func__, walletTx.GetHash().GetHex());
return false;
@ -1402,8 +1396,9 @@ void CWallet::ChainTipAdded(const CBlockIndex *pindex,
SaplingMerkleTree saplingTree,
bool performOrchardWalletUpdates)
{
const auto chainParams = Params();
IncrementNoteWitnesses(
Params().GetConsensus(),
chainParams.GetConsensus(),
pindex, pblock,
sproutTree, saplingTree, performOrchardWalletUpdates);
UpdateSaplingNullifierNoteMapForBlock(pblock);
@ -1417,7 +1412,9 @@ void CWallet::ChainTipAdded(const CBlockIndex *pindex,
nLastSetChain = nNow;
}
if (++nSetChainUpdates >= WITNESS_WRITE_UPDATES ||
nLastSetChain + (int64_t)WITNESS_WRITE_INTERVAL * 1000000 < nNow) {
nLastSetChain + (int64_t)WITNESS_WRITE_INTERVAL * 1000000 < nNow ||
(chainParams.NetworkIDString() == CBaseChainParams::REGTEST && mapArgs.count("-regtestwalletsetbestchaineveryblock")))
{
nLastSetChain = nNow;
nSetChainUpdates = 0;
CBlockLocator loc;