Rework Sprout and Sapling witnesses increment and cache workflow, so it does not loop over the entire wallet txs map indiscriminately.
From: ``` 1) for-each tx in wallet : call CopyPreviousWitnesses. 2) for-each tx in block: a) for-each note in tx: a1) for-each tx in wallet : call AppendNoteCommitment. a2) if note is mine : call WitnessNoteIfMine. 3) for-each tx in wallet : call UpdateWitnessHeights. ``` To: ``` 1) for-each tx in block: a) gather note commitments in vecComm. b) witness note if ours. 2) for-each shield tx in wallet: a) copy the previous witness. b) append vecComm notes commitment. c) Update witness last processed height. ```
This commit is contained in:
parent
54b55ebae6
commit
8fb46c794e
|
@ -2598,11 +2598,10 @@ static void CopyPreviousWitnesses(NoteDataMap& noteDataMap, int indexHeight, int
|
|||
}
|
||||
}
|
||||
|
||||
template<typename NoteDataMap>
|
||||
static void AppendNoteCommitment(NoteDataMap& noteDataMap, int indexHeight, int64_t nWitnessCacheSize, const uint256& note_commitment)
|
||||
template<typename NoteData>
|
||||
static void AppendNoteCommitment(NoteData* nd, int indexHeight, int64_t nWitnessCacheSize, const uint256& note_commitment)
|
||||
{
|
||||
for (auto& item : noteDataMap) {
|
||||
auto* nd = &(item.second);
|
||||
// No empty witnesses can reach here. Before any append, the note must be already witnessed.
|
||||
if (nd->witnessHeight < indexHeight && nd->witnesses.size() > 0) {
|
||||
// Check the validity of the cache
|
||||
// See comment in CopyPreviousWitnesses about validity.
|
||||
|
@ -2610,13 +2609,11 @@ static void AppendNoteCommitment(NoteDataMap& noteDataMap, int indexHeight, int6
|
|||
nd->witnesses.front().append(note_commitment);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<typename OutPoint, typename NoteData, typename Witness>
|
||||
static void WitnessNoteIfMine(std::map<OutPoint, NoteData>& noteDataMap, int indexHeight, int64_t nWitnessCacheSize, const OutPoint& key, const Witness& witness)
|
||||
template<typename NoteData, typename Witness>
|
||||
static void WitnessNoteIfMine(NoteData* nd, int indexHeight, int64_t nWitnessCacheSize, const Witness& witness)
|
||||
{
|
||||
if (noteDataMap.count(key) && noteDataMap[key].witnessHeight < indexHeight) {
|
||||
auto* nd = &(noteDataMap[key]);
|
||||
if (nd->witnessHeight < indexHeight) {
|
||||
if (nd->witnesses.size() > 0) {
|
||||
// We think this can happen because we write out the
|
||||
// witness cache state after every block increment or
|
||||
|
@ -2626,8 +2623,8 @@ static void WitnessNoteIfMine(std::map<OutPoint, NoteData>& noteDataMap, int ind
|
|||
// to be called again on previously-cached blocks. This
|
||||
// doesn't affect existing cached notes because of the
|
||||
// NoteData::witnessHeight checks. See #1378 for details.
|
||||
LogPrintf("Inconsistent witness cache state found for %s\n- Cache size: %d\n- Top (height %d): %s\n- New (height %d): %s\n",
|
||||
key.ToString(), nd->witnesses.size(),
|
||||
LogPrintf("Inconsistent witness cache state found\n- Cache size: %d\n- Top (height %d): %s\n- New (height %d): %s\n",
|
||||
nd->witnesses.size(),
|
||||
nd->witnessHeight,
|
||||
nd->witnesses.front().root().GetHex(),
|
||||
indexHeight,
|
||||
|
@ -2638,11 +2635,11 @@ static void WitnessNoteIfMine(std::map<OutPoint, NoteData>& noteDataMap, int ind
|
|||
// Set height to one less than pindex so it gets incremented
|
||||
nd->witnessHeight = indexHeight - 1;
|
||||
// Check the validity of the cache
|
||||
// See comment in CopyPreviousWitnesses about validity.
|
||||
assert(nWitnessCacheSize >= nd->witnesses.size());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
template<typename NoteDataMap>
|
||||
static void UpdateWitnessHeights(NoteDataMap& noteDataMap, int indexHeight, int64_t nWitnessCacheSize)
|
||||
{
|
||||
|
@ -2670,6 +2667,39 @@ static void UpdateWitnessHeights(NoteDataMap& noteDataMap, int indexHeight, int6
|
|||
}
|
||||
}
|
||||
|
||||
template<typename NoteData, typename OutPoint>
|
||||
static void IncrementNoteWitnesses(std::map<OutPoint, NoteData>& noteDataMap,
|
||||
const std::vector<uint256>& noteCommitments,
|
||||
const std::vector<uint256>& nullifiers,
|
||||
int chainHeight,
|
||||
int prevWitCacheSize,
|
||||
int nWitnessCacheSize)
|
||||
{
|
||||
if (noteDataMap.empty()) return; // Nothing to do
|
||||
|
||||
// Update spentness information for notes. This will never, in practice,
|
||||
// prune witnesses for new notes witnessed in this block.
|
||||
for (const auto& nullifier : nullifiers) {
|
||||
::UpdateSpentHeightAndMaybePruneWitnesses(noteDataMap, chainHeight, nullifier);
|
||||
}
|
||||
|
||||
// For any notes that still have stored witnesses (and thus are still being
|
||||
// incremented), copy their previous witness so we have a starting point to
|
||||
// which we can append this block's commitments.
|
||||
::CopyPreviousWitnesses(noteDataMap, chainHeight, prevWitCacheSize);
|
||||
|
||||
// Append new notes commitments.
|
||||
for (const auto& noteComm : noteCommitments) {
|
||||
for (auto& item : noteDataMap) {
|
||||
::AppendNoteCommitment(&(item.second), chainHeight, nWitnessCacheSize, noteComm);
|
||||
}
|
||||
}
|
||||
|
||||
// Set last processed height.
|
||||
::UpdateWitnessHeights(noteDataMap, chainHeight, nWitnessCacheSize);
|
||||
}
|
||||
|
||||
|
||||
void CWallet::IncrementNoteWitnesses(
|
||||
const Consensus::Params& consensus,
|
||||
const CBlockIndex* pindex,
|
||||
|
@ -2678,6 +2708,13 @@ void CWallet::IncrementNoteWitnesses(
|
|||
bool performOrchardWalletUpdates)
|
||||
{
|
||||
LOCK(cs_wallet);
|
||||
int chainHeight = pindex->nHeight;
|
||||
|
||||
// Set the update cache flag.
|
||||
int64_t prevWitCacheSize = nWitnessCacheSize;
|
||||
if (nWitnessCacheSize < WITNESS_CACHE_SIZE) {
|
||||
nWitnessCacheSize += 1;
|
||||
}
|
||||
|
||||
// Read the block from disk if we don't already have it.
|
||||
const CBlock* pblock {pblockIn};
|
||||
|
@ -2690,88 +2727,127 @@ void CWallet::IncrementNoteWitnesses(
|
|||
pblock = █
|
||||
}
|
||||
|
||||
// Update the wallet's note maps for spentness changes.
|
||||
for (const CTransaction& tx : pblock->vtx) {
|
||||
if (!tx.vJoinSplit.empty() || !tx.vShieldedSpend.empty()) {
|
||||
for (std::pair<const uint256, CWalletTx>& wtxItem : mapWallet) {
|
||||
// Sprout
|
||||
for (const auto& jsdesc : tx.vJoinSplit) {
|
||||
for (const uint256& nullifier : jsdesc.nullifiers) {
|
||||
::UpdateSpentHeightAndMaybePruneWitnesses(
|
||||
wtxItem.second.mapSproutNoteData, pindex->nHeight, nullifier);
|
||||
}
|
||||
}
|
||||
// Sapling
|
||||
for (const auto& spend : tx.vShieldedSpend) {
|
||||
::UpdateSpentHeightAndMaybePruneWitnesses(
|
||||
wtxItem.second.mapSaplingNoteData, pindex->nHeight, spend.nullifier);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// For any notes that still have stored witnesses (and thus are still being
|
||||
// incremented), copy their previous witness so we have a starting point to
|
||||
// which we can append this block's commitments.
|
||||
for (std::pair<const uint256, CWalletTx>& wtxItem : mapWallet) {
|
||||
::CopyPreviousWitnesses(wtxItem.second.mapSproutNoteData, pindex->nHeight, nWitnessCacheSize);
|
||||
::CopyPreviousWitnesses(wtxItem.second.mapSaplingNoteData, pindex->nHeight, nWitnessCacheSize);
|
||||
}
|
||||
|
||||
if (performOrchardWalletUpdates && consensus.NetworkUpgradeActive(pindex->nHeight, Consensus::UPGRADE_NU5)) {
|
||||
if (!orchardWallet.GetLastCheckpointHeight().has_value()) {
|
||||
orchardWallet.InitNoteCommitmentTree(frontiers.orchard);
|
||||
}
|
||||
assert(orchardWallet.CheckpointNoteCommitmentTree(pindex->nHeight));
|
||||
}
|
||||
|
||||
if (nWitnessCacheSize < WITNESS_CACHE_SIZE) {
|
||||
nWitnessCacheSize += 1;
|
||||
}
|
||||
|
||||
// We want to minimise the number of times we loop over both the entire block,
|
||||
// and the entire wallet. The strategy we use to achieve this is to first loop
|
||||
// over the block, witnessing new notes as we go, and at the same time we cache
|
||||
// the information necessary to increment the witnesses for existing notes.
|
||||
// This costs us memory (bounded by the block size) in exchange for only needing
|
||||
// to loop over mapWallet in a single location (plus some lookups that are
|
||||
// sublinear in the size of the wallet).
|
||||
std::vector<uint256> noteCommitmentsSprout;
|
||||
std::vector<uint256> nullifiersSprout;
|
||||
std::vector<std::pair<CWalletTx*, SproutNoteData*>> inBlockNotesSprout;
|
||||
std::vector<uint256> noteCommitmentsSapling;
|
||||
std::vector<uint256> nullifiersSapling;
|
||||
std::vector<std::pair<CWalletTx*, SaplingNoteData*>> inBlockNotesSapling;
|
||||
|
||||
// 1) Loop over the block txs and gather the note commitments ordered.
|
||||
// If the tx is from this wallet, witness it and append the next block note commitments on top.
|
||||
for (const CTransaction& tx : pblock->vtx) {
|
||||
if (tx.vJoinSplit.empty() && tx.vShieldedSpend.empty() && tx.vShieldedOutput.empty()) continue;
|
||||
auto hash = tx.GetHash();
|
||||
bool txIsOurs = mapWallet.count(hash);
|
||||
auto txInWallet = mapWallet.find(hash);
|
||||
|
||||
// Sprout
|
||||
for (size_t i = 0; i < tx.vJoinSplit.size(); i++) {
|
||||
const JSDescription& jsdesc = tx.vJoinSplit[i];
|
||||
for (uint8_t j = 0; j < jsdesc.commitments.size(); j++) {
|
||||
const uint256& note_commitment = jsdesc.commitments[j];
|
||||
frontiers.sprout.append(note_commitment);
|
||||
noteCommitmentsSprout.emplace_back(note_commitment);
|
||||
nullifiersSprout.emplace_back(jsdesc.nullifiers[j]);
|
||||
|
||||
// Increment existing witnesses
|
||||
for (std::pair<const uint256, CWalletTx>& wtxItem : mapWallet) {
|
||||
::AppendNoteCommitment(wtxItem.second.mapSproutNoteData, pindex->nHeight, nWitnessCacheSize, note_commitment);
|
||||
// Append note commitment to the notes belonging to the wallet found in this block.
|
||||
// This is done here to append only the notes that occur after the witness.
|
||||
for (auto& item : inBlockNotesSprout) {
|
||||
::AppendNoteCommitment(item.second, pindex->nHeight, nWitnessCacheSize, note_commitment);
|
||||
}
|
||||
|
||||
// If this is our note, witness it
|
||||
if (txIsOurs) {
|
||||
JSOutPoint jsoutpt {hash, i, j};
|
||||
::WitnessNoteIfMine(mapWallet[hash].mapSproutNoteData, pindex->nHeight, nWitnessCacheSize, jsoutpt, frontiers.sprout.witness());
|
||||
// For each note in the transaction that is for this wallet, witness it for the
|
||||
// first time and add it to the list of notes we're tracking from this block.
|
||||
if (txInWallet != mapWallet.end()) {
|
||||
CWalletTx* wtx = &txInWallet->second;
|
||||
auto ndIt = wtx->mapSproutNoteData.find({hash, i, j});
|
||||
if (ndIt != wtx->mapSproutNoteData.end()) {
|
||||
SproutNoteData* nd = &ndIt->second;
|
||||
::WitnessNoteIfMine(nd, chainHeight, nWitnessCacheSize, frontiers.sprout.witness());
|
||||
inBlockNotesSprout.emplace_back(std::make_pair(wtx, nd));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Sapling
|
||||
for (const auto& spend : tx.vShieldedSpend) {
|
||||
nullifiersSapling.emplace_back(spend.nullifier);
|
||||
}
|
||||
for (uint32_t i = 0; i < tx.vShieldedOutput.size(); i++) {
|
||||
const uint256& note_commitment = tx.vShieldedOutput[i].cmu;
|
||||
frontiers.sapling.append(note_commitment);
|
||||
noteCommitmentsSapling.emplace_back(note_commitment);
|
||||
|
||||
// Increment existing witnesses
|
||||
for (std::pair<const uint256, CWalletTx>& wtxItem : mapWallet) {
|
||||
::AppendNoteCommitment(wtxItem.second.mapSaplingNoteData, pindex->nHeight, nWitnessCacheSize, note_commitment);
|
||||
// Append note commitment to the notes belonging to the wallet found in this block.
|
||||
// This is done here to append only the notes that occur after the witness.
|
||||
for (auto& item : inBlockNotesSapling) {
|
||||
::AppendNoteCommitment(item.second, chainHeight, nWitnessCacheSize, note_commitment);
|
||||
}
|
||||
|
||||
// If this is our note, witness it
|
||||
if (txIsOurs) {
|
||||
SaplingOutPoint outPoint {hash, i};
|
||||
::WitnessNoteIfMine(mapWallet[hash].mapSaplingNoteData, pindex->nHeight, nWitnessCacheSize, outPoint, frontiers.sapling.witness());
|
||||
// For each note in the transaction that is for this wallet, witness it for the
|
||||
// first time and add it to the list of notes we're tracking from this block.
|
||||
if (txInWallet != mapWallet.end()) {
|
||||
CWalletTx* wtx = &txInWallet->second;
|
||||
auto ndIt = wtx->mapSaplingNoteData.find({hash, i});
|
||||
if (ndIt != wtx->mapSaplingNoteData.end()) {
|
||||
SaplingNoteData* nd = &ndIt->second;
|
||||
::WitnessNoteIfMine(nd, chainHeight, nWitnessCacheSize, frontiers.sapling.witness());
|
||||
inBlockNotesSapling.emplace_back(std::make_pair(wtx, nd));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we're at or beyond NU5 activation, update the Orchard note commitment tree.
|
||||
// 2) Update witness heights for notes witnessed in this block. This means
|
||||
// that when we run the incrementing logic again over the entire wallet
|
||||
// below, the notes we found in this wallet will be skipped, due to the
|
||||
// same witnessHeight logic we use to skip existing notes when rescanning.
|
||||
for (auto& item : inBlockNotesSapling) {
|
||||
::UpdateWitnessHeights(item.first->mapSaplingNoteData, chainHeight, nWitnessCacheSize);
|
||||
}
|
||||
for (auto& item : inBlockNotesSprout) {
|
||||
::UpdateWitnessHeights(item.first->mapSproutNoteData, chainHeight, nWitnessCacheSize);
|
||||
}
|
||||
|
||||
// 3) Apply the information we collected to the existing notes in the
|
||||
// wallet that we are tracking. Step (2) above ensures that we won't
|
||||
// attempt to re-update the notes discovered in this block even though
|
||||
// we iterate over all of mapWallet.
|
||||
for (auto& it : mapWallet) {
|
||||
CWalletTx& wtx = it.second;
|
||||
// Sprout
|
||||
::IncrementNoteWitnesses(wtx.mapSproutNoteData,
|
||||
noteCommitmentsSprout,
|
||||
nullifiersSprout,
|
||||
chainHeight,
|
||||
prevWitCacheSize,
|
||||
nWitnessCacheSize);
|
||||
// Sapling
|
||||
::IncrementNoteWitnesses(wtx.mapSaplingNoteData,
|
||||
noteCommitmentsSapling,
|
||||
nullifiersSapling,
|
||||
chainHeight,
|
||||
prevWitCacheSize,
|
||||
nWitnessCacheSize);
|
||||
}
|
||||
|
||||
// If we're at or beyond NU5 activation, initialize if necessary and then
|
||||
// update the Orchard note commitment tree.
|
||||
if (performOrchardWalletUpdates && consensus.NetworkUpgradeActive(pindex->nHeight, Consensus::UPGRADE_NU5)) {
|
||||
if (!orchardWallet.GetLastCheckpointHeight().has_value()) {
|
||||
orchardWallet.InitNoteCommitmentTree(frontiers.orchard);
|
||||
}
|
||||
assert(orchardWallet.CheckpointNoteCommitmentTree(pindex->nHeight));
|
||||
|
||||
assert(orchardWallet.AppendNoteCommitments(pindex->nHeight, *pblock));
|
||||
|
||||
// This assertion slows scanning for blocks with few shielded transactions by an
|
||||
// order of magnitude. It is only intended as a consistency check between the node
|
||||
// and wallet computing trees. Commented out until we have figured out what is
|
||||
|
@ -2780,12 +2856,6 @@ void CWallet::IncrementNoteWitnesses(
|
|||
//assert(pindex->hashFinalOrchardRoot == orchardWallet.GetLatestAnchor());
|
||||
}
|
||||
|
||||
// Update witness heights
|
||||
for (std::pair<const uint256, CWalletTx>& wtxItem : mapWallet) {
|
||||
::UpdateWitnessHeights(wtxItem.second.mapSproutNoteData, pindex->nHeight, nWitnessCacheSize);
|
||||
::UpdateWitnessHeights(wtxItem.second.mapSaplingNoteData, pindex->nHeight, nWitnessCacheSize);
|
||||
}
|
||||
|
||||
// For performance reasons, we write out the witness cache in
|
||||
// CWallet::SetBestChain() (which also ensures that overall consistency
|
||||
// of the wallet.dat is maintained).
|
||||
|
|
Loading…
Reference in New Issue