zecfoundation
/
zcashd
mirror of https://github.com/ZcashFoundation/zcashd.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Rework Sprout and Sapling witnesses increment and cache workflow, so it does not loop over the entire wallet txs map indiscriminately. 

From:
```
1) for-each tx in wallet : call CopyPreviousWitnesses.
2) for-each tx in block:
   a) for-each note in tx:
      a1) for-each tx in wallet : call AppendNoteCommitment.
      a2) if note is mine : call WitnessNoteIfMine.
3) for-each tx in wallet : call UpdateWitnessHeights.
```

To:

```
1) for-each tx in block:
   a) gather note commitments in vecComm.
   b) witness note if ours.
2) for-each shield tx in wallet:
   a) copy the previous witness.
   b) append vecComm notes commitment.
   c) Update witness last processed height.
```
This commit is contained in:
furszy 2021-08-31 23:25:26 -03:00 committed by Kris Nuttycombe
parent 54b55ebae6
commit 8fb46c794e
1 changed files with 143 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

216
src/wallet/wallet.cpp
View File

@ -2598,25 +2598,22 @@ static void CopyPreviousWitnesses(NoteDataMap& noteDataMap, int indexHeight, int
}
}
template<typename NoteDataMap>
static void AppendNoteCommitment(NoteDataMap& noteDataMap, int indexHeight, int64_t nWitnessCacheSize, const uint256& note_commitment)
template<typename NoteData>
static void AppendNoteCommitment(NoteData* nd, int indexHeight, int64_t nWitnessCacheSize, const uint256& note_commitment)
{
for (auto& item : noteDataMap) {
auto* nd = &(item.second);
// No empty witnesses can reach here. Before any append, the note must be already witnessed.
if (nd->witnessHeight < indexHeight && nd->witnesses.size() > 0) {
// Check the validity of the cache
// See comment in CopyPreviousWitnesses about validity.
assert(nWitnessCacheSize >= nd->witnesses.size());
nd->witnesses.front().append(note_commitment);
}
}
}
template<typename OutPoint, typename NoteData, typename Witness>
static void WitnessNoteIfMine(std::map<OutPoint, NoteData>& noteDataMap, int indexHeight, int64_t nWitnessCacheSize, const OutPoint& key, const Witness& witness)
template<typename NoteData, typename Witness>
static void WitnessNoteIfMine(NoteData* nd, int indexHeight, int64_t nWitnessCacheSize, const Witness& witness)
{
if (noteDataMap.count(key) && noteDataMap[key].witnessHeight < indexHeight) {
auto* nd = &(noteDataMap[key]);
if (nd->witnessHeight < indexHeight) {
if (nd->witnesses.size() > 0) {
// We think this can happen because we write out the
// witness cache state after every block increment or
@ -2626,8 +2623,8 @@ static void WitnessNoteIfMine(std::map<OutPoint, NoteData>& noteDataMap, int ind
// to be called again on previously-cached blocks. This
// doesn't affect existing cached notes because of the
// NoteData::witnessHeight checks. See #1378 for details.
LogPrintf("Inconsistent witness cache state found for %s\n- Cache size: %d\n- Top (height %d): %s\n- New (height %d): %s\n",
key.ToString(), nd->witnesses.size(),
LogPrintf("Inconsistent witness cache state found\n- Cache size: %d\n- Top (height %d): %s\n- New (height %d): %s\n",
nd->witnesses.size(),
nd->witnessHeight,
nd->witnesses.front().root().GetHex(),
indexHeight,
@ -2638,11 +2635,11 @@ static void WitnessNoteIfMine(std::map<OutPoint, NoteData>& noteDataMap, int ind
// Set height to one less than pindex so it gets incremented
nd->witnessHeight = indexHeight - 1;
// Check the validity of the cache
// See comment in CopyPreviousWitnesses about validity.
assert(nWitnessCacheSize >= nd->witnesses.size());
}
}
template<typename NoteDataMap>
static void UpdateWitnessHeights(NoteDataMap& noteDataMap, int indexHeight, int64_t nWitnessCacheSize)
{
@ -2670,6 +2667,39 @@ static void UpdateWitnessHeights(NoteDataMap& noteDataMap, int indexHeight, int6
}
}
template<typename NoteData, typename OutPoint>
static void IncrementNoteWitnesses(std::map<OutPoint, NoteData>& noteDataMap,
const std::vector<uint256>& noteCommitments,
const std::vector<uint256>& nullifiers,
int chainHeight,
int prevWitCacheSize,
int nWitnessCacheSize)
{
if (noteDataMap.empty()) return; // Nothing to do
// Update spentness information for notes. This will never, in practice,
// prune witnesses for new notes witnessed in this block.
for (const auto& nullifier : nullifiers) {
::UpdateSpentHeightAndMaybePruneWitnesses(noteDataMap, chainHeight, nullifier);
}
// For any notes that still have stored witnesses (and thus are still being
// incremented), copy their previous witness so we have a starting point to
// which we can append this block's commitments.
::CopyPreviousWitnesses(noteDataMap, chainHeight, prevWitCacheSize);
// Append new notes commitments.
for (const auto& noteComm : noteCommitments) {
for (auto& item : noteDataMap) {
::AppendNoteCommitment(&(item.second), chainHeight, nWitnessCacheSize, noteComm);
}
}
// Set last processed height.
::UpdateWitnessHeights(noteDataMap, chainHeight, nWitnessCacheSize);
}
void CWallet::IncrementNoteWitnesses(
const Consensus::Params& consensus,
const CBlockIndex* pindex,
@ -2678,6 +2708,13 @@ void CWallet::IncrementNoteWitnesses(
bool performOrchardWalletUpdates)
{
LOCK(cs_wallet);
int chainHeight = pindex->nHeight;
// Set the update cache flag.
int64_t prevWitCacheSize = nWitnessCacheSize;
if (nWitnessCacheSize < WITNESS_CACHE_SIZE) {
nWitnessCacheSize += 1;
}
// Read the block from disk if we don't already have it.
const CBlock* pblock {pblockIn};
@ -2690,88 +2727,127 @@ void CWallet::IncrementNoteWitnesses(
pblock = &block;
}
// Update the wallet's note maps for spentness changes.
for (const CTransaction& tx : pblock->vtx) {
if (!tx.vJoinSplit.empty() || !tx.vShieldedSpend.empty()) {
for (std::pair<const uint256, CWalletTx>& wtxItem : mapWallet) {
// Sprout
for (const auto& jsdesc : tx.vJoinSplit) {
for (const uint256& nullifier : jsdesc.nullifiers) {
::UpdateSpentHeightAndMaybePruneWitnesses(
wtxItem.second.mapSproutNoteData, pindex->nHeight, nullifier);
}
}
// Sapling
for (const auto& spend : tx.vShieldedSpend) {
::UpdateSpentHeightAndMaybePruneWitnesses(
wtxItem.second.mapSaplingNoteData, pindex->nHeight, spend.nullifier);
}
}
}
}
// For any notes that still have stored witnesses (and thus are still being
// incremented), copy their previous witness so we have a starting point to
// which we can append this block's commitments.
for (std::pair<const uint256, CWalletTx>& wtxItem : mapWallet) {
::CopyPreviousWitnesses(wtxItem.second.mapSproutNoteData, pindex->nHeight, nWitnessCacheSize);
::CopyPreviousWitnesses(wtxItem.second.mapSaplingNoteData, pindex->nHeight, nWitnessCacheSize);
}
if (performOrchardWalletUpdates && consensus.NetworkUpgradeActive(pindex->nHeight, Consensus::UPGRADE_NU5)) {
if (!orchardWallet.GetLastCheckpointHeight().has_value()) {
orchardWallet.InitNoteCommitmentTree(frontiers.orchard);
}
assert(orchardWallet.CheckpointNoteCommitmentTree(pindex->nHeight));
}
if (nWitnessCacheSize < WITNESS_CACHE_SIZE) {
nWitnessCacheSize += 1;
}
// We want to minimise the number of times we loop over both the entire block,
// and the entire wallet. The strategy we use to achieve this is to first loop
// over the block, witnessing new notes as we go, and at the same time we cache
// the information necessary to increment the witnesses for existing notes.
// This costs us memory (bounded by the block size) in exchange for only needing
// to loop over mapWallet in a single location (plus some lookups that are
// sublinear in the size of the wallet).
std::vector<uint256> noteCommitmentsSprout;
std::vector<uint256> nullifiersSprout;
std::vector<std::pair<CWalletTx*, SproutNoteData*>> inBlockNotesSprout;
std::vector<uint256> noteCommitmentsSapling;
std::vector<uint256> nullifiersSapling;
std::vector<std::pair<CWalletTx*, SaplingNoteData*>> inBlockNotesSapling;
// 1) Loop over the block txs and gather the note commitments ordered.
// If the tx is from this wallet, witness it and append the next block note commitments on top.
for (const CTransaction& tx : pblock->vtx) {
if (tx.vJoinSplit.empty() && tx.vShieldedSpend.empty() && tx.vShieldedOutput.empty()) continue;
auto hash = tx.GetHash();
bool txIsOurs = mapWallet.count(hash);
auto txInWallet = mapWallet.find(hash);
// Sprout
for (size_t i = 0; i < tx.vJoinSplit.size(); i++) {
const JSDescription& jsdesc = tx.vJoinSplit[i];
for (uint8_t j = 0; j < jsdesc.commitments.size(); j++) {
const uint256& note_commitment = jsdesc.commitments[j];
frontiers.sprout.append(note_commitment);
noteCommitmentsSprout.emplace_back(note_commitment);
nullifiersSprout.emplace_back(jsdesc.nullifiers[j]);
// Increment existing witnesses
for (std::pair<const uint256, CWalletTx>& wtxItem : mapWallet) {
::AppendNoteCommitment(wtxItem.second.mapSproutNoteData, pindex->nHeight, nWitnessCacheSize, note_commitment);
// Append note commitment to the notes belonging to the wallet found in this block.
// This is done here to append only the notes that occur after the witness.
for (auto& item : inBlockNotesSprout) {
::AppendNoteCommitment(item.second, pindex->nHeight, nWitnessCacheSize, note_commitment);
}
// If this is our note, witness it
if (txIsOurs) {
JSOutPoint jsoutpt {hash, i, j};
::WitnessNoteIfMine(mapWallet[hash].mapSproutNoteData, pindex->nHeight, nWitnessCacheSize, jsoutpt, frontiers.sprout.witness());
// For each note in the transaction that is for this wallet, witness it for the
// first time and add it to the list of notes we're tracking from this block.
if (txInWallet != mapWallet.end()) {
CWalletTx* wtx = &txInWallet->second;
auto ndIt = wtx->mapSproutNoteData.find({hash, i, j});
if (ndIt != wtx->mapSproutNoteData.end()) {
SproutNoteData* nd = &ndIt->second;
::WitnessNoteIfMine(nd, chainHeight, nWitnessCacheSize, frontiers.sprout.witness());
inBlockNotesSprout.emplace_back(std::make_pair(wtx, nd));
}
}
}
}
// Sapling
for (const auto& spend : tx.vShieldedSpend) {
nullifiersSapling.emplace_back(spend.nullifier);
}
for (uint32_t i = 0; i < tx.vShieldedOutput.size(); i++) {
const uint256& note_commitment = tx.vShieldedOutput[i].cmu;
frontiers.sapling.append(note_commitment);
noteCommitmentsSapling.emplace_back(note_commitment);
// Increment existing witnesses
for (std::pair<const uint256, CWalletTx>& wtxItem : mapWallet) {
::AppendNoteCommitment(wtxItem.second.mapSaplingNoteData, pindex->nHeight, nWitnessCacheSize, note_commitment);
// Append note commitment to the notes belonging to the wallet found in this block.
// This is done here to append only the notes that occur after the witness.
for (auto& item : inBlockNotesSapling) {
::AppendNoteCommitment(item.second, chainHeight, nWitnessCacheSize, note_commitment);
}
// If this is our note, witness it
if (txIsOurs) {
SaplingOutPoint outPoint {hash, i};
::WitnessNoteIfMine(mapWallet[hash].mapSaplingNoteData, pindex->nHeight, nWitnessCacheSize, outPoint, frontiers.sapling.witness());
// For each note in the transaction that is for this wallet, witness it for the
// first time and add it to the list of notes we're tracking from this block.
if (txInWallet != mapWallet.end()) {
CWalletTx* wtx = &txInWallet->second;
auto ndIt = wtx->mapSaplingNoteData.find({hash, i});
if (ndIt != wtx->mapSaplingNoteData.end()) {
SaplingNoteData* nd = &ndIt->second;
::WitnessNoteIfMine(nd, chainHeight, nWitnessCacheSize, frontiers.sapling.witness());
inBlockNotesSapling.emplace_back(std::make_pair(wtx, nd));
}
}
}
}
// If we're at or beyond NU5 activation, update the Orchard note commitment tree.
// 2) Update witness heights for notes witnessed in this block. This means
// that when we run the incrementing logic again over the entire wallet
// below, the notes we found in this wallet will be skipped, due to the
// same witnessHeight logic we use to skip existing notes when rescanning.
for (auto& item : inBlockNotesSapling) {
::UpdateWitnessHeights(item.first->mapSaplingNoteData, chainHeight, nWitnessCacheSize);
}
for (auto& item : inBlockNotesSprout) {
::UpdateWitnessHeights(item.first->mapSproutNoteData, chainHeight, nWitnessCacheSize);
}
// 3) Apply the information we collected to the existing notes in the
// wallet that we are tracking. Step (2) above ensures that we won't
// attempt to re-update the notes discovered in this block even though
// we iterate over all of mapWallet.
for (auto& it : mapWallet) {
CWalletTx& wtx = it.second;
// Sprout
::IncrementNoteWitnesses(wtx.mapSproutNoteData,
noteCommitmentsSprout,
nullifiersSprout,
chainHeight,
prevWitCacheSize,
nWitnessCacheSize);
// Sapling
::IncrementNoteWitnesses(wtx.mapSaplingNoteData,
noteCommitmentsSapling,
nullifiersSapling,
chainHeight,
prevWitCacheSize,
nWitnessCacheSize);
}
// If we're at or beyond NU5 activation, initialize if necessary and then
// update the Orchard note commitment tree.
if (performOrchardWalletUpdates && consensus.NetworkUpgradeActive(pindex->nHeight, Consensus::UPGRADE_NU5)) {
if (!orchardWallet.GetLastCheckpointHeight().has_value()) {
orchardWallet.InitNoteCommitmentTree(frontiers.orchard);
}
assert(orchardWallet.CheckpointNoteCommitmentTree(pindex->nHeight));
assert(orchardWallet.AppendNoteCommitments(pindex->nHeight, *pblock));
// This assertion slows scanning for blocks with few shielded transactions by an
// order of magnitude. It is only intended as a consistency check between the node
// and wallet computing trees. Commented out until we have figured out what is
@ -2780,12 +2856,6 @@ void CWallet::IncrementNoteWitnesses(
//assert(pindex->hashFinalOrchardRoot == orchardWallet.GetLatestAnchor());
}
// Update witness heights
for (std::pair<const uint256, CWalletTx>& wtxItem : mapWallet) {
::UpdateWitnessHeights(wtxItem.second.mapSproutNoteData, pindex->nHeight, nWitnessCacheSize);
::UpdateWitnessHeights(wtxItem.second.mapSaplingNoteData, pindex->nHeight, nWitnessCacheSize);
}
// For performance reasons, we write out the witness cache in
// CWallet::SetBestChain() (which also ensures that overall consistency
// of the wallet.dat is maintained).