zecfoundation
/
zcashd
mirror of https://github.com/ZcashFoundation/zcashd.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Avoid storing the Equihash solution in a CBlockIndex object once it has been written 

to the leveldb database.

Co-authored-by: Jack Grigg <thestr4d@gmail.com>
Signed-off-by: Daira Hopwood <daira@jacaranda.org>
This commit is contained in:
Daira Hopwood 2022-10-07 14:40:07 +01:00
parent 2ee172fe15
commit aee8d5c0ec
8 changed files with 145 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
doc/release-notes.md
View File

@ -9,3 +9,9 @@ Fixed
This release fixes an error "Assertion `uResultHeight == rewindHeight` failed" (#5958)
that could sometimes happen when restarting a node.
Memory Usage Improvement
------------------------
The memory usage of zcashd has been reduced by not keeping Equihash solutions for all
block headers in memory.

43
src/chain.cpp
View File

@ -6,6 +6,9 @@
#include "chain.h"
#include "main.h"
#include "txdb.h"
/**
* CChain implementation
*/
@ -58,6 +61,46 @@ const CBlockIndex *CChain::FindFork(const CBlockIndex *pindex) const {
return pindex;
}
void CBlockIndex::TrimSolution()
{
AssertLockHeld(cs_main);
// We can correctly trim a solution as soon as the block index entry has been added
// to leveldb. Updates to the block index entry (to update validity status) will be
// handled by re-reading the solution from the existing db entry. It does not help to
// try to avoid these reads by gating trimming on the validity status: the re-reads are
// efficient anyway because of caching in leveldb, and most of them are unavoidable.
if (HasSolution()) {
MetricsIncrementCounter("zcashd.trimmed_equihash_solutions");
std::vector<unsigned char> empty;
nSolution.swap(empty);
}
}
CBlockHeader CBlockIndex::GetBlockHeader() const
{
AssertLockHeld(cs_main);
CBlockHeader header;
header.nVersion = nVersion;
if (pprev) {
header.hashPrevBlock = pprev->GetBlockHash();
}
header.hashMerkleRoot = hashMerkleRoot;
header.hashBlockCommitments = hashBlockCommitments;
header.nTime = nTime;
header.nBits = nBits;
header.nNonce = nNonce;
if (HasSolution()) {
header.nSolution = nSolution;
} else {
CDiskBlockIndex dbindex;
assert(pblocktree->ReadDiskBlockIndex(GetBlockHash(), dbindex));
header.nSolution = dbindex.GetSolution();
}
return header;
}
/** Turn the lowest '1' bit in the binary representation of a number into a '0'. */
int static inline InvertLowestOne(int n) { return n & (n - 1); }

81
src/chain.h
View File

@ -12,6 +12,7 @@
#include "pow.h"
#include "tinyformat.h"
#include "uint256.h"
#include "util/strencodings.h"
#include <optional>
#include <vector>
@ -308,8 +309,13 @@ public:
unsigned int nTime;
unsigned int nBits;
uint256 nNonce;
protected:
// The Equihash solution, if it is stored. Once we know that the block index
// entry is present in leveldb, this field can be cleared via the TrimSolution
// method to save memory.
std::vector<unsigned char> nSolution;
public:
//! (memory only) Sequential id assigned to distinguish order in which blocks are received.
uint32_t nSequenceId;
@ -386,20 +392,11 @@ public:
return ret;
}
CBlockHeader GetBlockHeader() const
{
CBlockHeader block;
block.nVersion = nVersion;
if (pprev)
block.hashPrevBlock = pprev->GetBlockHash();
block.hashMerkleRoot = hashMerkleRoot;
block.hashBlockCommitments = hashBlockCommitments;
block.nTime = nTime;
block.nBits = nBits;
block.nNonce = nNonce;
block.nSolution = nSolution;
return block;
}
//! Get the block header for this block index. Requires cs_main.
CBlockHeader GetBlockHeader() const;
//! Clear the Equihash solution to save memory. Requires cs_main.
void TrimSolution();
uint256 GetBlockHash() const
{
@ -430,10 +427,11 @@ public:
std::string ToString() const
{
return strprintf("CBlockIndex(pprev=%p, nHeight=%d, merkle=%s, hashBlock=%s)",
return strprintf("CBlockIndex(pprev=%p, nHeight=%d, merkle=%s, hashBlock=%s, HasSolution=%s)",
pprev, nHeight,
hashMerkleRoot.ToString(),
phashBlock ? GetBlockHash().ToString() : "(nil)");
phashBlock ? GetBlockHash().ToString() : "(nil)",
HasSolution());
}
//! Check whether this block index entry is valid up to the passed validity level.
@ -445,6 +443,12 @@ public:
return ((nStatus & BLOCK_VALID_MASK) >= nUpTo);
}
//! Is the Equihash solution stored?
bool HasSolution() const
{
return !nSolution.empty();
}
//! Raise the validity level of this block index entry.
//! Returns true if the validity was changed.
bool RaiseValidity(enum BlockStatus nUpTo)
@ -483,8 +487,11 @@ public:
hashPrev = uint256();
}
explicit CDiskBlockIndex(const CBlockIndex* pindex) : CBlockIndex(*pindex) {
explicit CDiskBlockIndex(const CBlockIndex* pindex, std::function<std::vector<unsigned char>()> getSolution) : CBlockIndex(*pindex) {
hashPrev = (pprev ? pprev->GetBlockHash() : uint256());
if (!HasSolution()) {
nSolution = getSolution();
}
}
ADD_SERIALIZE_METHODS;
@ -565,20 +572,31 @@ public:
// them to CBlockTreeDB::LoadBlockIndexGuts() in txdb.cpp :)
}
uint256 GetBlockHash() const
//! Get the block header for this block index.
CBlockHeader GetBlockHeader() const
{
CBlockHeader block;
block.nVersion = nVersion;
block.hashPrevBlock = hashPrev;
block.hashMerkleRoot = hashMerkleRoot;
block.hashBlockCommitments = hashBlockCommitments;
block.nTime = nTime;
block.nBits = nBits;
block.nNonce = nNonce;
block.nSolution = nSolution;
return block.GetHash();
CBlockHeader header;
header.nVersion = nVersion;
header.hashPrevBlock = hashPrev;
header.hashMerkleRoot = hashMerkleRoot;
header.hashBlockCommitments = hashBlockCommitments;
header.nTime = nTime;
header.nBits = nBits;
header.nNonce = nNonce;
header.nSolution = nSolution;
return header;
}
uint256 GetBlockHash() const
{
return GetBlockHeader().GetHash();
}
std::vector<unsigned char> GetSolution() const
{
assert(HasSolution());
return nSolution;
}
std::string ToString() const
{
@ -589,6 +607,13 @@ public:
hashPrev.ToString());
return str;
}
private:
//! This method should not be called on a CDiskBlockIndex.
void TrimSolution()
{
assert(!"called CDiskBlockIndex::TrimSolution");
}
};
/** An in-memory indexed chain of blocks. */

8
src/main.cpp
View File

@ -3715,7 +3715,7 @@ bool static FlushStateToDisk(
vFiles.push_back(make_pair(*it, &vinfoBlockFile[*it]));
it = setDirtyFileInfo.erase(it);
}
std::vector<const CBlockIndex*> vBlocks;
std::vector<CBlockIndex*> vBlocks;
vBlocks.reserve(setDirtyBlockIndex.size());
for (set<CBlockIndex*>::iterator it = setDirtyBlockIndex.begin(); it != setDirtyBlockIndex.end(); ) {
vBlocks.push_back(*it);
@ -3724,6 +3724,12 @@ bool static FlushStateToDisk(
if (!pblocktree->WriteBatchSync(vFiles, nLastBlockFile, vBlocks)) {
return AbortNode(state, "Files to write to block index database");
}
// Now that we have written the block indices to the database, we do not
// need to store solutions for these CBlockIndex objects in memory.
// cs_main must be held here.
for (CBlockIndex *pblockindex : vBlocks) {
pblockindex->TrimSolution();
}
}
// Finally remove any pruned files
if (fFlushForPrune)

10
src/rest.cpp
View File

@ -141,6 +141,7 @@ static bool rest_headers(HTTPRequest* req,
std::vector<const CBlockIndex *> headers;
headers.reserve(count);
CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION);
{
LOCK(cs_main);
BlockMap::const_iterator it = mapBlockIndex.find(hash);
@ -151,11 +152,12 @@ static bool rest_headers(HTTPRequest* req,
break;
pindex = chainActive.Next(pindex);
}
}
CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION);
for (const CBlockIndex *pindex : headers) {
ssHeader << pindex->GetBlockHeader();
if (rf == RF_BINARY || rf == RF_HEX) {
for (const CBlockIndex *pindex : headers) {
ssHeader << pindex->GetBlockHeader();
}
}
}
switch (rf) {

2
src/rpc/blockchain.cpp
View File

@ -118,7 +118,7 @@ UniValue blockheaderToJSON(const CBlockIndex* blockindex)
result.pushKV("finalsaplingroot", blockindex->hashFinalSaplingRoot.GetHex());
result.pushKV("time", (int64_t)blockindex->nTime);
result.pushKV("nonce", blockindex->nNonce.GetHex());
result.pushKV("solution", HexStr(blockindex->nSolution));
result.pushKV("solution", HexStr(blockindex->GetBlockHeader().nSolution));
result.pushKV("bits", strprintf("%08x", blockindex->nBits));
result.pushKV("difficulty", GetDifficulty(blockindex));
result.pushKV("chainwork", blockindex->nChainWork.GetHex());

31
src/txdb.cpp
View File

@ -382,14 +382,29 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const {
return true;
}
bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) {
bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<CBlockIndex*>& blockinfo) {
CDBBatch batch(*this);
for (std::vector<std::pair<int, const CBlockFileInfo*> >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) {
batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second);
}
batch.Write(DB_LAST_BLOCK, nLastFile);
for (std::vector<const CBlockIndex*>::const_iterator it=blockinfo.begin(); it != blockinfo.end(); it++) {
batch.Write(make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash()), CDiskBlockIndex(*it));
std::pair<char, uint256> key = make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash());
try {
CDiskBlockIndex dbindex {*it, [this, &key]() {
// It can happen that the index entry is written, then the Equihash solution is cleared from memory,
// then the index entry is rewritten. In that case we must read the solution from the old entry.
CDiskBlockIndex dbindex_old;
if (!Read(key, dbindex_old)) {
LogPrintf("%s: Failed to read index entry", __func__);
throw runtime_error("Failed to read index entry");
}
return dbindex_old.GetSolution();
}};
batch.Write(key, dbindex);
} catch (const runtime_error&) {
return false;
}
}
return WriteBatch(batch, true);
}
@ -402,6 +417,10 @@ bool CBlockTreeDB::EraseBatchSync(const std::vector<const CBlockIndex*>& blockin
return WriteBatch(batch, true);
}
bool CBlockTreeDB::ReadDiskBlockIndex(const uint256 &blockhash, CDiskBlockIndex &dbindex) {
return Read(make_pair(DB_BLOCK_INDEX, blockhash), dbindex);
}
bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) {
return Read(make_pair(DB_TXINDEX, txid), pos);
}
@ -599,7 +618,7 @@ bool CBlockTreeDB::LoadBlockIndexGuts(
pindexNew->nTime = diskindex.nTime;
pindexNew->nBits = diskindex.nBits;
pindexNew->nNonce = diskindex.nNonce;
pindexNew->nSolution = diskindex.nSolution;
// the Equihash solution will be loaded lazily from the dbindex entry
pindexNew->nStatus = diskindex.nStatus;
pindexNew->nCachedBranchId = diskindex.nCachedBranchId;
pindexNew->nTx = diskindex.nTx;
@ -612,7 +631,11 @@ bool CBlockTreeDB::LoadBlockIndexGuts(
pindexNew->hashAuthDataRoot = diskindex.hashAuthDataRoot;
// Consistency checks
auto header = pindexNew->GetBlockHeader();
CBlockHeader header;
{
LOCK(cs_main);
header = pindexNew->GetBlockHeader();
}
if (header.GetHash() != diskindex.GetBlockHash())
return error("LoadBlockIndex(): inconsistent header vs diskindex hash: header hash = %s, diskindex hash = %s",
header.GetHash().ToString(), diskindex.GetBlockHash().ToString());

3
src/txdb.h
View File

@ -117,12 +117,13 @@ private:
CBlockTreeDB(const CBlockTreeDB&);
void operator=(const CBlockTreeDB&);
public:
bool WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo);
bool WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<CBlockIndex*>& blockinfo);
bool EraseBatchSync(const std::vector<const CBlockIndex*>& blockinfo);
bool ReadBlockFileInfo(int nFile, CBlockFileInfo &info);
bool ReadLastBlockFile(int &nFile);
bool WriteReindexing(bool fReindexing);
bool ReadReindexing(bool &fReindexing);
bool ReadDiskBlockIndex(const uint256 &blockhash, CDiskBlockIndex &dbindex);
bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos);
bool WriteTxIndex(const std::vector<std::pair<uint256, CDiskTxPos> > &vect);