zecfoundation
/
zcashd
mirror of https://github.com/ZcashFoundation/zcashd.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #6192 from daira/reduce-equihash-solution-memory 

Reduce memory usage of CBlockIndex
This commit is contained in:
str4d 2022-11-21 22:25:30 +00:00 committed by GitHub
parent 27ffefed12 fb66589022
commit c267c3ee26
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 204 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
doc/release-notes.md
View File

@ -9,3 +9,9 @@ Fixed
This release fixes an error "Assertion `uResultHeight == rewindHeight` failed" (#5958)
that could sometimes happen when restarting a node.
Memory Usage Improvement
------------------------
The memory usage of zcashd has been reduced by not keeping Equihash solutions for all
block headers in memory.

4
qa/rpc-tests/test_framework/util.py
View File

@ -214,7 +214,7 @@ def wait_for_bitcoind_start(process, url, i):
'''
while True:
if process.poll() is not None:
raise Exception('%s exited with status %i during initialization' % (zcashd_binary(), process.returncode))
raise Exception('%s node %d exited with status %i during initialization' % (zcashd_binary(), i, process.returncode))
try:
rpc = get_rpc_proxy(url, i)
rpc.getblockcount()
@ -433,7 +433,7 @@ def assert_start_raises_init_error(i, dirname, extra_args=None, expected_msg=Non
node = start_node(i, dirname, extra_args, stderr=log_stderr)
stop_node(node, i)
except Exception as e:
assert ("%s exited" % (zcashd_binary(),)) in str(e) #node must have shutdown
assert ("%s node %d exited" % (zcashd_binary(), i)) in str(e) # node must have shutdown
if expected_msg is not None:
log_stderr.seek(0)
stderr = log_stderr.read().decode('utf-8')

49
src/chain.cpp
View File

@ -6,6 +6,11 @@
#include "chain.h"
#include "main.h"
#include "txdb.h"
#include <rust/metrics.h>
/**
* CChain implementation
*/
@ -58,6 +63,50 @@ const CBlockIndex *CChain::FindFork(const CBlockIndex *pindex) const {
return pindex;
}
void CBlockIndex::TrimSolution()
{
AssertLockHeld(cs_main);
// We can correctly trim a solution as soon as the block index entry has been added
// to leveldb. Updates to the block index entry (to update validity status) will be
// handled by re-reading the solution from the existing db entry. It does not help to
// try to avoid these reads by gating trimming on the validity status: the re-reads are
// efficient anyway because of caching in leveldb, and most of them are unavoidable.
if (HasSolution()) {
MetricsIncrementCounter("zcashd.debug.memory.trimmed_equihash_solutions");
std::vector<unsigned char> empty;
nSolution.swap(empty);
}
}
CBlockHeader CBlockIndex::GetBlockHeader() const
{
AssertLockHeld(cs_main);
CBlockHeader header;
header.nVersion = nVersion;
if (pprev) {
header.hashPrevBlock = pprev->GetBlockHash();
}
header.hashMerkleRoot = hashMerkleRoot;
header.hashBlockCommitments = hashBlockCommitments;
header.nTime = nTime;
header.nBits = nBits;
header.nNonce = nNonce;
if (HasSolution()) {
header.nSolution = nSolution;
} else {
MetricsIncrementCounter("zcashd.debug.blocktree.trimmed_equihash_read_dbindex");
CDiskBlockIndex dbindex;
if (!pblocktree->ReadDiskBlockIndex(GetBlockHash(), dbindex)) {
LogPrintf("%s: Failed to read index entry", __func__);
throw std::runtime_error("Failed to read index entry");
}
header.nSolution = dbindex.GetSolution();
}
return header;
}
/** Turn the lowest '1' bit in the binary representation of a number into a '0'. */
int static inline InvertLowestOne(int n) { return n & (n - 1); }

85
src/chain.h
View File

@ -12,10 +12,13 @@
#include "pow.h"
#include "tinyformat.h"
#include "uint256.h"
#include "util/strencodings.h"
#include <optional>
#include <vector>
#include <rust/metrics.h>
static const int SPROUT_VALUE_VERSION = 1001400;
static const int SAPLING_VALUE_VERSION = 1010100;
static const int CHAIN_HISTORY_ROOT_VERSION = 2010200;
@ -308,8 +311,13 @@ public:
unsigned int nTime;
unsigned int nBits;
uint256 nNonce;
protected:
// The Equihash solution, if it is stored. Once we know that the block index
// entry is present in leveldb, this field can be cleared via the TrimSolution
// method to save memory.
std::vector<unsigned char> nSolution;
public:
//! (memory only) Sequential id assigned to distinguish order in which blocks are received.
uint32_t nSequenceId;
@ -366,6 +374,7 @@ public:
nBits = block.nBits;
nNonce = block.nNonce;
nSolution = block.nSolution;
MetricsIncrementCounter("zcashd.debug.memory.allocated_equihash_solutions");
}
CDiskBlockPos GetBlockPos() const {
@ -386,23 +395,15 @@ public:
return ret;
}
CBlockHeader GetBlockHeader() const
{
CBlockHeader block;
block.nVersion = nVersion;
if (pprev)
block.hashPrevBlock = pprev->GetBlockHash();
block.hashMerkleRoot = hashMerkleRoot;
block.hashBlockCommitments = hashBlockCommitments;
block.nTime = nTime;
block.nBits = nBits;
block.nNonce = nNonce;
block.nSolution = nSolution;
return block;
}
//! Get the block header for this block index. Requires cs_main.
CBlockHeader GetBlockHeader() const;
//! Clear the Equihash solution to save memory. Requires cs_main.
void TrimSolution();
uint256 GetBlockHash() const
{
assert(phashBlock);
return *phashBlock;
}
@ -429,10 +430,11 @@ public:
std::string ToString() const
{
return strprintf("CBlockIndex(pprev=%p, nHeight=%d, merkle=%s, hashBlock=%s)",
return strprintf("CBlockIndex(pprev=%p, nHeight=%d, merkle=%s, hashBlock=%s, HasSolution=%s)",
pprev, nHeight,
hashMerkleRoot.ToString(),
GetBlockHash().ToString());
phashBlock ? GetBlockHash().ToString() : "(nil)",
HasSolution());
}
//! Check whether this block index entry is valid up to the passed validity level.
@ -444,6 +446,12 @@ public:
return ((nStatus & BLOCK_VALID_MASK) >= nUpTo);
}
//! Is the Equihash solution stored?
bool HasSolution() const
{
return !nSolution.empty();
}
//! Raise the validity level of this block index entry.
//! Returns true if the validity was changed.
bool RaiseValidity(enum BlockStatus nUpTo)
@ -482,8 +490,11 @@ public:
hashPrev = uint256();
}
explicit CDiskBlockIndex(const CBlockIndex* pindex) : CBlockIndex(*pindex) {
explicit CDiskBlockIndex(const CBlockIndex* pindex, std::function<std::vector<unsigned char>()> getSolution) : CBlockIndex(*pindex) {
hashPrev = (pprev ? pprev->GetBlockHash() : uint256());
if (!HasSolution()) {
nSolution = getSolution();
}
}
ADD_SERIALIZE_METHODS;
@ -564,20 +575,31 @@ public:
// them to CBlockTreeDB::LoadBlockIndexGuts() in txdb.cpp :)
}
uint256 GetBlockHash() const
//! Get the block header for this block index.
CBlockHeader GetBlockHeader() const
{
CBlockHeader block;
block.nVersion = nVersion;
block.hashPrevBlock = hashPrev;
block.hashMerkleRoot = hashMerkleRoot;
block.hashBlockCommitments = hashBlockCommitments;
block.nTime = nTime;
block.nBits = nBits;
block.nNonce = nNonce;
block.nSolution = nSolution;
return block.GetHash();
CBlockHeader header;
header.nVersion = nVersion;
header.hashPrevBlock = hashPrev;
header.hashMerkleRoot = hashMerkleRoot;
header.hashBlockCommitments = hashBlockCommitments;
header.nTime = nTime;
header.nBits = nBits;
header.nNonce = nNonce;
header.nSolution = nSolution;
return header;
}
uint256 GetBlockHash() const
{
return GetBlockHeader().GetHash();
}
std::vector<unsigned char> GetSolution() const
{
assert(HasSolution());
return nSolution;
}
std::string ToString() const
{
@ -588,6 +610,13 @@ public:
hashPrev.ToString());
return str;
}
private:
//! This method should not be called on a CDiskBlockIndex.
void TrimSolution()
{
assert(!"called CDiskBlockIndex::TrimSolution");
}
};
/** An in-memory indexed chain of blocks. */

14
src/main.cpp
View File

@ -3715,7 +3715,7 @@ bool static FlushStateToDisk(
vFiles.push_back(make_pair(*it, &vinfoBlockFile[*it]));
it = setDirtyFileInfo.erase(it);
}
std::vector<const CBlockIndex*> vBlocks;
std::vector<CBlockIndex*> vBlocks;
vBlocks.reserve(setDirtyBlockIndex.size());
for (set<CBlockIndex*>::iterator it = setDirtyBlockIndex.begin(); it != setDirtyBlockIndex.end(); ) {
vBlocks.push_back(*it);
@ -3724,6 +3724,12 @@ bool static FlushStateToDisk(
if (!pblocktree->WriteBatchSync(vFiles, nLastBlockFile, vBlocks)) {
return AbortNode(state, "Files to write to block index database");
}
// Now that we have written the block indices to the database, we do not
// need to store solutions for these CBlockIndex objects in memory.
// cs_main must be held here.
for (CBlockIndex *pblockindex : vBlocks) {
pblockindex->TrimSolution();
}
}
// Finally remove any pruned files
if (fFlushForPrune)
@ -6128,7 +6134,11 @@ void static CheckBlockIndex(const Consensus::Params& consensusParams)
}
}
}
// assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow
// try {
// assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow
// } catch (const runtime_error&) {
// assert(!"Failed to read index entry");
// }
// End: actual consistency checks.
// Try descending into the first subnode.

14
src/rest.cpp
View File

@ -141,6 +141,7 @@ static bool rest_headers(HTTPRequest* req,
std::vector<const CBlockIndex *> headers;
headers.reserve(count);
CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION);
{
LOCK(cs_main);
BlockMap::const_iterator it = mapBlockIndex.find(hash);
@ -151,11 +152,16 @@ static bool rest_headers(HTTPRequest* req,
break;
pindex = chainActive.Next(pindex);
}
}
CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION);
for (const CBlockIndex *pindex : headers) {
ssHeader << pindex->GetBlockHeader();
if (rf == RF_BINARY || rf == RF_HEX) {
try {
for (const CBlockIndex *pindex : headers) {
ssHeader << pindex->GetBlockHeader();
}
} catch (const std::runtime_error&) {
return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Failed to read index entry");
}
}
}
switch (rf) {

21
src/rpc/blockchain.cpp
View File

@ -118,7 +118,7 @@ UniValue blockheaderToJSON(const CBlockIndex* blockindex)
result.pushKV("finalsaplingroot", blockindex->hashFinalSaplingRoot.GetHex());
result.pushKV("time", (int64_t)blockindex->nTime);
result.pushKV("nonce", blockindex->nNonce.GetHex());
result.pushKV("solution", HexStr(blockindex->nSolution));
result.pushKV("solution", HexStr(blockindex->GetBlockHeader().nSolution));
result.pushKV("bits", strprintf("%08x", blockindex->nBits));
result.pushKV("difficulty", GetDifficulty(blockindex));
result.pushKV("chainwork", blockindex->nChainWork.GetHex());
@ -684,15 +684,18 @@ UniValue getblockheader(const UniValue& params, bool fHelp)
CBlockIndex* pblockindex = mapBlockIndex[hash];
if (!fVerbose)
{
CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION);
ssBlock << pblockindex->GetBlockHeader();
std::string strHex = HexStr(ssBlock.begin(), ssBlock.end());
return strHex;
try {
if (!fVerbose) {
CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION);
ssBlock << pblockindex->GetBlockHeader();
std::string strHex = HexStr(ssBlock.begin(), ssBlock.end());
return strHex;
} else {
return blockheaderToJSON(pblockindex);
}
} catch (const runtime_error&) {
throw JSONRPCError(RPC_DATABASE_ERROR, "Failed to read index entry");
}
return blockheaderToJSON(pblockindex);
}
UniValue getblock(const UniValue& params, bool fHelp)

59
src/txdb.cpp
View File

@ -17,6 +17,8 @@
#include <boost/thread.hpp>
#include <rust/metrics.h>
using namespace std;
// NOTE: Per issue #3277, do not use the prefix 'X' or 'x' as they were
@ -316,7 +318,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins,
CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) {
}
bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) {
bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) const {
return Read(make_pair(DB_BLOCK_FILES, nFile), info);
}
@ -327,12 +329,12 @@ bool CBlockTreeDB::WriteReindexing(bool fReindexing) {
return Erase(DB_REINDEX_FLAG);
}
bool CBlockTreeDB::ReadReindexing(bool &fReindexing) {
bool CBlockTreeDB::ReadReindexing(bool &fReindexing) const {
fReindexing = Exists(DB_REINDEX_FLAG);
return true;
}
bool CBlockTreeDB::ReadLastBlockFile(int &nFile) {
bool CBlockTreeDB::ReadLastBlockFile(int &nFile) const {
return Read(DB_LAST_BLOCK, nFile);
}
@ -382,14 +384,31 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const {
return true;
}
bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) {
bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<CBlockIndex*>& blockinfo) {
MetricsIncrementCounter("zcashd.debug.blocktree.write_batch");
CDBBatch batch(*this);
for (std::vector<std::pair<int, const CBlockFileInfo*> >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) {
batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second);
}
batch.Write(DB_LAST_BLOCK, nLastFile);
for (std::vector<const CBlockIndex*>::const_iterator it=blockinfo.begin(); it != blockinfo.end(); it++) {
batch.Write(make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash()), CDiskBlockIndex(*it));
std::pair<char, uint256> key = make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash());
try {
CDiskBlockIndex dbindex {*it, [this, &key]() {
MetricsIncrementCounter("zcashd.debug.blocktree.write_batch_read_dbindex");
// It can happen that the index entry is written, then the Equihash solution is cleared from memory,
// then the index entry is rewritten. In that case we must read the solution from the old entry.
CDiskBlockIndex dbindex_old;
if (!Read(key, dbindex_old)) {
LogPrintf("%s: Failed to read index entry", __func__);
throw runtime_error("Failed to read index entry");
}
return dbindex_old.GetSolution();
}};
batch.Write(key, dbindex);
} catch (const runtime_error&) {
return false;
}
}
return WriteBatch(batch, true);
}
@ -402,7 +421,11 @@ bool CBlockTreeDB::EraseBatchSync(const std::vector<const CBlockIndex*>& blockin
return WriteBatch(batch, true);
}
bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) {
bool CBlockTreeDB::ReadDiskBlockIndex(const uint256 &blockhash, CDiskBlockIndex &dbindex) const {
return Read(make_pair(DB_BLOCK_INDEX, blockhash), dbindex);
}
bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) const {
return Read(make_pair(DB_TXINDEX, txid), pos);
}
@ -491,7 +514,7 @@ bool CBlockTreeDB::ReadAddressIndex(
return true;
}
bool CBlockTreeDB::ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) {
bool CBlockTreeDB::ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) const {
return Read(make_pair(DB_SPENTINDEX, key), value);
}
@ -547,7 +570,7 @@ bool CBlockTreeDB::WriteTimestampBlockIndex(const CTimestampBlockIndexKey &block
return WriteBatch(batch);
}
bool CBlockTreeDB::ReadTimestampBlockIndex(const uint256 &hash, unsigned int &ltimestamp)
bool CBlockTreeDB::ReadTimestampBlockIndex(const uint256 &hash, unsigned int &ltimestamp) const
{
CTimestampBlockIndexValue(lts);
if (!Read(std::make_pair(DB_BLOCKHASHINDEX, hash), lts))
@ -562,7 +585,7 @@ bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) {
return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0');
}
bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) {
bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) const {
char ch;
if (!Read(std::make_pair(DB_FLAG, name), ch))
return false;
@ -599,7 +622,7 @@ bool CBlockTreeDB::LoadBlockIndexGuts(
pindexNew->nTime = diskindex.nTime;
pindexNew->nBits = diskindex.nBits;
pindexNew->nNonce = diskindex.nNonce;
pindexNew->nSolution = diskindex.nSolution;
// the Equihash solution will be loaded lazily from the dbindex entry
pindexNew->nStatus = diskindex.nStatus;
pindexNew->nCachedBranchId = diskindex.nCachedBranchId;
pindexNew->nTx = diskindex.nTx;
@ -612,10 +635,22 @@ bool CBlockTreeDB::LoadBlockIndexGuts(
pindexNew->hashAuthDataRoot = diskindex.hashAuthDataRoot;
// Consistency checks
auto header = pindexNew->GetBlockHeader();
CBlockHeader header;
{
LOCK(cs_main);
try {
header = pindexNew->GetBlockHeader();
} catch (const runtime_error&) {
return error("LoadBlockIndex(): failed to read index entry: diskindex hash = %s",
diskindex.GetBlockHash().ToString());
}
}
if (header.GetHash() != diskindex.GetBlockHash())
return error("LoadBlockIndex(): inconsistent header vs diskindex hash: header hash = %s, diskindex hash = %s",
header.GetHash().ToString(), diskindex.GetBlockHash().ToString());
if (header.GetHash() != pindexNew->GetBlockHash())
return error("LoadBlockIndex(): block header inconsistency detected: on-disk = %s, in-memory = %s",
diskindex.ToString(), pindexNew->ToString());
diskindex.ToString(), pindexNew->ToString());
if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, Params().GetConsensus()))
return error("LoadBlockIndex(): CheckProofOfWork failed: %s", pindexNew->ToString());

17
src/txdb.h
View File

@ -117,13 +117,14 @@ private:
CBlockTreeDB(const CBlockTreeDB&);
void operator=(const CBlockTreeDB&);
public:
bool WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo);
bool WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<CBlockIndex*>& blockinfo);
bool EraseBatchSync(const std::vector<const CBlockIndex*>& blockinfo);
bool ReadBlockFileInfo(int nFile, CBlockFileInfo &info);
bool ReadLastBlockFile(int &nFile);
bool ReadBlockFileInfo(int nFile, CBlockFileInfo &info) const;
bool ReadLastBlockFile(int &nFile) const;
bool WriteReindexing(bool fReindexing);
bool ReadReindexing(bool &fReindexing);
bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos);
bool ReadReindexing(bool &fReindexing) const;
bool ReadDiskBlockIndex(const uint256 &blockhash, CDiskBlockIndex &dbindex) const;
bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) const;
bool WriteTxIndex(const std::vector<std::pair<uint256, CDiskTxPos> > &vect);
// START insightexplorer
@ -132,18 +133,18 @@ public:
bool WriteAddressIndex(const std::vector<CAddressIndexDbEntry> &vect);
bool EraseAddressIndex(const std::vector<CAddressIndexDbEntry> &vect);
bool ReadAddressIndex(uint160 addressHash, int type, std::vector<CAddressIndexDbEntry> &addressIndex, int start = 0, int end = 0);
bool ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value);
bool ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) const;
bool UpdateSpentIndex(const std::vector<CSpentIndexDbEntry> &vect);
bool WriteTimestampIndex(const CTimestampIndexKey &timestampIndex);
bool ReadTimestampIndex(unsigned int high, unsigned int low,
const bool fActiveOnly, std::vector<std::pair<uint256, unsigned int> > &vect);
bool WriteTimestampBlockIndex(const CTimestampBlockIndexKey &blockhashIndex,
const CTimestampBlockIndexValue &logicalts);
bool ReadTimestampBlockIndex(const uint256 &hash, unsigned int &logicalTS);
bool ReadTimestampBlockIndex(const uint256 &hash, unsigned int &logicalTS) const;
// END insightexplorer
bool WriteFlag(const std::string &name, bool fValue);
bool ReadFlag(const std::string &name, bool &fValue);
bool ReadFlag(const std::string &name, bool &fValue) const;
bool LoadBlockIndexGuts(
std::function<CBlockIndex*(const uint256&)> insertBlockIndex,
const CChainParams& chainParams);