From bde5c8b0f1e45e276818b6ea6f3cd1ddf69cb1a6 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Thu, 9 Jul 2015 13:56:31 -0400 Subject: [PATCH 1/5] Implement accurate memory accounting for mempool --- src/memusage.h | 67 +++++++++++++++++++++++++++++++++- src/primitives/transaction.cpp | 5 +++ src/primitives/transaction.h | 9 +++++ src/rpcblockchain.cpp | 2 + src/script/script.cpp | 5 +++ src/script/script.h | 3 ++ src/txmempool.cpp | 14 ++++++- src/txmempool.h | 7 ++++ 8 files changed, 110 insertions(+), 2 deletions(-) diff --git a/src/memusage.h b/src/memusage.h index 9f7de9e2..7a831e6d 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -20,12 +21,28 @@ namespace memusage /** Compute the total memory used by allocating alloc bytes. */ static size_t MallocUsage(size_t alloc); +/** Dynamic memory usage for built-in types is zero. */ +static inline size_t DynamicUsage(const int8_t& v) { return 0; } +static inline size_t DynamicUsage(const uint8_t& v) { return 0; } +static inline size_t DynamicUsage(const int16_t& v) { return 0; } +static inline size_t DynamicUsage(const uint16_t& v) { return 0; } +static inline size_t DynamicUsage(const int32_t& v) { return 0; } +static inline size_t DynamicUsage(const uint32_t& v) { return 0; } +static inline size_t DynamicUsage(const int64_t& v) { return 0; } +static inline size_t DynamicUsage(const uint64_t& v) { return 0; } +static inline size_t DynamicUsage(const float& v) { return 0; } +static inline size_t DynamicUsage(const double& v) { return 0; } +template static inline size_t DynamicUsage(X * const &v) { return 0; } +template static inline size_t DynamicUsage(const X * const &v) { return 0; } +template static inline size_t DynamicUsage(std::pair &p) { return 0; } + /** Compute the memory used for dynamically allocated but owned data structures. * For generic data types, this is *not* recursive. DynamicUsage(vector >) * will compute the memory used for the vector's, but not for the ints inside. * This is for efficiency reasons, as these functions are intended to be fast. If * application data structures require more accurate inner accounting, they should - * do the recursion themselves, or use more efficient caching + updating on modification. + * use RecursiveDynamicUsage, iterate themselves, or use more efficient caching + + * updating on modification. */ template static size_t DynamicUsage(const std::vector& v); template static size_t DynamicUsage(const std::set& s); @@ -34,6 +51,12 @@ template static size_t DynamicUsage(const boost::unorder template static size_t DynamicUsage(const boost::unordered_map& s); template static size_t DynamicUsage(const X& x); +template static size_t RecursiveDynamicUsage(const std::vector& v); +template static size_t RecursiveDynamicUsage(const std::set& v); +template static size_t RecursiveDynamicUsage(const std::map& v); +template static size_t RecursiveDynamicUsage(const std::pair& v); +template static size_t RecursiveDynamicUsage(const X& v); + static inline size_t MallocUsage(size_t alloc) { // Measured on libc6 2.19 on Linux. @@ -65,18 +88,54 @@ static inline size_t DynamicUsage(const std::vector& v) return MallocUsage(v.capacity() * sizeof(X)); } +template +static inline size_t RecursiveDynamicUsage(const std::vector& v) +{ + size_t usage = DynamicUsage(v); + BOOST_FOREACH(const X& x, v) { + usage += RecursiveDynamicUsage(x); + } + return usage; +} + template static inline size_t DynamicUsage(const std::set& s) { return MallocUsage(sizeof(stl_tree_node)) * s.size(); } +template +static inline size_t RecursiveDynamicUsage(const std::set& v) +{ + size_t usage = DynamicUsage(v); + BOOST_FOREACH(const X& x, v) { + usage += RecursiveDynamicUsage(x); + } + return usage; +} + template static inline size_t DynamicUsage(const std::map& m) { return MallocUsage(sizeof(stl_tree_node >)) * m.size(); } +template +static inline size_t RecursiveDynamicUsage(const std::map& v) +{ + size_t usage = DynamicUsage(v); + for (typename std::map::const_iterator it = v.begin(); it != v.end(); it++) { + usage += RecursiveDynamicUsage(*it); + } + return usage; +} + +template +static inline size_t RecursiveDynamicUsage(const std::pair& v) +{ + return RecursiveDynamicUsage(v.first) + RecursiveDynamicUsage(v.second); +} + // Boost data structures template @@ -106,6 +165,12 @@ static inline size_t DynamicUsage(const X& x) return x.DynamicMemoryUsage(); } +template +static inline size_t RecursiveDynamicUsage(const X& x) +{ + return DynamicUsage(x); +} + } #endif diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index f6236a2f..dc0133be 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -159,6 +159,11 @@ void CTransaction::UpdateHash() const *const_cast(&hash) = SerializeHash(*this); } +size_t CTransaction::DynamicMemoryUsage() const +{ + return memusage::RecursiveDynamicUsage(vin) + memusage::RecursiveDynamicUsage(vout); +} + CTransaction::CTransaction() : nVersion(CTransaction::MIN_CURRENT_VERSION), vin(), vout(), nLockTime(0), vjoinsplit(), joinSplitPubKey(), joinSplitSig() { } CTransaction::CTransaction(const CMutableTransaction &tx) : nVersion(tx.nVersion), vin(tx.vin), vout(tx.vout), nLockTime(tx.nLockTime), vjoinsplit(tx.vjoinsplit), diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 111237cb..7866a755 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -7,6 +7,7 @@ #define BITCOIN_PRIMITIVES_TRANSACTION_H #include "amount.h" +#include "memusage.h" #include "random.h" #include "script/script.h" #include "serialize.h" @@ -179,6 +180,8 @@ public: } std::string ToString() const; + + size_t DynamicMemoryUsage() const { return 0; } }; /** An input of a transaction. It contains the location of the previous @@ -227,6 +230,8 @@ public: } std::string ToString() const; + + size_t DynamicMemoryUsage() const { return scriptSig.DynamicMemoryUsage(); } }; /** An output of a transaction. It contains the public key that the next input @@ -300,6 +305,8 @@ public: } std::string ToString() const; + + size_t DynamicMemoryUsage() const { return scriptPubKey.DynamicMemoryUsage(); } }; struct CMutableTransaction; @@ -403,6 +410,8 @@ public: } std::string ToString() const; + + size_t DynamicMemoryUsage() const; }; /** A mutable version of CTransaction. */ diff --git a/src/rpcblockchain.cpp b/src/rpcblockchain.cpp index 24bff3f4..0ac4d230 100644 --- a/src/rpcblockchain.cpp +++ b/src/rpcblockchain.cpp @@ -787,6 +787,7 @@ UniValue getmempoolinfo(const UniValue& params, bool fHelp) "{\n" " \"size\": xxxxx (numeric) Current tx count\n" " \"bytes\": xxxxx (numeric) Sum of all tx sizes\n" + " \"usage\": xxxxx (numeric) Total memory usage for the mempool\n" "}\n" "\nExamples:\n" + HelpExampleCli("getmempoolinfo", "") @@ -796,6 +797,7 @@ UniValue getmempoolinfo(const UniValue& params, bool fHelp) UniValue ret(UniValue::VOBJ); ret.push_back(Pair("size", (int64_t) mempool.size())); ret.push_back(Pair("bytes", (int64_t) mempool.GetTotalTxSize())); + ret.push_back(Pair("usage", (int64_t) mempool.DynamicMemoryUsage())); return ret; } diff --git a/src/script/script.cpp b/src/script/script.cpp index fd339247..b1d2ceeb 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -260,3 +260,8 @@ std::string CScript::ToString() const } return str; } + +size_t CScript::DynamicMemoryUsage() const +{ + return memusage::DynamicUsage(*(static_cast*>(this))); +} diff --git a/src/script/script.h b/src/script/script.h index e0e89185..3b6fb5ef 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -6,6 +6,7 @@ #ifndef BITCOIN_SCRIPT_SCRIPT_H #define BITCOIN_SCRIPT_SCRIPT_H +#include "memusage.h" #include "crypto/common.h" #include @@ -582,6 +583,8 @@ public: // The default std::vector::clear() does not release memory. std::vector().swap(*this); } + + size_t DynamicMemoryUsage() const; }; #endif // BITCOIN_SCRIPT_SCRIPT_H diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 3f0e3938..9ae57f80 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -18,7 +18,7 @@ using namespace std; CTxMemPoolEntry::CTxMemPoolEntry(): - nFee(0), nTxSize(0), nModSize(0), nTime(0), dPriority(0.0), hadNoDependencies(false) + nFee(0), nTxSize(0), nModSize(0), nUsageSize(0), nTime(0), dPriority(0.0), hadNoDependencies(false) { nHeight = MEMPOOL_HEIGHT; } @@ -31,6 +31,7 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTransaction& _tx, const CAmount& _nFee, { nTxSize = ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION); nModSize = tx.CalculateModifiedSize(nTxSize); + nUsageSize = tx.DynamicMemoryUsage(); } CTxMemPoolEntry::CTxMemPoolEntry(const CTxMemPoolEntry& other) @@ -106,6 +107,7 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, } nTransactionsUpdated++; totalTxSize += entry.GetTxSize(); + cachedInnerUsage += entry.DynamicMemoryUsage(); minerPolicyEstimator->processTransaction(entry, fCurrentEstimate); return true; @@ -156,6 +158,7 @@ void CTxMemPool::remove(const CTransaction &origTx, std::list& rem removed.push_back(tx); totalTxSize -= mapTx[hash].GetTxSize(); + cachedInnerUsage -= mapTx[hash].DynamicMemoryUsage(); mapTx.erase(hash); nTransactionsUpdated++; minerPolicyEstimator->removeTx(hash); @@ -275,6 +278,7 @@ void CTxMemPool::clear() mapTx.clear(); mapNextTx.clear(); totalTxSize = 0; + cachedInnerUsage = 0; ++nTransactionsUpdated; } @@ -286,6 +290,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const LogPrint("mempool", "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), (unsigned int)mapNextTx.size()); uint64_t checkTotal = 0; + uint64_t innerUsage = 0; CCoinsViewCache mempoolDuplicate(const_cast(pcoins)); @@ -294,6 +299,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const for (std::map::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { unsigned int i = 0; checkTotal += it->second.GetTxSize(); + innerUsage += it->second.DynamicMemoryUsage(); const CTransaction& tx = it->second.GetTx(); bool fDependsWait = false; BOOST_FOREACH(const CTxIn &txin, tx.vin) { @@ -379,6 +385,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const } assert(totalTxSize == checkTotal); + assert(innerUsage == cachedInnerUsage); } void CTxMemPool::queryHashes(vector& vtxid) @@ -506,3 +513,8 @@ bool CCoinsViewMemPool::GetCoins(const uint256 &txid, CCoins &coins) const { bool CCoinsViewMemPool::HaveCoins(const uint256 &txid) const { return mempool.exists(txid) || base->HaveCoins(txid); } + +size_t CTxMemPool::DynamicMemoryUsage() const { + LOCK(cs); + return memusage::DynamicUsage(mapTx) + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + cachedInnerUsage; +} diff --git a/src/txmempool.h b/src/txmempool.h index e6fd8e60..cd0f9a54 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -40,6 +40,7 @@ private: CAmount nFee; //! Cached to avoid expensive parent-transaction lookups size_t nTxSize; //! ... and avoid recomputing tx size size_t nModSize; //! ... and modified size for priority + size_t nUsageSize; //! ... and total memory usage int64_t nTime; //! Local time when entering the mempool double dPriority; //! Priority when entering the mempool unsigned int nHeight; //! Chain height when entering the mempool @@ -58,6 +59,7 @@ public: int64_t GetTime() const { return nTime; } unsigned int GetHeight() const { return nHeight; } bool WasClearAtEntry() const { return hadNoDependencies; } + size_t DynamicMemoryUsage() const { return nUsageSize; } }; class CBlockPolicyEstimator; @@ -73,6 +75,7 @@ public: CInPoint(const CTransaction* ptxIn, uint32_t nIn) { ptx = ptxIn; n = nIn; } void SetNull() { ptx = NULL; n = (uint32_t) -1; } bool IsNull() const { return (ptx == NULL && n == (uint32_t) -1); } + size_t DynamicMemoryUsage() const { return 0; } }; /** @@ -93,6 +96,7 @@ private: CBlockPolicyEstimator* minerPolicyEstimator; uint64_t totalTxSize = 0; //! sum of all mempool tx' byte sizes + uint64_t cachedInnerUsage; //! sum of dynamic memory usage of all the map elements (NOT the maps themselves) public: mutable CCriticalSection cs; @@ -141,6 +145,7 @@ public: LOCK(cs); return mapTx.size(); } + uint64_t GetTotalTxSize() { LOCK(cs); @@ -164,6 +169,8 @@ public: /** Write/Read estimates to disk */ bool WriteFeeEstimates(CAutoFile& fileout) const; bool ReadFeeEstimates(CAutoFile& filein); + + size_t DynamicMemoryUsage() const; }; /** From 6bd1d60c97d43879b16d79f5f0914f93a67f4575 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 17 Jul 2015 13:46:18 -0400 Subject: [PATCH 2/5] Separate core memory usage computation in core_memusage.h --- src/Makefile.am | 1 + src/coins.cpp | 14 ++++---- src/coins.h | 4 +-- src/core_memusage.h | 62 ++++++++++++++++++++++++++++++++ src/memusage.h | 66 +--------------------------------- src/primitives/transaction.cpp | 5 --- src/primitives/transaction.h | 9 ----- src/script/script.cpp | 5 --- src/script/script.h | 3 -- src/test/coins_tests.cpp | 4 +-- src/txmempool.cpp | 2 +- 11 files changed, 76 insertions(+), 99 deletions(-) create mode 100644 src/core_memusage.h diff --git a/src/Makefile.am b/src/Makefile.am index 8f638b47..20455c97 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -115,6 +115,7 @@ BITCOIN_CORE_H = \ consensus/params.h \ consensus/validation.h \ core_io.h \ + core_memusage.h \ eccryptoverify.h \ ecwrapper.h \ hash.h \ diff --git a/src/coins.cpp b/src/coins.cpp index 6a6855bd..5c388750 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -101,7 +101,7 @@ CCoinsMap::const_iterator CCoinsViewCache::FetchCoins(const uint256 &txid) const // version as fresh. ret->second.flags = CCoinsCacheEntry::FRESH; } - cachedCoinsUsage += memusage::DynamicUsage(ret->second.coins); + cachedCoinsUsage += ret->second.coins.DynamicMemoryUsage(); return ret; } @@ -224,7 +224,7 @@ CCoinsModifier CCoinsViewCache::ModifyCoins(const uint256 &txid) { ret.first->second.flags = CCoinsCacheEntry::FRESH; } } else { - cachedCoinUsage = memusage::DynamicUsage(ret.first->second.coins); + cachedCoinUsage = ret.first->second.coins.DynamicMemoryUsage(); } // Assume that whenever ModifyCoins is called, the entry will be modified. ret.first->second.flags |= CCoinsCacheEntry::DIRTY; @@ -284,7 +284,7 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, assert(it->second.flags & CCoinsCacheEntry::FRESH); CCoinsCacheEntry& entry = cacheCoins[it->first]; entry.coins.swap(it->second.coins); - cachedCoinsUsage += memusage::DynamicUsage(entry.coins); + cachedCoinsUsage += entry.coins.DynamicMemoryUsage(); entry.flags = CCoinsCacheEntry::DIRTY | CCoinsCacheEntry::FRESH; } } else { @@ -292,13 +292,13 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, // The grandparent does not have an entry, and the child is // modified and being pruned. This means we can just delete // it from the parent. - cachedCoinsUsage -= memusage::DynamicUsage(itUs->second.coins); + cachedCoinsUsage -= itUs->second.coins.DynamicMemoryUsage(); cacheCoins.erase(itUs); } else { // A normal modification. - cachedCoinsUsage -= memusage::DynamicUsage(itUs->second.coins); + cachedCoinsUsage -= itUs->second.coins.DynamicMemoryUsage(); itUs->second.coins.swap(it->second.coins); - cachedCoinsUsage += memusage::DynamicUsage(itUs->second.coins); + cachedCoinsUsage += itUs->second.coins.DynamicMemoryUsage(); itUs->second.flags |= CCoinsCacheEntry::DIRTY; } } @@ -498,6 +498,6 @@ CCoinsModifier::~CCoinsModifier() cache.cacheCoins.erase(it); } else { // If the coin still exists after the modification, add the new usage - cache.cachedCoinsUsage += memusage::DynamicUsage(it->second.coins); + cache.cachedCoinsUsage += it->second.coins.DynamicMemoryUsage(); } } diff --git a/src/coins.h b/src/coins.h index af78782e..ebc86188 100644 --- a/src/coins.h +++ b/src/coins.h @@ -7,6 +7,7 @@ #define BITCOIN_COINS_H #include "compressor.h" +#include "core_memusage.h" #include "memusage.h" #include "serialize.h" #include "uint256.h" @@ -258,8 +259,7 @@ public: size_t DynamicMemoryUsage() const { size_t ret = memusage::DynamicUsage(vout); BOOST_FOREACH(const CTxOut &out, vout) { - const std::vector *script = &out.scriptPubKey; - ret += memusage::DynamicUsage(*script); + ret += RecursiveDynamicUsage(out.scriptPubKey); } return ret; } diff --git a/src/core_memusage.h b/src/core_memusage.h new file mode 100644 index 00000000..711135bb --- /dev/null +++ b/src/core_memusage.h @@ -0,0 +1,62 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CORE_MEMUSAGE_H +#define BITCOIN_CORE_MEMUSAGE_H + +#include "primitives/transaction.h" +#include "primitives/block.h" +#include "memusage.h" + +static inline size_t RecursiveDynamicUsage(const CScript& script) { + return memusage::DynamicUsage(*static_cast*>(&script)); +} + +static inline size_t RecursiveDynamicUsage(const COutPoint& out) { + return 0; +} + +static inline size_t RecursiveDynamicUsage(const CTxIn& in) { + return RecursiveDynamicUsage(in.scriptSig) + RecursiveDynamicUsage(in.prevout); +} + +static inline size_t RecursiveDynamicUsage(const CTxOut& out) { + return RecursiveDynamicUsage(out.scriptPubKey); +} + +static inline size_t RecursiveDynamicUsage(const CTransaction& tx) { + size_t mem = memusage::DynamicUsage(tx.vin) + memusage::DynamicUsage(tx.vout); + for (std::vector::const_iterator it = tx.vin.begin(); it != tx.vin.end(); it++) { + mem += RecursiveDynamicUsage(*it); + } + for (std::vector::const_iterator it = tx.vout.begin(); it != tx.vout.end(); it++) { + mem += RecursiveDynamicUsage(*it); + } + return mem; +} + +static inline size_t RecursiveDynamicUsage(const CMutableTransaction& tx) { + size_t mem = memusage::DynamicUsage(tx.vin) + memusage::DynamicUsage(tx.vout); + for (std::vector::const_iterator it = tx.vin.begin(); it != tx.vin.end(); it++) { + mem += RecursiveDynamicUsage(*it); + } + for (std::vector::const_iterator it = tx.vout.begin(); it != tx.vout.end(); it++) { + mem += RecursiveDynamicUsage(*it); + } + return mem; +} + +static inline size_t RecursiveDynamicUsage(const CBlock& block) { + size_t mem = memusage::DynamicUsage(block.vtx) + memusage::DynamicUsage(block.vMerkleTree); + for (std::vector::const_iterator it = block.vtx.begin(); it != block.vtx.end(); it++) { + mem += RecursiveDynamicUsage(*it); + } + return mem; +} + +static inline size_t RecursiveDynamicUsage(const CBlockLocator& locator) { + return memusage::DynamicUsage(locator.vHave); +} + +#endif // BITCOIN_CORE_MEMUSAGE_H diff --git a/src/memusage.h b/src/memusage.h index 7a831e6d..be3964df 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -34,28 +34,14 @@ static inline size_t DynamicUsage(const float& v) { return 0; } static inline size_t DynamicUsage(const double& v) { return 0; } template static inline size_t DynamicUsage(X * const &v) { return 0; } template static inline size_t DynamicUsage(const X * const &v) { return 0; } -template static inline size_t DynamicUsage(std::pair &p) { return 0; } /** Compute the memory used for dynamically allocated but owned data structures. * For generic data types, this is *not* recursive. DynamicUsage(vector >) * will compute the memory used for the vector's, but not for the ints inside. * This is for efficiency reasons, as these functions are intended to be fast. If * application data structures require more accurate inner accounting, they should - * use RecursiveDynamicUsage, iterate themselves, or use more efficient caching + - * updating on modification. + * iterate themselves, or use more efficient caching + updating on modification. */ -template static size_t DynamicUsage(const std::vector& v); -template static size_t DynamicUsage(const std::set& s); -template static size_t DynamicUsage(const std::map& m); -template static size_t DynamicUsage(const boost::unordered_set& s); -template static size_t DynamicUsage(const boost::unordered_map& s); -template static size_t DynamicUsage(const X& x); - -template static size_t RecursiveDynamicUsage(const std::vector& v); -template static size_t RecursiveDynamicUsage(const std::set& v); -template static size_t RecursiveDynamicUsage(const std::map& v); -template static size_t RecursiveDynamicUsage(const std::pair& v); -template static size_t RecursiveDynamicUsage(const X& v); static inline size_t MallocUsage(size_t alloc) { @@ -88,54 +74,18 @@ static inline size_t DynamicUsage(const std::vector& v) return MallocUsage(v.capacity() * sizeof(X)); } -template -static inline size_t RecursiveDynamicUsage(const std::vector& v) -{ - size_t usage = DynamicUsage(v); - BOOST_FOREACH(const X& x, v) { - usage += RecursiveDynamicUsage(x); - } - return usage; -} - template static inline size_t DynamicUsage(const std::set& s) { return MallocUsage(sizeof(stl_tree_node)) * s.size(); } -template -static inline size_t RecursiveDynamicUsage(const std::set& v) -{ - size_t usage = DynamicUsage(v); - BOOST_FOREACH(const X& x, v) { - usage += RecursiveDynamicUsage(x); - } - return usage; -} - template static inline size_t DynamicUsage(const std::map& m) { return MallocUsage(sizeof(stl_tree_node >)) * m.size(); } -template -static inline size_t RecursiveDynamicUsage(const std::map& v) -{ - size_t usage = DynamicUsage(v); - for (typename std::map::const_iterator it = v.begin(); it != v.end(); it++) { - usage += RecursiveDynamicUsage(*it); - } - return usage; -} - -template -static inline size_t RecursiveDynamicUsage(const std::pair& v) -{ - return RecursiveDynamicUsage(v.first) + RecursiveDynamicUsage(v.second); -} - // Boost data structures template @@ -157,20 +107,6 @@ static inline size_t DynamicUsage(const boost::unordered_map& m) return MallocUsage(sizeof(boost_unordered_node >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); } -// Dispatch to class method as fallback - -template -static inline size_t DynamicUsage(const X& x) -{ - return x.DynamicMemoryUsage(); -} - -template -static inline size_t RecursiveDynamicUsage(const X& x) -{ - return DynamicUsage(x); -} - } #endif diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index dc0133be..f6236a2f 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -159,11 +159,6 @@ void CTransaction::UpdateHash() const *const_cast(&hash) = SerializeHash(*this); } -size_t CTransaction::DynamicMemoryUsage() const -{ - return memusage::RecursiveDynamicUsage(vin) + memusage::RecursiveDynamicUsage(vout); -} - CTransaction::CTransaction() : nVersion(CTransaction::MIN_CURRENT_VERSION), vin(), vout(), nLockTime(0), vjoinsplit(), joinSplitPubKey(), joinSplitSig() { } CTransaction::CTransaction(const CMutableTransaction &tx) : nVersion(tx.nVersion), vin(tx.vin), vout(tx.vout), nLockTime(tx.nLockTime), vjoinsplit(tx.vjoinsplit), diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 7866a755..111237cb 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -7,7 +7,6 @@ #define BITCOIN_PRIMITIVES_TRANSACTION_H #include "amount.h" -#include "memusage.h" #include "random.h" #include "script/script.h" #include "serialize.h" @@ -180,8 +179,6 @@ public: } std::string ToString() const; - - size_t DynamicMemoryUsage() const { return 0; } }; /** An input of a transaction. It contains the location of the previous @@ -230,8 +227,6 @@ public: } std::string ToString() const; - - size_t DynamicMemoryUsage() const { return scriptSig.DynamicMemoryUsage(); } }; /** An output of a transaction. It contains the public key that the next input @@ -305,8 +300,6 @@ public: } std::string ToString() const; - - size_t DynamicMemoryUsage() const { return scriptPubKey.DynamicMemoryUsage(); } }; struct CMutableTransaction; @@ -410,8 +403,6 @@ public: } std::string ToString() const; - - size_t DynamicMemoryUsage() const; }; /** A mutable version of CTransaction. */ diff --git a/src/script/script.cpp b/src/script/script.cpp index b1d2ceeb..fd339247 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -260,8 +260,3 @@ std::string CScript::ToString() const } return str; } - -size_t CScript::DynamicMemoryUsage() const -{ - return memusage::DynamicUsage(*(static_cast*>(this))); -} diff --git a/src/script/script.h b/src/script/script.h index 3b6fb5ef..e0e89185 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -6,7 +6,6 @@ #ifndef BITCOIN_SCRIPT_SCRIPT_H #define BITCOIN_SCRIPT_SCRIPT_H -#include "memusage.h" #include "crypto/common.h" #include @@ -583,8 +582,6 @@ public: // The default std::vector::clear() does not release memory. std::vector().swap(*this); } - - size_t DynamicMemoryUsage() const; }; #endif // BITCOIN_SCRIPT_SCRIPT_H diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 67b97a0b..2779557e 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -143,9 +143,9 @@ public: memusage::DynamicUsage(cacheAnchors) + memusage::DynamicUsage(cacheNullifiers); for (CCoinsMap::iterator it = cacheCoins.begin(); it != cacheCoins.end(); it++) { - ret += memusage::DynamicUsage(it->second.coins); + ret += it->second.coins.DynamicMemoryUsage(); } - BOOST_CHECK_EQUAL(memusage::DynamicUsage(*this), ret); + BOOST_CHECK_EQUAL(DynamicMemoryUsage(), ret); } }; diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 9ae57f80..ed565323 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -31,7 +31,7 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTransaction& _tx, const CAmount& _nFee, { nTxSize = ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION); nModSize = tx.CalculateModifiedSize(nTxSize); - nUsageSize = tx.DynamicMemoryUsage(); + nUsageSize = RecursiveDynamicUsage(tx); } CTxMemPoolEntry::CTxMemPoolEntry(const CTxMemPoolEntry& other) From 30c163d99d51c876f0ea23a03cc99a3362746793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Jan=C3=ADk?= Date: Thu, 6 Aug 2015 19:38:19 +0200 Subject: [PATCH 3/5] Implement REST mempool API, add test and documentation. --- qa/rpc-tests/rest.py | 13 ++++++ src/rest.cpp | 56 +++++++++++++++++++++++++ src/rpcblockchain.cpp | 97 +++++++++++++++++++++++-------------------- 3 files changed, 122 insertions(+), 44 deletions(-) diff --git a/qa/rpc-tests/rest.py b/qa/rpc-tests/rest.py index 0c67ceac..16489c09 100755 --- a/qa/rpc-tests/rest.py +++ b/qa/rpc-tests/rest.py @@ -301,6 +301,19 @@ class RESTTest (BitcoinTestFramework): txs.append(self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 1)) self.sync_all() + # check that there are exactly 3 transactions in the TX memory pool before generating the block + json_string = http_get_call(url.hostname, url.port, '/rest/mempool/info'+self.FORMAT_SEPARATOR+'json') + json_obj = json.loads(json_string) + assert_equal(json_obj['size'], 3) + # the size of the memory pool should be greater than 3x ~100 bytes + assert_greater_than(json_obj['bytes'], 300) + + # check that there are our submitted transactions in the TX memory pool + json_string = http_get_call(url.hostname, url.port, '/rest/mempool/contents'+self.FORMAT_SEPARATOR+'json') + json_obj = json.loads(json_string) + for tx in txs: + assert_equal(tx in json_obj, True) + # now mine the transactions newblockhash = self.nodes[1].generate(1) self.sync_all() diff --git a/src/rest.cpp b/src/rest.cpp index b670c1b3..92450dd1 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -64,6 +64,8 @@ public: extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); extern UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails = false); +extern UniValue mempoolInfoToJSON(); +extern UniValue mempoolToJSON(bool fVerbose = false); extern void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); extern UniValue blockheaderToJSON(const CBlockIndex* blockindex); @@ -292,6 +294,58 @@ static bool rest_chaininfo(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } +static bool rest_mempool_info(AcceptedConnection* conn, + const std::string& strURIPart, + const std::string& strRequest, + const std::map& mapHeaders, + bool fRun) +{ + vector params; + const RetFormat rf = ParseDataFormat(params, strURIPart); + + switch (rf) { + case RF_JSON: { + UniValue mempoolInfoObject = mempoolInfoToJSON(); + + string strJSON = mempoolInfoObject.write() + "\n"; + conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + return true; + } + default: { + throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: json)"); + } + } + + // not reached + return true; // continue to process further HTTP reqs on this cxn +} + +static bool rest_mempool_contents(AcceptedConnection* conn, + const std::string& strURIPart, + const std::string& strRequest, + const std::map& mapHeaders, + bool fRun) +{ + vector params; + const RetFormat rf = ParseDataFormat(params, strURIPart); + + switch (rf) { + case RF_JSON: { + UniValue mempoolObject = mempoolToJSON(true); + + string strJSON = mempoolObject.write() + "\n"; + conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + return true; + } + default: { + throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: json)"); + } + } + + // not reached + return true; // continue to process further HTTP reqs on this cxn +} + static bool rest_tx(AcceptedConnection* conn, const std::string& strURIPart, const std::string& strRequest, @@ -552,6 +606,8 @@ static const struct { {"/rest/block/notxdetails/", rest_block_notxdetails}, {"/rest/block/", rest_block_extended}, {"/rest/chaininfo", rest_chaininfo}, + {"/rest/mempool/info", rest_mempool_info}, + {"/rest/mempool/contents", rest_mempool_contents}, {"/rest/headers/", rest_headers}, {"/rest/getutxos", rest_getutxos}, }; diff --git a/src/rpcblockchain.cpp b/src/rpcblockchain.cpp index 0ac4d230..550da5ae 100644 --- a/src/rpcblockchain.cpp +++ b/src/rpcblockchain.cpp @@ -191,45 +191,8 @@ UniValue getdifficulty(const UniValue& params, bool fHelp) return GetNetworkDifficulty(); } - -UniValue getrawmempool(const UniValue& params, bool fHelp) +UniValue mempoolToJSON(bool fVerbose = false) { - if (fHelp || params.size() > 1) - throw runtime_error( - "getrawmempool ( verbose )\n" - "\nReturns all transaction ids in memory pool as a json array of string transaction ids.\n" - "\nArguments:\n" - "1. verbose (boolean, optional, default=false) true for a json object, false for array of transaction ids\n" - "\nResult: (for verbose = false):\n" - "[ (json array of string)\n" - " \"transactionid\" (string) The transaction id\n" - " ,...\n" - "]\n" - "\nResult: (for verbose = true):\n" - "{ (json object)\n" - " \"transactionid\" : { (json object)\n" - " \"size\" : n, (numeric) transaction size in bytes\n" - " \"fee\" : n, (numeric) transaction fee in bitcoins\n" - " \"time\" : n, (numeric) local time transaction entered pool in seconds since 1 Jan 1970 GMT\n" - " \"height\" : n, (numeric) block height when transaction entered pool\n" - " \"startingpriority\" : n, (numeric) priority when transaction entered pool\n" - " \"currentpriority\" : n, (numeric) transaction priority now\n" - " \"depends\" : [ (array) unconfirmed transactions used as inputs for this transaction\n" - " \"transactionid\", (string) parent transaction id\n" - " ... ]\n" - " }, ...\n" - "}\n" - "\nExamples\n" - + HelpExampleCli("getrawmempool", "true") - + HelpExampleRpc("getrawmempool", "true") - ); - - LOCK(cs_main); - - bool fVerbose = false; - if (params.size() > 0) - fVerbose = params[0].get_bool(); - if (fVerbose) { LOCK(mempool.cs); @@ -277,6 +240,47 @@ UniValue getrawmempool(const UniValue& params, bool fHelp) } } +UniValue getrawmempool(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() > 1) + throw runtime_error( + "getrawmempool ( verbose )\n" + "\nReturns all transaction ids in memory pool as a json array of string transaction ids.\n" + "\nArguments:\n" + "1. verbose (boolean, optional, default=false) true for a json object, false for array of transaction ids\n" + "\nResult: (for verbose = false):\n" + "[ (json array of string)\n" + " \"transactionid\" (string) The transaction id\n" + " ,...\n" + "]\n" + "\nResult: (for verbose = true):\n" + "{ (json object)\n" + " \"transactionid\" : { (json object)\n" + " \"size\" : n, (numeric) transaction size in bytes\n" + " \"fee\" : n, (numeric) transaction fee in bitcoins\n" + " \"time\" : n, (numeric) local time transaction entered pool in seconds since 1 Jan 1970 GMT\n" + " \"height\" : n, (numeric) block height when transaction entered pool\n" + " \"startingpriority\" : n, (numeric) priority when transaction entered pool\n" + " \"currentpriority\" : n, (numeric) transaction priority now\n" + " \"depends\" : [ (array) unconfirmed transactions used as inputs for this transaction\n" + " \"transactionid\", (string) parent transaction id\n" + " ... ]\n" + " }, ...\n" + "}\n" + "\nExamples\n" + + HelpExampleCli("getrawmempool", "true") + + HelpExampleRpc("getrawmempool", "true") + ); + + LOCK(cs_main); + + bool fVerbose = false; + if (params.size() > 0) + fVerbose = params[0].get_bool(); + + return mempoolToJSON(fVerbose); +} + UniValue getblockhash(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 1) @@ -777,6 +781,16 @@ UniValue getchaintips(const UniValue& params, bool fHelp) return res; } +UniValue mempoolInfoToJSON() +{ + UniValue ret(UniValue::VOBJ); + ret.push_back(Pair("size", (int64_t) mempool.size())); + ret.push_back(Pair("bytes", (int64_t) mempool.GetTotalTxSize())); + ret.push_back(Pair("usage", (int64_t) mempool.DynamicMemoryUsage())); + + return ret; +} + UniValue getmempoolinfo(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 0) @@ -794,12 +808,7 @@ UniValue getmempoolinfo(const UniValue& params, bool fHelp) + HelpExampleRpc("getmempoolinfo", "") ); - UniValue ret(UniValue::VOBJ); - ret.push_back(Pair("size", (int64_t) mempool.size())); - ret.push_back(Pair("bytes", (int64_t) mempool.GetTotalTxSize())); - ret.push_back(Pair("usage", (int64_t) mempool.DynamicMemoryUsage())); - - return ret; + return mempoolInfoToJSON(); } UniValue invalidateblock(const UniValue& params, bool fHelp) From 43a03e3bfc3cd000fbf73864a8ca0b006dd2364e Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 14 Mar 2017 17:42:54 +1300 Subject: [PATCH 4/5] Migrate IncrementalMerkleTree memory usage calls --- src/coins.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coins.cpp b/src/coins.cpp index 5c388750..1116f159 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -124,7 +124,7 @@ bool CCoinsViewCache::GetAnchorAt(const uint256 &rt, ZCIncrementalMerkleTree &tr CAnchorsMap::iterator ret = cacheAnchors.insert(std::make_pair(rt, CAnchorsCacheEntry())).first; ret->second.entered = true; ret->second.tree = tree; - cachedCoinsUsage += memusage::DynamicUsage(ret->second.tree); + cachedCoinsUsage += ret->second.tree.DynamicMemoryUsage(); return true; } @@ -163,7 +163,7 @@ void CCoinsViewCache::PushAnchor(const ZCIncrementalMerkleTree &tree) { if (insertRet.second) { // An insert took place - cachedCoinsUsage += memusage::DynamicUsage(ret->second.tree); + cachedCoinsUsage += ret->second.tree.DynamicMemoryUsage(); } hashAnchor = newrt; @@ -318,7 +318,7 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, entry.tree = child_it->second.tree; entry.flags = CAnchorsCacheEntry::DIRTY; - cachedCoinsUsage += memusage::DynamicUsage(entry.tree); + cachedCoinsUsage += entry.tree.DynamicMemoryUsage(); } else { if (parent_it->second.entered != child_it->second.entered) { // The parent may have removed the entry. From 542a7a40f1d7813e9ed998d37428f37370747880 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 16 Mar 2017 14:30:16 +1300 Subject: [PATCH 5/5] Add tests for getmempoolinfo --- qa/rpc-tests/mempool_spendcoinbase.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/qa/rpc-tests/mempool_spendcoinbase.py b/qa/rpc-tests/mempool_spendcoinbase.py index 7eab6f37..b2ec6937 100755 --- a/qa/rpc-tests/mempool_spendcoinbase.py +++ b/qa/rpc-tests/mempool_spendcoinbase.py @@ -54,15 +54,31 @@ class MempoolSpendCoinbaseTest(BitcoinTestFramework): assert_raises(JSONRPCException, self.nodes[0].sendrawtransaction, spends_raw[1]) # mempool should have just spend_101: + mempoolinfo = self.nodes[0].getmempoolinfo() + assert_equal(mempoolinfo['size'], 1) assert_equal(self.nodes[0].getrawmempool(), [ spend_101_id ]) + # the size of the memory pool should be greater than 1x ~100 bytes + assert_greater_than(mempoolinfo['bytes'], 100) + # the actual memory usage should be strictly greater than the size + # of the memory pool + assert_greater_than(mempoolinfo['usage'], mempoolinfo['bytes']) + # mine a block, spend_101 should get confirmed self.nodes[0].generate(1) + mempoolinfo = self.nodes[0].getmempoolinfo() + assert_equal(mempoolinfo['size'], 0) + assert_equal(mempoolinfo['bytes'], 0) + assert_equal(mempoolinfo['usage'], 0) assert_equal(set(self.nodes[0].getrawmempool()), set()) # ... and now height 102 can be spent: spend_102_id = self.nodes[0].sendrawtransaction(spends_raw[1]) + mempoolinfo = self.nodes[0].getmempoolinfo() + assert_equal(mempoolinfo['size'], 1) assert_equal(self.nodes[0].getrawmempool(), [ spend_102_id ]) + assert_greater_than(mempoolinfo['bytes'], 100) + assert_greater_than(mempoolinfo['usage'], mempoolinfo['bytes']) if __name__ == '__main__': MempoolSpendCoinbaseTest().main()