mapNextTx: use pointer as key, simplify value

Saves about 10% of application memory usage once the mempool warms up. Since the
mempool is DynamicUsage-regulated, this will translate to a larger mempool in
the same amount of space.

Map value type: eliminate the vin index; no users of the map need to know which
input of the transaction is spending the prevout.

Map key type: replace the COutPoint with a pointer to a COutPoint. A COutPoint
is 36 bytes, but each COutPoint is accessible from the same map entry's value.
A trivial DereferencingComparator functor allows indirect map keys, but the
resulting syntax is misleading: `map.find(&outpoint)`. Implement an indirectmap
that acts as a wrapper to a map that uses a DereferencingComparator, supporting
a syntax that accurately reflect the container's semantics: inserts and
iterators use pointers since they store pointers and need them to remain
constant and dereferenceable, but lookup functions take const references.
This commit is contained in:
Kaz Wesley 2016-04-30 21:45:26 -07:00
parent 03cf6e8675
commit 9805f4af7e
6 changed files with 96 additions and 41 deletions

View File

@ -94,6 +94,7 @@ BITCOIN_CORE_H = \
core_memusage.h \ core_memusage.h \
httprpc.h \ httprpc.h \
httpserver.h \ httpserver.h \
indirectmap.h \
init.h \ init.h \
key.h \ key.h \
keystore.h \ keystore.h \

52
src/indirectmap.h Normal file
View File

@ -0,0 +1,52 @@
#ifndef BITCOIN_INDIRECTMAP_H
#define BITCOIN_INDIRECTMAP_H
template <class T>
struct DereferencingComparator { bool operator()(const T a, const T b) const { return *a < *b; } };
/* Map whose keys are pointers, but are compared by their dereferenced values.
*
* Differs from a plain std::map<const K*, T, DereferencingComparator<K*> > in
* that methods that take a key for comparison take a K rather than taking a K*
* (taking a K* would be confusing, since it's the value rather than the address
* of the object for comparison that matters due to the dereferencing comparator).
*
* Objects pointed to by keys must not be modified in any way that changes the
* result of DereferencingComparator.
*/
template <class K, class T>
class indirectmap {
private:
typedef std::map<const K*, T, DereferencingComparator<const K*> > base;
base m;
public:
typedef typename base::iterator iterator;
typedef typename base::const_iterator const_iterator;
typedef typename base::size_type size_type;
typedef typename base::value_type value_type;
// passthrough (pointer interface)
std::pair<iterator, bool> insert(const value_type& value) { return m.insert(value); }
// pass address (value interface)
iterator find(const K& key) { return m.find(&key); }
const_iterator find(const K& key) const { return m.find(&key); }
iterator lower_bound(const K& key) { return m.lower_bound(&key); }
const_iterator lower_bound(const K& key) const { return m.lower_bound(&key); }
size_type erase(const K& key) { return m.erase(&key); }
size_type count(const K& key) const { return m.count(&key); }
// passthrough
bool empty() const { return m.empty(); }
size_type size() const { return m.size(); }
size_type max_size() const { return m.max_size(); }
void clear() { m.clear(); }
iterator begin() { return m.begin(); }
iterator end() { return m.end(); }
const_iterator begin() const { return m.begin(); }
const_iterator end() const { return m.end(); }
const_iterator cbegin() const { return m.cbegin(); }
const_iterator cend() const { return m.cend(); }
};
#endif // BITCOIN_INDIRECTMAP_H

View File

@ -1049,9 +1049,10 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C
LOCK(pool.cs); // protect pool.mapNextTx LOCK(pool.cs); // protect pool.mapNextTx
BOOST_FOREACH(const CTxIn &txin, tx.vin) BOOST_FOREACH(const CTxIn &txin, tx.vin)
{ {
if (pool.mapNextTx.count(txin.prevout)) auto itConflicting = pool.mapNextTx.find(txin.prevout);
if (itConflicting != pool.mapNextTx.end())
{ {
const CTransaction *ptxConflicting = pool.mapNextTx[txin.prevout].ptx; const CTransaction *ptxConflicting = itConflicting->second;
if (!setConflicts.count(ptxConflicting->GetHash())) if (!setConflicts.count(ptxConflicting->GetHash()))
{ {
// Allow opt-out of transaction replacement by setting // Allow opt-out of transaction replacement by setting

View File

@ -5,6 +5,8 @@
#ifndef BITCOIN_MEMUSAGE_H #ifndef BITCOIN_MEMUSAGE_H
#define BITCOIN_MEMUSAGE_H #define BITCOIN_MEMUSAGE_H
#include "indirectmap.h"
#include <stdlib.h> #include <stdlib.h>
#include <map> #include <map>
@ -106,6 +108,20 @@ static inline size_t IncrementalDynamicUsage(const std::map<X, Y, Z>& m)
return MallocUsage(sizeof(stl_tree_node<std::pair<const X, Y> >)); return MallocUsage(sizeof(stl_tree_node<std::pair<const X, Y> >));
} }
// indirectmap has underlying map with pointer as key
template<typename X, typename Y>
static inline size_t DynamicUsage(const indirectmap<X, Y>& m)
{
return MallocUsage(sizeof(stl_tree_node<std::pair<const X*, Y> >)) * m.size();
}
template<typename X, typename Y>
static inline size_t IncrementalDynamicUsage(const indirectmap<X, Y>& m)
{
return MallocUsage(sizeof(stl_tree_node<std::pair<const X*, Y> >));
}
// Boost data structures // Boost data structures
template<typename X> template<typename X>

View File

@ -147,11 +147,11 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector<uint256> &vHashes
if (it == mapTx.end()) { if (it == mapTx.end()) {
continue; continue;
} }
std::map<COutPoint, CInPoint>::iterator iter = mapNextTx.lower_bound(COutPoint(hash, 0)); auto iter = mapNextTx.lower_bound(COutPoint(hash, 0));
// First calculate the children, and update setMemPoolChildren to // First calculate the children, and update setMemPoolChildren to
// include them, and update their setMemPoolParents to include this tx. // include them, and update their setMemPoolParents to include this tx.
for (; iter != mapNextTx.end() && iter->first.hash == hash; ++iter) { for (; iter != mapNextTx.end() && iter->first->hash == hash; ++iter) {
const uint256 &childHash = iter->second.ptx->GetHash(); const uint256 &childHash = iter->second->GetHash();
txiter childIter = mapTx.find(childHash); txiter childIter = mapTx.find(childHash);
assert(childIter != mapTx.end()); assert(childIter != mapTx.end());
// We can skip updating entries we've encountered before or that // We can skip updating entries we've encountered before or that
@ -365,11 +365,11 @@ void CTxMemPool::pruneSpent(const uint256 &hashTx, CCoins &coins)
{ {
LOCK(cs); LOCK(cs);
std::map<COutPoint, CInPoint>::iterator it = mapNextTx.lower_bound(COutPoint(hashTx, 0)); auto it = mapNextTx.lower_bound(COutPoint(hashTx, 0));
// iterate over all COutPoints in mapNextTx whose hash equals the provided hashTx // iterate over all COutPoints in mapNextTx whose hash equals the provided hashTx
while (it != mapNextTx.end() && it->first.hash == hashTx) { while (it != mapNextTx.end() && it->first->hash == hashTx) {
coins.Spend(it->first.n); // and remove those outputs from coins coins.Spend(it->first->n); // and remove those outputs from coins
it++; it++;
} }
} }
@ -414,7 +414,7 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry,
const CTransaction& tx = newit->GetTx(); const CTransaction& tx = newit->GetTx();
std::set<uint256> setParentTransactions; std::set<uint256> setParentTransactions;
for (unsigned int i = 0; i < tx.vin.size(); i++) { for (unsigned int i = 0; i < tx.vin.size(); i++) {
mapNextTx[tx.vin[i].prevout] = CInPoint(&tx, i); mapNextTx.insert(std::make_pair(&tx.vin[i].prevout, &tx));
setParentTransactions.insert(tx.vin[i].prevout.hash); setParentTransactions.insert(tx.vin[i].prevout.hash);
} }
// Don't bother worrying about child transactions of this one. // Don't bother worrying about child transactions of this one.
@ -500,10 +500,10 @@ void CTxMemPool::removeRecursive(const CTransaction &origTx, std::list<CTransact
// happen during chain re-orgs if origTx isn't re-accepted into // happen during chain re-orgs if origTx isn't re-accepted into
// the mempool for any reason. // the mempool for any reason.
for (unsigned int i = 0; i < origTx.vout.size(); i++) { for (unsigned int i = 0; i < origTx.vout.size(); i++) {
std::map<COutPoint, CInPoint>::iterator it = mapNextTx.find(COutPoint(origTx.GetHash(), i)); auto it = mapNextTx.find(COutPoint(origTx.GetHash(), i));
if (it == mapNextTx.end()) if (it == mapNextTx.end())
continue; continue;
txiter nextit = mapTx.find(it->second.ptx->GetHash()); txiter nextit = mapTx.find(it->second->GetHash());
assert(nextit != mapTx.end()); assert(nextit != mapTx.end());
txToRemove.insert(nextit); txToRemove.insert(nextit);
} }
@ -561,9 +561,9 @@ void CTxMemPool::removeConflicts(const CTransaction &tx, std::list<CTransaction>
list<CTransaction> result; list<CTransaction> result;
LOCK(cs); LOCK(cs);
BOOST_FOREACH(const CTxIn &txin, tx.vin) { BOOST_FOREACH(const CTxIn &txin, tx.vin) {
std::map<COutPoint, CInPoint>::iterator it = mapNextTx.find(txin.prevout); auto it = mapNextTx.find(txin.prevout);
if (it != mapNextTx.end()) { if (it != mapNextTx.end()) {
const CTransaction &txConflict = *it->second.ptx; const CTransaction &txConflict = *it->second;
if (txConflict != tx) if (txConflict != tx)
{ {
removeRecursive(txConflict, removed); removeRecursive(txConflict, removed);
@ -671,10 +671,10 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const
assert(coins && coins->IsAvailable(txin.prevout.n)); assert(coins && coins->IsAvailable(txin.prevout.n));
} }
// Check whether its inputs are marked in mapNextTx. // Check whether its inputs are marked in mapNextTx.
std::map<COutPoint, CInPoint>::const_iterator it3 = mapNextTx.find(txin.prevout); auto it3 = mapNextTx.find(txin.prevout);
assert(it3 != mapNextTx.end()); assert(it3 != mapNextTx.end());
assert(it3->second.ptx == &tx); assert(it3->first == &txin.prevout);
assert(it3->second.n == i); assert(it3->second == &tx);
i++; i++;
} }
assert(setParentCheck == GetMemPoolParents(it)); assert(setParentCheck == GetMemPoolParents(it));
@ -701,10 +701,10 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const
// Check children against mapNextTx // Check children against mapNextTx
CTxMemPool::setEntries setChildrenCheck; CTxMemPool::setEntries setChildrenCheck;
std::map<COutPoint, CInPoint>::const_iterator iter = mapNextTx.lower_bound(COutPoint(it->GetTx().GetHash(), 0)); auto iter = mapNextTx.lower_bound(COutPoint(it->GetTx().GetHash(), 0));
int64_t childSizes = 0; int64_t childSizes = 0;
for (; iter != mapNextTx.end() && iter->first.hash == it->GetTx().GetHash(); ++iter) { for (; iter != mapNextTx.end() && iter->first->hash == it->GetTx().GetHash(); ++iter) {
txiter childit = mapTx.find(iter->second.ptx->GetHash()); txiter childit = mapTx.find(iter->second->GetHash());
assert(childit != mapTx.end()); // mapNextTx points to in-mempool transactions assert(childit != mapTx.end()); // mapNextTx points to in-mempool transactions
if (setChildrenCheck.insert(childit).second) { if (setChildrenCheck.insert(childit).second) {
childSizes += childit->GetTxSize(); childSizes += childit->GetTxSize();
@ -738,14 +738,12 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const
stepsSinceLastRemove = 0; stepsSinceLastRemove = 0;
} }
} }
for (std::map<COutPoint, CInPoint>::const_iterator it = mapNextTx.begin(); it != mapNextTx.end(); it++) { for (auto it = mapNextTx.cbegin(); it != mapNextTx.cend(); it++) {
uint256 hash = it->second.ptx->GetHash(); uint256 hash = it->second->GetHash();
indexed_transaction_set::const_iterator it2 = mapTx.find(hash); indexed_transaction_set::const_iterator it2 = mapTx.find(hash);
const CTransaction& tx = it2->GetTx(); const CTransaction& tx = it2->GetTx();
assert(it2 != mapTx.end()); assert(it2 != mapTx.end());
assert(&tx == it->second.ptx); assert(&tx == it->second);
assert(tx.vin.size() > it->second.n);
assert(it->first == it->second.ptx->vin[it->second.n].prevout);
} }
assert(totalTxSize == checkTotal); assert(totalTxSize == checkTotal);
@ -1044,8 +1042,8 @@ void CTxMemPool::TrimToSize(size_t sizelimit, std::vector<uint256>* pvNoSpendsRe
BOOST_FOREACH(const CTxIn& txin, tx.vin) { BOOST_FOREACH(const CTxIn& txin, tx.vin) {
if (exists(txin.prevout.hash)) if (exists(txin.prevout.hash))
continue; continue;
std::map<COutPoint, CInPoint>::iterator it = mapNextTx.lower_bound(COutPoint(txin.prevout.hash, 0)); auto it = mapNextTx.lower_bound(COutPoint(txin.prevout.hash, 0));
if (it == mapNextTx.end() || it->first.hash != txin.prevout.hash) if (it == mapNextTx.end() || it->first->hash != txin.prevout.hash)
pvNoSpendsRemaining->push_back(txin.prevout.hash); pvNoSpendsRemaining->push_back(txin.prevout.hash);
} }
} }

View File

@ -11,6 +11,7 @@
#include "amount.h" #include "amount.h"
#include "coins.h" #include "coins.h"
#include "indirectmap.h"
#include "primitives/transaction.h" #include "primitives/transaction.h"
#include "sync.h" #include "sync.h"
@ -305,20 +306,6 @@ struct ancestor_score {};
class CBlockPolicyEstimator; class CBlockPolicyEstimator;
/** An inpoint - a combination of a transaction and an index n into its vin */
class CInPoint
{
public:
const CTransaction* ptx;
uint32_t n;
CInPoint() { SetNull(); }
CInPoint(const CTransaction* ptxIn, uint32_t nIn) { ptx = ptxIn; n = nIn; }
void SetNull() { ptx = NULL; n = (uint32_t) -1; }
bool IsNull() const { return (ptx == NULL && n == (uint32_t) -1); }
size_t DynamicMemoryUsage() const { return 0; }
};
/** /**
* CTxMemPool stores valid-according-to-the-current-best-chain * CTxMemPool stores valid-according-to-the-current-best-chain
* transactions that may be included in the next block. * transactions that may be included in the next block.
@ -477,7 +464,7 @@ private:
void UpdateChild(txiter entry, txiter child, bool add); void UpdateChild(txiter entry, txiter child, bool add);
public: public:
std::map<COutPoint, CInPoint> mapNextTx; indirectmap<COutPoint, const CTransaction*> mapNextTx;
std::map<uint256, std::pair<double, CAmount> > mapDeltas; std::map<uint256, std::pair<double, CAmount> > mapDeltas;
/** Create a new CTxMemPool. /** Create a new CTxMemPool.