Auto merge of #1058 - ebfull:coin-cache, r=ebfull

Flush to disk more consistently by accounting memory usage of serials/anchors in cache.

Closes #626.

It's important that this at least *approximates* the memory usage, so that we flush the cache to disk as expected. It's okay that we overestimate. The serials are stored in keys in the `boost::unordered_map`, so we can simply use that map's `DynamicMemoryUsage`. The anchors are another story.

src/coins.cpp

@@ -80,7 +80,10 @@ CCoinsViewCache::~CCoinsViewCache()
 }
 
 size_t CCoinsViewCache::DynamicMemoryUsage() const {
-    return memusage::DynamicUsage(cacheCoins) + cachedCoinsUsage;
+    return memusage::DynamicUsage(cacheCoins) +
+           memusage::DynamicUsage(cacheAnchors) +
+           memusage::DynamicUsage(cacheSerials) +
+           cachedCoinsUsage;
 }
 
 CCoinsMap::const_iterator CCoinsViewCache::FetchCoins(const uint256 &txid) const {
@@ -120,6 +123,7 @@ bool CCoinsViewCache::GetAnchorAt(const uint256 &rt, ZCIncrementalMerkleTree &tr
     CAnchorsMap::iterator ret = cacheAnchors.insert(std::make_pair(rt, CAnchorsCacheEntry())).first;
     ret->second.entered = true;
     ret->second.tree = tree;
+    cachedCoinsUsage += memusage::DynamicUsage(ret->second.tree);
 
     return true;
 }
@@ -135,8 +139,6 @@ bool CCoinsViewCache::GetSerial(const uint256 &serial) const {
 
     cacheSerials.insert(std::make_pair(serial, entry));
 
-    // TODO: cache usage
-
     return tmp;
 }
 
@@ -151,12 +153,18 @@ void CCoinsViewCache::PushAnchor(const ZCIncrementalMerkleTree &tree) {
     // different way (make all blocks modify mapAnchors somehow)
     // but this is simpler to reason about.
     if (currentRoot != newrt) {
-        CAnchorsMap::iterator ret = cacheAnchors.insert(std::make_pair(newrt, CAnchorsCacheEntry())).first;
+        auto insertRet = cacheAnchors.insert(std::make_pair(newrt, CAnchorsCacheEntry()));
+        CAnchorsMap::iterator ret = insertRet.first;
 
         ret->second.entered = true;
         ret->second.tree = tree;
         ret->second.flags = CAnchorsCacheEntry::DIRTY;
 
+        if (insertRet.second) {
+            // An insert took place
+            cachedCoinsUsage += memusage::DynamicUsage(ret->second.tree);
+        }
+
         hashAnchor = newrt;
     }
 }
@@ -303,7 +311,7 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins,
                     entry.tree = child_it->second.tree;
                     entry.flags = CAnchorsCacheEntry::DIRTY;
 
-                    // TODO: cache usage
+                    cachedCoinsUsage += memusage::DynamicUsage(entry.tree);
                 }
             } else {
                 if (parent_it->second.entered != child_it->second.entered) {
@@ -331,8 +339,6 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins,
                     CSerialsCacheEntry& entry = cacheSerials[child_it->first];
                     entry.entered = true;
                     entry.flags = CSerialsCacheEntry::DIRTY;
-
-                    // TODO: cache usage
                 }
             } else {
                 if (parent_it->second.entered != child_it->second.entered) {

src/test/coins_tests.cpp

@@ -136,7 +136,9 @@ public:
     void SelfTest() const
     {
         // Manually recompute the dynamic usage of the whole data, and compare it.
-        size_t ret = memusage::DynamicUsage(cacheCoins);
+        size_t ret = memusage::DynamicUsage(cacheCoins) +
+                     memusage::DynamicUsage(cacheAnchors) +
+                     memusage::DynamicUsage(cacheSerials);
         for (CCoinsMap::iterator it = cacheCoins.begin(); it != cacheCoins.end(); it++) {
             ret += memusage::DynamicUsage(it->second.coins);
         }

src/zcash/IncrementalMerkleTree.hpp

@@ -60,6 +60,12 @@ public:
 
     IncrementalMerkleTree() { }
 
+    size_t DynamicMemoryUsage() const {
+        return 32 + // left
+               32 + // right
+               parents.size() * 32; // parents
+    }
+
     void append(Hash obj);
     Hash root() const {
         return root(Depth, std::deque<Hash>());