bitcoind: add lru caching for results

lib/services/bitcoind.js

@@ -9,6 +9,7 @@ var bitcore = require('bitcore-lib');
 var Address = bitcore.Address;
 var zmq = require('zmq');
 var async = require('async');
+var LRU = require('lru-cache');
 var BitcoinRPC = require('bitcoind-rpc');
 var $ = bitcore.util.preconditions;
 var _  = bitcore.deps._;
@@ -33,6 +34,18 @@ function Bitcoin(options) {
   this._reindex = false;
   this._reindexWait = 1000;
   Service.call(this, options);
+
+  // caches valid until there is a new block
+  this.txidsCache = LRU(50000);
+  this.balanceCache = LRU(50000);
+  this.summaryCache = LRU(50000);
+
+  // caches valid indefinetly
+  this.transactionCache = LRU(100000);
+  this.transactionInfoCache = LRU(100000);
+  this.blockCache = LRU(144);
+  this.blockHeaderCache = LRU(288);
+
   $.checkState(this.node.datadir, 'Node is missing datadir property');
 }
 
@@ -132,6 +145,12 @@ Bitcoin.prototype._loadConfiguration = function() {
 
 };
 
+Bitcoin.prototype._resetCaches = function() {
+  this.txidsCache.reset();
+  this.balanceCache.reset();
+  this.summaryCache.reset();
+};
+
 Bitcoin.prototype._registerEventHandlers = function() {
   var self = this;
 
@@ -143,6 +162,7 @@ Bitcoin.prototype._registerEventHandlers = function() {
     if (topicString === 'hashtx') {
       self.emit('tx', message.toString('hex'));
     } else if (topicString === 'hashblock') {
+      self._resetCaches();
       self.tiphash = message.toString('hex');
       self.client.getBlock(self.tiphash, function(err, response) {
         if (err) {
@@ -327,17 +347,26 @@ Bitcoin.prototype.syncPercentage = function(callback) {
 };
 
 Bitcoin.prototype.getAddressBalance = function(addressArg, options, callback) {
-  // TODO keep a cache and update the cache by a range of block heights
+  var self = this;
   var addresses = [addressArg];
   if (Array.isArray(addressArg)) {
     addresses = addressArg;
   }
-  this.client.getAddressBalance({addresses: addresses}, function(err, response) {
+  var cacheKey = addresses.join('');
-    if (err) {
+  var balance = self.balanceCache.get(cacheKey);
-      return callback(err);
+  if (balance) {
-    }
+    return setImmediate(function() {
-    callback(null, response.result);
+      callback(null, balance);
-  });
+    });
+  } else {
+    this.client.getAddressBalance({addresses: addresses}, function(err, response) {
+      if (err) {
+        return callback(err);
+      }
+      self.balanceCache.set(cacheKey, response.result);
+      callback(null, response.result);
+    });
+  }
 };
 
 Bitcoin.prototype.getAddressUnspentOutputs = function() {
@@ -345,17 +374,27 @@ Bitcoin.prototype.getAddressUnspentOutputs = function() {
 };
 
 Bitcoin.prototype.getAddressTxids = function(addressArg, options, callback) {
-  // TODO Keep a cache updated for queries
+  var self = this;
   var addresses = [addressArg];
   if (Array.isArray(addressArg)) {
     addresses = addressArg;
   }
-  this.client.getAddressTxids({addresses: addresses}, function(err, response) {
+  var cacheKey = addresses.join('');
-    if (err) {
+  var txids = self.txidsCache.get(cacheKey);
-      return callback(err);
+  if (txids) {
-    }
+    return setImmediate(function() {
-    return callback(null, response.result);
+      callback(null, txids);
-  });
+    });
+  } else {
+    self.client.getAddressTxids({addresses: addresses}, function(err, response) {
+      if (err) {
+        return callback(err);
+      }
+      response.result.reverse();
+      self.txidsCache.set(cacheKey, response.result);
+      return callback(null, response.result);
+    });
+  }
 };
 
 Bitcoin.prototype._getConfirmationsDetail = function(transaction) {
@@ -475,6 +514,19 @@ Bitcoin.prototype._getAddressStrings = function(addresses) {
   return addressStrings;
 };
 
+Bitcoin.prototype._paginateTxids = function(fullTxids, from, to) {
+  var totalCount = fullTxids.length;
+  var txids;
+  if (from >= 0 && to >= 0) {
+    var fromOffset = totalCount - from;
+    var toOffset = totalCount - to;
+    txids = fullTxids.slice(toOffset, fromOffset);
+  } else {
+    txids = fullTxids;
+  }
+  return txids;
+};
+
 Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) {
   var self = this;
   var addresses = [addressArg];
@@ -489,6 +541,10 @@ Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) {
     if (err) {
       return callback(err);
     }
+
+    var totalCount = txids.length;
+    txids = self._paginateTxids(txids, options.from, options.to);
+
     async.mapSeries(
       txids,
       function(txid, next) {
@@ -502,7 +558,7 @@ Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) {
           return callback(err);
         }
         callback(null, {
-          totalCount: txids.length,
+          totalCount: totalCount,
           items: transactions
         });
       }
@@ -514,48 +570,65 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) {
   // TODO: optional mempool
   var self = this;
   var summary = {};
+  var summaryTxids = [];
+
+  var addresses = [addressArg];
+  if (Array.isArray(addressArg)) {
+    addresses = addressArg;
+  }
+
+  var cacheKey = addresses.join('');
 
   if (_.isUndefined(options.queryMempool)) {
     options.queryMempool = true;
   }
 
-  function getBalance(done) {
+  function querySummary() {
-    self.getAddressBalance(addressArg, options, function(err, data) {
+    async.parallel([
-      if (err) {
+      function getTxList(done) {
-        return done(err);
+        self.getAddressTxids(addressArg, options, function(err, txids) {
+          if (err) {
+            return done(err);
+          }
+          summaryTxids = txids;
+          summary.appearances = txids.length;
+          done();
+        });
+      },
+      function getBalance(done) {
+        self.getAddressBalance(addressArg, options, function(err, data) {
+          if (err) {
+            return done(err);
+          }
+          summary.totalReceived = data.received;
+          summary.totalSpent = data.received - data.balance;
+          summary.balance = data.balance;
+          done();
+        });
       }
-      summary.totalReceived = data.received;
+    ], function(err) {
-      summary.totalSpent = data.received - data.balance;
+      if (err) {
-      summary.balance = data.balance;
+        return callback(err);
-      done();
+      }
+      self.summaryCache.set(cacheKey, summary);
+      if (!options.noTxList) {
+        summary.txids = summaryTxids;
+      }
+      callback(null, summary);
     });
   }
 
-  function getTxList(done) {
+  if (options.noTxList) {
-    self.getAddressTxids(addressArg, options, function(err, txids) {
+    var summaryCache = self.summaryCache.get(cacheKey);
-      if (err) {
+    if (summaryCache) {
-        return done(err);
+      callback(null, summaryCache);
-      }
+    } else {
-      summary.txids = txids;
+      querySummary();
-      summary.appearances = txids.length;
-      done();
-    });
-  }
-
-  var tasks = [];
-  if (!options.noBalance) {
-    tasks.push(getBalance);
-  }
-  if (!options.noTxList) {
-    tasks.push(getTxList);
-  }
-
-  async.parallel(tasks, function(err) {
-    if (err) {
-      return callback(err);
     }
-    callback(null, summary);
+  } else {
-  });
+    querySummary();
+  }
+
 };
 
 /**
@@ -564,29 +637,36 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) {
  */
 Bitcoin.prototype.getBlock = function(block, callback) {
   // TODO apply performance patch to the RPC method for raw data
-  // TODO keep a cache of results
   var self = this;
 
-  function queryHeader(blockhash) {
+  function queryBlock(blockhash) {
     self.client.getBlock(blockhash, false, function(err, response) {
       if (err) {
         return callback(err);
       }
-      var block = bitcore.Block.fromString(response.result);
+      var blockCache = bitcore.Block.fromString(response.result);
-      callback(null, block);
+      self.blockCache.set(block, blockCache);
+      callback(null, blockCache);
     });
   }
 
-  if (_.isNumber(block)) {
+  var cachedBlock = self.blockCache.get(block);
-    self.client.getBlockHash(block, function(err, response) {
+  if (cachedBlock) {
-      if (err) {
+    return setImmediate(function() {
-        return callback(err);
+      callback(null, cachedBlock);
-      }
-      var blockhash = response.result;
-      queryHeader(blockhash);
     });
   } else {
-    queryHeader(block);
+    if (_.isNumber(block)) {
+      self.client.getBlockHash(block, function(err, response) {
+        if (err) {
+          return callback(err);
+        }
+        var blockhash = response.result;
+        queryBlock(blockhash);
+      });
+    } else {
+      queryBlock(block);
+    }
   }
 
 };
@@ -614,7 +694,6 @@ Bitcoin.prototype.getBlockHashesByTimestamp = function(high, low, callback) {
  * @returns {Object}
  */
 Bitcoin.prototype.getBlockHeader = function(block, callback) {
-  // TODO keep a cache of queries
   var self = this;
 
   function queryHeader(blockhash) {
@@ -683,18 +762,26 @@ Bitcoin.prototype.sendTransaction = function(tx, allowAbsurdFees, callback) {
  * @param {Function} callback
  */
 Bitcoin.prototype.getTransaction = function(txid, queryMempool, callback) {
-  // TODO keep an LRU cache available of transactions
+  var self = this;
-  this.client.getRawTransaction(txid, function(err, response) {
+  var tx = self.transactionCache.get(txid);
-    if (err) {
+  if (tx) {
-      return callback(err);
+    return setImmediate(function() {
-    }
+      callback(null, tx);
-    if (!response.result) {
+    });
-      return callback(new errors.Transaction.NotFound());
+  } else {
-    }
+    self.client.getRawTransaction(txid, function(err, response) {
-    var tx = Transaction();
+      if (err) {
-    tx.fromString(response.result);
+        return callback(err);
-    callback(null, tx);
+      }
-  });
+      if (!response.result) {
+        return callback(new errors.Transaction.NotFound());
+      }
+      var tx = Transaction();
+      tx.fromString(response.result);
+      self.transactionCache.set(txid, tx);
+      callback(null, tx);
+    });
+  }
 };
 
 /**
@@ -710,22 +797,29 @@ Bitcoin.prototype.getTransaction = function(txid, queryMempool, callback) {
  * @param {Function} callback
  */
 Bitcoin.prototype.getTransactionWithBlockInfo = function(txid, queryMempool, callback) {
-  // TODO keep an LRU cache available of transactions
+  var self = this;
-  // TODO get information from txindex as an RPC method
+  var tx = self.transactionInfoCache.get(txid);
-  this.client.getRawTransaction(txid, 1, function(err, response) {
+  if (tx) {
-    if (err) {
+    return setImmediate(function() {
-      return callback(err);
+      callback(null, tx);
-    }
+    });
-    if (!response.result) {
+  } else {
-      return callback(new errors.Transaction.NotFound());
+    self.client.getRawTransaction(txid, 1, function(err, response) {
-    }
+      if (err) {
-    var tx = Transaction();
+        return callback(err);
-    tx.fromString(response.result.hex);
+      }
-    tx.__blockHash = response.result.blockhash;
+      if (!response.result) {
-    tx.__height = response.result.height;
+        return callback(new errors.Transaction.NotFound());
-    tx.__timestamp = response.result.time;
+      }
-    callback(null, tx);
+      var tx = Transaction();
-  });
+      tx.fromString(response.result.hex);
+      tx.__blockHash = response.result.blockhash;
+      tx.__height = response.result.height;
+      tx.__timestamp = response.result.time;
+      self.transactionInfoCache.set(txid, tx);
+      callback(null, tx);
+    });
+  }
 };
 
 /**
@@ -733,7 +827,6 @@ Bitcoin.prototype.getTransactionWithBlockInfo = function(txid, queryMempool, cal
  * @returns {String}
  */
 Bitcoin.prototype.getBestBlockHash = function(callback) {
-  // TODO keep an LRU cache available of transactions
   this.client.getBestBlockHash(function(err, response) {
     if (err) {
       return callback(err);

package.json

@@ -41,14 +41,15 @@
   "dependencies": {
     "async": "^1.3.0",
     "bindings": "^1.2.1",
-    "bitcore-lib": "^0.13.13",
     "bitcoind-rpc": "^0.3.0",
+    "bitcore-lib": "^0.13.13",
     "body-parser": "^1.13.3",
     "colors": "^1.1.2",
     "commander": "^2.8.1",
     "errno": "^0.1.4",
     "express": "^4.13.3",
     "liftoff": "^2.2.0",
+    "lru-cache": "^4.0.1",
     "memdown": "^1.0.0",
     "mkdirp": "0.5.0",
     "npm": "^2.14.1",