From 4954be9aa2370ec3234ddfff2dceea37614b2a64 Mon Sep 17 00:00:00 2001 From: Patrick Nagurny Date: Tue, 7 Jul 2015 10:51:35 -0400 Subject: [PATCH 1/2] cached headers --- lib/blockchain.js | 9 +++++ lib/node.js | 6 +++ lib/services/block.js | 87 ++++++++++++++++++++++++++++--------------- 3 files changed, 71 insertions(+), 31 deletions(-) diff --git a/lib/blockchain.js b/lib/blockchain.js index 2ba21d43..8945686c 100644 --- a/lib/blockchain.js +++ b/lib/blockchain.js @@ -9,6 +9,7 @@ var NULL = '0000000000000000000000000000000000000000000000000000000000000000'; function BlockChain() { this.tip = NULL; + this.header = {}; this.work = {}; this.work[NULL] = 0; this.height = {}; @@ -18,6 +19,7 @@ function BlockChain() { }; this.next = {}; this.prev = {}; + this.cachedHeaders = []; } BlockChain.NULL = NULL; @@ -44,6 +46,7 @@ BlockChain.prototype.addData = function(header) { var prevHash = BufferUtil.reverse(header.prevHash).toString('hex'); var hash = header.hash; + this.header[hash] = header; this.work[hash] = this.work[prevHash] + getWork(header.bits); this.prev[hash] = prevHash; }; @@ -112,6 +115,7 @@ BlockChain.prototype.confirm = function(hash) { this.next[prevHash] = hash; this.hashByHeight[height] = hash; this.height[hash] = height; + this.cachedHeaders.unshift(this.header[hash]); }; BlockChain.prototype.unconfirm = function(hash) { @@ -123,6 +127,7 @@ BlockChain.prototype.unconfirm = function(hash) { delete this.next[prevHash]; delete this.hashByHeight[height]; delete this.height[hash]; + this.cachedHeaders.shift(); }; BlockChain.prototype.hasData = function(hash) { @@ -178,4 +183,8 @@ BlockChain.prototype.getCurrentHeight = function() { return this.height[this.tip]; }; +BlockChain.prototype.getHeaders = function() { + return this.cachedHeaders; +}; + module.exports = BlockChain; diff --git a/lib/node.js b/lib/node.js index a81da25b..bc02f1ad 100644 --- a/lib/node.js +++ b/lib/node.js @@ -128,6 +128,12 @@ BitcoreNode.prototype.initialize = function() { delete self.blockCache[deleteHash]; } }) + .then(function() { + // Update header cache every 100 blocks + if(block.height % 100 === 0) { + return self.blockService.saveHeaders(self.blockchain.getHeaders()) + } + }) .catch(function(error) { self.stop(error); }); diff --git a/lib/services/block.js b/lib/services/block.js index 0032897e..5e8a95ed 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -38,7 +38,8 @@ var Index = { height: 'bh-', // bh- -> height (-1 means disconnected) tip: 'tip', // tip -> { hash: hex, height: int }, the latest tip work: 'wk-', // wk- -> amount of work for block - header: 'header-' // header- -> JSON for block header + header: 'header-', // header- -> JSON for block header + headersCached: 'hc-' }; _.extend(Index, { getNextBlock: helper(Index.next), @@ -48,7 +49,7 @@ _.extend(Index, { getBlockByTs: function(block) { return Index.timestamp + block.header.time; }, - getBlockHeader: helper(Index.header), + getBlockHeader: helper(Index.header) }); function BlockService(opts) { @@ -466,39 +467,63 @@ BlockService.prototype.getBlockchain = function() { var blockchain = new BlockChain(); var headers = []; - console.log('Fetching headers from db...'); - var fetchHeader = function(blockHash) { - if (blockHash === BlockChain.NULL) { - console.log('All headers fetched, total =', headers.length); - return; - } - var headerKey = Index.getBlockHeader(blockHash); - return self.database.getAsync(headerKey) - .then(function(json) { - return bitcore.Block.BlockHeader.fromJSON(json); - }) - .then(function(header) { - headers.push(header); - return fetchHeader(BufferUtil.reverse(header.prevHash).toString('hex')); - }); - }; + console.log('Fetching hashes from db...'); - return self._getLatestHash() - .then(function(tip) { - if (!tip) { - console.log('No tip found, syncing blockchain from genesis block'); - return null; + return self.database.getAsync(Index.headersCached, {valueEncoding: 'json'}) + .catch(function(err) { + if(err instanceof LevelUp.errors.NotFoundError) { + return []; } - console.log('Tip is', tip); - return fetchHeader(tip) - .then(function() { - while (headers.length !== 0) { - var header = headers.pop(); - blockchain.proposeNewHeader(header); + throw err; + }) + .then(function(cachedHeaders) { + console.log(cachedHeaders.length + ' headers cached'); + cachedHeaders = cachedHeaders.map(function(json) { + return bitcore.Block.BlockHeader.fromJSON(json); + }); + console.log('Cached headers parsed. Loading remaining headers from DB'); + + var fetchHeader = function(blockHash) { + if (blockHash === BlockChain.NULL) { + console.log('All headers fetched, total =', headers.length); + return Promise.resolve(); + } else if(cachedHeaders[0] && cachedHeaders[0].hash === blockHash) { + headers = headers.concat(cachedHeaders); + return Promise.resolve(); + } + var headerKey = Index.getBlockHeader(blockHash); + return self.database.getAsync(headerKey) + .then(function(json) { + return bitcore.Block.BlockHeader.fromJSON(json); + }) + .then(function(header) { + headers.push(header); + return fetchHeader(BufferUtil.reverse(header.prevHash).toString('hex')); + }); + }; + + return self._getLatestHash() + .then(function(tip) { + if (!tip) { + console.log('No tip found, syncing blockchain from genesis block'); + return null; } - return blockchain; - }); + console.log('Tip is', tip); + + return fetchHeader(tip) + .then(function() { + while (headers.length !== 0) { + var header = headers.pop(); + blockchain.proposeNewHeader(header); + } + return blockchain; + }); + }) }); }; +BlockService.prototype.saveHeaders = function(headers) { + return this.database.putAsync(Index.headersCached, JSON.stringify(headers)); +} + module.exports = BlockService; From 938eb53a77cb7f5e462a8be854f04d504f793186 Mon Sep 17 00:00:00 2001 From: Patrick Nagurny Date: Tue, 7 Jul 2015 13:26:12 -0400 Subject: [PATCH 2/2] more optimizations --- lib/blockchain.js | 28 +++++++++++++++++++++------- lib/node.js | 2 +- lib/services/block.js | 24 +++++++++++------------- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/lib/blockchain.js b/lib/blockchain.js index 8945686c..a792073d 100644 --- a/lib/blockchain.js +++ b/lib/blockchain.js @@ -9,7 +9,6 @@ var NULL = '0000000000000000000000000000000000000000000000000000000000000000'; function BlockChain() { this.tip = NULL; - this.header = {}; this.work = {}; this.work[NULL] = 0; this.height = {}; @@ -19,7 +18,7 @@ function BlockChain() { }; this.next = {}; this.prev = {}; - this.cachedHeaders = []; + this.cache = []; } BlockChain.NULL = NULL; @@ -46,7 +45,6 @@ BlockChain.prototype.addData = function(header) { var prevHash = BufferUtil.reverse(header.prevHash).toString('hex'); var hash = header.hash; - this.header[hash] = header; this.work[hash] = this.work[prevHash] + getWork(header.bits); this.prev[hash] = prevHash; }; @@ -115,7 +113,7 @@ BlockChain.prototype.confirm = function(hash) { this.next[prevHash] = hash; this.hashByHeight[height] = hash; this.height[hash] = height; - this.cachedHeaders.unshift(this.header[hash]); + this.cache.push(this.getCachedDataForHash(hash)); }; BlockChain.prototype.unconfirm = function(hash) { @@ -127,7 +125,7 @@ BlockChain.prototype.unconfirm = function(hash) { delete this.next[prevHash]; delete this.hashByHeight[height]; delete this.height[hash]; - this.cachedHeaders.shift(); + this.cache.pop(); }; BlockChain.prototype.hasData = function(hash) { @@ -183,8 +181,24 @@ BlockChain.prototype.getCurrentHeight = function() { return this.height[this.tip]; }; -BlockChain.prototype.getHeaders = function() { - return this.cachedHeaders; +BlockChain.prototype.loadFromCache = function(cache) { + for(var i = 0; i < cache.length; i++) { + this.prev[cache[i].hash] = cache[i].prevHash; + this.work[cache[i].hash] = cache[i].work; + this.confirm(cache[i].hash); + } +}; + +BlockChain.prototype.getCache = function() { + return this.cache; +}; + +BlockChain.prototype.getCachedDataForHash = function(hash) { + return { + hash: hash, + prevHash: this.prev[hash], + work: this.work[hash] + }; }; module.exports = BlockChain; diff --git a/lib/node.js b/lib/node.js index bc02f1ad..652069e6 100644 --- a/lib/node.js +++ b/lib/node.js @@ -131,7 +131,7 @@ BitcoreNode.prototype.initialize = function() { .then(function() { // Update header cache every 100 blocks if(block.height % 100 === 0) { - return self.blockService.saveHeaders(self.blockchain.getHeaders()) + return self.blockService.saveBlockchainCache(self.blockchain.getCache()); } }) .catch(function(error) { diff --git a/lib/services/block.js b/lib/services/block.js index 5e8a95ed..b676675c 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -39,7 +39,7 @@ var Index = { tip: 'tip', // tip -> { hash: hex, height: int }, the latest tip work: 'wk-', // wk- -> amount of work for block header: 'header-', // header- -> JSON for block header - headersCached: 'hc-' + blockchain: 'bc-' }; _.extend(Index, { getNextBlock: helper(Index.next), @@ -49,7 +49,7 @@ _.extend(Index, { getBlockByTs: function(block) { return Index.timestamp + block.header.time; }, - getBlockHeader: helper(Index.header) + getBlockHeader: helper(Index.header), }); function BlockService(opts) { @@ -469,26 +469,22 @@ BlockService.prototype.getBlockchain = function() { console.log('Fetching hashes from db...'); - return self.database.getAsync(Index.headersCached, {valueEncoding: 'json'}) + return self.database.getAsync(Index.blockchain, {valueEncoding: 'json'}) .catch(function(err) { if(err instanceof LevelUp.errors.NotFoundError) { return []; } throw err; }) - .then(function(cachedHeaders) { - console.log(cachedHeaders.length + ' headers cached'); - cachedHeaders = cachedHeaders.map(function(json) { - return bitcore.Block.BlockHeader.fromJSON(json); - }); - console.log('Cached headers parsed. Loading remaining headers from DB'); + .then(function(blockchainCache) { + console.log(blockchainCache.length + ' headers cached'); var fetchHeader = function(blockHash) { if (blockHash === BlockChain.NULL) { console.log('All headers fetched, total =', headers.length); return Promise.resolve(); - } else if(cachedHeaders[0] && cachedHeaders[0].hash === blockHash) { - headers = headers.concat(cachedHeaders); + } else if(blockchainCache.length && blockchainCache[blockchainCache.length - 1].hash === blockHash) { + console.log(headers.length + ' non-cached headers loaded'); return Promise.resolve(); } var headerKey = Index.getBlockHeader(blockHash); @@ -512,6 +508,8 @@ BlockService.prototype.getBlockchain = function() { return fetchHeader(tip) .then(function() { + blockchain.loadFromCache(blockchainCache); + while (headers.length !== 0) { var header = headers.pop(); blockchain.proposeNewHeader(header); @@ -522,8 +520,8 @@ BlockService.prototype.getBlockchain = function() { }); }; -BlockService.prototype.saveHeaders = function(headers) { - return this.database.putAsync(Index.headersCached, JSON.stringify(headers)); +BlockService.prototype.saveBlockchainCache = function(cache) { + return this.database.putAsync(Index.blockchain, JSON.stringify(cache)); } module.exports = BlockService;