zcash-hackworks
/
bitcore-node-zcash
mirror of https://github.com/zcash-hackworks/bitcore-node-zcash.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #197 from braydonf/feature/better-reorgs 

Feature/better reorgs
This commit is contained in:
Patrick Nagurny 2015-09-04 13:58:48 -04:00
parent d068681283 50925d1e0f
commit f72c478492
2 changed files with 111 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

218
lib/services/db.js
View File

@ -16,9 +16,6 @@ var errors = index.errors;
var log = index.log; var log = index.log;
var Transaction = require('../transaction'); var Transaction = require('../transaction');
var Service = require('../service'); var Service = require('../service');
var utils = require('../utils');
var MAX_STACK_DEPTH = 1000;
/** /**
* Represents the current state of the bitcoin blockchain. Other services * Represents the current state of the bitcoin blockchain. Other services
@ -48,13 +45,6 @@ function DB(options) {
this._setDataPath(); this._setDataPath();
this.cache = {
hashes: {}, // dictionary of hash -> prevHash
chainHashes: {}
};
this.lastSavedMetadata = null;
this.lastSavedMetadataThreshold = 0; // Set this during syncing for faster performance
this.levelupStore = leveldown; this.levelupStore = leveldown;
if (options.store) { if (options.store) {
this.levelupStore = options.store; this.levelupStore = options.store;
@ -125,19 +115,19 @@ DB.prototype.start = function(callback) {
}); });
} else { } else {
metadata.tip = metadata.tip;
self.getBlock(metadata.tip, function(err, tip) { self.getBlock(metadata.tip, function(err, tip) {
if(err) { if(err) {
return callback(err); return callback(err);
} }
self.tip = tip; self.tip = tip;
self.tip.__height = metadata.tipHeight; var blockIndex = self.node.services.bitcoind.getBlockIndex(self.tip.hash);
self.cache = metadata.cache; if (!blockIndex) {
return callback(new Error('Could not get height for tip.'));
}
self.tip.__height = blockIndex.height;
self.sync(); self.sync();
self.emit('ready'); self.emit('ready');
setImmediate(callback); setImmediate(callback);
}); });
} }
}); });
@ -303,19 +293,10 @@ DB.prototype.saveMetadata = function(callback) {
callback = callback || defaultCallback; callback = callback || defaultCallback;
var threshold = self.lastSavedMetadataThreshold;
if (self.lastSavedMetadata && Date.now() < self.lastSavedMetadata.getTime() + threshold) {
return callback();
}
var metadata = { var metadata = {
tip: self.tip ? self.tip.hash : null, tip: self.tip ? self.tip.hash : null
tipHeight: self.tip && self.tip.__height ? self.tip.__height : 0,
cache: self.cache
}; };
self.lastSavedMetadata = new Date();
this.store.put('metadata', JSON.stringify(metadata), {}, callback); this.store.put('metadata', JSON.stringify(metadata), {}, callback);
}; };
@ -406,76 +387,6 @@ DB.prototype.runAllBlockHandlers = function(block, add, callback) {
); );
}; };
/**
* Will get an array of hashes all the way to the genesis block for
* the chain based on "block hash" as the tip.
*
* @param {String} block hash - a block hash
* @param {Function} callback - A function that accepts: Error and Array of hashes
*/
DB.prototype.getHashes = function getHashes(tipHash, callback) {
var self = this;
$.checkArgument(utils.isHash(tipHash));
var hashes = [];
var depth = 0;
function getHashAndContinue(err, hash) {
/* jshint maxstatements: 20 */
if (err) {
return callback(err);
}
depth++;
hashes.unshift(hash);
if (hash === self.genesis.hash) {
// Stop at the genesis block
self.cache.chainHashes[tipHash] = hashes;
callback(null, hashes);
} else if(self.cache.chainHashes[hash]) {
hashes.shift();
hashes = self.cache.chainHashes[hash].concat(hashes);
self.cache.chainHashes[tipHash] = hashes;
if(hash !== tipHash) {
delete self.cache.chainHashes[hash];
}
callback(null, hashes);
} else {
// Continue with the previous hash
// check cache first
var prevHash = self.cache.hashes[hash];
if(prevHash) {
// Don't let the stack get too deep. Otherwise we will crash.
if(depth >= MAX_STACK_DEPTH) {
depth = 0;
return setImmediate(function() {
getHashAndContinue(null, prevHash);
});
} else {
return getHashAndContinue(null, prevHash);
}
} else {
// do a db call if we don't have it
self.getPrevHash(hash, function(err, prevHash) {
if(err) {
return callback(err);
}
return getHashAndContinue(null, prevHash);
});
}
}
}
getHashAndContinue(null, tipHash);
};
/** /**
* This function will find the common ancestor between the current chain and a forked block, * This function will find the common ancestor between the current chain and a forked block,
* by moving backwards from the forked block until it meets the current chain. * by moving backwards from the forked block until it meets the current chain.
@ -486,41 +397,61 @@ DB.prototype.findCommonAncestor = function(block, done) {
var self = this; var self = this;
// The current chain of hashes will likely already be available in a cache. var mainPosition = self.tip.hash;
self.getHashes(self.tip.hash, function(err, currentHashes) { var forkPosition = block.hash;
if (err) {
done(err); var mainHashesMap = {};
var forkHashesMap = {};
mainHashesMap[mainPosition] = true;
forkHashesMap[forkPosition] = true;
var commonAncestor = null;
async.whilst(
function() {
return !commonAncestor;
},
function(next) {
if(mainPosition) {
var mainBlockIndex = self.node.services.bitcoind.getBlockIndex(mainPosition);
if(mainBlockIndex && mainBlockIndex.prevHash) {
mainHashesMap[mainBlockIndex.prevHash] = true;
mainPosition = mainBlockIndex.prevHash;
} else {
mainPosition = null;
}
}
if(forkPosition) {
var forkBlockIndex = self.node.services.bitcoind.getBlockIndex(forkPosition);
if(forkBlockIndex && forkBlockIndex.prevHash) {
forkHashesMap[forkBlockIndex.prevHash] = true;
forkPosition = forkBlockIndex.prevHash;
} else {
forkPosition = null;
}
}
if(forkPosition && mainHashesMap[forkPosition]) {
commonAncestor = forkPosition;
}
if(mainPosition && forkHashesMap[mainPosition]) {
commonAncestor = mainPosition;
}
if(!mainPosition && !forkPosition) {
return next(new Error('Unknown common ancestor'));
}
setImmediate(next);
},
function(err) {
done(err, commonAncestor);
} }
);
// Create a hash map for faster lookups
var currentHashesMap = {};
var length = currentHashes.length;
for (var i = 0; i < length; i++) {
currentHashesMap[currentHashes[i]] = true;
}
// TODO: expose prevHash as a string from bitcore
var ancestorHash = BufferUtil.reverse(block.header.prevHash).toString('hex');
// We only need to go back until we meet the main chain for the forked block
// and thus don't need to find the entire chain of hashes.
while(ancestorHash && !currentHashesMap[ancestorHash]) {
var blockIndex = self.node.services.bitcoind.getBlockIndex(ancestorHash);
ancestorHash = blockIndex ? blockIndex.prevHash : null;
}
// Hash map is no-longer needed, quickly let
// scavenging garbage collection know to cleanup
currentHashesMap = null;
if (!ancestorHash) {
return done(new Error('Unknown common ancestor.'));
}
done(null, ancestorHash);
});
}; };
/** /**
@ -537,6 +468,7 @@ DB.prototype.syncRewind = function(block, done) {
if (err) { if (err) {
return done(err); return done(err);
} }
log.warn('Reorg common ancestor found:', ancestorHash);
// Rewind the chain to the common ancestor // Rewind the chain to the common ancestor
async.whilst( async.whilst(
function() { function() {
@ -594,7 +526,6 @@ DB.prototype.sync = function() {
} }
self.bitcoindSyncing = true; self.bitcoindSyncing = true;
self.lastSavedMetadataThreshold = 30000;
var height; var height;
@ -620,28 +551,18 @@ DB.prototype.sync = function() {
// Populate height // Populate height
block.__height = self.tip.__height + 1; block.__height = self.tip.__height + 1;
// Update cache.hashes // Create indexes
self.cache.hashes[block.hash] = prevHash; self.connectBlock(block, function(err) {
// Update cache.chainHashes
self.getHashes(block.hash, function(err, hashes) {
if (err) { if (err) {
return done(err); return done(err);
} }
// Create indexes self.tip = block;
self.connectBlock(block, function(err) { log.debug('Saving metadata');
if (err) { self.saveMetadata();
return done(err); log.debug('Chain added block to main chain');
} self.emit('addblock', block);
self.tip = block; setImmediate(done);
log.debug('Saving metadata');
self.saveMetadata();
log.debug('Chain added block to main chain');
self.emit('addblock', block);
setImmediate(done);
});
}); });
} else { } else {
// This block doesn't progress the current tip, so we'll attempt // This block doesn't progress the current tip, so we'll attempt
// to rewind the chain to the common ancestor of the block and // to rewind the chain to the common ancestor of the block and
@ -668,7 +589,6 @@ DB.prototype.sync = function() {
} }
self.bitcoindSyncing = false; self.bitcoindSyncing = false;
self.lastSavedMetadataThreshold = 0;
self.saveMetadata(); self.saveMetadata();
// If bitcoind is completely synced // If bitcoind is completely synced

113
test/services/db.unit.js
View File

@ -162,19 +162,20 @@ describe('DB Service', function() {
}); });
it('metadata from the database if it exists', function(done) { it('metadata from the database if it exists', function(done) {
var tipHash = '00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206';
var node = { var node = {
network: Networks.testnet, network: Networks.testnet,
datadir: 'testdir', datadir: 'testdir',
services: { services: {
bitcoind: { bitcoind: {
genesisBuffer: genesisBuffer, genesisBuffer: genesisBuffer,
getBlockIndex: sinon.stub().returns({tip:tipHash}),
on: sinon.stub() on: sinon.stub()
} }
} }
}; };
var tip = Block.fromBuffer(genesisBuffer); var tip = Block.fromBuffer(genesisBuffer);
var db = new TestDB({node: node}); var db = new TestDB({node: node});
var tipHash = '00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206';
db.getMetadata = sinon.stub().callsArgWith(0, null, { db.getMetadata = sinon.stub().callsArgWith(0, null, {
tip: tipHash, tip: tipHash,
tipHeight: 0 tipHeight: 0
@ -535,12 +536,7 @@ describe('DB Service', function() {
put: function(key, value, options, callback) { put: function(key, value, options, callback) {
key.should.equal('metadata'); key.should.equal('metadata');
JSON.parse(value).should.deep.equal({ JSON.parse(value).should.deep.equal({
tip: '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f', tip: '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f'
tipHeight: 0,
cache: {
hashes: {},
chainHashes: {}
}
}); });
options.should.deep.equal({}); options.should.deep.equal({});
callback.should.be.a('function'); callback.should.be.a('function');
@ -549,22 +545,6 @@ describe('DB Service', function() {
}; };
db.saveMetadata(); db.saveMetadata();
}); });
it('will not call store with threshold', function(done) {
var db = new DB(baseConfig);
db.lastSavedMetadata = new Date();
db.lastSavedMetadataThreshold = 30000;
var put = sinon.stub();
db.store = {
put: put
};
db.saveMetadata(function(err) {
if (err) {
throw err;
}
put.callCount.should.equal(0);
done();
});
});
}); });
describe('#getMetadata', function() { describe('#getMetadata', function() {
@ -689,88 +669,79 @@ describe('DB Service', function() {
}); });
}); });
describe('#getHashes', function() {
it('should get an array of chain hashes', function(done) {
var blocks = {};
var genesisBlock = Block.fromBuffer(new Buffer(chainData[0], 'hex'));
var block1 = Block.fromBuffer(new Buffer(chainData[1], 'hex'));
var block2 = Block.fromBuffer(new Buffer(chainData[2], 'hex'));
blocks[genesisBlock.hash] = genesisBlock;
blocks[block1.hash] = block1;
blocks[block2.hash] = block2;
var db = new DB(baseConfig);
db.genesis = genesisBlock;
db.getPrevHash = function(blockHash, cb) {
// TODO: expose prevHash as a string from bitcore
var prevHash = BufferUtil.reverse(blocks[blockHash].header.prevHash).toString('hex');
cb(null, prevHash);
};
db.tip = block2;
// the test
db.getHashes(block2.hash, function(err, hashes) {
should.not.exist(err);
should.exist(hashes);
hashes.length.should.equal(3);
done();
});
});
});
describe('#findCommonAncestor', function() { describe('#findCommonAncestor', function() {
it('will find an ancestor 6 deep', function() { it('will find an ancestor 6 deep', function(done) {
var db = new DB(baseConfig); var db = new DB(baseConfig);
db.getHashes = function(tipHash, callback) {
callback(null, chainHashes);
};
db.tip = { db.tip = {
hash: chainHashes[chainHashes.length] hash: chainHashes[chainHashes.length - 1]
}; };
var expectedAncestor = chainHashes[chainHashes.length - 6]; var expectedAncestor = chainHashes[chainHashes.length - 6];
var mainBlocks = {};
for(var i = chainHashes.length - 1; i > chainHashes.length - 10; i--) {
var hash = chainHashes[i];
var prevHash = hexlebuf(chainHashes[i - 1]);
mainBlocks[hash] = {
header: {
prevHash: prevHash
}
};
}
var forkedBlocks = { var forkedBlocks = {
'd7fa6f3d5b2fe35d711e6aca5530d311b8c6e45f588a65c642b8baf4b4441d82': { 'd7fa6f3d5b2fe35d711e6aca5530d311b8c6e45f588a65c642b8baf4b4441d82': {
header: { header: {
prevHash: hexlebuf('76d920dbd83beca9fa8b2f346d5c5a81fe4a350f4b355873008229b1e6f8701a') prevHash: hexlebuf('76d920dbd83beca9fa8b2f346d5c5a81fe4a350f4b355873008229b1e6f8701a')
} },
hash: 'd7fa6f3d5b2fe35d711e6aca5530d311b8c6e45f588a65c642b8baf4b4441d82'
}, },
'76d920dbd83beca9fa8b2f346d5c5a81fe4a350f4b355873008229b1e6f8701a': { '76d920dbd83beca9fa8b2f346d5c5a81fe4a350f4b355873008229b1e6f8701a': {
header: { header: {
prevHash: hexlebuf('f0a0d76a628525243c8af7606ee364741ccd5881f0191bbe646c8a4b2853e60c') prevHash: hexlebuf('f0a0d76a628525243c8af7606ee364741ccd5881f0191bbe646c8a4b2853e60c')
} },
hash: '76d920dbd83beca9fa8b2f346d5c5a81fe4a350f4b355873008229b1e6f8701a'
}, },
'f0a0d76a628525243c8af7606ee364741ccd5881f0191bbe646c8a4b2853e60c': { 'f0a0d76a628525243c8af7606ee364741ccd5881f0191bbe646c8a4b2853e60c': {
header: { header: {
prevHash: hexlebuf('2f72b809d5ccb750c501abfdfa8c4c4fad46b0b66c088f0568d4870d6f509c31') prevHash: hexlebuf('2f72b809d5ccb750c501abfdfa8c4c4fad46b0b66c088f0568d4870d6f509c31')
} },
hash: 'f0a0d76a628525243c8af7606ee364741ccd5881f0191bbe646c8a4b2853e60c'
}, },
'2f72b809d5ccb750c501abfdfa8c4c4fad46b0b66c088f0568d4870d6f509c31': { '2f72b809d5ccb750c501abfdfa8c4c4fad46b0b66c088f0568d4870d6f509c31': {
header: { header: {
prevHash: hexlebuf('adf66e6ae10bc28fc22bc963bf43e6b53ef4429269bdb65038927acfe66c5453') prevHash: hexlebuf('adf66e6ae10bc28fc22bc963bf43e6b53ef4429269bdb65038927acfe66c5453')
} },
hash: '2f72b809d5ccb750c501abfdfa8c4c4fad46b0b66c088f0568d4870d6f509c31'
}, },
'adf66e6ae10bc28fc22bc963bf43e6b53ef4429269bdb65038927acfe66c5453': { 'adf66e6ae10bc28fc22bc963bf43e6b53ef4429269bdb65038927acfe66c5453': {
header: { header: {
prevHash: hexlebuf('3ea12707e92eed024acf97c6680918acc72560ec7112cf70ac213fb8bb4fa618') prevHash: hexlebuf('3ea12707e92eed024acf97c6680918acc72560ec7112cf70ac213fb8bb4fa618')
} },
hash: 'adf66e6ae10bc28fc22bc963bf43e6b53ef4429269bdb65038927acfe66c5453'
}, },
'3ea12707e92eed024acf97c6680918acc72560ec7112cf70ac213fb8bb4fa618': { '3ea12707e92eed024acf97c6680918acc72560ec7112cf70ac213fb8bb4fa618': {
header: { header: {
prevHash: hexlebuf(expectedAncestor) prevHash: hexlebuf(expectedAncestor)
} },
}, hash: '3ea12707e92eed024acf97c6680918acc72560ec7112cf70ac213fb8bb4fa618'
}
}; };
db.node.services = {}; db.node.services = {};
db.node.services.bitcoind = { db.node.services.bitcoind = {
getBlockIndex: function(hash) { getBlockIndex: function(hash) {
var block = forkedBlocks[hash]; var forkedBlock = forkedBlocks[hash];
var mainBlock = mainBlocks[hash];
var prevHash;
if (forkedBlock && forkedBlock.header.prevHash) {
prevHash = BufferUtil.reverse(forkedBlock.header.prevHash).toString('hex');
} else if (mainBlock && mainBlock.header.prevHash){
prevHash = BufferUtil.reverse(mainBlock.header.prevHash).toString('hex');
} else {
return null;
}
return { return {
prevHash: BufferUtil.reverse(block.header.prevHash).toString('hex') prevHash: prevHash
}; };
} }
}; };
@ -780,6 +751,7 @@ describe('DB Service', function() {
throw err; throw err;
} }
ancestorHash.should.equal(expectedAncestor); ancestorHash.should.equal(expectedAncestor);
done();
}); });
}); });
}); });
@ -854,7 +826,6 @@ describe('DB Service', function() {
__height: 0, __height: 0,
hash: lebufhex(block.header.prevHash) hash: lebufhex(block.header.prevHash)
}; };
db.getHashes = sinon.stub().callsArgWith(1, null);
db.saveMetadata = sinon.stub(); db.saveMetadata = sinon.stub();
db.emit = sinon.stub(); db.emit = sinon.stub();
db.cache = { db.cache = {