408 lines
10 KiB
JavaScript
408 lines
10 KiB
JavaScript
'use strict';
|
|
|
|
var imports = require('soop').imports();
|
|
var util = require('util');
|
|
var async = require('async');
|
|
|
|
var bitcore = require('bitcore');
|
|
var networks = bitcore.networks;
|
|
var config = imports.config || require('../config/config');
|
|
var Sync = require('./Sync');
|
|
var sockets = require('../app/controllers/socket.js');
|
|
var BlockExtractor = require('./BlockExtractor.js');
|
|
var buffertools = require('buffertools');
|
|
var bitcoreUtil = bitcore.util;
|
|
var logger = require('./logger').logger;
|
|
var info = logger.info;
|
|
var error = logger.error;
|
|
|
|
// var Deserialize = require('bitcore/Deserialize');
|
|
var BAD_GEN_ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:';
|
|
|
|
var BAD_GEN_ERROR_DB = 'Bad genesis block. Network mismatch between Insight and levelDB? Insight is configured for:';
|
|
function HistoricSync(opts) {
|
|
opts = opts || {};
|
|
this.shouldBroadcast = opts.shouldBroadcastSync;
|
|
|
|
this.network = config.network === 'testnet' ? networks.testnet: networks.livenet;
|
|
|
|
var genesisHashReversed = new Buffer(32);
|
|
this.network.genesisBlock.hash.copy(genesisHashReversed);
|
|
buffertools.reverse(genesisHashReversed);
|
|
this.genesis = genesisHashReversed.toString('hex');
|
|
|
|
var bitcore = require('bitcore');
|
|
var RpcClient = bitcore.RpcClient;
|
|
|
|
this.rpc = new RpcClient(config.bitcoind);
|
|
this.sync = new Sync(opts);
|
|
}
|
|
|
|
HistoricSync.prototype.showProgress = function() {
|
|
var self = this;
|
|
|
|
if ( self.status ==='syncing' &&
|
|
( self.syncedBlocks ) % self.step !== 1) return;
|
|
|
|
if (self.error)
|
|
error(self.error);
|
|
|
|
else {
|
|
self.updatePercentage();
|
|
info(util.format('status: [%d%%]', self.syncPercentage));
|
|
}
|
|
if (self.shouldBroadcast) {
|
|
sockets.broadcastSyncInfo(self.info());
|
|
}
|
|
//
|
|
// if (self.syncPercentage > 10) {
|
|
// process.exit(-1);
|
|
// }
|
|
};
|
|
|
|
|
|
HistoricSync.prototype.setError = function(err) {
|
|
var self = this;
|
|
self.error = err.message?err.message:err.toString();
|
|
self.status='error';
|
|
self.showProgress();
|
|
return err;
|
|
};
|
|
|
|
|
|
|
|
HistoricSync.prototype.close = function() {
|
|
this.sync.close();
|
|
};
|
|
|
|
|
|
HistoricSync.prototype.info = function() {
|
|
this.updatePercentage();
|
|
return {
|
|
status: this.status,
|
|
blockChainHeight: this.blockChainHeight,
|
|
syncPercentage: this.syncPercentage,
|
|
syncedBlocks: this.syncedBlocks,
|
|
syncTipHash: this.sync.tip,
|
|
error: this.error,
|
|
type: this.type,
|
|
startTs: this.startTs,
|
|
endTs: this.endTs,
|
|
};
|
|
};
|
|
|
|
HistoricSync.prototype.updatePercentage = function() {
|
|
var r = this.syncedBlocks / this.blockChainHeight;
|
|
this.syncPercentage = parseFloat(100 * r).toFixed(3);
|
|
if (this.syncPercentage > 100) this.syncPercentage = 100;
|
|
};
|
|
|
|
HistoricSync.prototype.getBlockFromRPC = function(cb) {
|
|
var self = this;
|
|
|
|
if (!self.currentRpcHash) return cb();
|
|
|
|
var blockInfo;
|
|
self.rpc.getBlock(self.currentRpcHash, function(err, ret) {
|
|
if (err) return cb(err);
|
|
if (ret) {
|
|
blockInfo = ret.result;
|
|
// this is to match block retreived from file
|
|
if (blockInfo.hash === self.genesis)
|
|
blockInfo.previousblockhash =
|
|
self.network.genesisBlock.prev_hash.toString('hex');
|
|
|
|
self.currentRpcHash = blockInfo.nextblockhash;
|
|
}
|
|
else {
|
|
blockInfo = null;
|
|
}
|
|
return cb(null, blockInfo);
|
|
});
|
|
};
|
|
|
|
HistoricSync.prototype.getStandardizedBlock = function(b) {
|
|
var self = this;
|
|
|
|
var block = {
|
|
hash: bitcoreUtil.formatHashFull(b.getHash()),
|
|
previousblockhash: bitcoreUtil.formatHashFull(b.prev_hash),
|
|
time: b.timestamp,
|
|
};
|
|
var isCoinBase = 1;
|
|
block.tx = b.txs.map(function(tx){
|
|
var ret = self.sync.txDb.getStandardizedTx(tx, b.timestamp, isCoinBase);
|
|
isCoinBase=0;
|
|
return ret;
|
|
});
|
|
return block;
|
|
};
|
|
|
|
HistoricSync.prototype.getBlockFromFile = function(cb) {
|
|
var self = this;
|
|
|
|
var blockInfo;
|
|
|
|
//get Info
|
|
self.blockExtractor.getNextBlock(function(err, b) {
|
|
if (err || ! b) return cb(err);
|
|
blockInfo = self.getStandardizedBlock(b);
|
|
self.sync.bDb.setLastFileIndex(self.blockExtractor.currentFileIndex, function(err) {
|
|
return cb(err,blockInfo);
|
|
});
|
|
});
|
|
};
|
|
|
|
HistoricSync.prototype.updateConnectedCountDB = function(cb) {
|
|
var self = this;
|
|
self.sync.bDb.countConnected(function(err, count) {
|
|
self.connectedCountDB = count || 0;
|
|
self.syncedBlocks = count || 0;
|
|
return cb(err);
|
|
});
|
|
};
|
|
|
|
|
|
HistoricSync.prototype.updateBlockChainHeight = function(cb) {
|
|
var self = this;
|
|
|
|
self.rpc.getBlockCount(function(err, res) {
|
|
self.blockChainHeight = res.result;
|
|
return cb(err);
|
|
});
|
|
};
|
|
|
|
|
|
HistoricSync.prototype.checkNetworkSettings = function(next) {
|
|
var self = this;
|
|
|
|
self.hasGenesis = false;
|
|
|
|
// check network config
|
|
self.rpc.getBlockHash(0, function(err, res){
|
|
if (!err && ( res && res.result !== self.genesis)) {
|
|
err = new Error(BAD_GEN_ERROR + config.network);
|
|
}
|
|
if (err) return next(err);
|
|
self.sync.bDb.has(self.genesis, function(err, b) {
|
|
if (!err && ( res && res.result !== self.genesis)) {
|
|
err = new Error(BAD_GEN_ERROR_DB + config.network);
|
|
}
|
|
self.hasGenesis = b?true:false;
|
|
return next(err);
|
|
});
|
|
});
|
|
};
|
|
|
|
HistoricSync.prototype.updateStartBlock = function(next) {
|
|
var self = this;
|
|
|
|
self.startBlock = self.genesis;
|
|
|
|
self.sync.bDb.getTip(function(err,tip) {
|
|
if (!tip) return next();
|
|
|
|
var blockInfo;
|
|
var oldtip;
|
|
|
|
//check that the tip is still on the mainchain
|
|
async.doWhilst(
|
|
function(cb) {
|
|
self.sync.bDb.fromHashWithInfo(tip, function(err, bi) {
|
|
blockInfo = bi ? bi.info : {};
|
|
if (oldtip)
|
|
self.sync.setBlockHeight(oldtip, -1, cb);
|
|
else
|
|
return cb();
|
|
});
|
|
},
|
|
function(err) {
|
|
if (err) return next(err);
|
|
var ret = false;
|
|
if ( self.blockChainHeight === blockInfo.height ||
|
|
blockInfo.confirmations > 0) {
|
|
ret = false;
|
|
}
|
|
else {
|
|
oldtip = tip;
|
|
if (!tip)
|
|
throw new Error('Previous blockchain tip was not found on bitcoind. Please reset Insight DB. Tip was:'+tip)
|
|
|
|
tip = blockInfo.previousblockhash;
|
|
info('Previous TIP is now orphan. Back to:' + tip);
|
|
ret = true;
|
|
}
|
|
return ret;
|
|
},
|
|
function(err) {
|
|
self.startBlock = tip;
|
|
info('Resuming sync from block:'+tip);
|
|
return next(err);
|
|
}
|
|
);
|
|
});
|
|
};
|
|
|
|
HistoricSync.prototype.prepareFileSync = function(opts, next) {
|
|
var self = this;
|
|
|
|
if ( opts.forceRPC || !config.bitcoind.dataDir ||
|
|
self.connectedCountDB > self.blockChainHeight * 0.9) return next();
|
|
|
|
|
|
try {
|
|
self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network);
|
|
} catch (e) {
|
|
info(e.message + '. Disabling file sync.');
|
|
return next();
|
|
}
|
|
|
|
self.getFn = self.getBlockFromFile;
|
|
self.allowReorgs = true;
|
|
self.sync.bDb.getLastFileIndex(function(err, idx) {
|
|
if (opts.forceStartFile)
|
|
self.blockExtractor.currentFileIndex = opts.forceStartFile;
|
|
else if (idx) self.blockExtractor.currentFileIndex = idx;
|
|
|
|
var h = self.genesis;
|
|
|
|
info('Seeking file to:' + self.startBlock);
|
|
//forward till startBlock
|
|
async.whilst(
|
|
function() {
|
|
return h !== self.startBlock;
|
|
},
|
|
function (w_cb) {
|
|
self.getBlockFromFile(function(err,b) {
|
|
if (!b) return w_cb('Could not find block ' + self.startBlock);
|
|
h=b.hash;
|
|
setImmediate(function(){
|
|
return w_cb(err);
|
|
});
|
|
});
|
|
}, next);
|
|
});
|
|
};
|
|
|
|
//NOP
|
|
HistoricSync.prototype.prepareRpcSync = function(opts, next) {
|
|
var self = this;
|
|
|
|
if (self.blockExtractor) return next();
|
|
self.getFn = self.getBlockFromRPC;
|
|
self.allowReorgs = true;
|
|
self.currentRpcHash = self.startBlock;
|
|
return next();
|
|
};
|
|
|
|
HistoricSync.prototype.showSyncStartMessage = function() {
|
|
var self = this;
|
|
|
|
info('Got ' + self.connectedCountDB +
|
|
' blocks in current DB, out of ' + self.blockChainHeight + ' block at bitcoind');
|
|
|
|
if (self.blockExtractor) {
|
|
info('bitcoind dataDir configured...importing blocks from .dat files');
|
|
info('First file index: ' + self.blockExtractor.currentFileIndex);
|
|
}
|
|
else {
|
|
info('syncing from RPC (slow)');
|
|
}
|
|
|
|
info('Starting from: ', self.startBlock);
|
|
self.showProgress();
|
|
};
|
|
|
|
|
|
HistoricSync.prototype.setupSyncStatus = function() {
|
|
var self = this;
|
|
|
|
var step = parseInt( (self.blockChainHeight - self.syncedBlocks) / 1000);
|
|
if (step < 10) step = 10;
|
|
|
|
self.step = step;
|
|
self.type = self.blockExtractor?'from .dat Files':'from RPC calls';
|
|
self.status = 'syncing';
|
|
self.startTs = Date.now();
|
|
self.endTs = null;
|
|
this.error = null;
|
|
this.syncPercentage = 0;
|
|
};
|
|
|
|
HistoricSync.prototype.prepareToSync = function(opts, next) {
|
|
var self = this;
|
|
|
|
self.status = 'starting';
|
|
async.series([
|
|
function(s_c) {
|
|
self.checkNetworkSettings(s_c);
|
|
},
|
|
function(s_c) {
|
|
self.updateConnectedCountDB(s_c);
|
|
},
|
|
function(s_c) {
|
|
self.updateBlockChainHeight(s_c);
|
|
},
|
|
function(s_c) {
|
|
self.updateStartBlock(s_c);
|
|
},
|
|
function(s_c) {
|
|
self.prepareFileSync(opts, s_c);
|
|
},
|
|
function(s_c) {
|
|
self.prepareRpcSync(opts, s_c);
|
|
},
|
|
],
|
|
function(err) {
|
|
if (err) return(self.setError(err));
|
|
|
|
self.showSyncStartMessage();
|
|
self.setupSyncStatus();
|
|
return next();
|
|
});
|
|
};
|
|
|
|
|
|
HistoricSync.prototype.start = function(opts, next) {
|
|
var self = this;
|
|
|
|
if (self.status==='starting' || self.status==='syncing') {
|
|
error('## Wont start to sync while status is %s', self.status);
|
|
return next();
|
|
}
|
|
|
|
self.prepareToSync(opts, function(err) {
|
|
if (err) return next(self.setError(err));
|
|
|
|
async.whilst(
|
|
function() {
|
|
self.showProgress();
|
|
return self.status === 'syncing';
|
|
},
|
|
function (w_cb) {
|
|
self.getFn(function(err,blockInfo) {
|
|
if (err) return w_cb(self.setError(err));
|
|
if (blockInfo && blockInfo.hash
|
|
&& (!opts.stopAt || opts.stopAt !== blockInfo.hash)
|
|
) {
|
|
self.syncedBlocks++;
|
|
self.sync.storeTipBlock(blockInfo, self.allowReorgs, function(err) {
|
|
if (err) return w_cb(self.setError(err));
|
|
setImmediate(function(){
|
|
return w_cb(err);
|
|
});
|
|
});
|
|
}
|
|
else {
|
|
self.endTs = Date.now();
|
|
self.status = 'finished';
|
|
console.log('Done Syncing', self.info());
|
|
return w_cb(err);
|
|
}
|
|
});
|
|
}, next);
|
|
});
|
|
};
|
|
|
|
module.exports = require('soop')(HistoricSync);
|