504 lines
13 KiB
JavaScript
504 lines
13 KiB
JavaScript
'use strict';
|
|
|
|
require('classtool');
|
|
|
|
|
|
|
|
function spec() {
|
|
var util = require('util');
|
|
var RpcClient = require('bitcore/RpcClient').class();
|
|
var bitutil = require('bitcore/util/util');
|
|
var Address = require('bitcore/Address').class();
|
|
var Script = require('bitcore/Script').class();
|
|
var networks = require('bitcore/networks');
|
|
var async = require('async');
|
|
var config = require('../config/config');
|
|
var Sync = require('./Sync').class();
|
|
var sockets = require('../app/controllers/socket.js');
|
|
var BlockExtractor = require('./BlockExtractor.js').class();
|
|
//var Deserialize = require('bitcore/Deserialize');
|
|
|
|
|
|
var BAD_GEN_ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:';
|
|
|
|
function HistoricSync() {
|
|
this.network = config.network === 'testnet' ? networks.testnet: networks.livenet;
|
|
|
|
var genesisHashReversed = new Buffer(32);
|
|
this.network.genesisBlock.hash.copy(genesisHashReversed);
|
|
this.genesis = genesisHashReversed.reverse().toString('hex');
|
|
|
|
|
|
//available status: starting / syncing / finished / aborted
|
|
this.status = 'starting';
|
|
this.error = null;
|
|
|
|
this.syncPercentage = 0;
|
|
this.syncedBlocks = 0;
|
|
this.skippedBlocks = 0;
|
|
this.orphanBlocks = 0;
|
|
this.type ='';
|
|
}
|
|
|
|
function p() {
|
|
var args = [];
|
|
Array.prototype.push.apply(args, arguments);
|
|
|
|
args.unshift('[historic_sync]');
|
|
/*jshint validthis:true */
|
|
console.log.apply(this, args);
|
|
}
|
|
|
|
HistoricSync.prototype.setError = function(err) {
|
|
var self = this;
|
|
self.error = err.toString();
|
|
self.status='error';
|
|
self.showProgress();
|
|
};
|
|
|
|
HistoricSync.prototype.init = function(opts, cb) {
|
|
|
|
var self = this;
|
|
self.opts = opts;
|
|
|
|
self.rpc = new RpcClient(config.bitcoind);
|
|
self.sync = new Sync(opts);
|
|
|
|
self.sync.init(opts, function(err) {
|
|
if (err) {
|
|
self.setError(err);
|
|
return cb(err);
|
|
}
|
|
else {
|
|
// check testnet?
|
|
self.rpc.getBlockHash(0, function(err, res){
|
|
if (!err && ( res && res.result !== self.genesis)) {
|
|
err = new Error(BAD_GEN_ERROR + config.network);
|
|
self.setError(err);
|
|
}
|
|
if (err) self.setError(err);
|
|
return cb(err);
|
|
});
|
|
}
|
|
});
|
|
|
|
};
|
|
|
|
HistoricSync.prototype.close = function() {
|
|
this.sync.close();
|
|
};
|
|
|
|
|
|
HistoricSync.prototype.info = function() {
|
|
return {
|
|
status: this.status,
|
|
blockChainHeight: this.blockChainHeight,
|
|
syncPercentage: this.syncPercentage,
|
|
skippedBlocks: this.skippedBlocks,
|
|
syncedBlocks: this.syncedBlocks,
|
|
orphanBlocks: this.orphanBlocks,
|
|
syncTipHash: this.sync.tip,
|
|
error: this.error,
|
|
type: this.type,
|
|
};
|
|
};
|
|
|
|
HistoricSync.prototype.showProgress = function() {
|
|
var self = this;
|
|
|
|
if ( ( self.syncedBlocks + self.skippedBlocks) % self.step !== 1) return;
|
|
|
|
if (self.error) {
|
|
p('ERROR: ' + self.error);
|
|
}
|
|
else {
|
|
self.syncPercentage = parseFloat(100 * (self.syncedBlocks + self.skippedBlocks) / self.blockChainHeight).toFixed(3);
|
|
if (self.syncPercentage > 100) self.syncPercentage = 100;
|
|
|
|
p(util.format('status: [%d%%] skipped: %d ', self.syncPercentage, self.skippedBlocks));
|
|
}
|
|
if (self.opts.shouldBroadcastSync) {
|
|
sockets.broadcastSyncInfo(self.info());
|
|
}
|
|
|
|
//TODO
|
|
// if (self.syncPercentage > 10) {
|
|
// process.exit(-1);
|
|
// }
|
|
};
|
|
|
|
HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, scanOpts, cb) {
|
|
var self = this;
|
|
|
|
// recursion end.
|
|
if (!blockHash) return cb();
|
|
|
|
var existed = false;
|
|
var blockInfo;
|
|
|
|
async.series([
|
|
// Already got it?
|
|
function(c) {
|
|
self.sync.bDb.has(blockHash, function(err, ret) {
|
|
if (err) {
|
|
p(err);
|
|
return c(err);
|
|
}
|
|
|
|
if (ret) existed = true;
|
|
return c();
|
|
});
|
|
},
|
|
//show some (inacurate) status
|
|
function(c) {
|
|
self.showProgress();
|
|
return c();
|
|
},
|
|
|
|
function(c) {
|
|
self.rpc.getBlock(blockHash, function(err, ret) {
|
|
if (err) return c(err);
|
|
if (ret) {
|
|
blockInfo = ret.result;
|
|
// this is to match block retreived from file
|
|
if (blockInfo.hash === self.genesis)
|
|
blockInfo.previousblockhash = self.network.genesisBlock.prev_hash.toString('hex');
|
|
}
|
|
else {
|
|
blockInfo = null;
|
|
}
|
|
|
|
return c();
|
|
});
|
|
},
|
|
//store it
|
|
function(c) {
|
|
if (existed) return c();
|
|
|
|
// When storing files from RPC recusively, reorgs are disabled
|
|
self.sync.storeTipBlock(blockInfo, false, function(err) {
|
|
return c(err);
|
|
});
|
|
}], function(err) {
|
|
|
|
if (err) {
|
|
self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]',
|
|
blockHash, err, self.syncedBlocks));
|
|
return cb(err);
|
|
}
|
|
else {
|
|
self.status = 'syncing';
|
|
}
|
|
|
|
if ( (scanOpts.upToExisting && existed &&
|
|
self.syncedBlocks >= self.blockChainHeight) ||
|
|
(blockEnd && blockEnd === blockHash)) {
|
|
self.status = 'finished';
|
|
p('DONE. Found block: ', blockHash);
|
|
self.showProgress();
|
|
return cb(err);
|
|
}
|
|
|
|
// Continue
|
|
if (blockInfo) {
|
|
|
|
if (existed)
|
|
self.skippedBlocks++;
|
|
else
|
|
self.syncedBlocks++;
|
|
|
|
// recursion
|
|
if (scanOpts.prev && blockInfo.previousblockhash)
|
|
return self.getPrevNextBlock(blockInfo.previousblockhash, blockEnd, scanOpts, cb);
|
|
|
|
if (scanOpts.next && blockInfo.nextblockhash)
|
|
return self.getPrevNextBlock(blockInfo.nextblockhash, blockEnd, scanOpts, cb);
|
|
}
|
|
return cb(err);
|
|
});
|
|
};
|
|
|
|
|
|
// TODO. replace with
|
|
// Script.prototype.getAddrStrs if that one get merged in bitcore
|
|
HistoricSync.prototype.getAddrStr = function(s) {
|
|
var self = this;
|
|
|
|
var addrStrs = [];
|
|
var type = s.classify();
|
|
var addr;
|
|
|
|
switch(type) {
|
|
case Script.TX_PUBKEY:
|
|
var chunk = s.captureOne();
|
|
addr = new Address(self.network.addressPubkey, bitutil.sha256ripe160(chunk));
|
|
addrStrs = [ addr.toString() ];
|
|
break;
|
|
case Script.TX_PUBKEYHASH:
|
|
addr = new Address(self.network.addressPubkey, s.captureOne());
|
|
addrStrs = [ addr.toString() ];
|
|
break;
|
|
case Script.TX_SCRIPTHASH:
|
|
addr = new Address(self.network.addressScript, s.captureOne());
|
|
addrStrs = [ addr.toString() ];
|
|
break;
|
|
case Script.TX_MULTISIG:
|
|
var chunks = s.capture();
|
|
chunks.forEach(function(chunk) {
|
|
var a = new Address(self.network.addressPubkey, bitutil.sha256ripe160(chunk));
|
|
addrStrs.push(a.toString());
|
|
});
|
|
break;
|
|
case Script.TX_UNKNOWN:
|
|
break;
|
|
}
|
|
|
|
return addrStrs;
|
|
};
|
|
|
|
HistoricSync.prototype.getBlockFromFile = function(cb) {
|
|
var self = this;
|
|
|
|
var blockInfo;
|
|
|
|
//get Info
|
|
self.blockExtractor.getNextBlock(function(err, b) {
|
|
if (err || ! b) return cb(err);
|
|
|
|
blockInfo = b.getStandardizedObject(b.txs, self.network);
|
|
// blockInfo.curWork = Deserialize.intFromCompact(b.bits);
|
|
// We keep the RPC field names
|
|
blockInfo.previousblockhash = blockInfo.prev_block;
|
|
|
|
var ti=0;
|
|
// Get TX Address
|
|
b.txs.forEach(function(t) {
|
|
|
|
|
|
var objTx = blockInfo.tx[ti++];
|
|
|
|
//add time from block
|
|
objTx.time = blockInfo.time;
|
|
|
|
var to=0;
|
|
t.outs.forEach( function(o) {
|
|
|
|
|
|
var s = new Script(o.s);
|
|
var addrs = self.getAddrStr(s);
|
|
|
|
// support only for p2pubkey p2pubkeyhash and p2sh
|
|
if (addrs.length === 1) {
|
|
objTx.out[to].addrStr = addrs[0];
|
|
}
|
|
to++;
|
|
});
|
|
});
|
|
|
|
return cb(err,blockInfo);
|
|
});
|
|
};
|
|
|
|
|
|
HistoricSync.prototype.nextBlockFromFile = function(scanOpts, cb) {
|
|
var self = this;
|
|
|
|
self.showProgress();
|
|
|
|
self.getBlockFromFile(function(err, blockInfo) {
|
|
if (err) {
|
|
self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]',
|
|
blockInfo ? blockInfo.hash : '-', err, self.syncedBlocks));
|
|
return cb(err);
|
|
}
|
|
|
|
self.sync.storeTipBlock(blockInfo, function(err) {
|
|
if (blockInfo && blockInfo.hash) {
|
|
self.syncedBlocks++;
|
|
} else
|
|
self.status = 'finished';
|
|
|
|
if (err) {
|
|
self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]',
|
|
blockInfo ? blockInfo.hash : '-', err, self.syncedBlocks));
|
|
}
|
|
return cb(err);
|
|
});
|
|
});
|
|
|
|
};
|
|
|
|
|
|
HistoricSync.prototype.countNotOrphan = function(cb) {
|
|
var self = this;
|
|
|
|
if (self.notOrphanCount) return cb(null, self.notOrphanCount);
|
|
|
|
|
|
self.sync.bDb.countNotOrphan(function(err, count) {
|
|
if (err) return cb(err);
|
|
self.notOrphanCount = count;
|
|
return cb(null, self.notOrphanCount);
|
|
});
|
|
};
|
|
|
|
|
|
HistoricSync.prototype.getBlockCount = function(cb) {
|
|
var self = this;
|
|
|
|
if (self.blockChainHeight) return cb();
|
|
|
|
self.rpc.getBlockCount(function(err, res) {
|
|
if (err) return cb(err);
|
|
self.blockChainHeight = res.result;
|
|
return cb();
|
|
});
|
|
};
|
|
|
|
HistoricSync.prototype.importHistory = function(scanOpts, next) {
|
|
var self = this;
|
|
|
|
var lastBlock;
|
|
|
|
async.series([
|
|
function(cb) {
|
|
if (scanOpts.destroy) {
|
|
p('Deleting DB...');
|
|
return self.sync.destroy(cb);
|
|
}
|
|
return cb();
|
|
},
|
|
// We are not using getBestBlockHash, because is not available in all clients
|
|
function (cb) { return self.getBlockCount(cb); },
|
|
function(cb) {
|
|
if (!scanOpts.reverse) return cb();
|
|
|
|
self.rpc.getBlockHash(self.blockChainHeight, function(err, res) {
|
|
if (err) return cb(err);
|
|
lastBlock = res.result;
|
|
|
|
return cb();
|
|
});
|
|
},
|
|
function(cb) {
|
|
if (scanOpts.upToExisting) {
|
|
// should be isOrphan = true or null to be more accurate.
|
|
self.countNotOrphan(function(err, count) {
|
|
if (err) return cb(err);
|
|
|
|
self.syncedBlocks = count || 0;
|
|
return cb();
|
|
});
|
|
}
|
|
else {
|
|
return cb();
|
|
}
|
|
},
|
|
], function(err) {
|
|
if (err) {
|
|
self.setError(err);
|
|
return next(err, 0);
|
|
}
|
|
|
|
|
|
// SETUP Sync params
|
|
var start, end;
|
|
|
|
if (!self.step) {
|
|
var step = parseInt( (self.blockChainHeight - self.syncedBlocks) / 1000);
|
|
|
|
if (self.opts.progressStep) {
|
|
step = self.opts.progressStep;
|
|
}
|
|
|
|
if (step < 10) step = 10;
|
|
self.step = step;
|
|
}
|
|
|
|
if (scanOpts.reverse) {
|
|
start = lastBlock;
|
|
end = self.genesis;
|
|
scanOpts.prev = true;
|
|
}
|
|
else {
|
|
start = self.genesis;
|
|
end = null;
|
|
scanOpts.next = true;
|
|
}
|
|
p('Starting from: ', start);
|
|
p(' to : ', end);
|
|
p(' scanOpts: ', JSON.stringify(scanOpts));
|
|
|
|
if (scanOpts.fromFiles) {
|
|
|
|
self.status = 'syncing';
|
|
self.type = 'from .dat Files';
|
|
|
|
|
|
async.whilst(function() {
|
|
return self.status === 'syncing';
|
|
}, function (w_cb) {
|
|
self.nextBlockFromFile(scanOpts, function(err) {
|
|
setImmediate(function(){
|
|
return w_cb(err);
|
|
});
|
|
});
|
|
}, function(err) {
|
|
return next(err);
|
|
});
|
|
}
|
|
else {
|
|
self.type = 'from RPC calls';
|
|
self.getPrevNextBlock(start, end, scanOpts, function(err) {
|
|
return next(err);
|
|
});
|
|
}
|
|
|
|
});
|
|
};
|
|
|
|
// upto if we have genesis block?
|
|
HistoricSync.prototype.smartImport = function(scanOpts, next) {
|
|
var self = this;
|
|
|
|
self.sync.bDb.has(self.genesis, function(err, b) {
|
|
if (err) return next(err);
|
|
self.countNotOrphan(function(err, count) {
|
|
if (err) return next(err);
|
|
self.getBlockCount(function(err) {
|
|
if (err) return next(err);
|
|
|
|
if (!b || scanOpts.destroy || count < self.blockChainHeight * 0.8 ) {
|
|
|
|
if (!b)
|
|
p('Could not find Genesis block. Running FULL SYNC');
|
|
else
|
|
p('Less that 80% of current blockchain is stored. Running FULL SYNC',
|
|
parseInt(count/self.blockChainHeight*100));
|
|
|
|
if (config.bitcoind.dataDir) {
|
|
p('bitcoind dataDir configured...importing blocks from .dat files');
|
|
scanOpts.fromFiles = true;
|
|
self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network);
|
|
}
|
|
else {
|
|
scanOpts.reverse = true;
|
|
}
|
|
}
|
|
else {
|
|
p('Genesis block found. Syncing upto known blocks.');
|
|
p('Got ' + count + ' out of ' + self.blockChainHeight + ' blocks');
|
|
scanOpts.reverse = true;
|
|
scanOpts.upToExisting = true;
|
|
}
|
|
return self.importHistory(scanOpts, next);
|
|
});
|
|
});
|
|
});
|
|
};
|
|
|
|
return HistoricSync;
|
|
}
|
|
module.defineClass(spec);
|
|
|