2014-01-16 11:11:20 -08:00
'use strict' ;
2014-03-05 18:03:56 -08:00
var imports = require ( 'soop' ) . imports ( ) ;
var util = require ( 'util' ) ;
2014-03-10 08:46:48 -07:00
var async = require ( 'async' ) ;
2014-05-07 05:45:44 -07:00
var bitcore = require ( 'bitcore' ) ;
var networks = bitcore . networks ;
2014-03-10 08:46:48 -07:00
var config = imports . config || require ( '../config/config' ) ;
2014-03-05 18:03:56 -08:00
var Sync = require ( './Sync' ) ;
var sockets = require ( '../app/controllers/socket.js' ) ;
var BlockExtractor = require ( './BlockExtractor.js' ) ;
var buffertools = require ( 'buffertools' ) ;
2014-05-23 17:23:44 -07:00
var bitcoreUtil = bitcore . util ;
var logger = require ( './logger' ) . logger ;
var info = logger . info ;
var error = logger . error ;
2014-01-21 14:13:21 -08:00
2014-03-05 18:03:56 -08:00
// var Deserialize = require('bitcore/Deserialize');
var BAD _GEN _ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:' ;
2014-01-21 14:13:21 -08:00
2014-03-05 18:03:56 -08:00
var BAD _GEN _ERROR _DB = 'Bad genesis block. Network mismatch between Insight and levelDB? Insight is configured for:' ;
function HistoricSync ( opts ) {
opts = opts || { } ;
2014-03-20 10:06:37 -07:00
this . shouldBroadcast = opts . shouldBroadcastSync ;
2014-01-21 14:13:21 -08:00
2014-03-05 18:03:56 -08:00
this . network = config . network === 'testnet' ? networks . testnet : networks . livenet ;
2014-02-17 22:40:55 -08:00
2014-03-05 18:03:56 -08:00
var genesisHashReversed = new Buffer ( 32 ) ;
this . network . genesisBlock . hash . copy ( genesisHashReversed ) ;
buffertools . reverse ( genesisHashReversed ) ;
this . genesis = genesisHashReversed . toString ( 'hex' ) ;
2014-01-17 11:36:34 -08:00
2014-05-07 05:45:44 -07:00
var bitcore = require ( 'bitcore' ) ;
var RpcClient = bitcore . RpcClient ;
2014-03-05 18:03:56 -08:00
this . rpc = new RpcClient ( config . bitcoind ) ;
this . sync = new Sync ( opts ) ;
2014-05-26 12:58:44 -07:00
this . height = 0 ;
2014-03-05 18:03:56 -08:00
}
2014-02-14 04:20:20 -08:00
2014-03-05 18:03:56 -08:00
HistoricSync . prototype . showProgress = function ( ) {
var self = this ;
2014-02-14 04:20:20 -08:00
2014-03-05 18:03:56 -08:00
if ( self . status === 'syncing' &&
2014-05-26 12:58:44 -07:00
( self . height ) % self . step !== 1 ) return ;
2014-02-14 04:20:20 -08:00
2014-05-23 17:23:44 -07:00
if ( self . error )
error ( self . error ) ;
2014-03-05 18:03:56 -08:00
else {
self . updatePercentage ( ) ;
2014-05-23 17:23:44 -07:00
info ( util . format ( 'status: [%d%%]' , self . syncPercentage ) ) ;
2014-03-05 18:03:56 -08:00
}
if ( self . shouldBroadcast ) {
sockets . broadcastSyncInfo ( self . info ( ) ) ;
}
2014-05-23 17:23:44 -07:00
//
// if (self.syncPercentage > 10) {
// process.exit(-1);
// }
2014-03-05 18:03:56 -08:00
} ;
2014-02-14 04:57:29 -08:00
2014-01-16 11:11:20 -08:00
2014-03-05 18:03:56 -08:00
HistoricSync . prototype . setError = function ( err ) {
var self = this ;
self . error = err . message ? err . message : err . toString ( ) ;
self . status = 'error' ;
self . showProgress ( ) ;
return err ;
} ;
2014-01-16 11:11:20 -08:00
2014-01-21 14:13:21 -08:00
2014-02-12 07:57:55 -08:00
2014-03-05 18:03:56 -08:00
HistoricSync . prototype . close = function ( ) {
this . sync . close ( ) ;
} ;
2014-01-16 11:11:20 -08:00
2014-03-05 18:03:56 -08:00
HistoricSync . prototype . info = function ( ) {
this . updatePercentage ( ) ;
return {
status : this . status ,
blockChainHeight : this . blockChainHeight ,
syncPercentage : this . syncPercentage ,
2014-05-26 12:58:44 -07:00
height : this . height ,
2014-03-05 18:03:56 -08:00
syncTipHash : this . sync . tip ,
error : this . error ,
type : this . type ,
startTs : this . startTs ,
endTs : this . endTs ,
2014-01-16 11:11:20 -08:00
} ;
2014-03-05 18:03:56 -08:00
} ;
HistoricSync . prototype . updatePercentage = function ( ) {
2014-05-26 12:58:44 -07:00
var r = this . height / this . blockChainHeight ;
2014-03-05 18:03:56 -08:00
this . syncPercentage = parseFloat ( 100 * r ) . toFixed ( 3 ) ;
if ( this . syncPercentage > 100 ) this . syncPercentage = 100 ;
} ;
HistoricSync . prototype . getBlockFromRPC = function ( cb ) {
var self = this ;
if ( ! self . currentRpcHash ) return cb ( ) ;
var blockInfo ;
self . rpc . getBlock ( self . currentRpcHash , function ( err , ret ) {
if ( err ) return cb ( err ) ;
if ( ret ) {
blockInfo = ret . result ;
// this is to match block retreived from file
if ( blockInfo . hash === self . genesis )
blockInfo . previousblockhash =
self . network . genesisBlock . prev _hash . toString ( 'hex' ) ;
self . currentRpcHash = blockInfo . nextblockhash ;
}
else {
blockInfo = null ;
}
return cb ( null , blockInfo ) ;
} ) ;
} ;
2014-01-16 11:11:20 -08:00
2014-05-23 17:23:44 -07:00
HistoricSync . prototype . getStandardizedBlock = function ( b ) {
var self = this ;
var block = {
hash : bitcoreUtil . formatHashFull ( b . getHash ( ) ) ,
previousblockhash : bitcoreUtil . formatHashFull ( b . prev _hash ) ,
time : b . timestamp ,
} ;
2014-05-25 13:34:49 -07:00
var isCoinBase = 1 ;
2014-05-23 17:23:44 -07:00
block . tx = b . txs . map ( function ( tx ) {
2014-05-25 19:54:08 -07:00
var ret = self . sync . txDb . getStandardizedTx ( tx , b . timestamp , isCoinBase ) ;
2014-05-25 13:34:49 -07:00
isCoinBase = 0 ;
return ret ;
2014-05-23 17:23:44 -07:00
} ) ;
return block ;
} ;
2014-03-05 18:03:56 -08:00
HistoricSync . prototype . getBlockFromFile = function ( cb ) {
var self = this ;
2014-02-08 05:57:37 -08:00
2014-03-05 18:03:56 -08:00
var blockInfo ;
2014-01-30 18:16:43 -08:00
2014-03-05 18:03:56 -08:00
//get Info
self . blockExtractor . getNextBlock ( function ( err , b ) {
if ( err || ! b ) return cb ( err ) ;
2014-05-23 17:23:44 -07:00
blockInfo = self . getStandardizedBlock ( b ) ;
2014-03-05 18:03:56 -08:00
self . sync . bDb . setLastFileIndex ( self . blockExtractor . currentFileIndex , function ( err ) {
return cb ( err , blockInfo ) ;
2014-02-08 05:57:37 -08:00
} ) ;
2014-03-05 18:03:56 -08:00
} ) ;
} ;
2014-02-08 05:57:37 -08:00
2014-03-05 18:03:56 -08:00
HistoricSync . prototype . updateBlockChainHeight = function ( cb ) {
var self = this ;
self . rpc . getBlockCount ( function ( err , res ) {
self . blockChainHeight = res . result ;
return cb ( err ) ;
} ) ;
} ;
2014-02-17 22:40:55 -08:00
2014-03-05 18:03:56 -08:00
HistoricSync . prototype . checkNetworkSettings = function ( next ) {
var self = this ;
2014-02-17 22:40:55 -08:00
2014-03-05 18:03:56 -08:00
self . hasGenesis = false ;
2014-02-08 11:20:51 -08:00
2014-03-05 18:03:56 -08:00
// check network config
self . rpc . getBlockHash ( 0 , function ( err , res ) {
if ( ! err && ( res && res . result !== self . genesis ) ) {
err = new Error ( BAD _GEN _ERROR + config . network ) ;
}
if ( err ) return next ( err ) ;
self . sync . bDb . has ( self . genesis , function ( err , b ) {
2014-02-17 22:40:55 -08:00
if ( ! err && ( res && res . result !== self . genesis ) ) {
2014-03-05 18:03:56 -08:00
err = new Error ( BAD _GEN _ERROR _DB + config . network ) ;
2014-02-17 22:40:55 -08:00
}
2014-03-05 18:03:56 -08:00
self . hasGenesis = b ? true : false ;
return next ( err ) ;
2014-02-17 22:40:55 -08:00
} ) ;
2014-03-05 18:03:56 -08:00
} ) ;
} ;
2014-02-17 22:40:55 -08:00
2014-03-05 18:03:56 -08:00
HistoricSync . prototype . updateStartBlock = function ( next ) {
var self = this ;
2014-02-17 22:40:55 -08:00
2014-03-05 18:03:56 -08:00
self . startBlock = self . genesis ;
2014-02-13 11:14:22 -08:00
2014-05-26 12:58:44 -07:00
self . sync . bDb . getTip ( function ( err , tip , height ) {
2014-03-05 18:03:56 -08:00
if ( ! tip ) return next ( ) ;
2014-02-13 11:14:22 -08:00
2014-03-05 18:03:56 -08:00
var blockInfo ;
var oldtip ;
//check that the tip is still on the mainchain
async . doWhilst (
function ( cb ) {
self . sync . bDb . fromHashWithInfo ( tip , function ( err , bi ) {
blockInfo = bi ? bi . info : { } ;
if ( oldtip )
2014-05-23 17:23:44 -07:00
self . sync . setBlockHeight ( oldtip , - 1 , cb ) ;
2014-03-05 18:03:56 -08:00
else
return cb ( ) ;
} ) ;
} ,
function ( err ) {
if ( err ) return next ( err ) ;
var ret = false ;
2014-05-26 12:58:44 -07:00
var d = Math . abs ( height - blockInfo . height ) ;
if ( d > 6 ) {
error ( 'Previous Tip block tip height differs by %d. Please delete and resync (-D)' , d ) ;
process . exit ( 1 ) ;
}
2014-03-05 18:03:56 -08:00
if ( self . blockChainHeight === blockInfo . height ||
blockInfo . confirmations > 0 ) {
ret = false ;
2014-02-17 22:40:55 -08:00
}
2014-03-05 18:03:56 -08:00
else {
oldtip = tip ;
2014-05-23 17:23:44 -07:00
if ( ! tip )
throw new Error ( 'Previous blockchain tip was not found on bitcoind. Please reset Insight DB. Tip was:' + tip )
2014-03-05 18:03:56 -08:00
tip = blockInfo . previousblockhash ;
2014-05-23 17:23:44 -07:00
info ( 'Previous TIP is now orphan. Back to:' + tip ) ;
2014-03-05 18:03:56 -08:00
ret = true ;
}
return ret ;
} ,
function ( err ) {
self . startBlock = tip ;
2014-05-26 12:58:44 -07:00
self . height = height ;
info ( 'Resuming sync from block: %s #%d' , tip , height ) ;
2014-03-05 18:03:56 -08:00
return next ( err ) ;
}
) ;
} ) ;
} ;
2014-02-04 21:48:54 -08:00
2014-03-05 18:03:56 -08:00
HistoricSync . prototype . prepareFileSync = function ( opts , next ) {
var self = this ;
2014-02-17 22:40:55 -08:00
2014-03-05 18:03:56 -08:00
if ( opts . forceRPC || ! config . bitcoind . dataDir ||
2014-05-26 12:58:44 -07:00
self . height > self . blockChainHeight * 0.9 ) return next ( ) ;
2014-02-04 21:48:54 -08:00
2014-03-05 18:03:56 -08:00
try {
self . blockExtractor = new BlockExtractor ( config . bitcoind . dataDir , config . network ) ;
} catch ( e ) {
2014-05-23 17:23:44 -07:00
info ( e . message + '. Disabling file sync.' ) ;
2014-02-17 22:40:55 -08:00
return next ( ) ;
2014-03-05 18:03:56 -08:00
}
2014-02-17 22:40:55 -08:00
2014-03-05 18:03:56 -08:00
self . getFn = self . getBlockFromFile ;
self . allowReorgs = true ;
self . sync . bDb . getLastFileIndex ( function ( err , idx ) {
2014-05-26 12:58:44 -07:00
2014-03-05 18:03:56 -08:00
if ( opts . forceStartFile )
self . blockExtractor . currentFileIndex = opts . forceStartFile ;
else if ( idx ) self . blockExtractor . currentFileIndex = idx ;
2014-02-04 08:06:05 -08:00
2014-03-05 18:03:56 -08:00
var h = self . genesis ;
2014-01-30 18:16:43 -08:00
2014-05-23 17:23:44 -07:00
info ( 'Seeking file to:' + self . startBlock ) ;
2014-03-05 18:03:56 -08:00
//forward till startBlock
async . whilst (
function ( ) {
return h !== self . startBlock ;
} ,
function ( w _cb ) {
self . getBlockFromFile ( function ( err , b ) {
if ( ! b ) return w _cb ( 'Could not find block ' + self . startBlock ) ;
h = b . hash ;
setImmediate ( function ( ) {
return w _cb ( err ) ;
} ) ;
} ) ;
2014-05-26 12:58:44 -07:00
} , function ( err ) {
return next ( err ) ;
} ) ;
2014-03-05 18:03:56 -08:00
} ) ;
} ;
//NOP
HistoricSync . prototype . prepareRpcSync = function ( opts , next ) {
var self = this ;
if ( self . blockExtractor ) return next ( ) ;
self . getFn = self . getBlockFromRPC ;
2014-05-21 11:51:24 -07:00
self . allowReorgs = true ;
2014-03-05 18:03:56 -08:00
self . currentRpcHash = self . startBlock ;
return next ( ) ;
} ;
HistoricSync . prototype . showSyncStartMessage = function ( ) {
var self = this ;
2014-05-26 12:58:44 -07:00
info ( 'Got ' + self . height +
2014-03-05 18:03:56 -08:00
' blocks in current DB, out of ' + self . blockChainHeight + ' block at bitcoind' ) ;
if ( self . blockExtractor ) {
2014-05-23 17:23:44 -07:00
info ( 'bitcoind dataDir configured...importing blocks from .dat files' ) ;
info ( 'First file index: ' + self . blockExtractor . currentFileIndex ) ;
2014-03-05 18:03:56 -08:00
}
else {
2014-05-23 17:23:44 -07:00
info ( 'syncing from RPC (slow)' ) ;
2014-03-05 18:03:56 -08:00
}
2014-02-17 22:40:55 -08:00
2014-05-23 17:23:44 -07:00
info ( 'Starting from: ' , self . startBlock ) ;
2014-03-05 18:03:56 -08:00
self . showProgress ( ) ;
} ;
HistoricSync . prototype . setupSyncStatus = function ( ) {
var self = this ;
2014-05-26 12:58:44 -07:00
var step = parseInt ( ( self . blockChainHeight - self . height ) / 1000 ) ;
2014-03-05 18:03:56 -08:00
if ( step < 10 ) step = 10 ;
self . step = step ;
self . type = self . blockExtractor ? 'from .dat Files' : 'from RPC calls' ;
self . status = 'syncing' ;
self . startTs = Date . now ( ) ;
self . endTs = null ;
this . error = null ;
this . syncPercentage = 0 ;
} ;
2014-05-26 14:49:03 -07:00
HistoricSync . prototype . checkDBVersion = function ( cb ) {
this . sync . txDb . checkVersion02 ( function ( isOk ) {
if ( ! isOk ) {
console . log ( '\n#############################\n\n ## Insight API DB is older that v0.2. Please resync using:\n $ util/sync.js -D\n More information at Insight API\'s Readme.md' ) ;
process . exit ( 1 ) ;
}
// Add more test here in future changes.
return cb ( ) ;
} ) ;
} ;
2014-03-05 18:03:56 -08:00
HistoricSync . prototype . prepareToSync = function ( opts , next ) {
var self = this ;
self . status = 'starting' ;
async . series ( [
2014-05-26 11:16:38 -07:00
function ( s _c ) {
2014-05-26 14:49:03 -07:00
self . checkDBVersion ( s _c ) ;
2014-05-26 11:16:38 -07:00
} ,
2014-03-05 18:03:56 -08:00
function ( s _c ) {
self . checkNetworkSettings ( s _c ) ;
} ,
function ( s _c ) {
self . updateBlockChainHeight ( s _c ) ;
} ,
function ( s _c ) {
self . updateStartBlock ( s _c ) ;
} ,
function ( s _c ) {
self . prepareFileSync ( opts , s _c ) ;
} ,
function ( s _c ) {
self . prepareRpcSync ( opts , s _c ) ;
} ,
] ,
function ( err ) {
if ( err ) return ( self . setError ( err ) ) ;
self . showSyncStartMessage ( ) ;
self . setupSyncStatus ( ) ;
return next ( ) ;
} ) ;
} ;
2014-01-16 11:11:20 -08:00
2014-03-05 18:03:56 -08:00
HistoricSync . prototype . start = function ( opts , next ) {
var self = this ;
2014-01-16 11:11:20 -08:00
2014-03-05 18:03:56 -08:00
if ( self . status === 'starting' || self . status === 'syncing' ) {
2014-05-23 17:23:44 -07:00
error ( '## Wont start to sync while status is %s' , self . status ) ;
2014-03-05 18:03:56 -08:00
return next ( ) ;
}
2014-02-17 22:40:55 -08:00
2014-03-05 18:03:56 -08:00
self . prepareToSync ( opts , function ( err ) {
if ( err ) return next ( self . setError ( err ) ) ;
2014-02-17 10:04:01 -08:00
2014-03-05 18:03:56 -08:00
async . whilst (
function ( ) {
self . showProgress ( ) ;
return self . status === 'syncing' ;
2014-02-13 12:12:19 -08:00
} ,
2014-03-05 18:03:56 -08:00
function ( w _cb ) {
self . getFn ( function ( err , blockInfo ) {
if ( err ) return w _cb ( self . setError ( err ) ) ;
2014-05-26 12:58:44 -07:00
if ( blockInfo && blockInfo . hash && ( ! opts . stopAt || opts . stopAt !== blockInfo . hash ) ) {
self . sync . storeTipBlock ( blockInfo , self . allowReorgs , function ( err , height ) {
2014-03-05 18:03:56 -08:00
if ( err ) return w _cb ( self . setError ( err ) ) ;
2014-05-27 20:17:42 -07:00
if ( height >= 0 ) self . height = height ;
2014-05-20 14:07:25 -07:00
setImmediate ( function ( ) {
return w _cb ( err ) ;
2014-02-17 22:40:55 -08:00
} ) ;
2014-03-05 18:03:56 -08:00
} ) ;
}
else {
self . endTs = Date . now ( ) ;
self . status = 'finished' ;
console . log ( 'Done Syncing' , self . info ( ) ) ;
return w _cb ( err ) ;
}
} ) ;
} , next ) ;
} ) ;
} ;
2014-01-16 11:11:20 -08:00
2014-03-05 18:03:56 -08:00
module . exports = require ( 'soop' ) ( HistoricSync ) ;