diff --git a/Gruntfile.js b/Gruntfile.js index 715c647e..e83a5145 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -48,7 +48,7 @@ module.exports = function(grunt) { }, jshint: { all: { - src: ['Gruntfile.js', 'server.js', 'app/**/*.js', 'public/js/**'], + src: ['Gruntfile.js', 'server.js', 'app/**/*.js', 'public/js/**','lib/*.js'], options: { jshintrc: true } @@ -66,7 +66,7 @@ module.exports = function(grunt) { options: { file: 'server.js', args: [], - ignoredFiles: ['public/**', 'test/**','util/**','lib/**'], + ignoredFiles: ['public/**', 'test/**','util/**'], watchedExtensions: ['js'], // nodeArgs: ['--debug'], delayTime: 1, diff --git a/app/models/Transaction.js b/app/models/Transaction.js index d8971f7a..fbb3f35a 100644 --- a/app/models/Transaction.js +++ b/app/models/Transaction.js @@ -141,7 +141,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, time, cb) { } }, function (err) { - if (err) console.log (err); + if (err && !err.message.match(/E11000/) ) console.log (err); async.forEachLimit(info.vout, CONCURRENCY, function(o, next_out) { /* diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js new file mode 100644 index 00000000..f3b99f14 --- /dev/null +++ b/lib/HistoricSync.js @@ -0,0 +1,248 @@ +'use strict'; + +require('classtool'); + + +function spec() { + var mongoose = require('mongoose'); + var util = require('util'); + var RpcClient = require('bitcore/RpcClient').class(); + var networks = require('bitcore/networks'); + var async = require('async'); + var config = require('../config/config'); + var Block = require('../app/models/Block'); + var Transaction = require('../app/models/Transaction'); + var TransactionItem = require('../app/models/TransactionItem'); + var Sync = require('./Sync').class(); + var sockets = require('../app/views/sockets/main.js'); + var CONCURRENCY = 5; + + + function HistoricSync(opts) { + this.block_count= 0; + this.block_total= 0; + this.network = config.network === 'testnet' ? networks.testnet: networks.livenet; + this.sync = new Sync(opts); + } + + function p() { + var params = Array.prototype.slice.call(arguments); + + params.unshift('[historic_sync]'); + console.log.apply(this,params); + } + + var progress_bar = function(string, current, total) { + p(util.format('%s %d/%d [%d%%]', string, current, total, parseInt(100 * current / total))); + }; + + HistoricSync.prototype.init = function(opts,cb) { + this.rpc = new RpcClient(config.bitcoind); + this.opts = opts; + this.sync.init(opts, cb); + }; + + HistoricSync.prototype.close = function() { + this.sync.close(); + }; + + HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) { + + var that = this; + + // recursion end. + if (!blockHash || (blockEnd && blockEnd === blockHash) ) { + return cb(); + } + + var existed = 0; + var blockInfo; + var blockObj; + + async.series([ + // Already got it? + function(c) { + Block.findOne({hash:blockHash}, function(err,block){ + if (err) { p(err); return c(err); }; + if (block) { + existed = 1; + blockObj = block; + } + + return c(); + }); + }, + //show some (inacurate) status + function(c) { + if (that.block_count++ % 1000 === 0) { + progress_bar('sync status:', that.block_count, that.block_total); + } + return c(); + }, + //get Info from RPC + function(c) { + + // TODO: if we store prev/next, no need to go to RPC + // if (blockObj && blockObj.nextBlockHash) return c(); + + that.rpc.getBlock(blockHash, function(err, ret) { + if (err) return c(err); + + blockInfo = ret; + return c(); + }); + }, + //store it + function(c) { + if (existed) return c(); + + that.sync.storeBlock(blockInfo.result, function(err, block) { + existed = err && err.toString().match(/E11000/); + if (err && ! existed) return c(err); + return c(); + }); + }, + /* TODO: Should Start to sync backwards? (this is for partial syncs) + function(c) { + + if (blockInfo.result.prevblockhash != current.blockHash) { + p("reorg?"); + opts.prev = 1; + } + return c(); + } + */ + ], + function (err){ + + if (err) + p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, that.block_count); + + if (blockInfo && blockInfo.result) { + if (opts.prev && blockInfo.result.previousblockhash) { + return that.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb); + } + + if (opts.next && blockInfo.result.nextblockhash) + return that.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb); + } + return cb(err); + }); + }; + + HistoricSync.prototype.syncBlocks = function(start, end, isForward, cb) { + var that = this; + + p('Syncing Blocks, starting from: %s end: %s isForward:', + start, end, isForward); + + + return that.getPrevNextBlock( start, end, + isForward ? { next: 1 } : { prev: 1}, cb); + }; + + HistoricSync.prototype.do_import_history = function(opts, next) { + var that = this; + + var retry_attemps = 100; + var retry_secs = 2; + + var block_best; + var block_height; + + async.series([ + function(cb) { + if (opts.destroy) { + p('Deleting Blocks...'); + that.db.collections.blocks.drop(cb); + } else { + return cb(); + } + }, + function(cb) { + if (opts.destroy) { + p('Deleting TXs...'); + that.db.collections.transactions.drop(cb); + } else { + return cb(); + } + }, + function(cb) { + if (opts.destroy) { + p('Deleting TXItems...'); + that.db.collections.transactionitems.drop(cb); + } else { + return cb(); + } + }, + function(cb) { + that.rpc.getInfo(function(err, res) { + if (err) cb(err); + + that.block_total = res.result.blocks; + return cb(); + }); + }, + // We are not using getBestBlockHash, because is not available in all clients + function(cb) { + if (!opts.reverse) return cb(); + + that.rpc.getBlockCount(function(err, res) { + if (err) cb(err); + block_height = res.result; + return cb(); + }); + }, + function(cb) { + if (!opts.reverse) return cb(); + + that.rpc.getBlockHash(block_height, function(err, res) { + if (err) cb(err); + + block_best = res.result; + return cb(); + }); + }, + ], + function(err) { + + function sync() { + var start, end, isForward; + + if (opts.reverse) { + start = block_best; + end = that.network.genesisBlock.hash.reverse().toString('hex'); + isForward = false; + } + else { + start = that.network.genesisBlock.hash.reverse().toString('hex'); + end = null; + isForward = true; + } + + that.syncBlocks(start, end, isForward, function(err) { + + if (err && err.message.match(/ECONNREFUSED/) && retry_attemps--){ + setTimeout(function() { + p("Retrying in %d secs ", retry_secs); + sync(); + }, retry_secs * 1000); + } + else + return next(err, that.block_count); + }); + } + sync(); + }); + } + + HistoricSync.prototype.import_history = function(opts, next) { + var that = this; + that.do_import_history(opts, next); + }; + + + return HistoricSync; +} +module.defineClass(spec); + diff --git a/lib/PeerSync.js b/lib/PeerSync.js index 4704fde8..7f0d1100 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -11,23 +11,27 @@ function spec() { var peerdb_fn = 'peerdb.json'; function PeerSync() {} - PeerSync.prototype.init = function(config) { + + + PeerSync.prototype.init = function(config, cb) { + + var that = this; var network = config && (config.network || 'testnet'); - this.peerdb = undefined; - this.sync = new Sync({ + that.peerdb = undefined; + that.sync = new Sync({ networkName: network }); - this.sync.init(config); - - this.PeerManager = require('bitcore/PeerManager').createClass({ - config: { - network: network - } + that.sync.init(config, function() { + that.PeerManager = require('bitcore/PeerManager').createClass({ + config: { + network: network + } + }); + that.load_peers(); + return cb(); }); - this.load_peers(); - }; PeerSync.prototype.load_peers = function() { @@ -51,6 +55,7 @@ function spec() { console.log('[p2p_sync] Handle inv for a ' + CoinConst.MSG.to_str(inv.type)); }); // TODO: should limit the invs to objects we haven't seen yet + console.log('getdata'); info.conn.sendGetData(invs); }; diff --git a/public/views/transaction.html b/public/views/transaction.html index b0cc4225..7ba47449 100644 --- a/public/views/transaction.html +++ b/public/views/transaction.html @@ -23,10 +23,13 @@