From e0647e71d0c6edb250b580b498d52ccff06dace6 Mon Sep 17 00:00:00 2001 From: Bechi Date: Mon, 17 Feb 2014 16:14:14 -0300 Subject: [PATCH 01/23] block page --- public/src/css/common.css | 17 ++++++++--------- public/views/block.html | 2 +- public/views/includes/currency.html | 17 +++++++++-------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/public/src/css/common.css b/public/src/css/common.css index 9d3496a3..e5e9ecf8 100644 --- a/public/src/css/common.css +++ b/public/src/css/common.css @@ -59,6 +59,7 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 { .m10v { margin: 10px 0; } .m50v { margin: 50px 0; } .m10b { margin-bottom: 10px; } +.m10l { margin-left: 10px; } .vm { vertical-align: middle; } @@ -190,6 +191,7 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 { } .col-gray-fixed { + margin-top: 15px; position: fixed; width: 250px; border: 1px solid #eee; @@ -239,11 +241,11 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 { width: 165px; color: #fff; text-align: center; - font-size: 80px; } .block-id span { - margin-top: 10px; + font-size: 40px; + margin: 30px 0; } .block-id h2 { @@ -295,9 +297,12 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 { color: #373D42; } +.btn-default:hover, .btn-default:focus, .btn-default:active, .btn-default.active, .open .dropdown-toggle.btn-default { + background-color: #fff; +} + .btn-default { background-color: #E7E7E7; - border: 2px solid #DCDCDC; } .btn-success { @@ -358,12 +363,6 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 { overflow: hidden; } -.currency a.active { - color: #000; - text-decoration: underline; - cursor: default; -} - #footer a.insight { font-size: 20px; text-decoration: none; diff --git a/public/views/block.html b/public/views/block.html index e983dbd1..21c26440 100644 --- a/public/views/block.html +++ b/public/views/block.html @@ -4,7 +4,7 @@

Block

- +

#{{block.height}}

diff --git a/public/views/includes/currency.html b/public/views/includes/currency.html index c9acb46c..0a026080 100644 --- a/public/views/includes/currency.html +++ b/public/views/includes/currency.html @@ -1,8 +1,9 @@ -
- - Currency: - USD · - BTC · - mBTC - -
+
+ Currency: +
+ + USD + BTC + mBTC +
+
\ No newline at end of file From b8d872e2d63cc6920d97b72e305891d888ebe7ea Mon Sep 17 00:00:00 2001 From: Bechi Date: Mon, 17 Feb 2014 17:19:22 -0300 Subject: [PATCH 02/23] adres-tx-page --- public/src/css/common.css | 11 ++++++++++- public/views/address.html | 1 + public/views/transaction.html | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/public/src/css/common.css b/public/src/css/common.css index e5e9ecf8..95f9d211 100644 --- a/public/src/css/common.css +++ b/public/src/css/common.css @@ -490,6 +490,11 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 { text-align: center; width: 30px; } + +.btn-copy:hover, .btn-expand:hover { + background-color: #f2f2f2; +} + .btn-copy { background: #fff url('/img/icons/copy.png') center center no-repeat; margin-left: 5px; @@ -520,10 +525,14 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 { margin: 0 auto; width: 165px; color: #FFFFFF; - font-size: 80px; text-align: center; } +.tx-id span { + font-size: 40px; + margin: 30px 0; +} + .transaction-vin-vout .ellipsis { margin-bottom: 10px; } .transaction-vin-vout .btc-value { margin-left: 15px; } diff --git a/public/views/address.html b/public/views/address.html index d2f050ac..df0af133 100644 --- a/public/views/address.html +++ b/public/views/address.html @@ -5,6 +5,7 @@

Address

+

{{$root.currency.getConvertion(address.balance)}}

diff --git a/public/views/transaction.html b/public/views/transaction.html index 5105dde0..b9c30827 100644 --- a/public/views/transaction.html +++ b/public/views/transaction.html @@ -4,7 +4,7 @@

Transaction

- +
From b8f95b7a0ebaf9bb2344017d7e6643888913d7ea Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 18 Feb 2014 03:40:55 -0300 Subject: [PATCH 03/23] refactor of historic sync. Handles all know scenarios --- dev-util/stats | 56 +++++ insight.js | 35 +-- lib/BlockDb.js | 2 +- lib/HistoricSync.js | 554 ++++++++++++++++++------------------------- lib/PeerSync.js | 32 ++- lib/PoolMatch.js | 1 + lib/Sync.js | 35 ++- test/lib/PeerSync.js | 12 +- util/p2p.js | 10 +- util/sync.js | 22 +- 10 files changed, 342 insertions(+), 417 deletions(-) create mode 100644 dev-util/stats diff --git a/dev-util/stats b/dev-util/stats new file mode 100644 index 00000000..f68bf9e1 --- /dev/null +++ b/dev-util/stats @@ -0,0 +1,56 @@ + + +first 5% + + => with data + mongo + w/RPC for blocks: 48.8s + => with RPC + mongo: 2m26s + => with files + mongo + wo/RPC for blocks: 36.7s + => with files + mongo + wo/RPC for blocks + wo/mongoIndexes: + + +first 10% + + => sin RPC, sin Tx, sin store block => 0.7s + => sin RPC, sin grabar, procesando TX => 8.5s + => sin RPC, sin TX processing, sin grabar => 12s28 + => con RPC, TX processing, sin Grabar Tx, grabando bloques => 29s + => con RPC, sin TX processing, sin Grabar Tx, grabando bloques => 35s + => con RPC, TX processing, sin Grabar Tx, grabando bloques => 43s + + => TX processing, sin RPC, sin saves TX, y blocks => 11.6s + => TX processing, CON RPC, sin saves TX, y blocks => 35s + => con RPC, TX processing, sin saves TX => 45s + => con RPC, TX processing, Grabarndo todo => 78s + => con RPC, TX processing, Grabarndo todo => 78s + (18k blocks, 36k txouts) + +//LEVEL DB + => sin RPC, TX processing, todo en level => 14s + => con RPC, TX processing, todo en level => 39.7s + => con RPC, TX processing, tx mongo, blocks en level => 64s + + + => sin RPC, TX processing, todo en level, handling REORGs, more data => 28s + => sin RPC, TX processing, todo en level, handling REORGs, more data, tx ts => 34t s + + +//FROM blk00002.dat (more txs), 5% + + => now total : 1m13s + => removing block writes => 1m8s + => sacando los contenidos adentro de getblock from file de => 4.5s!! + + => con base58 cpp => 21s + => toda la testnet => 17m + +10% de blk2 + => 50s con base58cpp + => 41s commentando todo addr + => 5s commentando todo get HistoricSync.prototype.getBlockFromFile = function(cb) { + => 15s commentando todo get HistoricSync.prototype.getBlockFromFile = function(cb) { + +10% de blk 1 + => 59s + => 15s comentando desde b.getStandardizedObject() + => 39s comentando dps b.getStandardizedObject() + diff --git a/insight.js b/insight.js index 42b0b977..0cf64a02 100644 --- a/insight.js +++ b/insight.js @@ -44,48 +44,29 @@ walk(models_path); * p2pSync process */ -var peerSync = new PeerSync(); +var peerSync = new PeerSync({shouldBroadcast: true}); if (!config.disableP2pSync) { - var ps = peerSync; - ps.init({ - shouldBroadcast: true, - }, function() { - ps.run(); - }); + peerSync.run(); } /** * historic_sync process */ -var historicSync = {}; +var historicSync = new HistoricSync({ shouldBroadcastSync: true }); +peerSync.historicSync = historicSync; if (!config.disableHistoricSync) { - historicSync = new HistoricSync(); - - historicSync.init({ - shouldBroadcastSync: true, - }, function(err) { + historicSync.start({}, function(err){ if (err) { var txt = 'ABORTED with error: ' + err.message; console.log('[historic_sync] ' + txt); } - else { - historicSync.smartImport({}, function(err){ - var txt = 'ended.'; - if (err) - txt = 'ABORTED with error: ' + err.message; - else if (ps) { - ps.allowReorgs = true; - ps.historicSync = historicSync; - } - - console.log('[historic_sync] ' + txt, historicSync.info()); - }); - } + if (peerSync) peerSync.allowReorgs = true; }); } - +else + if (peerSync) peerSync.allowReorgs = true; //express settings diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 76a118c1..8cd3abe1 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -136,7 +136,7 @@ function spec(b) { }); }; - BlockDb.prototype.countNotOrphan = function(cb) { + BlockDb.prototype.countConnected = function(cb) { var c = 0; console.log('Counting connected blocks. This could take some minutes'); db.createReadStream({start: MAIN_PREFIX, end: MAIN_PREFIX + '~' }) diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index aa557ac3..2168299c 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -6,6 +6,7 @@ require('classtool'); function spec() { var util = require('util'); + var assert = require('assert'); var RpcClient = require('bitcore/RpcClient').class(); var Script = require('bitcore/Script').class(); var networks = require('bitcore/networks'); @@ -20,14 +21,19 @@ function spec() { var BAD_GEN_ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:'; - function HistoricSync() { + var BAD_GEN_ERROR_DB = 'Bad genesis block. Network mismatch between Insight and levelDB? Insight is configured for:'; + function HistoricSync(opts) { + opts = opts || {}; + this.network = config.network === 'testnet' ? networks.testnet: networks.livenet; var genesisHashReversed = new Buffer(32); this.network.genesisBlock.hash.copy(genesisHashReversed); this.genesis = genesisHashReversed.reverse().toString('hex'); - + this.rpc = new RpcClient(config.bitcoind); + this.shouldBroadcast = opts.shouldBroadcastSync; + this.sync = new Sync(opts); } function p() { @@ -52,7 +58,7 @@ function spec() { self.updatePercentage(); p(util.format('status: [%d%%]', self.syncPercentage)); } - if (self.opts.shouldBroadcastSync) { + if (self.shouldBroadcast) { sockets.broadcastSyncInfo(self.info()); } @@ -67,38 +73,10 @@ function spec() { self.error = err.toString(); self.status='error'; self.showProgress(); + return err; }; - HistoricSync.prototype.init = function(opts, cb) { - var self = this; - self.opts = opts; - - self.rpc = new RpcClient(config.bitcoind); - self.sync = new Sync(opts); - - self.sync.init(opts, function(err) { - if (err) { - self.setError(err); - return cb(err); - } - else { - - self.startTs = parseInt(Date.now()); - - // check testnet? - self.rpc.getBlockHash(0, function(err, res){ - if (!err && ( res && res.result !== self.genesis)) { - err = new Error(BAD_GEN_ERROR + config.network); - self.setError(err); - } - if (err) self.setError(err); - return cb(err); - }); - } - }); - - }; HistoricSync.prototype.close = function() { this.sync.close(); @@ -112,7 +90,6 @@ function spec() { blockChainHeight: this.blockChainHeight, syncPercentage: this.syncPercentage, syncedBlocks: this.syncedBlocks, - orphanBlocks: this.orphanBlocks, syncTipHash: this.sync.tip, error: this.error, type: this.type, @@ -127,100 +104,30 @@ function spec() { if (this.syncPercentage > 100) this.syncPercentage = 100; }; - HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, scanOpts, cb) { + HistoricSync.prototype.getBlockFromRPC = function(cb) { var self = this; - // recursion end. - if (!blockHash) return cb(); + if (!self.currentRpcHash) return cb(); - var existed = false; var blockInfo; + self.rpc.getBlock(self.currentRpcHash, function(err, ret) { + if (err) return cb(err); + if (ret) { + blockInfo = ret.result; + // this is to match block retreived from file + if (blockInfo.hash === self.genesis) + blockInfo.previousblockhash = + self.network.genesisBlock.prev_hash.toString('hex'); - async.series([ - // Already got it? - function(c) { - self.sync.bDb.has(blockHash, function(err, ret) { - if (err) { - p(err); - return c(err); - } - - if (ret) existed = true; - return c(); - }); - }, - //show some (inacurate) status - function(c) { - self.showProgress(); - return c(); - }, - - function(c) { - self.rpc.getBlock(blockHash, function(err, ret) { - if (err) return c(err); - if (ret) { - blockInfo = ret.result; - // this is to match block retreived from file - if (blockInfo.hash === self.genesis) - blockInfo.previousblockhash = self.network.genesisBlock.prev_hash.toString('hex'); - } - else { - blockInfo = null; - } - - return c(); - }); - }, - //store it - function(c) { - if (existed) return c(); - - // When storing files from RPC recusively, reorgs are disabled - self.sync.storeTipBlock(blockInfo, false, function(err) { - return c(err); - }); - }], function(err) { - - if (err) { - self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', - blockHash, err, self.syncedBlocks)); - return cb(err); + self.currentRpcHash = blockInfo.nextblockhash; } else { - self.status = 'syncing'; + blockInfo = null; } - - if ( blockEnd && blockEnd === blockHash) { - p('blockEnd found!:' + blockEnd); - self.found=1; - } - - if ( self.found && self.syncedBlocks >= self.blockChainHeight ) { - - self.endTs = parseInt(Date.now()); - self.status = 'finished'; - p('DONE. Height: ' , self.syncedBlocks); - return cb(err); - } - - // Continue - if (blockInfo) { - - self.syncedBlocks++; - - // recursion - if (scanOpts.prev && blockInfo.previousblockhash) - return self.getPrevNextBlock(blockInfo.previousblockhash, blockEnd, scanOpts, cb); - - if (scanOpts.next && blockInfo.nextblockhash) - return self.getPrevNextBlock(blockInfo.nextblockhash, blockEnd, scanOpts, cb); - } - return cb(err); + return cb(null, blockInfo); }); }; - -//var a=1; HistoricSync.prototype.getBlockFromFile = function(cb) { var self = this; @@ -228,12 +135,8 @@ function spec() { //get Info self.blockExtractor.getNextBlock(function(err, b) { -// a++; -//return cb(null,{previousblockhash:a.toString(), hash:(a-1).toString() }); if (err || ! b) return cb(err); blockInfo = b.getStandardizedObject(b.txs, self.network); - // blockInfo.curWork = Deserialize.intFromCompact(b.bits); - // We keep the RPC field names blockInfo.previousblockhash = blockInfo.prev_block; var ti=0; @@ -260,241 +163,258 @@ function spec() { to++; }); }); - - return cb(err,blockInfo); - }); - }; - - - HistoricSync.prototype.nextBlockFromFile = function(scanOpts, cb) { - var self = this; - - self.showProgress(); - self.getBlockFromFile(function(err, blockInfo) { - if (err) { - self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', - blockInfo ? blockInfo.hash : '-', err, self.syncedBlocks)); - return cb(err); - } - - self.sync.storeTipBlock(blockInfo, function(err) { - if (err) { - self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', - blockInfo ? blockInfo.hash : '-', err, self.syncedBlocks)); - return cb(err); - } - - self.sync.bDb.setLastFileIndex(self.blockExtractor.currentFileIndex, function(err) { - if (err) return cb(err); - - if (blockInfo && blockInfo.hash) { - self.syncedBlocks++; - } - else { - self.endTs = parseInt(Date.now()); - self.status = 'finished'; - } - - return cb(err); - }); + self.sync.bDb.setLastFileIndex(self.blockExtractor.currentFileIndex, function(err) { + return cb(err,blockInfo); }); }); }; - - HistoricSync.prototype.countNotOrphan = function(cb) { + HistoricSync.prototype.updateConnectedCountDB = function(cb) { var self = this; - - if (self.notOrphanCount) return cb(null, self.notOrphanCount); - - - self.sync.bDb.countNotOrphan(function(err, count) { - if (err) return cb(err); - self.notOrphanCount = count; - return cb(null, self.notOrphanCount); + self.sync.bDb.countConnected(function(err, count) { + self.connectedCountDB = count || 0; + self.syncedBlocks = count || 0; + return cb(err); }); }; - HistoricSync.prototype.updateBlockCount = function(cb) { + HistoricSync.prototype.updateBlockChainHeight = function(cb) { var self = this; - if (self.blockChainHeight) return cb(); - self.rpc.getBlockCount(function(err, res) { - if (err) return cb(err); self.blockChainHeight = res.result; - return cb(); + return cb(err); }); }; - HistoricSync.prototype.smartImport = function(scanOpts, next) { + + HistoricSync.prototype.checkNetworkSettings = function(next) { var self = this; - var genesis, count; - var lastBlock; - var tip; + self.hasGenesis = false; - //available status: starting / syncing / finished / aborted - this.status = 'starting'; - this.error = null; - - this.syncPercentage = 0; - this.syncedBlocks = 0; - this.orphanBlocks = 0; - this.type =''; - - async.series([ - function(s_c) { - if (!scanOpts.destroy) return s_c(); - - p('Deleting DB...'); - return self.sync.destroy(s_c); - }, - function(s_c) { - self.sync.bDb.has(self.genesis, function(err, b) { - genesis = b; - return s_c(err); - }); - }, - function(s_c) { - self.countNotOrphan(function(err, c) { - count = c; - return s_c(err); - }); - }, - function(s_c) { - if (!config.bitcoind.dataDir) return s_c(); - if (scanOpts.startFile) { - self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); - self.blockExtractor.currentFileIndex = scanOpts.startFile; - return s_c(); + // check network config + self.rpc.getBlockHash(0, function(err, res){ + if (!err && ( res && res.result !== self.genesis)) { + err = new Error(BAD_GEN_ERROR + config.network); + } + if (err) return next(err); + self.sync.bDb.has(self.genesis, function(err, b) { + if (!err && ( res && res.result !== self.genesis)) { + err = new Error(BAD_GEN_ERROR_DB + config.network); } - self.sync.bDb.getLastFileIndex(function(err, idx) { - self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); - if (idx) self.blockExtractor.currentFileIndex = idx; - return s_c(err); - }); - }, - function(s_c) { - self.updateBlockCount(s_c); - }, + self.hasGenesis = b?true:false; + return next(err); + }); + }); + }; - // define sync strategy - function(s_c) { - if (!genesis || scanOpts.destroy || count < self.blockChainHeight * 0.9 ) { + HistoricSync.prototype.updateStartBlock = function(next) { + var self = this; - // Full sync. - if (!genesis) - p('Could not find Genesis block. Running FULL SYNC'); - else - p('Less that 90% of current blockchain is stored. Running FULL SYNC', - parseInt(count/self.blockChainHeight*100)); + self.startBlock = self.genesis; - if (config.bitcoind.dataDir) { - p('bitcoind dataDir configured...importing blocks from .dat files'); - p('Starting from file: ' + self.blockExtractor.currentFileIndex); - scanOpts.fromFiles = true; + self.sync.bDb.getTip(function(err,tip) { + if (!tip) return next(); + + var blockInfo; + var oldtip; + + //check that the tip is still on the mainchain + async.doWhilst( + function(cb) { + self.sync.bDb.fromHashWithInfo(tip, function(err, bi) { + blockInfo = bi ? bi.info : {}; + if (oldtip) + self.sync.setBlockMain(oldtip, false, cb); + else + return cb(); + }); + }, + function(err) { + if (err) return next(err); + var ret = false; + if ( self.blockChainHeight === blockInfo.height || + blockInfo.confirmations > 0) { + ret = false; } else { - scanOpts.reverse = true; + oldtip = tip; + tip = blockInfo.previousblockhash; + assert(tip); + p('Previous TIP is now orphan. Back to:' + tip); + ret = true; } + return ret; + }, + function(err) { + self.startBlock = tip; + p('Resuming sync from block:'+tip); + return next(err); } - else { - p('Genesis block found. Syncing upto old TIP.'); - p('Got ' + count + ' out of ' + self.blockChainHeight + ' blocks'); - scanOpts.reverse = true; - } + ); + }); + }; - if (!scanOpts.reverse) return s_c(); + HistoricSync.prototype.prepareFileSync = function(opts, next) { + var self = this; - self.rpc.getBlockHash(self.blockChainHeight, function(err, res) { - if (err) return s_c(err); - lastBlock = res.result; - return s_c(); - }); - }, - function(s_c) { - if (!scanOpts.reverse) return s_c(); - self.sync.bDb.getTip(function(err, res) { - if (err) return s_c(err); - tip = res; - - console.log('Old Tip:', tip); - return s_c(); - }); - }, - function(s_c) { - if (!scanOpts.reverse) return s_c(); - self.countNotOrphan(function(err, count) { - if (err) return s_c(err); - - self.syncedBlocks = count || 0; - return s_c(); - }); - }], - function(err) { - // SETUP Sync params - var start, end; - if (err) { - self.setError(err); - return next(err, 0); - } - - if (!self.step) { - var step = parseInt( (self.blockChainHeight - self.syncedBlocks) / 1000); - - if (self.opts.progressStep) { - step = self.opts.progressStep; - } - - if (step < 10) step = 10; - self.step = step; - } - - if (scanOpts.reverse) { - start = lastBlock; - end = tip || self.genesis; - scanOpts.prev = true; - } - else { - start = self.genesis; - end = null; - scanOpts.next = true; - } - p('Starting from: ', start); - p(' to : ', end); - p(' scanOpts: ', JSON.stringify(scanOpts)); - - if (scanOpts.fromFiles) { - - self.status = 'syncing'; - self.type = 'from .dat Files'; + if ( + opts.forceRPC || !config.bitcoind.dataDir || + self.connectedCountDB > self.blockChainHeight > 0.9) return next(); - async.whilst(function() { - return self.status === 'syncing'; - }, function (w_cb) { - self.nextBlockFromFile(scanOpts, function(err) { + self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); + + self.getFn = self.getBlockFromFile; + self.allowReorgs = true; + self.sync.bDb.getLastFileIndex(function(err, idx) { + if (opts.forceStartFile) + self.blockExtractor.currentFileIndex = opts.forceStartFile; + else if (idx) self.blockExtractor.currentFileIndex = idx; + + var h = self.genesis; + + p('Seeking file to:' + self.startBlock); + //forward till startBlock + async.whilst( + function() { + return h !== self.startBlock; + }, + function (w_cb) { + self.getBlockFromFile(function(err,b) { + h=b.hash; setImmediate(function(){ return w_cb(err); }); }); - }, function(err) { - self.showProgress(); - return next(err); - }); - } - else { - self.type = 'from RPC calls'; - self.getPrevNextBlock(start, end, scanOpts, function(err) { - self.showProgress(); - return next(err); - }); - } + }, next); }); }; + //NOP + HistoricSync.prototype.prepareRpcSync = function(opts, next) { + var self = this; + + if (self.blockExtractor) return next(); + self.getFn = self.getBlockFromRPC; + self.currentRpcHash = self.startBlock; + self.allowReorgs = false; + return next(); + }; + + HistoricSync.prototype.showSyncStartMessage = function() { + var self = this; + + p('Got ' + self.connectedCountDB + + ' blocks in current DB, out of ' + self.blockChainHeight + ' block at bitcoind'); + + if (self.blockExtractor) { + p('bitcoind dataDir configured...importing blocks from .dat files'); + p('First file index: ' + self.blockExtractor.currentFileIndex); + } + else { + p('syncing from RPC (slow)'); + } + + p('Starting from: ', self.startBlock); + self.showProgress(); + }; + + + HistoricSync.prototype.setupSyncStatus = function() { + var self = this; + + var step = parseInt( (self.blockChainHeight - self.syncedBlocks) / 1000); + if (step < 10) step = 10; + + self.step = step; + self.type = self.blockExtractor?'from .dat Files':'from RPC calls'; + self.status = 'syncing'; + self.startTs = Date.now(); + self.endTs = null; + this.error = null; + this.syncPercentage = 0; + }; + + HistoricSync.prototype.prepareToSync = function(opts, next) { + var self = this; + + self.status = 'starting'; + async.series([ + function(s_c) { + self.checkNetworkSettings(s_c); + }, + function(s_c) { + self.updateConnectedCountDB(s_c); + }, + function(s_c) { + self.updateBlockChainHeight(s_c); + }, + function(s_c) { + self.updateStartBlock(s_c); + }, + function(s_c) { + self.prepareFileSync(opts, s_c); + }, + function(s_c) { + self.prepareRpcSync(opts, s_c); + }, + ], + function(err) { + if (err) return(self.setError(err)); + + self.showSyncStartMessage(); + self.setupSyncStatus(); + return next(); + }); + }; + + + HistoricSync.prototype.start = function(opts, next) { + var self = this; + + if (self.status==='starting' || self.status==='syncing') { + p('## Wont start to sync while status is %s', self.status); + return next(); + } + + self.prepareToSync(opts, function(err) { + if (err) return next(self.setError(err)); + + async.whilst( + function() { + return self.status === 'syncing'; + }, + function (w_cb) { + self.getFn(function(err,blockInfo) { + if (err) return w_cb(self.setError(err)); + + self.showProgress(); + if (blockInfo && blockInfo.hash) { + self.syncedBlocks++; + self.sync.storeTipBlock(blockInfo, self.allowReorgs, function(err) { + if (err) return w_cb(self.setError(err)); + + self.sync.bDb.setTip(blockInfo.hash, function(err) { + if (err) return w_cb(self.setError(err)); + + setImmediate(function(){ + return w_cb(err); + }); + }); + }); + } + else { + self.endTs = Date.now(); + self.status = 'finished'; + return w_cb(err); + } + }); + }, next); + }); + }; return HistoricSync; } module.defineClass(spec); diff --git a/lib/PeerSync.js b/lib/PeerSync.js index 15b1438e..7122fe4c 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -3,8 +3,7 @@ require('classtool'); function spec() { var fs = require('fs'); - var CoinConst = require('bitcore/const'); - var coinUtil = require('bitcore/util/util'); + var bitcoreUtil = require('bitcore/util/util'); var Sync = require('./Sync').class(); var Peer = require('bitcore/Peer').class(); var config = require('../config/config'); @@ -12,26 +11,17 @@ function spec() { var peerdb_fn = 'peerdb.json'; - function PeerSync() {} - - - PeerSync.prototype.init = function(opts, cb) { - if (!opts) opts = {}; + function PeerSync(opts) { this.connected = false; this.peerdb = undefined; this.allowReorgs = false; - - this.sync = new Sync(); this.PeerManager = require('bitcore/PeerManager').createClass({ network: (config.network === 'testnet' ? networks.testnet : networks.livenet) }); this.peerman = new this.PeerManager(); this.load_peers(); - this.sync.init(opts, function() { - if (!cb) return; - return cb(); - }); - }; + this.sync = new Sync(opts); + } PeerSync.prototype.load_peers = function() { this.peerdb = [{ @@ -72,20 +62,26 @@ function spec() { PeerSync.prototype.handleBlock = function(info) { var self = this; var block = info.message.block; - var blockHash = coinUtil.formatHashFull(block.calcHash()); + var blockHash = bitcoreUtil.formatHashFull(block.calcHash()); console.log('[p2p_sync] Handle block: %s (allowReorgs: %s)', blockHash, self.allowReorgs); var tx_hashes = block.txs.map(function(tx) { - return coinUtil.formatHashFull(tx.hash); + return bitcoreUtil.formatHashFull(tx.hash); }); this.sync.storeTipBlock({ 'hash': blockHash, 'tx': tx_hashes, - 'previousblockhash': coinUtil.formatHashFull(block.prev_hash), + 'previousblockhash': bitcoreUtil.formatHashFull(block.prev_hash), }, self.allowReorgs, function(err) { - if (err) { + if (err && err.message.match(/NEED_SYNC/) && self.historicSync) { + console.log('[p2p_sync] Orphan block received. Triggering sync'); + self.historicSync.start({}, function(){ + console.log('[p2p_sync] Done resync.'); + }); + } + else if (err) { console.log('[p2p_sync] Error in handle Block: ' + err); } }); diff --git a/lib/PoolMatch.js b/lib/PoolMatch.js index 8c8db665..c32bc1f4 100644 --- a/lib/PoolMatch.js +++ b/lib/PoolMatch.js @@ -25,6 +25,7 @@ function spec(b) { self.stringsKl = self.stringsK.length; }; + PoolMatch.prototype.match = function(buffer) { var self = this; diff --git a/lib/Sync.js b/lib/Sync.js index 6a21627b..a712304d 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -13,20 +13,15 @@ function spec() { var async = require('async'); - function Sync() {} - - Sync.prototype.init = function(opts, cb) { - var self = this; - self.opts = opts; + function Sync(opts) { + this.opts = opts || {}; this.bDb = new BlockDb(opts); this.txDb = new TransactionDb(opts); this.txDb.on('tx_for_address', this.handleTxForAddress.bind(this)); this.txDb.on('new_tx', this.handleNewTx.bind(this)); this.bDb.on('new_block', this.handleNewBlock.bind(this)); - this.network = config.network === 'testnet' ? networks.testnet : networks.livenet; - return cb(); - }; + } Sync.prototype.close = function(cb) { var self = this; @@ -74,7 +69,8 @@ function spec() { * * A-B-C-D-E(TIP) ... NEW * - * NEW is ignored + * NEW is ignored (if allowReorgs is false) + * * */ @@ -100,15 +96,11 @@ function spec() { }, function(c) { if (!allowReorgs) return c(); - self.bDb.has(newPrev, function(err, val) { if (!val && newPrev.match(/^0+$/)) return c(); -/// AQUI! -> return is reor... - // - // needSync - // => + return c(err || - (!val ? new Error('WARN: Ignoring block with non existing prev:' + b.hash) : null)); + (!val ? new Error('NEED_SYNC Ignoring block with non existing prev:' + b.hash) : null)); }); }, function(c) { @@ -126,7 +118,6 @@ function spec() { }, function(c) { if (!needReorg) return c(); - self.bDb.getNext(newPrev, function(err, val) { if (err) return c(err); oldNext = val; @@ -142,13 +133,17 @@ function spec() { self.processReorg(oldTip, oldNext, newPrev, c); }, function(c) { + if (!allowReorgs) return c(); self.bDb.setTip(b.hash, function(err) { - if (err) return c(err); - self.bDb.setNext(newPrev, b.hash, function(err) { - return c(err); - }); + return c(err); + }); + }, + function(c) { + self.bDb.setNext(newPrev, b.hash, function(err) { + return c(err); }); } + ], function(err) { if (err && err.toString().match(/WARN/)) { diff --git a/test/lib/PeerSync.js b/test/lib/PeerSync.js index d5100f95..c4dc9734 100644 --- a/test/lib/PeerSync.js +++ b/test/lib/PeerSync.js @@ -9,22 +9,12 @@ describe('PeerSync', function() { beforeEach(function(done) { ps = new PeerSync(); - ps.init({ - verbose: false - }, done); + done(); }); afterEach(function() { ps.close(); }); - describe('#init()', function() { - it('should return with no errors', function() { - var other_ps = new PeerSync(); - expect(other_ps.init.bind(other_ps)).not.to. - throw (Error); - other_ps.close(); - }); - }); describe('#handleInv()', function() { var inv_info = { diff --git a/util/p2p.js b/util/p2p.js index 6954dde2..c7a11b5e 100755 --- a/util/p2p.js +++ b/util/p2p.js @@ -10,17 +10,9 @@ var program = require('commander'); program .version(PROGRAM_VERSION) - .option('-N --network [testnet]', 'Set bitcoin network [testnet]', 'testnet') - .option('-V --verbose', 'Verbose', 1) .parse(process.argv); var ps = new PeerSync(); -ps.init(program, function(err){ - if (err) { - console.log(err); - process.exit(1); - } - ps.run(); -}); +ps.run(); diff --git a/util/sync.js b/util/sync.js index 61be7e88..05087ba3 100755 --- a/util/sync.js +++ b/util/sync.js @@ -14,34 +14,28 @@ program .version(SYNC_VERSION) .option('-D --destroy', 'Remove current DB (and start from there)', 0) .option('-S --startfile', 'Number of file from bitcoind to start(default=0)') + .option('-R --rpc', 'Force sync with RPC') .option('-v --verbose', 'Verbose 0/1', 0) .parse(process.argv); var historicSync = new HistoricSync(); -/* TODO: Sure? -if (program.remove) { -} -*/ async.series([ function(cb) { - historicSync.init(program, cb); + if (!program.destroy) return cb(); + console.log('Deleting Sync DB...'); + historicSync.sync.destroy(cb); }, function(cb) { - historicSync.smartImport({ - destroy: program.destroy, - startFile: program.startfile, + historicSync.start({ + forceStartFile: program.startfile, + forceRPC: program.rpc, },cb); }, ], function(err) { historicSync.close(); - if (err) { - console.log('CRITICAL ERROR: ', historicSync.info()); - } - else { - console.log('Finished.\n Status:\n', historicSync.info()); - } + if (err) console.log('CRITICAL ERROR: ', historicSync.info()); }); From 21cdca537a9db2d1cdbefac08dbc86d8f440a812 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 18 Feb 2014 10:49:52 -0300 Subject: [PATCH 04/23] fix typo for sync strategy --- lib/HistoricSync.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 2168299c..5f3c4b61 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -260,9 +260,8 @@ function spec() { HistoricSync.prototype.prepareFileSync = function(opts, next) { var self = this; - if ( - opts.forceRPC || !config.bitcoind.dataDir || - self.connectedCountDB > self.blockChainHeight > 0.9) return next(); + if ( opts.forceRPC || !config.bitcoind.dataDir || + self.connectedCountDB > self.blockChainHeight * 0.9) return next(); self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); From 9bc952a274215007b7b8f9db2f4c47bc89699d1e Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 18 Feb 2014 10:50:07 -0300 Subject: [PATCH 05/23] fix status page sync error --- public/views/status.html | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/public/views/status.html b/public/views/status.html index 2fc3ac9a..a04b019a 100644 --- a/public/views/status.html +++ b/public/views/status.html @@ -10,23 +10,26 @@

Sync Status

- - {{sync.error}} - - + From d7a2da8f7bf38e4c6aa487e82e7a00b52a40e0c2 Mon Sep 17 00:00:00 2001 From: "Ryan X. Charles" Date: Tue, 18 Feb 2014 09:34:07 -0500 Subject: [PATCH 06/23] update version for npm - and hopefully fix description issue --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 7cb573d4..face300d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,7 @@ { "name": "insight-bitcore", - "version": "0.1.0", + "description": "An open-source bitcoin blockchain API. The Insight API provides you with a convenient, powerful and simple way to query and broadcast data on the bitcoin network and build your own services with it.", + "version": "0.1.1", "author": { "name": "Ryan X Charles", "email": "ryan@bitpay.com" From f21d28c60186a7dc5d207755c5a8a49d4475dc4a Mon Sep 17 00:00:00 2001 From: Rich Morgan Date: Tue, 18 Feb 2014 10:00:50 -0500 Subject: [PATCH 07/23] Added missing /td in Height row --- public/views/block.html | 1 + 1 file changed, 1 insertion(+) diff --git a/public/views/block.html b/public/views/block.html index 21c26440..60d1ee58 100644 --- a/public/views/block.html +++ b/public/views/block.html @@ -44,6 +44,7 @@ From a820594b8b2c18c115dbfb697bc2c43e88f03f6b Mon Sep 17 00:00:00 2001 From: Rich Morgan Date: Tue, 18 Feb 2014 10:28:40 -0500 Subject: [PATCH 08/23] Added missing /p in pagination row. --- public/views/block_list.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/views/block_list.html b/public/views/block_list.html index 99d9ffd4..f8656f64 100644 --- a/public/views/block_list.html +++ b/public/views/block_list.html @@ -14,7 +14,7 @@

 

Today

-

{{humanSince(pagination.currentTs)}} +

{{humanSince(pagination.currentTs)}}

 

From 6430f05d303b6817154d7f02fae78c9309971361 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 18 Feb 2014 12:50:27 -0300 Subject: [PATCH 09/23] broadcast last sync status --- lib/HistoricSync.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 5f3c4b61..c1e7302d 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -70,7 +70,7 @@ function spec() { HistoricSync.prototype.setError = function(err) { var self = this; - self.error = err.toString(); + self.error = err.message?err.message:err.toString(); self.status='error'; self.showProgress(); return err; @@ -384,13 +384,12 @@ function spec() { async.whilst( function() { + self.showProgress(); return self.status === 'syncing'; }, function (w_cb) { self.getFn(function(err,blockInfo) { if (err) return w_cb(self.setError(err)); - - self.showProgress(); if (blockInfo && blockInfo.hash) { self.syncedBlocks++; self.sync.storeTipBlock(blockInfo, self.allowReorgs, function(err) { @@ -408,6 +407,7 @@ function spec() { else { self.endTs = Date.now(); self.status = 'finished'; + self.shouldBroadcast return w_cb(err); } }); From 67a68ee3ca220e1045f36c72420eacf2cfdf2295 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 18 Feb 2014 12:51:22 -0300 Subject: [PATCH 10/23] broadcast last sync status --- lib/HistoricSync.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index c1e7302d..27383def 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -407,7 +407,6 @@ function spec() { else { self.endTs = Date.now(); self.status = 'finished'; - self.shouldBroadcast return w_cb(err); } }); From f5d8891317e5a4a14b62bb4704afce856eea827a Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 18 Feb 2014 14:01:18 -0300 Subject: [PATCH 11/23] failover to RPC sync if File sync fails --- app/controllers/socket.js | 4 ++++ insight.js | 4 +++- lib/HistoricSync.js | 6 ++++++ lib/Sync.js | 1 + util/sync.js | 4 +++- 5 files changed, 17 insertions(+), 2 deletions(-) diff --git a/app/controllers/socket.js b/app/controllers/socket.js index 825713b3..cdf41cfe 100644 --- a/app/controllers/socket.js +++ b/app/controllers/socket.js @@ -19,6 +19,8 @@ module.exports.broadcastTx = function(tx) { if (ios) { var t = {}; if (typeof tx === 'string') { + +console.log('[socket.js.22]',tx); //TODO t = { txid: tx }; @@ -31,6 +33,8 @@ module.exports.broadcastTx = function(tx) { }); t.valueOut = parseInt(valueOut) / util.COIN; + +console.log('[socket.js.36]',t); //TODO } ios.sockets. in ('inv').emit('tx', t); } diff --git a/insight.js b/insight.js index 0cf64a02..d4c2891d 100644 --- a/insight.js +++ b/insight.js @@ -53,7 +53,9 @@ if (!config.disableP2pSync) { /** * historic_sync process */ -var historicSync = new HistoricSync({ shouldBroadcastSync: true }); +var historicSync = new HistoricSync({ + shouldBroadcastSync: true +}); peerSync.historicSync = historicSync; if (!config.disableHistoricSync) { diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 27383def..f29e77bc 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -264,7 +264,12 @@ function spec() { self.connectedCountDB > self.blockChainHeight * 0.9) return next(); + try { self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); + } catch (e) { + p(e.message + '. Disabling file sync.'); + return next(); + } self.getFn = self.getBlockFromFile; self.allowReorgs = true; @@ -407,6 +412,7 @@ function spec() { else { self.endTs = Date.now(); self.status = 'finished'; + console.log('Done Syncing', self.info()); return w_cb(err); } }); diff --git a/lib/Sync.js b/lib/Sync.js index a712304d..06954853 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -265,6 +265,7 @@ function spec() { Sync.prototype.handleNewTx = function(data) { if (this.opts.shouldBroadcast) { +console.log('[Sync.js.268:shouldBroadcast:]', data.tx); //TODO sockets.broadcastTx(data.tx); } }; diff --git a/util/sync.js b/util/sync.js index 05087ba3..ee29e65a 100755 --- a/util/sync.js +++ b/util/sync.js @@ -18,7 +18,9 @@ program .option('-v --verbose', 'Verbose 0/1', 0) .parse(process.argv); -var historicSync = new HistoricSync(); +var historicSync = new HistoricSync({ + shouldBroadcastSync: true, +}); async.series([ From db064e9527ef268f46bb798424d11f1b112ef9cd Mon Sep 17 00:00:00 2001 From: Mario Colque Date: Tue, 18 Feb 2014 14:35:54 -0300 Subject: [PATCH 12/23] added Minig Pool Detection --- app/controllers/blocks.js | 7 +++++-- lib/BlockDb.js | 21 +++++++++++++++++++-- lib/PoolMatch.js | 27 +++++++++------------------ public/views/block.html | 4 ++++ 4 files changed, 37 insertions(+), 22 deletions(-) diff --git a/app/controllers/blocks.js b/app/controllers/blocks.js index 5d828dc5..409679f2 100644 --- a/app/controllers/blocks.js +++ b/app/controllers/blocks.js @@ -17,8 +17,11 @@ exports.block = function(req, res, next, hash) { if (err || !block) return common.handleErrors(err, res, next); else { - req.block = block.info; - return next(); + bdb.getPoolInfo(block.info.tx[0], function(info) { + block.info.poolInfo = info; + req.block = block.info; + return next(); + }); } }); }; diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 76a118c1..3fdb3ff6 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -25,6 +25,9 @@ function spec(b) { var Rpc = b.rpc || require('./Rpc').class(); var PoolMatch = b.poolMatch || require('./PoolMatch').class(config); + var buffertools = require('buffertools'); + var TransactionDb = require('./TransactionDb.js').class(); + var BlockDb = function() { BlockDb.super(this, arguments); this.poolMatch = new PoolMatch(); @@ -57,7 +60,7 @@ function spec(b) { return db.batch() .put(time_key, b.hash) .put(MAIN_PREFIX + b.hash, 1) - .put(PREV_PREFIX + b.hash, b.previousblockhash) + .put(PREV_PREFIX + b.hash, b.previousblockhash) .write(function(err){ if (!err) { self.emit('new_block', {blockid: b.hash}); @@ -164,6 +167,21 @@ function spec(b) { }); }; + BlockDb.prototype.getPoolInfo = function(tx, cb) { + var tr = new TransactionDb(); + var self = this; + + tr._getInfo(tx, function(e, a) { + if (e) return cb(false); + + if (a.isCoinBase) { + var coinbaseHexBuffer = new Buffer(a.vin[0].coinbase, 'hex'); + var a = self.poolMatch.match(coinbaseHexBuffer); + + return cb(a); + } + }); + }; BlockDb.prototype.fromHashWithInfo = function(hash, cb) { var self = this; @@ -175,7 +193,6 @@ function spec(b) { if (err) return cb(err); info.isMainChain = val ? true : false; -// info.poolInfo = self.poolMatch.match(info.hex); return cb(null, { hash: hash, diff --git a/lib/PoolMatch.js b/lib/PoolMatch.js index 8c8db665..83aa70aa 100644 --- a/lib/PoolMatch.js +++ b/lib/PoolMatch.js @@ -11,36 +11,27 @@ function spec(b) { var PoolMatch = function() { var self = this; - self.strings={}; + self.strings = {}; db.forEach(function(pool) { pool.searchStrings.forEach(function(s) { - if (!self.strings[s]) self.strings[s] = []; - self.strings[s].push(pool); + self.strings[s] = { + poolName: pool.poolName, + url: pool.url + }; }); }); - Object.keys( self.strings, function(s) { - delete self.strings[s].searchStrings; - }); - self.stringsK = Object.keys(self.strings); - self.stringsKl = self.stringsK.length; }; PoolMatch.prototype.match = function(buffer) { var self = this; - - var match; - var i =0; - while (!match && i < self.stringsKl) { - var k = self.stringsK[i++]; - if ( buffertools.indexOf(buffer,self.strings[k]) >= 0 ) { - match = self.strings[k]; + for(var k in self.strings) { + if (buffertools.indexOf(buffer, k) >= 0) { + return self.strings[k]; } - } - return match; + }; }; return PoolMatch; } module.defineClass(spec); - diff --git a/public/views/block.html b/public/views/block.html index e983dbd1..e68ea072 100644 --- a/public/views/block.html +++ b/public/views/block.html @@ -53,6 +53,10 @@
+ + + - diff --git a/public/views/block_list.html b/public/views/block_list.html index 99d9ffd4..ea6d8419 100644 --- a/public/views/block_list.html +++ b/public/views/block_list.html @@ -35,6 +35,7 @@ + @@ -46,6 +47,7 @@ + diff --git a/public/views/index.html b/public/views/index.html index 585e5fd7..96dbeaee 100644 --- a/public/views/index.html +++ b/public/views/index.html @@ -13,6 +13,7 @@ + @@ -24,6 +25,7 @@ + From 91652d854de906afd1a034e68fceea52291a38b5 Mon Sep 17 00:00:00 2001 From: Mario Colque Date: Tue, 18 Feb 2014 16:40:14 -0300 Subject: [PATCH 20/23] #356 Fixed copy button if have not flash installed --- lib/HistoricSync.js | 1 + public/src/js/directives.js | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index f29e77bc..691969dc 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -288,6 +288,7 @@ function spec() { }, function (w_cb) { self.getBlockFromFile(function(err,b) { + if (!b) return w_cb('Could not find block ' + self.startBlock); h=b.hash; setImmediate(function(){ return w_cb(err); diff --git a/public/src/js/directives.js b/public/src/js/directives.js index e5762456..39a7c77f 100644 --- a/public/src/js/directives.js +++ b/public/src/js/directives.js @@ -49,14 +49,15 @@ angular.module('insight') }; client.on('mousedown', onMousedown); - client.on('noflash wrongflash', function() { - return elm.remove(); - }); scope.$on('$destroy', function() { client.off('mousedown', onMousedown); }); }); + + clip.on('noFlash wrongflash', function() { + return elm.remove(); + }); } }; }); From 9d71a26766d45ca7b01e7967796197a3f9a7efb0 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 18 Feb 2014 19:31:08 -0300 Subject: [PATCH 21/23] fix spents in first part of key matching errors --- app/models/Address.js | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/app/models/Address.js b/app/models/Address.js index 6f9e06df..9c804524 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -77,20 +77,27 @@ function spec() { var db = new TransactionDb(); async.series([ function (cb) { + var seen={}; db.fromAddr(self.addrStr, function(err,txOut){ if (err) return cb(err); txOut.forEach(function(txItem){ - + var add=0, addSpend=0; var v = txItem.value_sat; - txs.push({txid: txItem.txid, ts: txItem.ts}); + if ( !seen[txItem.txid] ) { + txs.push({txid: txItem.txid, ts: txItem.ts}); + seen[txItem.txid]=1; + add=1; + } - if (txItem.spentTxId) { + if (txItem.spentTxId && !seen[txItem.spentTxId] ) { txs.push({txid: txItem.spentTxId, ts: txItem.spentTs}); + seen[txItem.spentTxId]=1; + addSpend=1; } if (txItem.isConfirmed) { - self.txApperances += 1; + self.txApperances += add; self.totalReceivedSat += v; if (! txItem.spentTxId ) { //unspent @@ -100,17 +107,17 @@ function spec() { // unspent self.balanceSat += v; self.unconfirmedBalanceSat -= v; - self.unconfirmedTxApperances += 1; + self.unconfirmedTxApperances += addSpend; } else { // spent self.totalSentSat += v; - self.txApperances += 1; + self.txApperances += addSpend; } } else { self.unconfirmedBalanceSat += v; - self.unconfirmedTxApperances += 1; + self.unconfirmedTxApperances += add; } }); return cb(); From c2221a9ece77918c1345b5501faddd6d3d9c96df Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 18 Feb 2014 19:31:23 -0300 Subject: [PATCH 22/23] fix repeating TXs in addr pages --- lib/TransactionDb.js | 17 ++++++++--------- public/src/js/controllers/transactions.js | 3 +-- public/views/transaction/tx.html | 17 +++++++++++++---- test/integration/addr.json | 22 +++++++++++++++++----- 4 files changed, 39 insertions(+), 20 deletions(-) diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 2a407b71..00c9faeb 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -137,7 +137,7 @@ function spec(b) { }) .on('end', function() { - var k = SPENT_PREFIX + txid; + var k = SPENT_PREFIX + txid + '-'; db.createReadStream({ start: k, end: k + '~' @@ -166,7 +166,7 @@ function spec(b) { if (!info) return cb(); - var k = SPENT_PREFIX + info.txid; + var k = SPENT_PREFIX + info.txid + '-'; db.createReadStream({ start: k, end: k + '~' @@ -311,7 +311,7 @@ isspent */ // spent? - var k = SPENT_PREFIX + txid + '-' + n; + var k = SPENT_PREFIX + txid + '-' + n + '-'; db.createReadStream({ start: k, end: k + '~' @@ -365,7 +365,7 @@ isspent TransactionDb.prototype.fromAddr = function(addr, cb) { var self = this; - var k = ADDR_PREFIX + addr; + var k = ADDR_PREFIX + addr + '-'; var ret = []; db.createReadStream({ @@ -388,7 +388,7 @@ isspent .on('end', function() { async.eachLimit(ret, CONCURRENCY, function(o, e_c) { - var k = SPENT_PREFIX + o.txid + '-' + o.index; + var k = SPENT_PREFIX + o.txid + '-' + o.index + '-'; db.createReadStream({ start: k, end: k + '~' @@ -415,14 +415,13 @@ isspent }; - TransactionDb.prototype.removeFromTxId = function(txid, cb) { async.series([ function(c) { db.createReadStream({ - start: OUTS_PREFIX + txid, + start: OUTS_PREFIX + txid + '-', end: OUTS_PREFIX + txid + '~', }).pipe( db.createWriteStream({ @@ -432,7 +431,7 @@ isspent }, function(c) { db.createReadStream({ - start: SPENT_PREFIX + txid, + start: SPENT_PREFIX + txid + '-', end: SPENT_PREFIX + txid + '~' }) .pipe( @@ -578,7 +577,7 @@ isspent addrs.push(addr); } - // existed? + // existed? var k = OUTS_PREFIX + tx.txid + '-' + o.n; db.get(k, function(err) { if (err && err.notFound) { diff --git a/public/src/js/controllers/transactions.js b/public/src/js/controllers/transactions.js index 72f15d48..e22897cb 100644 --- a/public/src/js/controllers/transactions.js +++ b/public/src/js/controllers/transactions.js @@ -59,16 +59,15 @@ function($scope, $rootScope, $routeParams, $location, Global, Transaction, Trans tmp[addr].unconfirmedInput += items[i].unconfirmedInput; tmp[addr].dbError = tmp[addr].dbError || items[i].dbError; tmp[addr].valueSat += items[i].value * COIN; - tmp[addr].value = tmp[addr].valueSat / COIN; tmp[addr].items.push(items[i]); tmp[addr].notAddr = notAddr; tmp[addr].count++; } angular.forEach(tmp, function(v) { + v.value = parseInt(v.valueSat) / COIN; ret.push(v); }); - return ret; }; diff --git a/public/views/transaction/tx.html b/public/views/transaction/tx.html index 65a465d0..f38005f9 100644 --- a/public/views/transaction/tx.html +++ b/public/views/transaction/tx.html @@ -26,7 +26,8 @@
-
+
+
{{$root.currency.getConvertion(vin.value)}}
@@ -41,7 +42,10 @@
-
+
+
+
{{$root.currency.getConvertion(vin.value)}}
@@ -67,6 +71,7 @@
+
-
+
+
@@ -97,7 +103,9 @@
-
+
+
+
{{$root.currency.getConvertion(vout.value)}} (U) @@ -126,6 +134,7 @@
+
diff --git a/test/integration/addr.json b/test/integration/addr.json index b68d913a..15c47af9 100644 --- a/test/integration/addr.json +++ b/test/integration/addr.json @@ -1,5 +1,4 @@ [ - { "addr": "mp3Rzxx9s1A21SY3sjJ3CQoa2Xjph7e5eS", "balance": 0, @@ -37,7 +36,7 @@ }, { "addr": "mgqvRGJMwR9JU5VhJ3x9uX9MTkzTsmmDgQ", - "txApperances": 28, + "txApperances": 27, "balance": 0, "totalReceived": 54.81284116 }, @@ -46,11 +45,11 @@ "balance": 1199.74393853, "totalReceived": 1199.74393853, "totalSent": 0, - "txApperances": 6049 + "txApperances": 6048 }, { "addr": "mjRmkmYzvZN3cA3aBKJgYJ65epn3WCG84H", - "txApperances": 13343, + "txApperances": 7164, "balance": 46413.0, "totalReceived": 357130.17644359, "totalSent": 310717.17644359 @@ -68,7 +67,7 @@ "balance": 10580.50027254, "totalReceived": 12157.65075053, "totalSent": 1577.15047799, - "txApperances": 459, + "txApperances": 441, "transactions": [ "91800d80bb4c69b238c9bfd94eb5155ab821e6b25cae5c79903d12853bbb4ed5", "f6e80d4fd1a2377406856c67d0cee5ac7e5120993ff97e617ca9aac33b4c6b1e", @@ -111,7 +110,20 @@ "a3789e113041db907a1217ddb5c3aaf0eff905cc3d913e68d977e1ab4d19acea", "80b460922faf0ad1e8b8a55533654c9a9f3039bfff0fff2bcf8536b8adf95939" ] + }, + { + "addr": "mtA6woo1wjCeu1dLkWgpSD3tRnRfrHt3FL", + "balance": 349.845, + "totalReceived": 349.845, + "totalSent": 0, + "txApperances": 13, + "transactions": [ + "794eafc0ad68a3576034eb137d7d20d3bdf1777ecf27e0e20e96e1adcfc66659", + "0130721f29f50b773858c3c9081678bdddebcd18078c5fa2436d979b54ed5cef", + "fb1024947b48d90255aedd3f0f1df3673a7e98d06346bb2ac89b116aa19c5db4" + ] } + ] From 5230a79114a7f2d25e406510a1a55a0d7047b022 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 18 Feb 2014 19:40:37 -0300 Subject: [PATCH 23/23] update to buffertools 2.0 --- lib/HistoricSync.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index f29e77bc..43e46386 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -15,6 +15,8 @@ function spec() { var Sync = require('./Sync').class(); var sockets = require('../app/controllers/socket.js'); var BlockExtractor = require('./BlockExtractor.js').class(); + var buffertools = require('buffertools'); + // var bitcoreUtil = require('bitcore/util/util'); // var Deserialize = require('bitcore/Deserialize'); @@ -29,7 +31,8 @@ function spec() { var genesisHashReversed = new Buffer(32); this.network.genesisBlock.hash.copy(genesisHashReversed); - this.genesis = genesisHashReversed.reverse().toString('hex'); + buffertools.reverse(genesisHashReversed); + this.genesis = genesisHashReversed.toString('hex'); this.rpc = new RpcClient(config.bitcoind); this.shouldBroadcast = opts.shouldBroadcastSync;
Sync Progress
- {{sync.syncPercentage}}% Complete + {{sync.syncPercentage}}% Complete
Current Sync Status{{sync.status}} + {{sync.status}} + + + {{sync.error}} + +
Start Date{{block.height}} (Mainchain) (Orphaned) +
Block RewardTimestamp {{block.time * 1000 | date:'medium'}}
Relayed by{{block.poolInfo.poolName}} +
Merkle Root From 1e4a6179defb79fc918367b31283a341e69ecc74 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 18 Feb 2014 14:51:45 -0300 Subject: [PATCH 13/23] fix tx broadcast --- app/controllers/socket.js | 17 +++++++++-------- lib/Sync.js | 1 - 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/controllers/socket.js b/app/controllers/socket.js index cdf41cfe..9a11f5b5 100644 --- a/app/controllers/socket.js +++ b/app/controllers/socket.js @@ -17,24 +17,25 @@ module.exports.init = function(app, io_ext) { module.exports.broadcastTx = function(tx) { if (ios) { - var t = {}; + var t; if (typeof tx === 'string') { - -console.log('[socket.js.22]',tx); //TODO t = { txid: tx }; - } else { - t = tx; + } + + else { + t = { + txid: tx.txid, + size: tx.size, + }; // Outputs var valueOut = 0; - t.vout.forEach(function(o) { + tx.vout.forEach(function(o) { valueOut += o.value * util.COIN; }); t.valueOut = parseInt(valueOut) / util.COIN; - -console.log('[socket.js.36]',t); //TODO } ios.sockets. in ('inv').emit('tx', t); } diff --git a/lib/Sync.js b/lib/Sync.js index 06954853..a712304d 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -265,7 +265,6 @@ function spec() { Sync.prototype.handleNewTx = function(data) { if (this.opts.shouldBroadcast) { -console.log('[Sync.js.268:shouldBroadcast:]', data.tx); //TODO sockets.broadcastTx(data.tx); } }; From 941caaafd5d861af389b79acc841fc86be5beead Mon Sep 17 00:00:00 2001 From: Bechi Date: Tue, 18 Feb 2014 15:09:11 -0300 Subject: [PATCH 14/23] navbar-footer --- public/src/css/common.css | 42 ++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/public/src/css/common.css b/public/src/css/common.css index 95f9d211..0c3bc4ab 100644 --- a/public/src/css/common.css +++ b/public/src/css/common.css @@ -15,10 +15,10 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 { /* Styling for the ngProgress itself */ #ngProgress { - background-color: #373D42 !important; + background-color: #6C9032 !important; box-shadow: none !important; color: #373D42 !important; - height: 4px !important; + height: 3px !important; margin: 0; opacity: 0; padding: 0; @@ -36,7 +36,7 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 { position: fixed; margin: 0; padding: 0; - top: 50px; + top: 63px; left: 0; right: 0; z-index: 99999; @@ -63,22 +63,32 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 { .vm { vertical-align: middle; } +.navbar-default .navbar-toggle { + border-color: #fff; + margin-top: 15px; +} + +.navbar-default .navbar-toggle .icon-bar { background-color: #fff; } +.navbar-default .navbar-toggle:hover, .navbar-default .navbar-toggle:focus {background-color: #373D42;} + .navbar-default { background-color: #8DC429; - border-bottom: 4px solid #64920F; } .navbar-default .navbar-nav>li>a { color: #F4FBE8; font-family: 'Ubuntu', sans-serif; + padding-left: 25px; + padding-right: 25px; } -.navbar-default .navbar-nav>.active>a, .navbar-default .navbar-nav>.active>a:focus, .navbar-default .navbar-nav>li>a:hover { +.navbar-default .navbar-nav>.active>a, .navbar-default .navbar-nav>.active>a:focus { + background-color: #6C9032; + color: #fff; +} + +.navbar-default .navbar-nav>li>a:hover, .navbar-default .navbar-nav>.active>a:hover { background-color: #fff; - -webkit-box-shadow: 0px 4px 0px 0px rgba(55,61,56,1); - -moz-box-shadow: 0px 4px 0px 0px rgba(55,61,56,1); - box-shadow: 0px 4px 0px 0px rgba(55,61,56,1); - color: #373D42; } .navbar-form .form-group { @@ -86,7 +96,9 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 { } .navbar-form { + margin-left: 20px; width: 35%; + margin-top: 15px; } @media (max-width: 991px) { @@ -121,6 +133,8 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 { .navbar-default .navbar-brand { color: #FFFFFF; + margin-right: 20px; + padding: 22px 15px; } .navbar-form .form-control { @@ -134,6 +148,11 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 { box-shadow: 1px 1px 0px 0px rgba(255,255,255,0.41), inset 1px 1px 3px 0px rgba(0,0,0,0.10); } +.navbar-nav>li>a { + padding-top: 22px; + padding-bottom: 22px; +} + #search { color: #fff; font-family: 'Ubuntu', sans-serif; @@ -166,7 +185,7 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 { -webkit-border-radius: 3px; background-color: #597338; border-radius: 3px; - margin: 8px 0; + margin: 15px 0; padding: 8px 10px; font-size: 12px; color: #eee; @@ -357,7 +376,6 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 { /* Set the fixed height of the footer here */ #footer { background-color: #373D42; - border-top: 4px solid #656E76; color: #fff; height: 51px; overflow: hidden; @@ -390,7 +408,7 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 { -------------------------------------------------- */ /* Not required for template or sticky footer method. */ -#wrap > .container { padding: 60px 15px 0; } +#wrap > .container { padding: 70px 15px 0; } #footer > .container { padding: auto 15px; } From 1e8bee4f18a1d71ab8dddd75905269b3fde8440f Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Sun, 16 Feb 2014 13:38:34 -0300 Subject: [PATCH 15/23] fix working for spent, also in var names --- public/views/transaction/tx.html | 7 ------- 1 file changed, 7 deletions(-) diff --git a/public/views/transaction/tx.html b/public/views/transaction/tx.html index d84c8954..65a465d0 100644 --- a/public/views/transaction/tx.html +++ b/public/views/transaction/tx.html @@ -100,15 +100,8 @@
{{$root.currency.getConvertion(vout.value)}} -<<<<<<< HEAD (U)    -======= - (U) -    ->>>>>>> 8e733340d31053254750053c5ca686eac87c5139 - -
From ed4f01e5bfff44bcecd8da76c9ff5e2ca185fcc9 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 18 Feb 2014 15:13:55 -0300 Subject: [PATCH 16/23] npm version 0.1.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index face300d..68da80e2 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "insight-bitcore", "description": "An open-source bitcoin blockchain API. The Insight API provides you with a convenient, powerful and simple way to query and broadcast data on the bitcoin network and build your own services with it.", - "version": "0.1.1", + "version": "0.1.2", "author": { "name": "Ryan X Charles", "email": "ryan@bitpay.com" From e8cfebf832f4fae20283f7bcc402599a6119e3e6 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 18 Feb 2014 15:17:25 -0300 Subject: [PATCH 17/23] update bechis name --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 68da80e2..5150cf8c 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "email": "cmgustavo83@gmail.com" }, { - "name": "Juan Sosa", + "name": "Juan Ignasio Sosa Lopez", "email": "bechilandia@gmail.com" } ], From 5bef701d11d96c29ef7b617129cfd39d0c19a183 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 18 Feb 2014 15:17:36 -0300 Subject: [PATCH 18/23] update bechis name --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5150cf8c..48df32f9 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "email": "cmgustavo83@gmail.com" }, { - "name": "Juan Ignasio Sosa Lopez", + "name": "Juan Ignacio Sosa Lopez", "email": "bechilandia@gmail.com" } ], From 36f3d6684a4ac2f11f8787c5dc766b1f12e46e81 Mon Sep 17 00:00:00 2001 From: Mario Colque Date: Tue, 18 Feb 2014 15:44:02 -0300 Subject: [PATCH 19/23] #340 added mining pool detection on homepage and block list page --- app/controllers/blocks.js | 8 +++++++- public/views/block.html | 4 +++- public/views/block_list.html | 2 ++ public/views/index.html | 2 ++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/app/controllers/blocks.js b/app/controllers/blocks.js index 409679f2..def4e359 100644 --- a/app/controllers/blocks.js +++ b/app/controllers/blocks.js @@ -66,7 +66,12 @@ var getBlock = function(blockhash, cb) { isOrphan: 1, }; } - return cb(err, block.info); + + bdb.getPoolInfo(block.info.tx[0], function(info) { + block.info.poolInfo = info; + return cb(err, block.info); + }); + }); }; @@ -130,6 +135,7 @@ exports.list = function(req, res) { hash: b.hash, time: b.ts || info.time, txlength: info.tx.length, + poolInfo: info.poolInfo }); }); }, function(err, allblocks) { diff --git a/public/views/block.html b/public/views/block.html index e68ea072..41f012e2 100644 --- a/public/views/block.html +++ b/public/views/block.html @@ -55,7 +55,9 @@
Relayed by{{block.poolInfo.poolName}} + + {{block.poolInfo.poolName}} +
Merkle RootHeight Timestamp TransactionsRelayed by Size
{{b.height}} {{b.time * 1000 | date:'medium'}} {{b.txlength}}{{b.poolInfo.poolName}} {{b.size}}
Height Age TransactionsRelayed by Size
{{humanSince(b.time)}} {{b.txlength}}{{b.poolInfo.poolName}} {{b.size}} bytes