From 444249763800fb8473d5d7b1457e0ac89669347b Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Fri, 17 Jan 2014 10:22:00 -0300 Subject: [PATCH 01/17] implements uptosync --- lib/HistoricSync.js | 32 +++++++++++++++++++------------- util/sync.js | 12 +++++++++--- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 2c432d9c..908f3414 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -111,6 +111,11 @@ function spec() { ], function (err){ + if (opts.uptoexisting && existed) { + p('DONE. Found existing block: %s ', blockHash); + return cb(err); + } + if (err) p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, that.block_count); @@ -126,19 +131,17 @@ function spec() { }); }; - HistoricSync.prototype.syncBlocks = function(start, end, isForward, cb) { + HistoricSync.prototype.syncBlocks = function(start, end, opts, cb) { var that = this; p('Starting from: ', start); p(' to : ', end); - p(' isForward: ', isForward); + p(' opts: ', JSON.stringify(opts)); - - return that.getPrevNextBlock( start, end, - isForward ? { next: 1 } : { prev: 1}, cb); + return that.getPrevNextBlock( start, end, opts , cb); }; - HistoricSync.prototype.do_import_history = function(opts, next) { + HistoricSync.prototype.import_history = function(opts, next) { var that = this; var retry_attemps = 100; @@ -203,21 +206,20 @@ function spec() { ], function(err) { + var start, end; function sync() { - var start, end, isForward; - if (opts.reverse) { start = block_best; end = that.network.genesisBlock.hash.reverse().toString('hex'); - isForward = false; + opts.prev = true; } else { start = that.network.genesisBlock.hash.reverse().toString('hex'); end = null; - isForward = true; + opts.next = true; } - that.syncBlocks(start, end, isForward, function(err) { + that.syncBlocks(start, end, opts, function(err) { if (err && err.message.match(/ECONNREFUSED/) && retry_attemps--){ setTimeout(function() { @@ -236,9 +238,13 @@ function spec() { }); }; - HistoricSync.prototype.import_history = function(opts, next) { + // Reverse Imports (upto if we have genesis block?) + HistoricSync.prototype.smart_import = function(next) { var that = this; - that.do_import_history(opts, next); + var opts = { + prev: 1, + }; + that.import_history(opts, next); }; diff --git a/util/sync.js b/util/sync.js index 569ca782..a4e3f441 100755 --- a/util/sync.js +++ b/util/sync.js @@ -16,6 +16,7 @@ program .option('-N --network [livenet]', 'Set bitcoin network [testnet]', 'testnet') .option('-D --destroy', 'Remove current DB (and start from there)', 0) .option('-R --reverse', 'Sync backwards', 0) + .option('-U --uptoexisting', 'Sync only until an existing block is found', 0) .parse(process.argv); var historicSync = new HistoricSync({ @@ -23,7 +24,7 @@ var historicSync = new HistoricSync({ }); if (program.remove) { - + // TODO: Sure? } async.series([ @@ -31,12 +32,17 @@ async.series([ historicSync.init(program, cb); }, function(cb) { - historicSync.import_history(program, function(err, count) { + historicSync.import_history({ + network: program.network, + destroy: program.destroy, + reverse: program.reverse, + uptoexisting: program.uptoexisting, + }, function(err, count) { if (err) { console.log('CRITICAL ERROR: ', err); } else { - console.log('Done! [%d blocks]', count, err); + console.log('Finished. [%d blocks]', count); } cb(); }); From 82beb27a3cd8f241dec123d74d15013dd3775c88 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Fri, 17 Jan 2014 10:44:14 -0300 Subject: [PATCH 02/17] better block_total accounting --- lib/HistoricSync.js | 12 ++++++------ util/sync.js | 2 ++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 908f3414..49d67630 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -70,7 +70,7 @@ function spec() { }, //show some (inacurate) status function(c) { - if (that.block_count++ % 1000 === 0) { + if (that.block_count % 1000 === 1) { progress_bar('sync status:', that.block_count, that.block_total); } return c(); @@ -91,7 +91,6 @@ function spec() { //store it function(c) { if (existed) return c(); - that.sync.storeBlock(blockInfo.result, function(err) { existed = err && err.toString().match(/E11000/); if (err && ! existed) return c(err); @@ -112,7 +111,7 @@ function spec() { function (err){ if (opts.uptoexisting && existed) { - p('DONE. Found existing block: %s ', blockHash); + p('DONE. Found existing block: ', blockHash); return cb(err); } @@ -120,6 +119,7 @@ function spec() { p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, that.block_count); if (blockInfo && blockInfo.result) { + block_total++; if (opts.prev && blockInfo.result.previousblockhash) { return that.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb); } @@ -177,7 +177,7 @@ function spec() { }, function(cb) { that.rpc.getInfo(function(err, res) { - if (err) cb(err); + if (err) return cb(err); that.block_total = res.result.blocks; return cb(); @@ -188,7 +188,7 @@ function spec() { if (!opts.reverse) return cb(); that.rpc.getBlockCount(function(err, res) { - if (err) cb(err); + if (err) return cb(err); block_height = res.result; return cb(); }); @@ -197,7 +197,7 @@ function spec() { if (!opts.reverse) return cb(); that.rpc.getBlockHash(block_height, function(err, res) { - if (err) cb(err); + if (err) return cb(err); block_best = res.result; return cb(); diff --git a/util/sync.js b/util/sync.js index a4e3f441..6e531200 100755 --- a/util/sync.js +++ b/util/sync.js @@ -14,6 +14,7 @@ var async = require('async'); program .version(SYNC_VERSION) .option('-N --network [livenet]', 'Set bitcoin network [testnet]', 'testnet') + .option('-S --smart', 'genesis stored? uptoexisting = 1', 1) .option('-D --destroy', 'Remove current DB (and start from there)', 0) .option('-R --reverse', 'Sync backwards', 0) .option('-U --uptoexisting', 'Sync only until an existing block is found', 0) @@ -37,6 +38,7 @@ async.series([ destroy: program.destroy, reverse: program.reverse, uptoexisting: program.uptoexisting, + smart: program.smart, }, function(err, count) { if (err) { console.log('CRITICAL ERROR: ', err); From afeec834a8e6ee840d17e5f3d4de3b92defff8bf Mon Sep 17 00:00:00 2001 From: Mario Colque Date: Fri, 17 Jan 2014 15:25:26 -0300 Subject: [PATCH 03/17] added new template for 404 --- public/js/config.js | 2 +- public/views/404.html | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 public/views/404.html diff --git a/public/js/config.js b/public/js/config.js index 2d074393..65028fd9 100755 --- a/public/js/config.js +++ b/public/js/config.js @@ -26,7 +26,7 @@ angular.module('insight').config(['$routeProvider', templateUrl: 'views/status.html' }). otherwise({ - redirectTo: '/' + templateUrl: 'views/404.html' }); } ]); diff --git a/public/views/404.html b/public/views/404.html new file mode 100644 index 00000000..f056178e --- /dev/null +++ b/public/views/404.html @@ -0,0 +1,5 @@ +
+

Ooops!

+

Page not found :(

+

Go to home

+
From 73f9f6921d664524739392356db975f4f61eaec2 Mon Sep 17 00:00:00 2001 From: Mario Colque Date: Fri, 17 Jan 2014 15:32:41 -0300 Subject: [PATCH 04/17] using values in BTC instead satoshis --- public/views/address.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/public/views/address.html b/public/views/address.html index 539dca16..1e21aec7 100644 --- a/public/views/address.html +++ b/public/views/address.html @@ -15,15 +15,15 @@ Total Received - {{address.totalReceivedSat / 100000000}} BTC + {{address.totalReceived}} BTC Total Sent - {{address.totalSentSat / 100000000}} BTC + {{address.totalSent}} BTC Final Balance - {{address.balanceSat / 100000000}} BTC + {{address.balance}} BTC No. Transactions @@ -38,7 +38,7 @@ -
+

Transactions Transactions contained within this block

From 4a664f71237acf283802dd455ed0dc5f9b474409 Mon Sep 17 00:00:00 2001 From: Bechi Date: Fri, 17 Jan 2014 16:18:27 -0300 Subject: [PATCH 05/17] transaction page --- public/css/common.css | 53 ++++++++-- public/views/transaction.html | 183 +++++++++++++++------------------- 2 files changed, 130 insertions(+), 106 deletions(-) diff --git a/public/css/common.css b/public/css/common.css index 3fc2670a..b069739f 100644 --- a/public/css/common.css +++ b/public/css/common.css @@ -34,6 +34,16 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 { padding: 0 0 60px; } +.m10h {margin: 0 10px;} +.m20h {margin: 0 20px;} +.m20v {margin: 20px 0;} +.m50v {margin: 50px 0;} +.m10b {margin-bottom: 10px;} + +.vm { + vertical-align: middle; +} + .navbar-default { background-color: #8DC429; border-bottom: 4px solid #64920F; @@ -145,22 +155,45 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 { border-radius: 2px; background: #F4F4F4; margin: 20px 0; - padding: 15px; + padding: 20px; } -.btn-primary { +.btn { + margin: 0 5px; border-radius: 2px; - background: #64920F; - border: 2px solid #557F08; +} +.btn-primary { + background: #8DC429; + border: 2px solid #76AF0F; } -.btn-primary:hover, .btn-primary:focus, .btn-primary:active, .btn-primary.active, .open .dropdown-toggle.btn-primary { +.btn-primary:hover, .btn-primary:focus, .btn-primary:active, +.btn-primary.active, .open .dropdown-toggle.btn-primary, +.btn-success:hover, .btn-success:focus, .btn-success:active, +.btn-success.active, .open .dropdown-toggle.btn-success, +.btn-danger:hover, .btn-danger:focus, .btn-danger:active, +.btn-danger.active, .open .dropdown-toggle.btn-danger { background: #fff; border: 2px solid #ccc; color: #373D42; font-weight: 500; } +.btn-default { + background: #E7E7E7; + border: 2px solid #DCDCDC; +} + +.btn-success { + background: #2FA4D7; + border: 2px solid #237FA7; +} + +.btn-danger { + background: #AC0015; + border: 2px solid #6C0000; +} + /* Set the fixed height of the footer here */ #footer { height: 60px; @@ -217,7 +250,15 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 { opacity: 1; } - +.tx-bg { + background-color: #F4F4F4; + width: 100%; + min-height: 340px; + position: absolute; + top: 0; + left: 0; + z-index: -9999; +} .badge { padding: 1px 9px 2px; diff --git a/public/views/transaction.html b/public/views/transaction.html index 7ba47449..8bca73b5 100644 --- a/public/views/transaction.html +++ b/public/views/transaction.html @@ -1,35 +1,26 @@
-

- Transaction + Transaction View information about a bitcoin transaction

-
-
- {{tx.txid}} -
+
+
+ {{tx.txid}} +
- - - - - - - - - - - - - - - - - - - - -
Input Output
+
+ +
    @@ -39,89 +30,81 @@
-
  - - - - - - - -
{{vout.scriptPubKey.type}} - -
-
- - - -
+
-
-
-
-
-

Summary

-
-
- - - - - - - - - - - - - - - -
Size{{tx.size}} (bytes)
Received Time{{tx.time * 1000|date:'medium'}}
BlockBlock
+
+ +
+ +
+
+
+ {{vout.scriptPubKey.type}} +
+
+ +
-
-
-
-

Inputs and Outputs

-
-
- - - - - - - - - - - - - - - -
Total Input{{tx.valueIn}} BTC
Total Output{{tx.valueOut}} BTC
Fees{{tx.feeds}} BTC
-
+ +
+ Feeds: {{tx.feeds}} +
+ + +
+
+
+ +
+
+

Summary

+ + + + + + + + + + + + + + + +
Size {{tx.size}} (bytes)
Received Time {{tx.time * 1000|date:'medium'}}
Block Block
+
+
+

Inputs and Outputs

+ + + + + + + + + + + + + + + +
Total Input{{tx.valueIn}} BTC
Total Output{{tx.valueOut}} BTC
Fees{{tx.feeds}} BTC
From 80d3c2475567be9192e9a9c8f19796f560227957 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Fri, 17 Jan 2014 16:36:34 -0300 Subject: [PATCH 06/17] smart sync. Skips genesisTX --- app/models/Block.js | 1 + app/models/Transaction.js | 7 +++ lib/HistoricSync.js | 106 ++++++++++++++++++++++---------------- lib/Sync.js | 12 +---- server.js | 4 +- util/sync.js | 43 +++++++++------- 6 files changed, 97 insertions(+), 76 deletions(-) diff --git a/app/models/Block.js b/app/models/Block.js index edb80f98..709fbcf8 100644 --- a/app/models/Block.js +++ b/app/models/Block.js @@ -55,6 +55,7 @@ BlockSchema.statics.customCreate = function(block, cb) { newBlock.hash = block.hash; newBlock.nextBlockHash = block.nextBlockHash; + Transaction.createFromArray(block.tx, newBlock.time, function(err, inserted_txs) { if (err) return cb(err); diff --git a/app/models/Transaction.js b/app/models/Transaction.js index fbb3f35a..84c8614c 100644 --- a/app/models/Transaction.js +++ b/app/models/Transaction.js @@ -19,6 +19,9 @@ var mongoose = require('mongoose'), var CONCURRENCY = 5; +// TODO: use bitcore networks module +var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b'; + /** */ var TransactionSchema = new Schema({ @@ -113,6 +116,10 @@ TransactionSchema.statics.createFromArray = function(txs, time, next) { TransactionSchema.statics.explodeTransactionItems = function(txid, time, cb) { + // Is it from genesis block? (testnet==livenet) + // TODO: parse it from networks.genesisTX + if (txid === genesisTXID) return cb(); + this.queryInfo(txid, function(err, info) { if (err || !info) return cb(err); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 49d67630..fdadfccb 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -16,6 +16,10 @@ function spec() { this.block_count= 0; this.block_total= 0; this.network = config.network === 'testnet' ? networks.testnet: networks.livenet; + + var genesisHashReversed = new Buffer(32); + this.network.genesisBlock.hash.copy(genesisHashReversed); + this.genesis = genesisHashReversed.reverse().toString('hex'); this.sync = new Sync(opts); } @@ -44,12 +48,10 @@ function spec() { HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) { - var that = this; + var self = this; // recursion end. - if (!blockHash || (blockEnd && blockEnd === blockHash) ) { - return cb(); - } + if (!blockHash ) return cb(); var existed = 0; var blockInfo; @@ -61,17 +63,16 @@ function spec() { Block.findOne({hash:blockHash}, function(err,block){ if (err) { p(err); return c(err); } if (block) { - existed = 1; - blockObj = block; + existed =1; + blockObj =block; } - return c(); }); }, //show some (inacurate) status function(c) { - if (that.block_count % 1000 === 1) { - progress_bar('sync status:', that.block_count, that.block_total); + if (self.block_count % 1000 === 1) { + progress_bar('sync status:', self.block_count, self.block_total); } return c(); }, @@ -81,7 +82,7 @@ function spec() { // TODO: if we store prev/next, no need to go to RPC // if (blockObj && blockObj.nextBlockHash) return c(); - that.rpc.getBlock(blockHash, function(err, ret) { + self.rpc.getBlock(blockHash, function(err, ret) { if (err) return c(err); blockInfo = ret; @@ -91,8 +92,10 @@ function spec() { //store it function(c) { if (existed) return c(); - that.sync.storeBlock(blockInfo.result, function(err) { + self.sync.storeBlock(blockInfo.result, function(err) { + existed = err && err.toString().match(/E11000/); + if (err && ! existed) return c(err); return c(); }); @@ -110,39 +113,36 @@ function spec() { ], function (err){ + if (err) + p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, self.block_count); + if (opts.uptoexisting && existed) { p('DONE. Found existing block: ', blockHash); return cb(err); } - if (err) - p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, that.block_count); + if (blockEnd && blockEnd === blockHash) { + p('DONE. Found END block: ', blockHash); + return cb(err); + } + + // Continue if (blockInfo && blockInfo.result) { - block_total++; + self.block_count++; if (opts.prev && blockInfo.result.previousblockhash) { - return that.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb); + return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb); } if (opts.next && blockInfo.result.nextblockhash) - return that.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb); + return self.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb); } return cb(err); }); }; - HistoricSync.prototype.syncBlocks = function(start, end, opts, cb) { - var that = this; - - p('Starting from: ', start); - p(' to : ', end); - p(' opts: ', JSON.stringify(opts)); - - return that.getPrevNextBlock( start, end, opts , cb); - }; - HistoricSync.prototype.import_history = function(opts, next) { - var that = this; + var self = this; var retry_attemps = 100; var retry_secs = 2; @@ -154,7 +154,7 @@ function spec() { function(cb) { if (opts.destroy) { p('Deleting Blocks...'); - that.db.collections.blocks.drop(cb); + self.db.collections.blocks.drop(cb); } else { return cb(); } @@ -162,7 +162,7 @@ function spec() { function(cb) { if (opts.destroy) { p('Deleting TXs...'); - that.db.collections.transactions.drop(cb); + self.db.collections.transactions.drop(cb); } else { return cb(); } @@ -170,16 +170,16 @@ function spec() { function(cb) { if (opts.destroy) { p('Deleting TXItems...'); - that.db.collections.transactionitems.drop(cb); + self.db.collections.transactionitems.drop(cb); } else { return cb(); } }, function(cb) { - that.rpc.getInfo(function(err, res) { + self.rpc.getInfo(function(err, res) { if (err) return cb(err); - that.block_total = res.result.blocks; + self.block_total = res.result.blocks; return cb(); }); }, @@ -187,7 +187,7 @@ function spec() { function(cb) { if (!opts.reverse) return cb(); - that.rpc.getBlockCount(function(err, res) { + self.rpc.getBlockCount(function(err, res) { if (err) return cb(err); block_height = res.result; return cb(); @@ -196,7 +196,7 @@ function spec() { function(cb) { if (!opts.reverse) return cb(); - that.rpc.getBlockHash(block_height, function(err, res) { + self.rpc.getBlockHash(block_height, function(err, res) { if (err) return cb(err); block_best = res.result; @@ -210,17 +210,20 @@ function spec() { function sync() { if (opts.reverse) { start = block_best; - end = that.network.genesisBlock.hash.reverse().toString('hex'); + end = self.genesis; opts.prev = true; } else { - start = that.network.genesisBlock.hash.reverse().toString('hex'); + start = self.genesis; end = null; opts.next = true; } - that.syncBlocks(start, end, opts, function(err) { + p('Starting from: ', start); + p(' to : ', end); + p(' opts: ', JSON.stringify(opts)); + self.getPrevNextBlock( start, end, opts , function(err) { if (err && err.message.match(/ECONNREFUSED/) && retry_attemps--){ setTimeout(function() { p('Retrying in %d secs', retry_secs); @@ -228,7 +231,7 @@ function spec() { }, retry_secs * 1000); } else - return next(err, that.block_count); + return next(err, self.block_count); }); } if (!err) @@ -238,13 +241,28 @@ function spec() { }); }; - // Reverse Imports (upto if we have genesis block?) + // upto if we have genesis block? HistoricSync.prototype.smart_import = function(next) { - var that = this; - var opts = { - prev: 1, - }; - that.import_history(opts, next); + var self = this; + + Block.findOne({hash:self.genesis}, function(err, b){ + if (err) return next(err); + + + if (!b) { + p('Could not find Genesis block. Running FULL SYNC'); + } + else { + p('Genesis block found. Syncing upto know blocks.'); + } + + var opts = { + reverse: 1, + uptoexisting: b ? true: false, + }; + + return self.import_history(opts, next); + }); }; diff --git a/lib/Sync.js b/lib/Sync.js index 941ac37a..1cebb389 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -19,6 +19,7 @@ function spec() { var that = this; Block.customCreate(block, function(err, block, inserted_txs){ + if (err) return cb(err); if (block && that.opts.broadcast_blocks) { sockets.broadcast_block(block); @@ -56,17 +57,6 @@ function spec() { }; - Sync.prototype.syncBlocks = function(start, end, isForward, cb) { - var that = this; - - console.log('Syncing Blocks, starting \n\tfrom: %s \n\tend: %s \n\tisForward:', - start, end, isForward); - - - return that.getPrevNextBlock( start, end, - isForward ? { next: 1 } : { prev: 1}, cb); - }; - Sync.prototype.init = function(opts, cb) { var that = this; diff --git a/server.js b/server.js index 1d61a87a..004e6ea3 100644 --- a/server.js +++ b/server.js @@ -50,9 +50,7 @@ if (!config.disableHistoricSync) { skip_db_connection: true, networkName: config.network }, function() { - hs.import_history({ - reverse: 1, - }, function(){ + hs.smart_import(function(){ console.log('historic_sync finished!'); }); }); diff --git a/util/sync.js b/util/sync.js index 6e531200..beccf0f9 100755 --- a/util/sync.js +++ b/util/sync.js @@ -24,33 +24,40 @@ var historicSync = new HistoricSync({ networkName: program.network }); +/* TODO: Sure? if (program.remove) { - // TODO: Sure? + } +*/ async.series([ function(cb) { historicSync.init(program, cb); }, function(cb) { - historicSync.import_history({ - network: program.network, - destroy: program.destroy, - reverse: program.reverse, - uptoexisting: program.uptoexisting, - smart: program.smart, - }, function(err, count) { - if (err) { - console.log('CRITICAL ERROR: ', err); - } - else { - console.log('Finished. [%d blocks]', count); - } - cb(); - }); + if (program.smart) { + historicSync.smart_import(cb); + } + else { + historicSync.import_history({ + destroy: program.destroy, + reverse: program.reverse, + uptoexisting: program.uptoexisting, + }, cb); + } }, function(cb) { historicSync.close(); - cb(); -}]); + return cb(); + }, + ], + function(err, count) { + if (err) { + console.log('CRITICAL ERROR: ', err); + } + else { + console.log('Finished. [%d blocks synced]', count[1]); + } + return; +}); From b02e85c45da0fc2586a71a3a2be7de77b74f16c8 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Fri, 17 Jan 2014 16:42:07 -0300 Subject: [PATCH 07/17] fix message --- server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.js b/server.js index 004e6ea3..84e659a4 100644 --- a/server.js +++ b/server.js @@ -51,7 +51,7 @@ if (!config.disableHistoricSync) { networkName: config.network }, function() { hs.smart_import(function(){ - console.log('historic_sync finished!'); + console.log('[historic_sync] finished!'); }); }); } From 8906a2f24f4ad46908ce802dc620be3cfd5795ce Mon Sep 17 00:00:00 2001 From: Gustavo Cortez Date: Fri, 17 Jan 2014 17:57:28 -0300 Subject: [PATCH 08/17] footer with info from bitcoind. it updates when get new block --- app/views/includes/foot.jade | 5 ++--- public/js/controllers/footer.js | 20 ++++++++++++++++++++ public/js/services/global.js | 11 +---------- public/views/footer.html | 10 ++++++++++ 4 files changed, 33 insertions(+), 13 deletions(-) create mode 100644 public/js/controllers/footer.js create mode 100644 public/views/footer.html diff --git a/app/views/includes/foot.jade b/app/views/includes/foot.jade index eb69b669..735e8c0b 100755 --- a/app/views/includes/foot.jade +++ b/app/views/includes/foot.jade @@ -1,6 +1,4 @@ -#footer - .container - p.text-muted Place sticky footer content here. +#footer(data-ng-include="'views/footer.html'", role='navigation') //script(type='text/javascript', src='/lib/jquery/jquery.min.js') //script(type='text/javascript', src='/lib/bootstrap/dist/js/bootstrap.min.js') @@ -38,6 +36,7 @@ script(type='text/javascript', src='/js/services/socket.js') //Application Controllers script(type='text/javascript', src='/js/controllers/index.js') script(type='text/javascript', src='/js/controllers/header.js') +script(type='text/javascript', src='/js/controllers/footer.js') script(type='text/javascript', src='/js/controllers/blocks.js') script(type='text/javascript', src='/js/controllers/transactions.js') script(type='text/javascript', src='/js/controllers/address.js') diff --git a/public/js/controllers/footer.js b/public/js/controllers/footer.js new file mode 100644 index 00000000..56ee28fa --- /dev/null +++ b/public/js/controllers/footer.js @@ -0,0 +1,20 @@ +'use strict'; + +angular.module('insight.system').controller('FooterController', ['$scope', 'Global', 'socket', 'Status', function ($scope, Global, socket, Status) { + $scope.global = Global; + + socket.on('block', function(block) { +console.log('[footer.js:14]',block); //TODO + console.log('Block received! ' + JSON.stringify(block)); + }); + + $scope.getFooter = function() { + Status.get({ + q: 'getInfo' + }, function(d) { + $scope.info = d.info; + }); + }; + +}]); + diff --git a/public/js/services/global.js b/public/js/services/global.js index dfda2fca..73637022 100755 --- a/public/js/services/global.js +++ b/public/js/services/global.js @@ -1,14 +1,5 @@ 'use strict'; //Global service for global variables -angular.module('insight.system').factory('Global', [ - function() { - var _this = this; - _this._data = { - user: window.user, - authenticated: !! window.user - }; +angular.module('insight.system').factory('Global', [function() {}]); - return _this._data; - } -]); diff --git a/public/views/footer.html b/public/views/footer.html new file mode 100644 index 00000000..4e08b207 --- /dev/null +++ b/public/views/footer.html @@ -0,0 +1,10 @@ +
+
+

+ Blocks: {{info.blocks}} | + Connections: {{info.connections}} | + Difficulty: {{info.difficulty}} +

+
+
+ From d7ab549a5ed96817b1a14a4dde45707e54808965 Mon Sep 17 00:00:00 2001 From: Mario Colque Date: Fri, 17 Jan 2014 18:25:48 -0300 Subject: [PATCH 09/17] added error messages for bad addresses, txids, blockids --- public/js/controllers/address.js | 5 ++++- public/js/controllers/blocks.js | 5 ++++- public/js/controllers/index.js | 6 ++++-- public/js/controllers/transactions.js | 8 ++++---- public/views/404.html | 2 +- public/views/index.html | 4 +++- 6 files changed, 20 insertions(+), 10 deletions(-) diff --git a/public/js/controllers/address.js b/public/js/controllers/address.js index 926f3f88..46a441d9 100644 --- a/public/js/controllers/address.js +++ b/public/js/controllers/address.js @@ -1,6 +1,6 @@ 'use strict'; -angular.module('insight.address').controller('AddressController', ['$scope', '$routeParams', '$location', 'Global', 'Address', function ($scope, $routeParams, $location, Global, Address) { +angular.module('insight.address').controller('AddressController', ['$scope', '$rootScope', '$routeParams', '$location', 'Global', 'Address', function ($scope, $rootScope, $routeParams, $location, Global, Address) { $scope.global = Global; $scope.findOne = function() { @@ -8,6 +8,9 @@ angular.module('insight.address').controller('AddressController', ['$scope', '$r addrStr: $routeParams.addrStr }, function(address) { $scope.address = address; + }, function() { + $rootScope.flashMessage = 'Address Not Found'; + $location.path('/'); }); }; diff --git a/public/js/controllers/blocks.js b/public/js/controllers/blocks.js index fe8f0598..b398ef3d 100644 --- a/public/js/controllers/blocks.js +++ b/public/js/controllers/blocks.js @@ -1,6 +1,6 @@ 'use strict'; -angular.module('insight.blocks').controller('BlocksController', ['$scope', '$routeParams', '$location', 'Global', 'Block', 'Blocks', function ($scope, $routeParams, $location, Global, Block, Blocks) { +angular.module('insight.blocks').controller('BlocksController', ['$scope', '$rootScope', '$routeParams', '$location', 'Global', 'Block', 'Blocks', function ($scope, $rootScope, $routeParams, $location, Global, Block, Blocks) { $scope.global = Global; $scope.list = function() { @@ -17,6 +17,9 @@ angular.module('insight.blocks').controller('BlocksController', ['$scope', '$rou blockHash: $routeParams.blockHash }, function(block) { $scope.block = block; + }, function() { + $rootScope.flashMessage = 'Block Not Found'; + $location.path('/'); }); }; diff --git a/public/js/controllers/index.js b/public/js/controllers/index.js index 12eec9d3..5c03de61 100755 --- a/public/js/controllers/index.js +++ b/public/js/controllers/index.js @@ -2,9 +2,12 @@ var TRANSACTION_DISPLAYED = 5; var BLOCKS_DISPLAYED = 5; -angular.module('insight.system').controller('IndexController', ['$scope', 'Global', 'socket', 'Blocks', 'Transactions', function($scope, Global, socket, Blocks, Transactions) { +angular.module('insight.system').controller('IndexController', ['$scope', '$rootScope', 'Global', 'socket', 'Blocks', 'Transactions', function($scope, $rootScope, Global, socket, Blocks, Transactions) { $scope.global = Global; + //show errors + $scope.flashMessage = $rootScope.flashMessage || null; + socket.on('tx', function(tx) { console.log('Transaction received! ' + JSON.stringify(tx)); if ($scope.txs.length === TRANSACTION_DISPLAYED) { @@ -43,4 +46,3 @@ angular.module('insight.system').controller('IndexController', ['$scope', 'Globa $scope.txs = []; $scope.blocks = []; }]); - diff --git a/public/js/controllers/transactions.js b/public/js/controllers/transactions.js index 665929a6..e1272e83 100644 --- a/public/js/controllers/transactions.js +++ b/public/js/controllers/transactions.js @@ -1,6 +1,6 @@ 'use strict'; -angular.module('insight.transactions').controller('transactionsController', ['$scope', '$routeParams', '$location', 'Global', 'Transaction', 'TransactionsByBlock', 'TransactionsByAddress', function ($scope, $routeParams, $location, Global, Transaction, TransactionsByBlock, TransactionsByAddress) { +angular.module('insight.transactions').controller('transactionsController', ['$scope', '$rootScope', '$routeParams', '$location', 'Global', 'Transaction', 'TransactionsByBlock', 'TransactionsByAddress', '$rootScope', function ($scope, $rootScope, $routeParams, $location, Global, Transaction, TransactionsByBlock, TransactionsByAddress) { $scope.global = Global; $scope.findOne = function() { @@ -8,6 +8,9 @@ angular.module('insight.transactions').controller('transactionsController', ['$s txId: $routeParams.txId }, function(tx) { $scope.tx = tx; + }, function() { + $rootScope.flashMessage = 'Transaction Not Found'; + $location.path('/'); }); }; @@ -26,7 +29,4 @@ angular.module('insight.transactions').controller('transactionsController', ['$s $scope.txs = txs; }); }; - - }]); - diff --git a/public/views/404.html b/public/views/404.html index f056178e..bb3b23f0 100644 --- a/public/views/404.html +++ b/public/views/404.html @@ -1,5 +1,5 @@

Ooops!

-

Page not found :(

+

404 Page not found :(

Go to home

diff --git a/public/views/index.html b/public/views/index.html index b9268299..0e4de2bc 100644 --- a/public/views/index.html +++ b/public/views/index.html @@ -1,3 +1,6 @@ +
+ {{flashMessage}} +
@@ -20,4 +23,3 @@
- From 4d23191116830b6a50af230e5a4320e9128d9944 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Sat, 18 Jan 2014 18:28:24 -0300 Subject: [PATCH 10/17] better smart syncs --- app/models/Block.js | 1 + app/models/Transaction.js | 4 +- lib/HistoricSync.js | 110 +++++++++++++++++++++++++++----------- lib/Sync.js | 76 ++++++++++++++------------ server.js | 7 ++- util/sync.js | 11 ++-- 6 files changed, 132 insertions(+), 77 deletions(-) diff --git a/app/models/Block.js b/app/models/Block.js index 709fbcf8..e4d84d38 100644 --- a/app/models/Block.js +++ b/app/models/Block.js @@ -27,6 +27,7 @@ var BlockSchema = new Schema({ }, time: Number, nextBlockHash: String, + isOrphan: Boolean, }); /** diff --git a/app/models/Transaction.js b/app/models/Transaction.js index 84c8614c..fdad11e9 100644 --- a/app/models/Transaction.js +++ b/app/models/Transaction.js @@ -142,7 +142,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, time, cb) { } else { if ( !i.coinbase ) { - console.log ('TX: %s,%d could not parse INPUT', txid, i.n); + console.log ('WARN in TX: %s: could not parse INPUT %d', txid, i.n); } return next_in(); } @@ -165,7 +165,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, time, cb) { }, next_out); } else { - console.log ('TX: %s,%d could not parse OUTPUT', txid, o.n); + console.log ('WARN in TX: %s could not parse OUTPUT %d', txid, o.n); return next_out(); } }, diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index fdadfccb..fd64c678 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -13,14 +13,16 @@ function spec() { var Sync = require('./Sync').class(); function HistoricSync(opts) { - this.block_count= 0; - this.block_total= 0; this.network = config.network === 'testnet' ? networks.testnet: networks.livenet; var genesisHashReversed = new Buffer(32); this.network.genesisBlock.hash.copy(genesisHashReversed); this.genesis = genesisHashReversed.reverse().toString('hex'); this.sync = new Sync(opts); + + //available status: new / syncing / finished / aborted + this.status = 'new'; + this.syncInfo = {}; } function p() { @@ -32,8 +34,9 @@ function spec() { console.log.apply(this, args); } - var progress_bar = function(string, current, total) { - p(util.format('%s %d/%d [%d%%]', string, current, total, parseInt(100 * current / total))); + var printProgress = function(i) { + var per = parseInt(100 * i.syncedBlocks / i.blocksToSync); + p(util.format('status: %d/%d [%d%%]', i.syncedBlocks, i.blocksToSync, per)); }; HistoricSync.prototype.init = function(opts,cb) { @@ -47,7 +50,6 @@ function spec() { }; HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) { - var self = this; // recursion end. @@ -71,8 +73,11 @@ function spec() { }, //show some (inacurate) status function(c) { - if (self.block_count % 1000 === 1) { - progress_bar('sync status:', self.block_count, self.block_total); + var step = parseInt(self.syncInfo.blocksToSync / 100); + if (step < 10) step = 10; + + if (self.syncInfo.syncedBlocks % step === 1) { + printProgress(self.syncInfo); } return c(); }, @@ -113,15 +118,30 @@ function spec() { ], function (err){ - if (err) - p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, self.block_count); + if (err) { + self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncInfo.syncedBlocks); + self.status = 'aborted'; + p(self.err); + } - if (opts.uptoexisting && existed) { - p('DONE. Found existing block: ', blockHash); - return cb(err); + else { + self.err = null; + self.status = 'syncing'; + } + + if (opts.uptoexisting && existed ) { + if (self.syncInfo.blocksToSync <= self.syncInfo.syncedBlocks) { + self.status = 'finished'; + p('DONE. Found existing block: ', blockHash); + return cb(err); + } + else { + p('WARN found target block\n\tbut blockChain Height is still higher that ours. Previous light sync must be interrupted.\n\tWill keep syncing.', self.syncInfo.syncedBlocks); + } } if (blockEnd && blockEnd === blockHash) { + self.status = 'finished'; p('DONE. Found END block: ', blockHash); return cb(err); } @@ -129,7 +149,7 @@ function spec() { // Continue if (blockInfo && blockInfo.result) { - self.block_count++; + self.syncInfo.syncedBlocks++; if (opts.prev && blockInfo.result.previousblockhash) { return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb); } @@ -144,11 +164,10 @@ function spec() { HistoricSync.prototype.import_history = function(opts, next) { var self = this; - var retry_attemps = 100; var retry_secs = 2; - var block_best; - var block_height; + var bestBlock; + var blockChainHeight; async.series([ function(cb) { @@ -175,41 +194,56 @@ function spec() { return cb(); } }, - function(cb) { - self.rpc.getInfo(function(err, res) { - if (err) return cb(err); - self.block_total = res.result.blocks; - return cb(); - }); - }, // We are not using getBestBlockHash, because is not available in all clients function(cb) { if (!opts.reverse) return cb(); self.rpc.getBlockCount(function(err, res) { if (err) return cb(err); - block_height = res.result; + blockChainHeight = res.result; return cb(); }); }, function(cb) { if (!opts.reverse) return cb(); - self.rpc.getBlockHash(block_height, function(err, res) { + self.rpc.getBlockHash(blockChainHeight, function(err, res) { if (err) return cb(err); - block_best = res.result; + bestBlock = res.result; + return cb(); }); }, + function(cb) { + // This is only to inform progress. + if (!opts.uptoexisting) { + self.rpc.getInfo(function(err, res) { + if (err) return cb(err); + self.syncInfo.blocksToSync = res.result.blocks; + return cb(); + }); + } + else { + // should be isOrphan = true or null to be more accurate. + Block.count({ isOrphan: null}, function(err, count) { + if (err) return cb(err); + + self.syncInfo.blocksToSync = blockChainHeight - count; + if (self.syncInfo.blocksToSync < 1) self.syncInfo.blocksToSync = 1; + return cb(); + }); + } + }, ], function(err) { + var start, end; function sync() { if (opts.reverse) { - start = block_best; + start = bestBlock; end = self.genesis; opts.prev = true; } @@ -219,25 +253,38 @@ function spec() { opts.next = true; } + self.syncInfo = util._extend(self.syncInfo, { + start: start, + isStartGenesis: start === self.genesis, + end: end, + isEndGenesis: end === self.genesis, + scanningForward: opts.next, + scanningBackward: opts.prev, + uptoexisting: opts.uptoexisting, + syncedBlocks: 0, + }); + p('Starting from: ', start); p(' to : ', end); p(' opts: ', JSON.stringify(opts)); self.getPrevNextBlock( start, end, opts , function(err) { - if (err && err.message.match(/ECONNREFUSED/) && retry_attemps--){ + if (err && err.message.match(/ECONNREFUSED/)){ setTimeout(function() { p('Retrying in %d secs', retry_secs); sync(); }, retry_secs * 1000); } else - return next(err, self.block_count); + return next(err); }); } + if (!err) sync(); - else + else { return next(err, 0); + } }); }; @@ -246,6 +293,7 @@ function spec() { var self = this; Block.findOne({hash:self.genesis}, function(err, b){ + if (err) return next(err); @@ -253,7 +301,7 @@ function spec() { p('Could not find Genesis block. Running FULL SYNC'); } else { - p('Genesis block found. Syncing upto know blocks.'); + p('Genesis block found. Syncing upto known blocks.'); } var opts = { diff --git a/lib/Sync.js b/lib/Sync.js index ff2ffb8f..8561d4a4 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -15,6 +15,48 @@ function spec() { this.tx_count = 0; } + Sync.prototype.init = function(opts, cb) { + var that = this; + + that.opts = opts; + + if (!(opts && opts.skip_db_connection)) { + + + if (!mongoose.connection.readyState == 1) { + mongoose.connect(config.db, function(err) { + if (err) { + console.log('CRITICAL ERROR: connecting to mongoDB:',err); + return (err); + } + }); + } + + that.db = mongoose.connection; + + that.db.on('error', function(err) { + console.log('MongoDB ERROR:' + err); + return cb(err); + }); + + that.db.on('disconnect', function(err) { + console.log('MongoDB disconnect:' + err); + return cb(err); + }); + + return that.db.once('open', function(err) { + return cb(err); + }); + } + else return cb(); + }; + + Sync.prototype.close = function() { + if ( this.db && this.db.readyState ) { + this.db.close(); + } + }; + Sync.prototype.storeBlock = function(block, cb) { var that = this; @@ -55,40 +97,6 @@ function spec() { return cb(err); }); }; - - - Sync.prototype.init = function(opts, cb) { - var that = this; - - that.opts = opts; - - if (!(opts && opts.skip_db_connection)) { - if (!mongoose.connection) { - mongoose.connect(config.db, {server: {auto_reconnect: true}} ); - } - - this.db = mongoose.connection; - - this.db.on('error', function(err) { - console.log('connection error:' + err); - mongoose.disconnect(); - }); - - this.db.on('disconnect', function(err) { - console.log('disconnect:' + err); - mongoose.connect(config.db, {server: {auto_reconnect: true}} ); - }); - - return that.db.once('open', cb); - } - else return cb(); - }; - - Sync.prototype.close = function() { - if (!(this.opts && this.opts.skip_db_connection)) { - this.db.close(); - } - }; return Sync; } module.defineClass(spec); diff --git a/server.js b/server.js index 84e659a4..38f607fb 100644 --- a/server.js +++ b/server.js @@ -50,8 +50,11 @@ if (!config.disableHistoricSync) { skip_db_connection: true, networkName: config.network }, function() { - hs.smart_import(function(){ - console.log('[historic_sync] finished!'); + hs.smart_import(function(err){ + var txt= 'ended.'; + if (err) txt = 'ABORTED with error: ' + err.message; + + console.log('[historic_sync] ' + txt, hs.syncInfo); }); }); } diff --git a/util/sync.js b/util/sync.js index beccf0f9..01205d9d 100755 --- a/util/sync.js +++ b/util/sync.js @@ -29,7 +29,6 @@ if (program.remove) { } */ - async.series([ function(cb) { historicSync.init(program, cb); @@ -46,18 +45,14 @@ async.series([ }, cb); } }, - function(cb) { - historicSync.close(); - return cb(); - }, ], - function(err, count) { + function(err) { + historicSync.close(); if (err) { console.log('CRITICAL ERROR: ', err); } else { - console.log('Finished. [%d blocks synced]', count[1]); + console.log('Finished.\n Status:\n', historicSync.syncInfo); } - return; }); From 027935844a4715962b8c34ce55fb7164295d7402 Mon Sep 17 00:00:00 2001 From: Gustavo Cortez Date: Sun, 19 Jan 2014 05:30:22 -0300 Subject: [PATCH 11/17] refactory: status controller --- app/controllers/status.js | 71 ++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/app/controllers/status.js b/app/controllers/status.js index eb2f1ae3..6fa8e9b9 100644 --- a/app/controllers/status.js +++ b/app/controllers/status.js @@ -15,43 +15,44 @@ exports.show = function(req, res, next) { res.status(400).send('Bad Request'); } else { - var s = req.query.q; - var d = Status.new(); - - if (s === 'getInfo') { - d.getInfo(function(err) { - if (err) next(err); - res.jsonp(d); - }); - } - else if (s === 'getDifficulty') { - d.getDifficulty(function(err) { - if (err) next(err); - res.jsonp(d); - }); - } - else if (s === 'getTxOutSetInfo') { - d.getTxOutSetInfo(function(err) { - if (err) next(err); - res.jsonp(d); - }); - } - else if (s === 'getBestBlockHash') { - d.getBestBlockHash(function(err) { - if (err) next(err); - res.jsonp(d); - }); - } - else if (s === 'getLastBlockHash') { - d.getLastBlockHash(function(err) { - if (err) next(err); - res.jsonp(d); - }); - } + var option = req.query.q; + var statusObject = Status.new(); - else { - res.status(400).send('Bad Request'); + switch(option) { + case 'getInfo': + statusObject.getInfo(function(err) { + if (err) next(err); + res.jsonp(statusObject); + }); + break; + case 'getDifficulty': + statusObject.getDifficulty(function(err) { + if (err) next(err); + res.jsonp(statusObject); + }); + break; + case 'getTxOutSetInfo': + statusObject.getTxOutSetInfo(function(err) { + if (err) next(err); + res.jsonp(statusObject); + }); + break; + case 'getBestBlockHash': + statusObject.getBestBlockHash(function(err) { + if (err) next(err); + res.jsonp(statusObject); + }); + break; + case 'getLastBlockHash': + statusObject.getLastBlockHash(function(err) { + if (err) next(err); + res.jsonp(statusObject); + }); + break; + default: + res.status(400).send('Bad Request'); } } }; + From 08a54a40e28085d2cd5f332c55663468928befc8 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Sun, 19 Jan 2014 10:09:59 -0300 Subject: [PATCH 12/17] sync API --- .gitignore | 1 + app/controllers/status.js | 6 +++++- config/express.js | 14 ++++++++++++-- config/routes.js | 4 +++- lib/PeerSync.js | 4 ++-- lib/Sync.js | 2 +- server.js | 13 +++++++------ 7 files changed, 31 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 888b5ce4..aabec275 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ node_modules # extras *.swp +*.swo *~ .project peerdb.json diff --git a/app/controllers/status.js b/app/controllers/status.js index eb2f1ae3..d1cd93d7 100644 --- a/app/controllers/status.js +++ b/app/controllers/status.js @@ -48,10 +48,14 @@ exports.show = function(req, res, next) { res.jsonp(d); }); } - else { res.status(400).send('Bad Request'); } } }; +exports.sync = function(req, res, next) { + if (req.syncInfo) + res.jsonp(req.syncInfo); + next(); +}; diff --git a/config/express.js b/config/express.js index a25db3b9..0e5c58d4 100644 --- a/config/express.js +++ b/config/express.js @@ -7,7 +7,8 @@ var express = require('express'), helpers = require('view-helpers'), config = require('./config'); -module.exports = function(app, passport, db) { +module.exports = function(app, historicSync) { + app.set('showStackError', true); //Prettify HTML @@ -26,9 +27,17 @@ module.exports = function(app, passport, db) { app.set('view engine', 'jade'); //Enable jsonp - app.enable("jsonp callback"); + app.enable('jsonp callback'); + //custom middleware + function setHistoric(req, res, next) { + req.syncInfo = historicSync.syncInfo; + next(); + } + app.use('/api/sync', setHistoric); + app.configure(function() { + //cookieParser should be above session app.use(express.cookieParser()); @@ -43,6 +52,7 @@ module.exports = function(app, passport, db) { //routes should be at the last app.use(app.router); + //Setting the fav icon and static folder app.use(express.favicon()); app.use(express.static(config.root + '/public')); diff --git a/config/routes.js b/config/routes.js index 2f26dc55..fc27b789 100644 --- a/config/routes.js +++ b/config/routes.js @@ -1,6 +1,6 @@ 'use strict'; -module.exports = function(app) { +module.exports = function(app, historicSync) { //Home route var index = require('../app/controllers/index'); @@ -29,4 +29,6 @@ module.exports = function(app) { var st = require('../app/controllers/status'); app.get('/api/status', st.show); + app.get('/api/sync', st.sync); + }; diff --git a/lib/PeerSync.js b/lib/PeerSync.js index 49526cf8..3d7e89c7 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -15,8 +15,6 @@ function spec() { PeerSync.prototype.init = function(config, cb) { if (!config) config = {}; - var that = this; - var network = config && (config.network || 'testnet'); this.verbose = config.verbose; @@ -71,6 +69,8 @@ function spec() { } this.sync.storeTxs([tx.hash], null, function(err) { if (err) { +console.log('[PeerSync.js.71:err:]',err); //TODO + console.log('[p2p_sync] Error in handle TX: ' + JSON.stringify(err)); } }); diff --git a/lib/Sync.js b/lib/Sync.js index 8561d4a4..4a4ecdc7 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -23,7 +23,7 @@ function spec() { if (!(opts && opts.skip_db_connection)) { - if (!mongoose.connection.readyState == 1) { + if (mongoose.connection.readyState !== 1) { mongoose.connect(config.db, function(err) { if (err) { console.log('CRITICAL ERROR: connecting to mongoDB:',err); diff --git a/server.js b/server.js index 38f607fb..59303fd4 100644 --- a/server.js +++ b/server.js @@ -24,7 +24,7 @@ var express = require('express'), var config = require('./config/config'); //Bootstrap db connection -var db = mongoose.connect(config.db); +mongoose.connect(config.db); //Bootstrap models var models_path = __dirname + '/app/models'; @@ -44,17 +44,18 @@ var walk = function(path) { walk(models_path); // historic_sync process +var historicSync = {}; if (!config.disableHistoricSync) { - var hs = new HistoricSync(); - hs.init({ + historicSync = new HistoricSync(); + historicSync.init({ skip_db_connection: true, networkName: config.network }, function() { - hs.smart_import(function(err){ + historicSync.smart_import(function(err){ var txt= 'ended.'; if (err) txt = 'ABORTED with error: ' + err.message; - console.log('[historic_sync] ' + txt, hs.syncInfo); + console.log('[historic_sync] ' + txt, historicSync.syncInfo); }); }); } @@ -77,7 +78,7 @@ if (!config.disableP2pSync) { var app = express(); //express settings -require('./config/express')(app, db); +require('./config/express')(app, historicSync); //Bootstrap routes require('./config/routes')(app); From e73064e7ca33b52c920a05ba1da61c1454a36a74 Mon Sep 17 00:00:00 2001 From: Gustavo Cortez Date: Sun, 19 Jan 2014 11:29:59 -0300 Subject: [PATCH 13/17] refactory: no repeat same return function --- app/controllers/status.js | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/app/controllers/status.js b/app/controllers/status.js index 6fa8e9b9..22970a94 100644 --- a/app/controllers/status.js +++ b/app/controllers/status.js @@ -10,7 +10,7 @@ var Status = require('../models/Status'); * Status */ exports.show = function(req, res, next) { - + if (! req.query.q) { res.status(400).send('Bad Request'); } @@ -18,36 +18,26 @@ exports.show = function(req, res, next) { var option = req.query.q; var statusObject = Status.new(); + var returnJsonp = function (err) { + if(err) return next(err); + res.jsonp(statusObject); + }; + switch(option) { case 'getInfo': - statusObject.getInfo(function(err) { - if (err) next(err); - res.jsonp(statusObject); - }); + statusObject.getInfo(returnJsonp); break; case 'getDifficulty': - statusObject.getDifficulty(function(err) { - if (err) next(err); - res.jsonp(statusObject); - }); + statusObject.getDifficulty(returnJsonp); break; case 'getTxOutSetInfo': - statusObject.getTxOutSetInfo(function(err) { - if (err) next(err); - res.jsonp(statusObject); - }); + statusObject.getTxOutSetInfo(returnJsonp); break; case 'getBestBlockHash': - statusObject.getBestBlockHash(function(err) { - if (err) next(err); - res.jsonp(statusObject); - }); + statusObject.getBestBlockHash(returnJsonp); break; case 'getLastBlockHash': - statusObject.getLastBlockHash(function(err) { - if (err) next(err); - res.jsonp(statusObject); - }); + statusObject.getLastBlockHash(returnJsonp); break; default: res.status(400).send('Bad Request'); From 10972fd8ddd2d629773aafb8901ac8d2130fe851 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Sun, 19 Jan 2014 12:33:39 -0300 Subject: [PATCH 14/17] sync API not emits "sync" thru socket.io --- app/controllers/socket.js | 4 +++ lib/HistoricSync.js | 56 +++++++++++++++++---------------------- lib/Sync.js | 38 ++++++++++++++++---------- server.js | 5 ++-- util/sync.js | 2 +- 5 files changed, 57 insertions(+), 48 deletions(-) diff --git a/app/controllers/socket.js b/app/controllers/socket.js index 7bcad799..fa78f559 100644 --- a/app/controllers/socket.js +++ b/app/controllers/socket.js @@ -21,3 +21,7 @@ module.exports.broadcast_tx = function(tx) { module.exports.broadcast_block = function(block) { ios.sockets.emit('block', block); }; + +module.exports.broadcastSyncInfo = function(syncInfo) { + ios.sockets.emit('block', syncInfo); +}; diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index fd64c678..2735f44f 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -11,6 +11,7 @@ function spec() { var config = require('../config/config'); var Block = require('../app/models/Block'); var Sync = require('./Sync').class(); + var sockets = require('../app/controllers/socket.js'); function HistoricSync(opts) { this.network = config.network === 'testnet' ? networks.testnet: networks.livenet; @@ -20,6 +21,7 @@ function spec() { this.genesis = genesisHashReversed.reverse().toString('hex'); this.sync = new Sync(opts); + //available status: new / syncing / finished / aborted this.status = 'new'; this.syncInfo = {}; @@ -28,17 +30,14 @@ function spec() { function p() { var args = []; Array.prototype.push.apply( args, arguments ); + + args.unshift('[historic_sync]'); /*jshint validthis:true */ console.log.apply(this, args); } - var printProgress = function(i) { - var per = parseInt(100 * i.syncedBlocks / i.blocksToSync); - p(util.format('status: %d/%d [%d%%]', i.syncedBlocks, i.blocksToSync, per)); - }; - HistoricSync.prototype.init = function(opts,cb) { this.rpc = new RpcClient(config.bitcoind); this.opts = opts; @@ -49,6 +48,19 @@ function spec() { this.sync.close(); }; + + HistoricSync.prototype.showProgress = function() { + var self = this; + + var i = self.syncInfo; + var per = parseInt(100 * i.syncedBlocks / i.blocksToSync); + p(util.format('status: %d/%d [%d%%]', i.syncedBlocks, i.blocksToSync, per)); + if (self.opts.broadcast) { + sockets.broadcastSyncInfo(self.syncInfo); + } + }; + + HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) { var self = this; @@ -77,7 +89,7 @@ function spec() { if (step < 10) step = 10; if (self.syncInfo.syncedBlocks % step === 1) { - printProgress(self.syncInfo); + self.showProgress(); } return c(); }, @@ -129,7 +141,7 @@ function spec() { self.status = 'syncing'; } - if (opts.uptoexisting && existed ) { + if (opts.upToExisting && existed ) { if (self.syncInfo.blocksToSync <= self.syncInfo.syncedBlocks) { self.status = 'finished'; p('DONE. Found existing block: ', blockHash); @@ -172,29 +184,11 @@ function spec() { async.series([ function(cb) { if (opts.destroy) { - p('Deleting Blocks...'); - self.db.collections.blocks.drop(cb); - } else { - return cb(); + p('Deleting DB...'); + return self.sync.destroy(cb); } + return cb(); }, - function(cb) { - if (opts.destroy) { - p('Deleting TXs...'); - self.db.collections.transactions.drop(cb); - } else { - return cb(); - } - }, - function(cb) { - if (opts.destroy) { - p('Deleting TXItems...'); - self.db.collections.transactionitems.drop(cb); - } else { - return cb(); - } - }, - // We are not using getBestBlockHash, because is not available in all clients function(cb) { if (!opts.reverse) return cb(); @@ -218,7 +212,7 @@ function spec() { }, function(cb) { // This is only to inform progress. - if (!opts.uptoexisting) { + if (!opts.upToExisting) { self.rpc.getInfo(function(err, res) { if (err) return cb(err); self.syncInfo.blocksToSync = res.result.blocks; @@ -260,7 +254,7 @@ function spec() { isEndGenesis: end === self.genesis, scanningForward: opts.next, scanningBackward: opts.prev, - uptoexisting: opts.uptoexisting, + upToExisting: opts.upToExisting, syncedBlocks: 0, }); @@ -306,7 +300,7 @@ function spec() { var opts = { reverse: 1, - uptoexisting: b ? true: false, + upToExisting: b ? true: false, }; return self.import_history(opts, next); diff --git a/lib/Sync.js b/lib/Sync.js index 4a4ecdc7..3ff55549 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -9,6 +9,7 @@ function spec() { var Block = require('../app/models/Block'); var Transaction = require('../app/models/Transaction'); var sockets = require('../app/controllers/socket.js'); + var async = require('async'); function Sync() { @@ -16,12 +17,11 @@ function spec() { } Sync.prototype.init = function(opts, cb) { - var that = this; + var self = this; - that.opts = opts; - - if (!(opts && opts.skip_db_connection)) { + self.opts = opts; + if (!(opts && opts.skipDbConnection)) { if (mongoose.connection.readyState !== 1) { mongoose.connect(config.db, function(err) { @@ -32,19 +32,19 @@ function spec() { }); } - that.db = mongoose.connection; + self.db = mongoose.connection; - that.db.on('error', function(err) { + self.db.on('error', function(err) { console.log('MongoDB ERROR:' + err); return cb(err); }); - that.db.on('disconnect', function(err) { + self.db.on('disconnect', function(err) { console.log('MongoDB disconnect:' + err); return cb(err); }); - return that.db.once('open', function(err) { + return self.db.once('open', function(err) { return cb(err); }); } @@ -57,24 +57,34 @@ function spec() { } }; + + Sync.prototype.destroy = function(next) { + var self = this; + async.series([ + function(b) { return self.db.collections.blocks.drop(b);}, + function(b) { return self.db.collections.transactions.drop(b);}, + function(b) { return self.db.collections.transactionitems.drop(b);}, + ], next); + }; + Sync.prototype.storeBlock = function(block, cb) { - var that = this; + var self = this; Block.customCreate(block, function(err, block, inserted_txs){ if (err) return cb(err); - if (block && that.opts.broadcast_blocks) { + if (block && self.opts.broadcast_blocks) { sockets.broadcast_block(block); } - if (inserted_txs && that.opts.broadcast_txs) { + if (inserted_txs && self.opts.broadcast_txs) { inserted_txs.forEach(function(tx) { sockets.broadcast_tx(tx); }); } if (inserted_txs) - that.tx_count += inserted_txs.length; + self.tx_count += inserted_txs.length; return cb(); }); @@ -82,12 +92,12 @@ function spec() { Sync.prototype.storeTxs = function(txs, inTime, cb) { - var that = this; + var self = this; var time = inTime ? inTime : Math.round(new Date().getTime() / 1000); Transaction.createFromArray(txs, time, function(err, inserted_txs) { - if (!err && inserted_txs && that.opts.broadcast_txs) { + if (!err && inserted_txs && self.opts.broadcast_txs) { inserted_txs.forEach(function(tx) { sockets.broadcast_tx(tx); diff --git a/server.js b/server.js index 59303fd4..1f495031 100644 --- a/server.js +++ b/server.js @@ -48,7 +48,8 @@ var historicSync = {}; if (!config.disableHistoricSync) { historicSync = new HistoricSync(); historicSync.init({ - skip_db_connection: true, + skipDbConnection: true, + shouldBroadcast: true, networkName: config.network }, function() { historicSync.smart_import(function(err){ @@ -65,7 +66,7 @@ if (!config.disableHistoricSync) { if (!config.disableP2pSync) { var ps = new PeerSync(); ps.init({ - skip_db_connection: true, + skipDbConnection: true, broadcast_txs: true, broadcast_blocks: true }, function() { diff --git a/util/sync.js b/util/sync.js index 01205d9d..d399a1ef 100755 --- a/util/sync.js +++ b/util/sync.js @@ -41,7 +41,7 @@ async.series([ historicSync.import_history({ destroy: program.destroy, reverse: program.reverse, - uptoexisting: program.uptoexisting, + upToExisting: program.uptoexisting, }, cb); } }, From 73a1d64e0187f99b0ab84a19539eeca774c2cc5c Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Sun, 19 Jan 2014 12:39:34 -0300 Subject: [PATCH 15/17] README updated --- README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.md b/README.md index dc344c69..8c1ed487 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,12 @@ A REST API is provided at /api. The entry points are: /api/txs/?address=ADDR /api/txs/?address=mmhmMNfBiZZ37g1tgg2t8DDbNoEdqKVxAL + +### Sync status +``` + /api/sync +``` + ## Web Socket API The web socket API is served using [socket.io](http://socket.io) at: ``` @@ -160,6 +166,22 @@ Sample output: } ``` +'sync': every 1% increment on the sync task, this event will be triggered. + +Sample output: +``` +{ +blocksToSync: 164141, +syncedBlocks: 475, +upToExisting: true, +scanningBackward: true, +isEndGenesis: true, +end: "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943", +isStartGenesis: false, +start: "000000009f929800556a8f3cfdbe57c187f2f679e351b12f7011bfc276c41b6d" +} +``` + ## Troubleshooting If you did not get all library during grunt command, please use the follow command: From 8898e9b8c5038d435b1b806d2f41386a29676231 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Sun, 19 Jan 2014 21:43:10 -0300 Subject: [PATCH 16/17] add error messages to sync status --- app/controllers/status.js | 3 +-- lib/HistoricSync.js | 8 +++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/controllers/status.js b/app/controllers/status.js index 01b51b99..6652359e 100644 --- a/app/controllers/status.js +++ b/app/controllers/status.js @@ -45,8 +45,7 @@ exports.show = function(req, res, next) { } }; -exports.sync = function(req, res, next) { +exports.sync = function(req, res) { if (req.syncInfo) res.jsonp(req.syncInfo); - next(); }; diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 2735f44f..f9d3d0a9 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -274,11 +274,13 @@ function spec() { }); } - if (!err) - sync(); - else { + if (err) { + self.syncInfo = util._extend(self.syncInfo, { error: err.message }); return next(err, 0); } + else { + sync(); + } }); }; From ed0a8bb0e36ad862891d9b36ca1f0a963faac85b Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Mon, 20 Jan 2014 10:28:59 -0300 Subject: [PATCH 17/17] remove unnecesaries dependencies --- util/sync.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/util/sync.js b/util/sync.js index d399a1ef..b7f6e03c 100755 --- a/util/sync.js +++ b/util/sync.js @@ -4,8 +4,6 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; -require('buffertools').extend(); - var SYNC_VERSION = '0.1'; var program = require('commander'); var HistoricSync = require('../lib/HistoricSync').class();