Merge branch 'master' of github.com:bitpay/insight into feature/address-socket-api

Conflicts:
	app/controllers/socket.js
	app/models/Transaction.js
	lib/PeerSync.js
	public/js/controllers/address.js
	public/js/controllers/index.js
	public/js/controllers/transactions.js
	public/views/transaction.html

.gitignore

@@ -18,6 +18,7 @@ node_modules
 
 # extras
 *.swp
+*.swo
 *~
 .project
 peerdb.json

README.md

@@ -128,6 +128,12 @@ A REST API is provided at /api. The entry points are:
   /api/txs/?address=ADDR
   /api/txs/?address=mmhmMNfBiZZ37g1tgg2t8DDbNoEdqKVxAL
 
+
+### Sync status
+```
+  /api/sync
+```
+
 ## Web Socket API
 The web socket API is served using [socket.io](http://socket.io) at:
 ```
@@ -160,6 +166,22 @@ Sample output:
 }
 ```
 
+'sync': every 1% increment on the sync task, this event will be triggered.
+
+Sample output:
+```
+{
+blocksToSync: 164141,
+syncedBlocks: 475,
+upToExisting: true,
+scanningBackward: true,
+isEndGenesis: true,
+end: "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943",
+isStartGenesis: false,
+start: "000000009f929800556a8f3cfdbe57c187f2f679e351b12f7011bfc276c41b6d"
+}
+```
+
 ## Troubleshooting
 If you did not get all library during grunt command, please use the follow command:
 

app/controllers/socket.js

@@ -26,3 +26,6 @@ module.exports.broadcast_address_tx = function(address, tx) {
   ios.sockets.in(address).emit('tx', tx);
 };
 
+module.exports.broadcastSyncInfo = function(syncInfo) {
+  ios.sockets.emit('block', syncInfo);
+};

app/controllers/status.js

@@ -15,43 +15,37 @@ exports.show = function(req, res, next) {
     res.status(400).send('Bad Request');
   }
   else {
-    var s = req.query.q;
+    var option = req.query.q;
-    var d = Status.new();
+    var statusObject = Status.new();
 
-    if (s === 'getInfo') {
+    var returnJsonp = function (err) {
-      d.getInfo(function(err) {
+      if(err) return next(err);
-        if (err) next(err);
+      res.jsonp(statusObject);
-        res.jsonp(d);
+    };
-      });
-    }
-    else if (s === 'getDifficulty') {
-      d.getDifficulty(function(err) {
-        if (err) next(err);
-        res.jsonp(d);
-      });
-    }
-    else if (s === 'getTxOutSetInfo') {
-      d.getTxOutSetInfo(function(err) {
-        if (err) next(err);
-        res.jsonp(d);
-      });
-    }
-    else if (s === 'getBestBlockHash') {
-      d.getBestBlockHash(function(err) {
-        if (err) next(err);
-        res.jsonp(d);
-      });
-    }
-    else if (s === 'getLastBlockHash') {
-      d.getLastBlockHash(function(err) {
-        if (err) next(err);
-        res.jsonp(d);
-      });
-    }
 
-    else {
+    switch(option) {
+      case 'getInfo':
+        statusObject.getInfo(returnJsonp);
+        break;
+      case 'getDifficulty':
+        statusObject.getDifficulty(returnJsonp);
+        break;
+      case 'getTxOutSetInfo':
+        statusObject.getTxOutSetInfo(returnJsonp);
+        break;
+      case 'getBestBlockHash':
+        statusObject.getBestBlockHash(returnJsonp);
+        break;
+      case 'getLastBlockHash':
+        statusObject.getLastBlockHash(returnJsonp);
+        break;
+      default:
         res.status(400).send('Bad Request');
     }
   }
 };
 
+exports.sync = function(req, res) {
+  if (req.syncInfo)
+    res.jsonp(req.syncInfo);
+};

app/models/Block.js

@@ -27,6 +27,7 @@ var BlockSchema = new Schema({
   },
   time: Number,
   nextBlockHash: String,
+  isOrphan: Boolean,
 });
 
 /**
@@ -55,6 +56,7 @@ BlockSchema.statics.customCreate = function(block, cb) {
   newBlock.hash = block.hash;
   newBlock.nextBlockHash = block.nextBlockHash;
 
+
   Transaction.createFromArray(block.tx, newBlock.time, function(err, inserted_txs) {
     if (err) return cb(err);
 

app/models/Transaction.js

@@ -20,6 +20,9 @@ var mongoose        = require('mongoose'),
 
 var CONCURRENCY = 5;
 
+// TODO: use bitcore networks module
+var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b';
+
 /**
  */
 var TransactionSchema = new Schema({
@@ -120,6 +123,11 @@ TransactionSchema.statics.createFromArray = function(txs, time, next) {
 
 TransactionSchema.statics.explodeTransactionItems = function(txid, time,  cb) {
   var addrs = [];
+
+  // Is it from genesis block? (testnet==livenet)
+  // TODO: parse it from networks.genesisTX
+  if (txid === genesisTXID) return cb();
+
   this.queryInfo(txid, function(err, info) {
     if (err || !info) return cb(err);
 
@@ -144,7 +152,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, time,  cb) {
       }
       else {
         if ( !i.coinbase ) {
-            console.log ('TX: %s,%d could not parse INPUT', txid, i.n);
+            console.log ('WARN in TX: %s: could not parse INPUT %d', txid, i.n);
         }
         return next_in();
       }
@@ -166,7 +174,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, time,  cb) {
           }, next_out);
         }
         else {
-          console.log ('TX: %s,%d could not parse OUTPUT', txid, o.n);
+          console.log ('WARN in TX: %s could not parse OUTPUT %d', txid, o.n);
           return next_out();
         }
       },

app/views/includes/foot.jade

@@ -1,6 +1,4 @@
-#footer
+#footer(data-ng-include="'views/footer.html'", role='navigation')
-  .container
-    p.text-muted Place sticky footer content here.
 
 //script(type='text/javascript', src='/lib/jquery/jquery.min.js')
 //script(type='text/javascript', src='/lib/bootstrap/dist/js/bootstrap.min.js')
@@ -38,6 +36,7 @@ script(type='text/javascript', src='/js/services/socket.js')
 //Application Controllers
 script(type='text/javascript', src='/js/controllers/index.js')
 script(type='text/javascript', src='/js/controllers/header.js')
+script(type='text/javascript', src='/js/controllers/footer.js')
 script(type='text/javascript', src='/js/controllers/blocks.js')
 script(type='text/javascript', src='/js/controllers/transactions.js')
 script(type='text/javascript', src='/js/controllers/address.js')

config/express.js

@@ -7,7 +7,8 @@ var express = require('express'),
     helpers = require('view-helpers'),
     config = require('./config');
 
-module.exports = function(app, passport, db) {
+module.exports = function(app, historicSync) {
+
   app.set('showStackError', true);
 
   //Prettify HTML
@@ -26,9 +27,17 @@ module.exports = function(app, passport, db) {
   app.set('view engine', 'jade');
 
   //Enable jsonp
-  app.enable("jsonp callback");
+  app.enable('jsonp callback');
+
+ //custom middleware
+  function setHistoric(req, res, next) {
+    req.syncInfo = historicSync.syncInfo;
+    next();
+  }
+  app.use('/api/sync', setHistoric);
  
   app.configure(function() {
+
     //cookieParser should be above session
     app.use(express.cookieParser());
 
@@ -43,6 +52,7 @@ module.exports = function(app, passport, db) {
     //routes should be at the last
     app.use(app.router);
 
+
     //Setting the fav icon and static folder
     app.use(express.favicon());
     app.use(express.static(config.root + '/public'));

config/routes.js

@@ -1,6 +1,6 @@
 'use strict';
 
-module.exports = function(app) {
+module.exports = function(app, historicSync) {
 
   //Home route
   var index = require('../app/controllers/index');
@@ -29,4 +29,6 @@ module.exports = function(app) {
   var st = require('../app/controllers/status');
   app.get('/api/status', st.show);
 
+  app.get('/api/sync', st.sync);
+
 };

lib/HistoricSync.js

@@ -11,27 +11,33 @@ function spec() {
   var config          = require('../config/config');
   var Block           = require('../app/models/Block');
   var Sync            = require('./Sync').class();
+  var sockets         = require('../app/controllers/socket.js');
 
   function HistoricSync(opts) {
-    this.block_count= 0;
-    this.block_total= 0;
     this.network    = config.network === 'testnet' ? networks.testnet: networks.livenet;
+
+    var genesisHashReversed = new Buffer(32);
+    this.network.genesisBlock.hash.copy(genesisHashReversed);
+    this.genesis = genesisHashReversed.reverse().toString('hex');
     this.sync       = new Sync(opts);
+
+
+    //available status: new / syncing / finished / aborted
+    this.status   = 'new';
+    this.syncInfo = {};
   }
 
   function p() {
     var args = [];
     Array.prototype.push.apply( args, arguments );
 
+
+    
     args.unshift('[historic_sync]');
     /*jshint validthis:true */
     console.log.apply(this, args);
   }
 
-  var progress_bar = function(string, current, total) {
-    p(util.format('%s %d/%d [%d%%]', string, current, total, parseInt(100 * current / total)));
-  };
-
   HistoricSync.prototype.init = function(opts,cb) {
     this.rpc = new RpcClient(config.bitcoind);
     this.opts = opts;
@@ -42,14 +48,24 @@ function spec() {
     this.sync.close();
   };
 
-  HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) {
 
-    var that = this;
+  HistoricSync.prototype.showProgress = function() {
+    var self = this;
+
+    var i = self.syncInfo;
+    var per = parseInt(100 * i.syncedBlocks / i.blocksToSync);
+    p(util.format('status: %d/%d [%d%%]', i.syncedBlocks, i.blocksToSync, per));
+    if (self.opts.broadcast) {
+      sockets.broadcastSyncInfo(self.syncInfo);
+    }
+  };
+
+
+  HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) {
+    var self = this;
 
     // recursion end.
-    if (!blockHash || (blockEnd && blockEnd === blockHash) ) {
+    if (!blockHash ) return cb();
-      return cb();
-    }
 
     var existed = 0;
     var blockInfo;
@@ -64,14 +80,16 @@ function spec() {
             existed  =1;
             blockObj =block;
           }
-
           return c();
         });
       },
       //show some (inacurate) status
       function(c) {
-        if (that.block_count++ % 1000 === 0) {
+        var step = parseInt(self.syncInfo.blocksToSync / 100);
-          progress_bar('sync status:', that.block_count, that.block_total);
+        if (step < 10) step = 10;
+
+        if (self.syncInfo.syncedBlocks % step === 1) {
+          self.showProgress();
         }
         return c();
       },
@@ -81,7 +99,7 @@ function spec() {
         // TODO: if we store prev/next, no need to go to RPC
         // if (blockObj && blockObj.nextBlockHash) return c();
 
-        that.rpc.getBlock(blockHash, function(err, ret) {
+        self.rpc.getBlock(blockHash, function(err, ret) {
           if (err) return c(err);
 
           blockInfo = ret;
@@ -91,9 +109,10 @@ function spec() {
       //store it
       function(c) {
         if (existed) return c();
+        self.sync.storeBlock(blockInfo.result, function(err) {
 
-        that.sync.storeBlock(blockInfo.result, function(err) {
           existed = err && err.toString().match(/E11000/);
+
           if (err && ! existed)  return c(err);
           return c();
         });
@@ -111,134 +130,183 @@ function spec() {
       ],
       function (err){
 
-        if (err)
+        if (err) {
-          p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, that.block_count);
+          self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncInfo.syncedBlocks);
+          self.status = 'aborted';
+          p(self.err);
+        }
 
+        else {
+          self.err = null;
+          self.status = 'syncing';
+        }
+
+        if (opts.upToExisting && existed ) {
+          if (self.syncInfo.blocksToSync <= self.syncInfo.syncedBlocks) {
+            self.status = 'finished';
+            p('DONE. Found existing block: ', blockHash);
+            return cb(err);
+          }
+          else {
+            p('WARN found target block\n\tbut blockChain Height is still higher that ours. Previous light sync must be interrupted.\n\tWill keep syncing.', self.syncInfo.syncedBlocks);
+          }
+        }
+
+        if (blockEnd && blockEnd === blockHash) {
+          self.status = 'finished';
+          p('DONE. Found END block: ', blockHash);
+          return cb(err);
+        }
+
+
+        // Continue
         if (blockInfo && blockInfo.result) {
+          self.syncInfo.syncedBlocks++;
           if (opts.prev && blockInfo.result.previousblockhash) {
-            return that.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb);
+            return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb);
           }
 
           if (opts.next && blockInfo.result.nextblockhash)
-            return that.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb);
+            return self.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb);
         }
         return cb(err);
     });
   };
 
-  HistoricSync.prototype.syncBlocks = function(start, end, isForward, cb) {
+  HistoricSync.prototype.import_history = function(opts, next) {
-    var that = this;
+    var self = this;
 
-    p('Starting from: ', start);
-    p('         to  : ', end);
-    p('    isForward: ', isForward);
-
-
-    return that.getPrevNextBlock( start, end,
-                  isForward ? { next: 1 } : { prev: 1}, cb);
-  };
-
-  HistoricSync.prototype.do_import_history = function(opts, next) {
-    var that = this;
-
-    var retry_attemps = 100;
     var retry_secs    = 2;
 
-    var block_best;
+    var bestBlock;
-    var block_height;
+    var blockChainHeight;
 
     async.series([
       function(cb) {
         if (opts.destroy) {
-          p('Deleting Blocks...');
+          p('Deleting DB...');
-          that.db.collections.blocks.drop(cb);
+          return self.sync.destroy(cb);
-        } else {
-          return cb();
         }
-      },
-      function(cb) {
-        if (opts.destroy) {
-          p('Deleting TXs...');
-          that.db.collections.transactions.drop(cb);
-        } else {
         return cb();
-        }
-      },
-      function(cb) {
-        if (opts.destroy) {
-          p('Deleting TXItems...');
-          that.db.collections.transactionitems.drop(cb);
-        } else {
-          return cb();
-        }
-      },
-      function(cb) {
-        that.rpc.getInfo(function(err, res) {
-          if (err) cb(err);
-
-          that.block_total = res.result.blocks;
-          return cb();
-        });
       },
       // We are not using getBestBlockHash, because is not available in all clients
       function(cb) {
         if (!opts.reverse) return cb();
 
-        that.rpc.getBlockCount(function(err, res) {
+        self.rpc.getBlockCount(function(err, res) {
-          if (err) cb(err);
+          if (err) return cb(err);
-          block_height = res.result;
+          blockChainHeight = res.result;
           return cb();
         });
       },
       function(cb) {
         if (!opts.reverse) return cb();
 
-        that.rpc.getBlockHash(block_height, function(err, res) {
+        self.rpc.getBlockHash(blockChainHeight, function(err, res) {
-          if (err) cb(err);
+          if (err) return cb(err);
+
+          bestBlock = res.result;
 
-          block_best = res.result;
           return cb();
         });
       },
+      function(cb) {
+        // This is only to inform progress.
+        if (!opts.upToExisting) {
+          self.rpc.getInfo(function(err, res) {
+            if (err) return cb(err);
+            self.syncInfo.blocksToSync = res.result.blocks;
+            return cb();
+          });
+        }
+        else {
+          // should be isOrphan = true or null to be more accurate.
+          Block.count({ isOrphan: null}, function(err, count) {
+            if (err) return cb(err);
+
+            self.syncInfo.blocksToSync = blockChainHeight - count;
+            if (self.syncInfo.blocksToSync < 1) self.syncInfo.blocksToSync = 1;
+            return cb();
+          });
+         }
+      },
       ],
       function(err) {
 
-        function sync() {
-          var start, end, isForward;
 
+        var start, end;
+        function sync() {
           if (opts.reverse) {
-            start     = block_best;
+            start     = bestBlock;
-            end       = that.network.genesisBlock.hash.reverse().toString('hex');
+            end       = self.genesis;
-            isForward = false;
+            opts.prev      = true;
           }
           else {
-            start     = that.network.genesisBlock.hash.reverse().toString('hex');
+            start     = self.genesis;
             end       = null;
-            isForward = true;
+            opts.next      = true;
           }
 
-          that.syncBlocks(start, end, isForward, function(err) {
+          self.syncInfo = util._extend(self.syncInfo, {
+            start: start,
+            isStartGenesis: start === self.genesis,
+            end:   end,
+            isEndGenesis: end === self.genesis,
+            scanningForward: opts.next,
+            scanningBackward: opts.prev,
+            upToExisting: opts.upToExisting,
+            syncedBlocks: 0,
+          });
 
-            if (err && err.message.match(/ECONNREFUSED/) && retry_attemps--){
+          p('Starting from: ', start);
+          p('         to  : ', end);
+          p('         opts: ', JSON.stringify(opts));
+
+          self.getPrevNextBlock( start, end, opts , function(err) {
+            if (err && err.message.match(/ECONNREFUSED/)){
               setTimeout(function() {
                 p('Retrying in %d secs', retry_secs);
                 sync();
               }, retry_secs * 1000);
             }
             else
-              return next(err, that.block_count);
+              return next(err);
           });
         }
-        if (!err)
+
-          sync();
+        if (err) {
-        else
+          self.syncInfo = util._extend(self.syncInfo, { error: err.message });
           return next(err, 0);
+        }
+        else {
+          sync();
+        }
     });
   };
 
-  HistoricSync.prototype.import_history = function(opts, next) {
+  // upto if we have genesis block?
-    var that = this;
+  HistoricSync.prototype.smart_import = function(next) {
-    that.do_import_history(opts, next);
+    var self = this;
+
+    Block.findOne({hash:self.genesis}, function(err, b){
+
+      if (err) return next(err);
+
+
+      if (!b) {
+        p('Could not find Genesis block. Running FULL SYNC');
+      }
+      else {
+        p('Genesis block found. Syncing upto known blocks.');
+      }
+
+      var opts = {
+        reverse: 1,
+        upToExisting: b ? true: false,
+      };
+
+      return self.import_history(opts, next);
+    });
   };
 
 

lib/PeerSync.js

@@ -15,7 +15,6 @@ function spec() {
 
   PeerSync.prototype.init = function(config, cb) {
     if (!config) config = {};
-
     var network = config && (config.network || 'testnet');
 
     this.verbose = config.verbose;
@@ -70,6 +69,8 @@ function spec() {
     }
     this.sync.storeTxs([tx.hash], null, function(err) {
       if (err) {
+console.log('[PeerSync.js.71:err:]',err); //TODO
+
         console.log('[p2p_sync] Error in handle TX: ' + JSON.stringify(err));
       }
     });

lib/Sync.js

@@ -9,29 +9,82 @@ function spec() {
   var Block           = require('../app/models/Block');
   var Transaction     = require('../app/models/Transaction');
   var sockets         = require('../app/controllers/socket.js');
+  var async           = require('async');
 
 
   function Sync() {
     this.tx_count   = 0;
   }
 
+  Sync.prototype.init = function(opts, cb) {
+    var self = this;
+
+    self.opts = opts;
+
+    if (!(opts && opts.skipDbConnection)) {
+
+      if (mongoose.connection.readyState !== 1) {
+        mongoose.connect(config.db, function(err) {
+          if (err) {
+            console.log('CRITICAL ERROR: connecting to mongoDB:',err);
+            return (err);
+          }
+        });
+      }
+
+      self.db = mongoose.connection;
+
+      self.db.on('error', function(err) {
+          console.log('MongoDB ERROR:' + err);
+          return cb(err);
+      });
+
+      self.db.on('disconnect', function(err) {
+          console.log('MongoDB disconnect:' + err);
+          return cb(err);
+      });
+
+      return self.db.once('open', function(err) {
+        return cb(err);
+      });
+    }
+    else return cb();
+  };
+
+  Sync.prototype.close = function() {
+    if ( this.db && this.db.readyState ) {
+      this.db.close();
+    }
+  };
+
+
+  Sync.prototype.destroy = function(next) {
+    var self = this;
+    async.series([
+      function(b) { return self.db.collections.blocks.drop(b);},
+      function(b) { return self.db.collections.transactions.drop(b);},
+      function(b) { return self.db.collections.transactionitems.drop(b);},
+    ], next);
+  };
+
   Sync.prototype.storeBlock = function(block, cb) {
-    var that = this;
+    var self = this;
 
     Block.customCreate(block, function(err, block, inserted_txs){
+      if (err) return cb(err);
 
-      if (block && that.opts.broadcast_blocks) {
+      if (block && self.opts.broadcast_blocks) {
         sockets.broadcast_block(block);
       }
 
-      if (inserted_txs && that.opts.broadcast_txs) {
+      if (inserted_txs && self.opts.broadcast_txs) {
         inserted_txs.forEach(function(tx) {
           sockets.broadcast_tx(tx);
         });
       }
 
       if (inserted_txs)
-        that.tx_count += inserted_txs.length;
+        self.tx_count += inserted_txs.length;
 
       return cb();
     });
@@ -39,12 +92,12 @@ function spec() {
 
 
   Sync.prototype.storeTxs = function(txs, inTime, cb) {
-    var that = this;
+    var self = this;
 
     var time = inTime ? inTime : Math.round(new Date().getTime() / 1000);
 
     Transaction.createFromArray(txs, time, function(err, inserted_txs) {
-      if (!err && inserted_txs && that.opts.broadcast_txs) {
+      if (!err && inserted_txs && self.opts.broadcast_txs) {
 
         inserted_txs.forEach(function(tx) {
           sockets.broadcast_tx(tx);
@@ -54,51 +107,6 @@ function spec() {
       return cb(err);
     });
   };
-
-
-  Sync.prototype.syncBlocks = function(start, end, isForward, cb) {
-    var that = this;
-
-    console.log('Syncing Blocks, starting \n\tfrom: %s \n\tend: %s \n\tisForward:',
-                  start, end, isForward);
-
-
-    return that.getPrevNextBlock( start, end,
-                  isForward ? { next: 1 } : { prev: 1}, cb);
-  };
-
-  Sync.prototype.init = function(opts, cb) {
-    var that = this;
-
-    that.opts = opts;
-
-    if (!(opts && opts.skip_db_connection)) {
-      if (!mongoose.connection) {
-        mongoose.connect(config.db, {server: {auto_reconnect: true}} );
-      }
-
-      this.db = mongoose.connection;
-
-      this.db.on('error', function(err) {
-          console.log('connection error:' + err);
-          mongoose.disconnect();
-      });
-
-      this.db.on('disconnect', function(err) {
-          console.log('disconnect:' + err);
-          mongoose.connect(config.db, {server: {auto_reconnect: true}} );
-      });
-
-      return that.db.once('open', cb);
-    }
-    else return cb();
-  };
-
-  Sync.prototype.close = function() {
-    if (!(this.opts && this.opts.skip_db_connection)) {
-      this.db.close();
-    }
-  };
   return Sync;
 }
 module.defineClass(spec);

public/css/common.css

@@ -34,6 +34,16 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 {
   padding: 0 0 60px;
 }
 
+.m10h {margin: 0 10px;}
+.m20h {margin: 0 20px;}
+.m20v {margin: 20px 0;}
+.m50v {margin: 50px 0;}
+.m10b {margin-bottom: 10px;}
+
+.vm {
+  vertical-align: middle;
+}
+
 .navbar-default {
   background-color: #8DC429;
   border-bottom: 4px solid #64920F;
@@ -145,22 +155,45 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 {
   border-radius: 2px;
   background: #F4F4F4;
   margin: 20px 0;
-  padding: 15px;
+  padding: 20px;
 }
 
-.btn-primary {
+.btn {
+  margin: 0 5px;
   border-radius: 2px;
-  background: #64920F;
+}
-  border: 2px solid #557F08;
+.btn-primary {
+  background: #8DC429;
+  border: 2px solid #76AF0F;
 }
 
-.btn-primary:hover, .btn-primary:focus, .btn-primary:active, .btn-primary.active, .open .dropdown-toggle.btn-primary {
+.btn-primary:hover, .btn-primary:focus, .btn-primary:active,
+.btn-primary.active, .open .dropdown-toggle.btn-primary,
+.btn-success:hover, .btn-success:focus, .btn-success:active,
+.btn-success.active, .open .dropdown-toggle.btn-success,
+.btn-danger:hover, .btn-danger:focus, .btn-danger:active,
+.btn-danger.active, .open .dropdown-toggle.btn-danger {
   background: #fff;
   border: 2px solid #ccc;
   color: #373D42;
   font-weight: 500;
 }
 
+.btn-default {
+  background: #E7E7E7;
+  border: 2px solid #DCDCDC;
+}
+
+.btn-success {
+  background: #2FA4D7;
+  border: 2px solid #237FA7;
+}
+
+.btn-danger {
+  background: #AC0015;
+  border: 2px solid #6C0000;
+}
+
 /* Set the fixed height of the footer here */
 #footer {
   height: 60px;
@@ -217,7 +250,15 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 {
   opacity: 1;
 }
 
-
+.tx-bg {
+  background-color: #F4F4F4;
+  width: 100%;
+  min-height: 340px;
+  position: absolute;
+  top: 0;
+  left: 0;
+  z-index: -9999;
+}
 
 .badge {
   padding: 1px 9px 2px;

public/js/config.js

@@ -26,7 +26,7 @@ angular.module('insight').config(['$routeProvider',
       templateUrl: 'views/status.html'
     }).
     otherwise({
-      redirectTo: '/'
+      templateUrl: 'views/404.html'
     });
   }
 ]);

public/js/controllers/address.js

@@ -2,12 +2,13 @@
 
 angular.module('insight.address').controller('AddressController',
     ['$scope',
+    '$rootScope',
     '$routeParams',
     '$location',
     'Global',
     'Address',
     'socket',
-    function ($scope, $routeParams, $location, Global, Address, socket) {
+    function ($scope, $rootScope, $routeParams, $location, Global, Address, socket) {
     $scope.global = Global;
 
     $scope.findOne = function() {
@@ -15,6 +16,9 @@ angular.module('insight.address').controller('AddressController',
         addrStr: $routeParams.addrStr
       }, function(address) {
         $scope.address = address;
+      }, function() {
+        $rootScope.flashMessage = 'Address Not Found';
+        $location.path('/');
       });
     };
     socket.on('connect', function() {

public/js/controllers/blocks.js

@@ -1,6 +1,6 @@
 'use strict';
 
-angular.module('insight.blocks').controller('BlocksController', ['$scope', '$routeParams', '$location', 'Global', 'Block', 'Blocks', function ($scope, $routeParams, $location, Global, Block, Blocks) {
+angular.module('insight.blocks').controller('BlocksController', ['$scope', '$rootScope', '$routeParams', '$location', 'Global', 'Block', 'Blocks', function ($scope, $rootScope, $routeParams, $location, Global, Block, Blocks) {
   $scope.global = Global;
 
   $scope.list = function() {
@@ -17,6 +17,9 @@ angular.module('insight.blocks').controller('BlocksController', ['$scope', '$rou
       blockHash: $routeParams.blockHash
     }, function(block) {
       $scope.block = block;
+    }, function() {
+      $rootScope.flashMessage = 'Block Not Found';
+      $location.path('/');
     });
   };
 

public/js/controllers/footer.js

@@ -0,0 +1,20 @@
+'use strict';
+
+angular.module('insight.system').controller('FooterController', ['$scope', 'Global', 'socket', 'Status', function ($scope, Global, socket, Status) {
+  $scope.global = Global;
+
+  socket.on('block', function(block) {
+console.log('[footer.js:14]',block); //TODO
+    console.log('Block received! ' + JSON.stringify(block));
+  });
+
+  $scope.getFooter = function() {
+    Status.get({
+     q: 'getInfo'
+    }, function(d) {
+      $scope.info = d.info;
+    });
+  };
+
+}]);
+

public/js/controllers/index.js

@@ -4,16 +4,21 @@ var TRANSACTION_DISPLAYED = 5;
 var BLOCKS_DISPLAYED = 5;
 angular.module('insight.system').controller('IndexController',
     ['$scope',
+    '$rootScope',
     'Global',
     'socket',
     'Blocks',
     'Transactions',
-    function($scope, Global, socket, Blocks, Transactions) {
+    function($scope, $rootScope, Global, socket, Blocks, Transactions) {
   $scope.global = Global;
 
   socket.on('connect', function() {
     socket.emit('subscribe', 'inv');
   });
+
+  //show errors
+  $scope.flashMessage = $rootScope.flashMessage || null;
+
   socket.on('tx', function(tx) {
     console.log('Transaction received! ' + JSON.stringify(tx));
     if ($scope.txs.length === TRANSACTION_DISPLAYED) {
@@ -52,4 +57,3 @@ angular.module('insight.system').controller('IndexController',
   $scope.txs = [];
   $scope.blocks = [];
 }]);
-

public/js/controllers/transactions.js

@@ -12,7 +12,6 @@ angular.module('insight.transactions').controller('transactionsController',
     function ($scope, $routeParams, $location, Global, Transaction, TransactionsByBlock, TransactionsByAddress, socket) {
   $scope.global = Global;
 
-
   $scope.findThis = function() {
     $scope.findTx($routeParams.txId);
   };
@@ -23,6 +22,9 @@ angular.module('insight.transactions').controller('transactionsController',
     }, function(tx) {
       $scope.tx = tx;
       $scope.txs.push(tx);
+    }, function() {
+      $rootScope.flashMessage = 'Transaction Not Found';
+      $location.path('/');
     });
   };
 
@@ -48,6 +50,4 @@ angular.module('insight.transactions').controller('transactionsController',
 
   $scope.txs = [];
 
-
 }]);
-

public/js/services/global.js

@@ -1,14 +1,5 @@
 'use strict';
 
 //Global service for global variables
-angular.module('insight.system').factory('Global', [
+angular.module('insight.system').factory('Global', [function() {}]);
-  function() {
-    var _this = this;
-    _this._data = {
-      user: window.user,
-      authenticated: !! window.user
-    };
 
-    return _this._data;
-  }
-]);

public/views/404.html

@@ -0,0 +1,5 @@
+<div class="jumbotron">
+  <h1>Ooops!</h1>
+  <h2 class="text-muted">404 Page not found :(</h2>
+  <p><a href="/" class="pull-right">Go to home</a></p>
+</div>

public/views/address.html

@@ -15,15 +15,15 @@
         </tr>
         <tr>
           <td>Total Received</td>
-          <td>{{address.totalReceivedSat / 100000000}} BTC</td>
+          <td>{{address.totalReceived}} BTC</td>
         </tr>
         <tr>
           <td>Total Sent</td>
-          <td>{{address.totalSentSat / 100000000}} BTC</td>
+          <td>{{address.totalSent}} BTC</td>
         </tr>
         <tr>
           <td>Final Balance</td>
-          <td>{{address.balanceSat / 100000000}} BTC</td>
+          <td>{{address.balance}} BTC</td>
         </tr>
         <tr>
           <td>No. Transactions</td>

public/views/footer.html

@@ -0,0 +1,10 @@
+<div data-ng-controller="FooterController" data-ng-init="getFooter()">
+  <div class="container">
+    <p class="text-muted text-right" data-ng-show="info.blocks">
+      Blocks: {{info.blocks}} | 
+      Connections: {{info.connections}} |
+      Difficulty: {{info.difficulty}}
+    </p>
+  </div>
+</div>
+

public/views/index.html

@@ -1,3 +1,6 @@
+<div class="alert alert-danger"  ng:show="flashMessage">
+  {{flashMessage}}
+</div>
 <section data-ng-controller="IndexController" data-ng-init="index()">
   <div class="container">
     <div class="row">
@@ -20,4 +23,3 @@
     </div>
   </div>
 </section>
-

public/views/transaction.html

@@ -1,35 +1,26 @@
 <section data-ng-controller="transactionsController" data-ng-init="findThis()">
-  <div class="page-header">
     <h1>
       Transaction
       <small>View information about a bitcoin transaction</small>
     </h1>
+
+  <div class="block-tx">
+    <div class="line-bot">
+      <a href="/#!/tx/{{tx.txid}}">{{tx.txid}}</a>
     </div> 
 
-  <div class="well well-sm">
+    <div class="row m10b">
-    {{tx.txid}} 
-  </div>  
 
-  <table class="table table-striped">
+      <div class="col-md-5">
-    <thead>
-      <tr>
-        <th>Input</th>
-        <th>&nbsp;</th>
-        <th>Output</th>
-      </tr>
-    </thead>
-    <tbody>
-    <tr>
-      <td width="45%"> 
         <ul class="list-unstyled" data-ng-repeat="vin in tx.vin" data-ng-show="!tx.isCoinBase">
           <li>
+            <a class="m10h vm lead glyphicon glyphicon-circle-arrow-left" href="/#!/tx/{{vin.txid}}" alt="Outpoint: {{vin.txid}},{{vin.vout}}" data-toggle="tooltip" title="Outpoint: {{vin.txid}},{{vin.vout}}">
+            </a>
             <span data-ng-show="!vin.addr">Address could not be parsed</span>  
             <a data-ng-show="vin.addr" href="/#!/address/{{vin.addr}}">{{vin.addr}}</a>  
             <span class="pull-right badge">{{vin.value}} BTC</span>
           </li>
 
-          <li>
-          Outpoint: <a href="/#!/tx/{{vin.txid}}">{{vin.txid}},{{vin.vout}}</a>
         </ul>
         <div data-ng-show="tx.isCoinBase">
           <ul class="list-unstyled" data-ng-repeat="vinn in tx.vin">
@@ -39,90 +30,82 @@
             </li>
           </ul>
         </div>
-      </td>
+      </div>
-      <td width="10%" style="text-align: center;"><span class="glyphicon glyphicon-chevron-right">&nbsp;</span></td>
+
-      <td width="45%">
+      <div class="col-md-2 text-center">
-        <table class="table table-condensed">
+        <span class="glyphicon glyphicon-chevron-right lead"></span>
-          <tbody>
+      </div>
-          <tr data-ng-repeat="vout in tx.vout">
+
-            <td><b>{{vout.scriptPubKey.type}}</b></td>
+      <div class="col-md-5" data-ng-repeat="vout in tx.vout">
-            <td>
+        <div class="row m10b">
+          <div class="col-md-3">
+            <b>{{vout.scriptPubKey.type}}</b>
+          </div>
+          <div class="col-md-9">
             <ul class="list-unstyled" data-ng-repeat="addr in vout.scriptPubKey.addresses">
-                <li><a href="/#!/address/{{addr}}">{{addr}}</a> <span class="pull-right badge">{{vout.value}} BTC</span></li>
+              <li>
+                <a class="ellipsis pull-left" style="width:200px;" href="/#!/address/{{addr}}">{{addr}}</a>
+                <span class="pull-right badge">{{vout.value}} BTC</span>
+              </li>
             </ul>
-            </td>
+          </div>
-          </tr>
+        </div>
-          </tbody>
+      </div>
-        </table>
+    </div>
-      </td>
+
-    </tr>
+    <div class="line-top">
-    </tbody>
+      <small class="text-muted">Feeds: {{tx.feeds}}</small>
-    <tfoot>
+      <div class="pull-right">
-      <tr>
+        <button data-ng-show="tx.confirmations" type="button" class="btn btn-success">
-        <td colspan="3" style="text-align: right;">
-          <button data-ng-show="tx.confirmations" type="button" class="btn btn-primary">
           {{tx.confirmations}} Confirmations
         </button>
         <button data-ng-show="!tx.confirmations"  type="button" class="btn btn-danger">
           Unconfirmed Transaction!
         </button>
-          <button type="button" class="btn btn-success">{{tx.valueOut}} BTC</button>
+        <button type="button" class="btn btn-primary">{{tx.valueOut}} BTC</button>
-        </td>
+      </div>
-      </tr>
+    </div> 
-    </tfoot>
+  </div><!--  END OF BLOCK-TX -->
-  </table>
 
-  <div class="row">
+  <div class="row m50v">
-    <div class="col-md-6">
+    <div data-ng-class="{'col-md-6':!tx.isCoinBase}">
-      <div class="panel panel-default">
+      <h3>Summary</h3>
-        <div class="panel-heading">
+      <table class="table">
-          <h3 class="panel-title">Summary</h3>
-        </div>
-        <div class="panel-body">
-          <table class="table table-striped">
         <tbody>
         <tr>
-              <td>Size</td>
+          <td><strong> Size </strong></td>
-              <td>{{tx.size}} (bytes)</td>
+          <td class="text-muted text-right">{{tx.size}} (bytes)</td>
         </tr>
         <tr>
-              <td>Received Time</td>
+          <td><strong>Received Time </strong></td>
-              <td>{{tx.time * 1000|date:'medium'}}</td>
+          <td class="text-muted text-right">{{tx.time * 1000|date:'medium'}}</td>
         </tr>
         <tr>
-              <td>Block</td>
+          <td><strong>Block </strong></td>
-              <td><a href="/#!/block/{{tx.blockhash}}">Block</a></td>
+          <td class="text-muted text-right"><a href="/#!/block/{{tx.blockhash}}">Block</a></td>
         </tr>
         </tbody>
       </table>
     </div>
-      </div>
+    <div class="col-md-6"  data-ng-show="!tx.isCoinBase">
-    </div>
+      <h3>Inputs and Outputs</h3>
-    <div class="col-md-6">
+        <table class="table">
-      <div class="panel panel-default" data-ng-show="!tx.isCoinBase">
-        <div class="panel-heading">
-          <h3 class="panel-title">Inputs and Outputs</h3>
-        </div>
-        <div class="panel-body">
-          <table class="table table-striped">
           <tbody>
           <tr>
-              <td>Total Input</td>
+            <td><strong>Total Input</strong></td>
-              <td>{{tx.valueIn}} BTC</td>
+            <td class="text-muted text-right">{{tx.valueIn}} BTC</td>
           </tr>
           <tr>
-              <td>Total Output</td>
+            <td><strong>Total Output</strong></td>
-              <td>{{tx.valueOut}} BTC</td>
+            <td class="text-muted text-right">{{tx.valueOut}} BTC</td>
           </tr>
           <tr>
-              <td>Fees</td>
+            <td><strong>Fees</strong></td>
-              <td>{{tx.feeds}} BTC</td>
+            <td class="text-muted text-right">{{tx.feeds}} BTC</td>
           </tr>
           </tbody>
         </table>
     </div>
   </div>
-    </div>
-  </div>
 </section>
 

server.js

@@ -24,7 +24,7 @@ var express = require('express'),
 var config = require('./config/config');
 
 //Bootstrap db connection
-var db = mongoose.connect(config.db);
+mongoose.connect(config.db);
 
 //Bootstrap models
 var models_path = __dirname + '/app/models';
@@ -44,16 +44,19 @@ var walk = function(path) {
 walk(models_path);
 
 // historic_sync process
+var historicSync = {};
 if (!config.disableHistoricSync) {
-  var hs = new HistoricSync();
+  historicSync = new HistoricSync();
-  hs.init({
+  historicSync.init({
-    skip_db_connection: true,
+    skipDbConnection: true,
+    shouldBroadcast: true,
     networkName: config.network
   }, function() {
-    hs.import_history({
+    historicSync.smart_import(function(err){
-      reverse: 1,
+      var txt= 'ended.';
-    }, function(){
+      if (err) txt = 'ABORTED with error: ' + err.message;
-      console.log('historic_sync finished!');
+
+      console.log('[historic_sync] ' + txt, historicSync.syncInfo);
     });
   });
 }
@@ -63,7 +66,7 @@ if (!config.disableHistoricSync) {
 if (!config.disableP2pSync) {
   var ps = new PeerSync();
   ps.init({
-    skip_db_connection: true,
+    skipDbConnection: true,
     broadcast_txs: true,
     broadcast_blocks: true
   }, function() {
@@ -76,7 +79,7 @@ if (!config.disableP2pSync) {
 var app = express();
 
 //express settings
-require('./config/express')(app, db);
+require('./config/express')(app, historicSync);
 
 //Bootstrap routes
 require('./config/routes')(app);

util/sync.js

@@ -4,8 +4,6 @@
 
 process.env.NODE_ENV = process.env.NODE_ENV || 'development';
 
-require('buffertools').extend();
-
 var SYNC_VERSION = '0.1';
 var program = require('commander');
 var HistoricSync = require('../lib/HistoricSync').class();
@@ -14,35 +12,45 @@ var async = require('async');
 program
   .version(SYNC_VERSION)
   .option('-N --network [livenet]', 'Set bitcoin network [testnet]', 'testnet')
+  .option('-S --smart', 'genesis stored? uptoexisting = 1', 1)
   .option('-D --destroy', 'Remove current DB (and start from there)', 0)
   .option('-R --reverse', 'Sync backwards', 0)
+  .option('-U --uptoexisting', 'Sync only until an existing block is found', 0)
   .parse(process.argv);
 
 var historicSync = new HistoricSync({
   networkName: program.network
 });
 
+/*  TODO: Sure?
 if (program.remove) {
 
 }
-
+*/
 async.series([
   function(cb) {
     historicSync.init(program, cb);
   },
   function(cb) {
-    historicSync.import_history(program, function(err, count) {
+    if (program.smart) {
+      historicSync.smart_import(cb);
+    }
+    else {
+      historicSync.import_history({
+        destroy: program.destroy,
+        reverse: program.reverse,
+        upToExisting: program.uptoexisting,
+      }, cb);
+    }
+  },
+  ],
+  function(err) {
+    historicSync.close();
     if (err) {
       console.log('CRITICAL ERROR: ', err);
     }
     else {
-        console.log('Done! [%d blocks]', count, err);
+      console.log('Finished.\n Status:\n', historicSync.syncInfo);
     }
-      cb();
 });
-  },
-  function(cb) {
-    historicSync.close();
-    cb();
-}]);