From b689fd1c513ed2020dd5373349da47dc3d792925 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 16 Jan 2014 13:01:10 -0300 Subject: [PATCH 1/7] test skeleton --- test/lib/PeerSync.js | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/test/lib/PeerSync.js b/test/lib/PeerSync.js index de6b49b..230a02f 100644 --- a/test/lib/PeerSync.js +++ b/test/lib/PeerSync.js @@ -2,13 +2,30 @@ var assert = require('assert'); var PeerSync = require('../../lib/PeerSync.js').class(); describe('Unit testing PeerSync', function() { - var ps = new PeerSync(); + var ps; + beforeEach(function() { + ps = new PeerSync(); + }); describe('#init()', function() { it('should return with no errors', function() { - assert.doesNotThrow(function(){ + assert.doesNotThrow(function() { ps.init(); }); }); }); + describe('#handle_inv()', function() { + it('should return with no errors'); + it('should call sendGetData'); + }); + describe('#handle_tx()', function() { + it('should call storeTxs'); + }); + describe('#handle_block()', function() { + it('should call storeBlock'); + it('should call storeTxs for each transaction'); + }); + describe('#run()', function() { + it('should setup peerman'); + }); }); From e70631ee1f25708b9adb79e9e0eb4e81862786f5 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 16 Jan 2014 13:14:38 -0300 Subject: [PATCH 2/7] chai testing framework added --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index b136f9b..d295bc6 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,8 @@ "view-helpers": "latest", "socket.io": "~0.9.16", "moment": "~2.5.0", - "sinon": "~1.7.3" + "sinon": "~1.7.3", + "chai": "~1.8.1" }, "devDependencies": { "grunt-contrib-watch": "latest", From 87913a8561aa809377d8fd53a815caccf83c7fd4 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 16 Jan 2014 15:35:36 -0300 Subject: [PATCH 3/7] fix typo --- lib/Sync.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Sync.js b/lib/Sync.js index 92b89af..a29a40b 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -274,7 +274,7 @@ function spec() { this.db.on('error', function(err) { console.log('connection error:' + err); - moogose.disconnect(); + mongoose.disconnect(); }); this.db.on('disconnect', function(err) { From 1df72a818ca814500b0d1c9f3360db32ce54c93c Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 16 Jan 2014 16:12:32 -0300 Subject: [PATCH 4/7] handle_inv tests --- .jshintrc | 1 - lib/PeerSync.js | 6 ++++++ lib/Sync.js | 8 ++++---- test/lib/PeerSync.js | 34 ++++++++++++++++++++++++++-------- 4 files changed, 36 insertions(+), 13 deletions(-) diff --git a/.jshintrc b/.jshintrc index 948f3b6..48c745a 100644 --- a/.jshintrc +++ b/.jshintrc @@ -32,7 +32,6 @@ "afterEach", "it", "inject", - "expect", "$", "io", "app", diff --git a/lib/PeerSync.js b/lib/PeerSync.js index fff731b..4704fde 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -111,6 +111,12 @@ function spec() { peerman.start(); }; + + PeerSync.prototype.close = function() { + this.sync.close(); + }; + + return PeerSync; } diff --git a/lib/Sync.js b/lib/Sync.js index a29a40b..e56cb6f 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -266,7 +266,7 @@ function spec() { this.rpc = new RpcClient(config.bitcoind); - if (!(opts && opts.skip_db_connection)) { + if (!(opts && opts.skip_db_connection) && !mongoose.connection) { mongoose.connect(config.db, {server: {auto_reconnect: true}} ); } this.opts = opts; @@ -364,8 +364,9 @@ function spec() { sync(); }, retry_secs * 1000); } - else - return next(err, that.block_count); + else { + return next(err, that.block_count); + } }); } @@ -377,7 +378,6 @@ function spec() { }; Sync.prototype.close = function() { - console.log("closing connection"); this.db.close(); }; return Sync; diff --git a/test/lib/PeerSync.js b/test/lib/PeerSync.js index 230a02f..15f3a58 100644 --- a/test/lib/PeerSync.js +++ b/test/lib/PeerSync.js @@ -1,21 +1,39 @@ 'use strict'; -var assert = require('assert'); +var chai = require('chai'), + expect = chai.expect, + sinon = require('sinon'); + var PeerSync = require('../../lib/PeerSync.js').class(); -describe('Unit testing PeerSync', function() { - var ps; +describe('PeerSync', function() { + var ps, inv_info; beforeEach(function() { ps = new PeerSync(); + ps.init(); + }); + afterEach(function(){ + ps.close(); }); describe('#init()', function() { it('should return with no errors', function() { - assert.doesNotThrow(function() { - ps.init(); - }); + var other_ps = new PeerSync(); + expect(other_ps.init.bind(other_ps)).not.to.throw(Error); + other_ps.close(); }); }); describe('#handle_inv()', function() { - it('should return with no errors'); - it('should call sendGetData'); + inv_info = { + message: {invs: []}, + conn: {sendGetData: sinon.spy()} + }; + it('should return with no errors', function(){ + expect(function() { + ps.handle_inv(inv_info); + }).not.to.throw(Error); + }); + it('should call sendGetData', function() { + ps.handle_inv(inv_info); + expect(inv_info.conn.calledOnce); + }); }); describe('#handle_tx()', function() { it('should call storeTxs'); From d51df9f0876ea77fd8a08bb1e1949dc98377163a Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 16 Jan 2014 16:47:06 -0300 Subject: [PATCH 5/7] handle_tx() tests --- lib/PeerSync.js | 1 - test/lib/PeerSync.js | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/PeerSync.js b/lib/PeerSync.js index 7f0d110..389c76c 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -55,7 +55,6 @@ function spec() { console.log('[p2p_sync] Handle inv for a ' + CoinConst.MSG.to_str(inv.type)); }); // TODO: should limit the invs to objects we haven't seen yet - console.log('getdata'); info.conn.sendGetData(invs); }; diff --git a/test/lib/PeerSync.js b/test/lib/PeerSync.js index f19aa50..f4b3562 100644 --- a/test/lib/PeerSync.js +++ b/test/lib/PeerSync.js @@ -35,7 +35,7 @@ describe('PeerSync', function() { }); it('should call sendGetData', function() { ps.handle_inv(inv_info); - expect(inv_info.conn.sendGetData.calledOnce).to.be.ok; + expect(inv_info.conn.sendGetData.calledTwice).to.be.ok; }); }); From aeb95aa427de085d10732860cb8ab27900854f7c Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 16 Jan 2014 17:00:11 -0300 Subject: [PATCH 6/7] revert conflicts --- lib/Sync.js | 317 ++++------------------------------------------------ 1 file changed, 20 insertions(+), 297 deletions(-) diff --git a/lib/Sync.js b/lib/Sync.js index e56cb6f..084a156 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -7,7 +7,6 @@ function spec() { var mongoose = require('mongoose'); var util = require('util'); var RpcClient = require('bitcore/RpcClient').class(); - var networks = require('bitcore/networks'); var async = require('async'); var config = require('../config/config'); var Block = require('../app/models/Block'); @@ -19,100 +18,8 @@ function spec() { function Sync(config) { this.tx_count = 0; - this.block_count= 0; - this.block_total= 0; - this.network = config.networkName === 'testnet' ? networks.testnet: networks.livenet; } - var progress_bar = function(string, current, total) { - console.log(util.format('\t%s %d/%d [%d%%]', string, current, total, parseInt(100 * current / total))); - }; - - Sync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) { - - var that = this; - - // recursion end. - if (!blockHash || (blockEnd && blockEnd == blockHash) ) { - console.log("Reach end:", blockHash, blockEnd); - return cb(); - } - - var existed = 0; - var blockInfo; - var blockObj; - - async.series([ - // Already got it? - function(c) { - Block.findOne({hash:blockHash}, function(err,block){ - if (err) { console.log(err); return c(err); }; - if (block) { - existed = 1; - blockObj = block; - } - - return c(); - }); - }, - //show some (inacurate) status - function(c) { - if (that.block_count++ % 1000 === 0) { - progress_bar('Historic sync status:', that.block_count, that.block_total); - } - return c(); - }, - //get Info from RPC - function(c) { - - // TODO: if we store prev/next, no need to go to RPC - // if (blockObj && blockObj.nextBlockHash) return c(); - - that.rpc.getBlock(blockHash, function(err, ret) { - if (err) return c(err); - - blockInfo = ret; - return c(); - }); - }, - //store it - function(c) { - if (existed) return c(); - - that.storeBlock(blockInfo.result, function(err, block) { - existed = err && err.toString().match(/E11000/); - if (err && ! existed) return c(err); - return c(); - }); - }, - /* TODO: Should Start to sync backwards? (this is for partial syncs) - function(c) { - - if (blockInfo.result.prevblockhash != current.blockHash) { - console.log("reorg?"); - opts.prev = 1; - } - return c(); - } - */ - ], - function (err){ - - if (err) - console.log("ERROR: @%s: %s [count: block_count: %d]", blockHash, err, that.block_count); - - if (blockInfo && blockInfo.result) { - if (opts.prev && blockInfo.result.previousblockhash) { - return that.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb); - } - - if (opts.next && blockInfo.result.nextblockhash) - return that.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb); - } - return cb(err); - }); - }; - Sync.prototype.storeBlock = function(block, cb) { var that = this; @@ -165,220 +72,36 @@ function spec() { isForward ? { next: 1 } : { prev: 1}, cb); }; - // This is not currently used. Transactions are represented by txid only - // in mongodb - Sync.prototype.syncTXs = function(cb) { - + Sync.prototype.init = function(opts, cb) { var that = this; - console.log('Syncing TXs...'); + that.opts = opts; - Transaction.find({ - blockhash: null - }, - function(err, txs) { - if (err) return cb(err); - - var read = 0; - var pull = 0; - var write = 0; - var total = txs.length; - console.log('\tneed to pull %d txs', total); - - if (!total) return cb(); - - async.each(txs, function(tx, next) { - if (!tx.txid) { - console.log('NO TXID skipping...', tx); - return next(); - } - - if (read++ % 1000 === 0) progress_bar('read', read, total); - - that.rpc.getRawTransaction(tx.txid, 1, function(err, txInfo) { - - if (pull++ % 1000 === 0) progress_bar('\tpull', pull, total); - - if (!err && txInfo) { - Transaction.update({ - txid: tx.txid - }, - txInfo.result, function(err) { - if (err) return next(err); - - if (write++ % 1000 === 0) progress_bar('\t\twrite', write, total); - - return next(); - }); - } - else return next(); - }); - }, - function(err) { - if (err) return cb(err); - return cb(err); - }); - }); - }; - - - // Not used - Sync.prototype.processTXs = function(reindex, cb) { - - var that = this; - - console.log('Syncing TXs...'); - - var filter = reindex ? {} : { processed: false } ; - - Transaction.find(filter, function(err, txs) { - if (err) return cb(err); - - var read = 0, - pull = 0, - proc = 0, - total = txs.length; - - console.log('\tneed to pull %d txs', total); - - if (!total) return cb(); - - - async.forEachLimit(txs, CONCURRENCY, function(tx, next) { - if (read++ % 1000 === 0) progress_bar('read', read, total); - - if (!tx.txid) { - console.log('NO TXID skipping...', tx); - return next(); - } - - // This will trigger an RPC call - Transaction.explodeTransactionItems( tx.txid, tx.time, function(err) { - if (proc++ % 1000 === 0) progress_bar('\tproc', pull, total); - next(err); - }); - }, - cb); - }); - }; - - Sync.prototype.init = function(opts) { - this.rpc = new RpcClient(config.bitcoind); - - - if (!(opts && opts.skip_db_connection) && !mongoose.connection) { + if (!(opts && opts.skip_db_connection)) { mongoose.connect(config.db, {server: {auto_reconnect: true}} ); - } - this.opts = opts; - this.db = mongoose.connection; - this.db.on('error', function(err) { - console.log('connection error:' + err); - mongoose.disconnect(); - }); + this.db = mongoose.connection; - this.db.on('disconnect', function(err) { - console.log('disconnect:' + err); - mongoose.connect(config.db, {server: {auto_reconnect: true}} ); - }); - - - }; - - Sync.prototype.import_history = function(opts, next) { - - var that = this; - - var retry_attemps = 100; - var retry_secs = 2; - - var block_best; - - this.db.once('open', function() { - async.series([ - function(cb) { - if (opts.destroy) { - console.log('Deleting Blocks...'); - that.db.collections.blocks.drop(cb); - } else { - return cb(); - } - }, - function(cb) { - if (opts.destroy) { - console.log('Deleting TXs...'); - that.db.collections.transactions.drop(cb); - } else { - return cb(); - } - }, - function(cb) { - if (opts.destroy) { - console.log('Deleting TXItems...'); - that.db.collections.transactionitems.drop(cb); - } else { - return cb(); - } - }, - function(cb) { - that.rpc.getInfo(function(err, res) { - if (err) cb(err); - - that.block_total = res.result.blocks; - return cb(); - }); - }, - function(cb) { - if (!opts.reverse) return cb(); - - that.rpc.getBestBlockHash(function(err, res) { - if (err) cb(err); - - block_best = res.result; - return cb(); - }); - }, - ], function(err) { - - - function sync() { - - var start, end, isForward; - - if (opts.reverse) { - start = block_best; - end = that.network.genesisBlock.hash.reverse().toString('hex'); - isForward = false; - } - else { - start = that.network.genesisBlock.hash.reverse().toString('hex'); - end = null; - isForward = true; - } - - that.syncBlocks(start, end, isForward, function(err) { - - if (err && err.message.match(/ECONNREFUSED/) && retry_attemps--){ - setTimeout(function() { - console.log("Retrying in %d secs ", retry_secs); - sync(); - }, retry_secs * 1000); - } - else { - return next(err, that.block_count); - } - }); - } - - if (!opts.skip_blocks) { - sync(); - } + this.db.on('error', function(err) { + console.log('connection error:' + err); + moogose.disconnect(); }); - }); + + this.db.on('disconnect', function(err) { + console.log('disconnect:' + err); + mongoose.connect(config.db, {server: {auto_reconnect: true}} ); + }); + + return that.db.once('open', cb); + } + else return cb(); }; Sync.prototype.close = function() { - this.db.close(); + if (!(this.opts && this.opts.skip_db_connection)) { + console.log("closing connection"); + this.db.close(); + } }; return Sync; } From 62d469be0cf123363e74f3a07d0937c7239bb0ed Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 16 Jan 2014 21:08:50 -0300 Subject: [PATCH 7/7] finished tests for PeerSync.js --- lib/PeerSync.js | 52 ++++++++++++++++++++++++++------------------ lib/Sync.js | 7 +++--- test/lib/PeerSync.js | 27 ++++++++++++++++++----- 3 files changed, 56 insertions(+), 30 deletions(-) diff --git a/lib/PeerSync.js b/lib/PeerSync.js index 389c76c..49526cf 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -14,22 +14,25 @@ function spec() { PeerSync.prototype.init = function(config, cb) { - + if (!config) config = {}; var that = this; var network = config && (config.network || 'testnet'); - that.peerdb = undefined; - that.sync = new Sync({ + this.verbose = config.verbose; + this.peerdb = undefined; + this.sync = new Sync({ networkName: network }); - that.sync.init(config, function() { - that.PeerManager = require('bitcore/PeerManager').createClass({ - config: { - network: network - } - }); - that.load_peers(); + + this.PeerManager = require('bitcore/PeerManager').createClass({ + config: { + network: network + } + }); + this.peerman = new this.PeerManager(); + this.load_peers(); + this.sync.init(config, function() { return cb(); }); }; @@ -50,9 +53,12 @@ function spec() { }; PeerSync.prototype.handle_inv = function(info) { + var self = this; var invs = info.message.invs; invs.forEach(function(inv) { - console.log('[p2p_sync] Handle inv for a ' + CoinConst.MSG.to_str(inv.type)); + if (self.verbose) { + console.log('[p2p_sync] Handle inv for a ' + CoinConst.MSG.to_str(inv.type)); + } }); // TODO: should limit the invs to objects we haven't seen yet info.conn.sendGetData(invs); @@ -60,19 +66,22 @@ function spec() { PeerSync.prototype.handle_tx = function(info) { var tx = info.message.tx.getStandardizedObject(); - console.log('[p2p_sync] Handle tx: ' + tx.hash); + if (this.verbose) { + console.log('[p2p_sync] Handle tx: ' + tx.hash); + } this.sync.storeTxs([tx.hash], null, function(err) { if (err) { - console.log('[p2p_sync] Error in handle TX: ' + err); + console.log('[p2p_sync] Error in handle TX: ' + JSON.stringify(err)); } }); }; PeerSync.prototype.handle_block = function(info) { - var self = this; var block = info.message.block; var blockHash = coinUtil.formatHashFull(block.calcHash()); - console.log('[p2p_sync] Handle block: ' + blockHash); + if (this.verbose) { + console.log('[p2p_sync] Handle block: ' + blockHash); + } var tx_hashes = block.txs.map(function(tx) { @@ -94,26 +103,27 @@ function spec() { PeerSync.prototype.handle_connected = function(data) { var peerman = data.pm; var peers_n = peerman.peers.length; - console.log('[p2p_sync] Connected to ' + peers_n + ' peer' + (peers_n !== 1 ? 's': '')); + if (this.verbose) { + console.log('[p2p_sync] Connected to ' + peers_n + ' peer' + (peers_n !== 1 ? 's': '')); + } }; PeerSync.prototype.run = function() { var self = this; - var peerman = new this.PeerManager(); this.peerdb.forEach(function(datum) { var peer = new Peer(datum.ipv4, datum.port); - peerman.addPeer(peer); + self.peerman.addPeer(peer); }); - peerman.on('connection', function(conn) { + this.peerman.on('connection', function(conn) { conn.on('inv', self.handle_inv.bind(self)); conn.on('block', self.handle_block.bind(self)); conn.on('tx', self.handle_tx.bind(self)); }); - peerman.on('connect', self.handle_connected.bind(self)); + this.peerman.on('connect', self.handle_connected.bind(self)); - peerman.start(); + this.peerman.start(); }; PeerSync.prototype.close = function() { diff --git a/lib/Sync.js b/lib/Sync.js index 084a156..eb34a48 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -78,13 +78,15 @@ function spec() { that.opts = opts; if (!(opts && opts.skip_db_connection)) { - mongoose.connect(config.db, {server: {auto_reconnect: true}} ); + if (!mongoose.connection) { + mongoose.connect(config.db, {server: {auto_reconnect: true}} ); + } this.db = mongoose.connection; this.db.on('error', function(err) { console.log('connection error:' + err); - moogose.disconnect(); + mongoose.disconnect(); }); this.db.on('disconnect', function(err) { @@ -99,7 +101,6 @@ function spec() { Sync.prototype.close = function() { if (!(this.opts && this.opts.skip_db_connection)) { - console.log("closing connection"); this.db.close(); } }; diff --git a/test/lib/PeerSync.js b/test/lib/PeerSync.js index f4b3562..c7c48e3 100644 --- a/test/lib/PeerSync.js +++ b/test/lib/PeerSync.js @@ -9,7 +9,7 @@ describe('PeerSync', function() { beforeEach(function() { ps = new PeerSync(); - ps.init(); + ps.init({verbose: false}); }); afterEach(function(){ ps.close(); @@ -42,22 +42,37 @@ describe('PeerSync', function() { describe('#handle_tx()', function() { var tx_info = { message: { tx: {getStandardizedObject: function(){ - return {hash: '00000000e3fe5b3b5416374d8d65560a0792a6da71546d67b00c9d37e8a4cf59'};}}} + return {hash: 'dac28b5c5e70c16942718f3a22438348c1b709e01d398795fce8fc455178b973'};}}} }; it('should call storeTxs', function(){ var spy = sinon.spy(ps.sync, 'storeTxs'); ps.handle_tx(tx_info); - expect(spy.calledOnce); + expect(spy.calledOnce).to.be.ok; }); }); describe('#handle_block()', function() { - it('should call storeBlock'); - it('should call storeTxs for each transaction'); + var block_info = { + message: { block: {calcHash: function(){ + return new Buffer('01234'); + }, txs: [{hash: new Buffer('cabafeca')}, {hash: new Buffer('bacacafe')}]}} + }; + it('should call storeBlock', function(){ + var spy = sinon.spy(ps.sync, 'storeBlock'); + ps.handle_block(block_info); + expect(spy.calledOnce).to.be.ok; + }); }); describe('#run()', function() { - it('should setup peerman'); + it('should setup peerman', function() { + var startSpy = sinon.spy(ps.peerman, 'start'); + var onSpy = sinon.spy(ps.peerman, 'on'); + ps.run(); + + expect(startSpy.called).to.be.ok; + expect(onSpy.called).to.be.ok; + }); }); });