From 6e8589e9b3c23761b9d06a0c36465368b65edada Mon Sep 17 00:00:00 2001 From: Gordon Hall Date: Sat, 19 Apr 2014 10:44:55 -0400 Subject: [PATCH 1/8] add extend dependency, use to extend defaults with config in PeerManager instead of overwiting all --- PeerManager.js | 4 +++- package.json | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/PeerManager.js b/PeerManager.js index 8e90317..4ee071e 100644 --- a/PeerManager.js +++ b/PeerManager.js @@ -1,5 +1,6 @@ var imports = require('soop').imports(); +var extend = imports.extend || require('extend'); var log = imports.log || require('./util/log'); var bitcoreDefaults = imports.config || require('./config'); var Connection = imports.Connection || require ('./Connection'); @@ -12,7 +13,8 @@ GetAdjustedTime = imports.GetAdjustedTime || function () { }; function PeerManager(config) { - this.config = config || bitcoreDefaults; + // extend defaults with config + this.config = extend(true, config || {}, bitcoreDefaults); this.active = false; this.timer = null; diff --git a/package.json b/package.json index 3bd9900..0592232 100644 --- a/package.json +++ b/package.json @@ -70,7 +70,8 @@ "socks5-client": "~0.3.6", "brfs": "=1.0.0", "chai": "=1.9.1", - "uglifyify": "=1.2.3" + "uglifyify": "=1.2.3", + "extend": "~1.2.1" }, "devDependencies": { "grunt-contrib-watch": "~0.5.3", From ba8542de62556e20caa04058821cf8c14629d95e Mon Sep 17 00:00:00 2001 From: Gordon Hall Date: Sat, 19 Apr 2014 11:33:26 -0400 Subject: [PATCH 2/8] add support for peer discovery to PeerManager and example of usage to examples/ --- PeerManager.js | 57 +++++++++++++++++++++++++++++++++++++++ examples/PeerDiscovery.js | 15 +++++++++++ networks.js | 18 +++++++++++-- 3 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 examples/PeerDiscovery.js diff --git a/PeerManager.js b/PeerManager.js index 4ee071e..47bdbe9 100644 --- a/PeerManager.js +++ b/PeerManager.js @@ -213,4 +213,61 @@ PeerManager.prototype.getActiveConnections = function () { return this.connections.slice(0); }; +PeerManager.prototype.discoverPeers = function(callback) { + var self = this; + var async = imports.async || require('async'); + var dns = imports.dns || require('dns'); + var networks = imports.networks || require('./networks')[this.config.network]; + var seeds = networks.dnsSeeds; + + // keep track of tried seeds and results + self.seeds = { + resolved: [], + failed: [], + results: {} + }; + + var dnsExecutor = seeds.map(function(seed) { + return function(done) { + // have we already resolved this seed? + if (~self.seeds.resolved.indexOf(seed)) { + // if so, just pass back cached peer list + return done(null, self.seeds.results[seed]); + } + // has this seed failed to resolve? + if (~self.seeds.resolved.indexOf(seed)) { + // if so, pass back empty results + return done(null, []); + } + // otherwise resolve the dns seed to get some peers + dns.resolve(seed, function(err, peers) { + if (err) { + self.seeds.failed.push(seed); + return done(null, []); + } + self.seeds.resolved.push(seed); + self.seeds.results[seed] = peers; + return done(null, peers); + }); + }; + }); + + // try resolving all seeds + async.parallel(dnsExecutor, function(err, results) { + var peers = []; + + // consolidate all resolved peers into one list + results.forEach(function(peerlist) { + peers = peers.concat(peerlist); + }); + + // transform that list into a list of Peer instances + peers = peers.map(function(ip) { + return new Peer(ip, networks.defaultClientPort); + }); + + callback(null, peers); + }); +}; + module.exports = require('soop')(PeerManager); diff --git a/examples/PeerDiscovery.js b/examples/PeerDiscovery.js new file mode 100644 index 0000000..b12cb3c --- /dev/null +++ b/examples/PeerDiscovery.js @@ -0,0 +1,15 @@ +var PeerManager = require('../PeerManager'); +var peerman = new PeerManager(); + +peerman.discoverPeers(function(err, peers) { + // we get an array of peer instances + console.log(peers); + // the peer manager caches the tried seeds and any results + console.log(peerman.seeds); + // we can use this array of peers to add to the manager + peers.forEach(function(p) { + peerman.addPeer(p); + }); + // then we can start the manager + peerman.start(); +}); diff --git a/networks.js b/networks.js index 27f60c4..787f719 100644 --- a/networks.js +++ b/networks.js @@ -19,7 +19,16 @@ exports.livenet = { prev_hash: buffertools.fill(new Buffer(32), 0), timestamp: 1231006505, bits: 486604799, - } + }, + dnsSeeds: [ + 'seed.bitcoin.sipa.be', + 'dnsseed.bluematt.me', + 'dnsseed.bitcoin.dashjr.org', + 'seed.bitcoinstats.com', + 'seed.bitnodes.io', + 'bitseed.xf2.org' + ], + defaultClientPort: 8333 }; exports.testnet = { @@ -39,5 +48,10 @@ exports.testnet = { prev_hash: buffertools.fill(new Buffer(32), 0), timestamp: 1296688602, bits: 486604799, - } + }, + dnsSeeds: [ + 'testnet-seed.bitcoin.petertodd.org', + 'testnet-seed.bluematt.me' + ], + defaultClientPort: 18333 }; From e82867007ec54c8a384202df42a9742f9045ed76 Mon Sep 17 00:00:00 2001 From: Gordon Hall Date: Sat, 19 Apr 2014 15:12:44 -0400 Subject: [PATCH 3/8] simplified example for peer discovery, added some debug output to discoverPeers(), made Peer reference the connection created from createConnection() --- examples/PeerDiscovery.js | 10 +++------- lib/Peer.js | 4 ++-- lib/PeerManager.js | 19 +++++++++++-------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/examples/PeerDiscovery.js b/examples/PeerDiscovery.js index 1663b2e..6440718 100644 --- a/examples/PeerDiscovery.js +++ b/examples/PeerDiscovery.js @@ -2,14 +2,10 @@ var PeerManager = require('../lib/PeerManager'); var peerman = new PeerManager(); peerman.discoverPeers(function(err, peers) { - // we get an array of peer instances - console.log(peers); - // the peer manager caches the tried seeds and any results - console.log(peerman.seeds); // we can use this array of peers to add to the manager - peers.forEach(function(p) { - peerman.addPeer(p); - }); + // but let's limit it to 6 connections for this example + var p = 0; + do { peerman.addPeer(peers[p]); p++; } while (p <= 6); // then we can start the manager peerman.start(); }); diff --git a/lib/Peer.js b/lib/Peer.js index 05be302..a73d66e 100644 --- a/lib/Peer.js +++ b/lib/Peer.js @@ -34,8 +34,8 @@ function Peer(host, port, services) { Peer.IPV6_IPV4_PADDING = new Buffer([0,0,0,0,0,0,0,0,0,0,255,255]); Peer.prototype.createConnection = function () { - var c = Net.createConnection(this.port, this.host); - return c; + this.connection = Net.createConnection(this.port, this.host); + return this.connection; }; Peer.prototype.getHostAsBuffer = function () { diff --git a/lib/PeerManager.js b/lib/PeerManager.js index a20774d..59182f4 100644 --- a/lib/PeerManager.js +++ b/lib/PeerManager.js @@ -26,6 +26,13 @@ function PeerManager(config) { this.interval = 5000; this.minConnections = 8; this.minKnownPeers = 10; + + // keep track of tried seeds and results + this.seeds = { + resolved: [], + failed: [], + results: {} + }; } PeerManager.parent = imports.parent || require('events').EventEmitter; @@ -219,13 +226,6 @@ PeerManager.prototype.discoverPeers = function(callback) { var networks = imports.networks || require('../networks')[this.config.network]; var seeds = networks.dnsSeeds; - // keep track of tried seeds and results - self.seeds = { - resolved: [], - failed: [], - results: {} - }; - var dnsExecutor = seeds.map(function(seed) { return function(done) { // have we already resolved this seed? @@ -234,16 +234,19 @@ PeerManager.prototype.discoverPeers = function(callback) { return done(null, self.seeds.results[seed]); } // has this seed failed to resolve? - if (~self.seeds.resolved.indexOf(seed)) { + if (~self.seeds.failed.indexOf(seed)) { // if so, pass back empty results return done(null, []); } // otherwise resolve the dns seed to get some peers + log.info('resolving dns seed '+ seed); dns.resolve(seed, function(err, peers) { if (err) { + log.err('failed to resolve dns seed '+ seed, err); self.seeds.failed.push(seed); return done(null, []); } + log.info('found '+ peers.length + ' peers from ' + seed); self.seeds.resolved.push(seed); self.seeds.results[seed] = peers; return done(null, peers); From 9cbab7f67ef98082399632829f3f0fbf60012950 Mon Sep 17 00:00:00 2001 From: Gordon Hall Date: Sat, 19 Apr 2014 15:26:34 -0400 Subject: [PATCH 4/8] fix networks var in discoverPeers() to match what might be imported --- lib/PeerManager.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/PeerManager.js b/lib/PeerManager.js index 59182f4..5e63188 100644 --- a/lib/PeerManager.js +++ b/lib/PeerManager.js @@ -223,8 +223,8 @@ PeerManager.prototype.discoverPeers = function(callback) { var self = this; var async = imports.async || require('async'); var dns = imports.dns || require('dns'); - var networks = imports.networks || require('../networks')[this.config.network]; - var seeds = networks.dnsSeeds; + var networks = imports.networks || require('../networks'); + var seeds = networks[self.config.network].dnsSeeds; var dnsExecutor = seeds.map(function(seed) { return function(done) { From f877b3e563c66ad2ab9b59f83942c055120241b1 Mon Sep 17 00:00:00 2001 From: Gordon Hall Date: Sat, 19 Apr 2014 15:32:20 -0400 Subject: [PATCH 5/8] some aesthetic changes --- lib/PeerManager.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/PeerManager.js b/lib/PeerManager.js index 5e63188..d18bd98 100644 --- a/lib/PeerManager.js +++ b/lib/PeerManager.js @@ -233,11 +233,13 @@ PeerManager.prototype.discoverPeers = function(callback) { // if so, just pass back cached peer list return done(null, self.seeds.results[seed]); } + // has this seed failed to resolve? if (~self.seeds.failed.indexOf(seed)) { // if so, pass back empty results return done(null, []); } + // otherwise resolve the dns seed to get some peers log.info('resolving dns seed '+ seed); dns.resolve(seed, function(err, peers) { @@ -251,6 +253,7 @@ PeerManager.prototype.discoverPeers = function(callback) { self.seeds.results[seed] = peers; return done(null, peers); }); + }; }); From 8b17591661dadbb02e3e9ab252482cc3d5d41617 Mon Sep 17 00:00:00 2001 From: Gordon Hall Date: Sat, 19 Apr 2014 16:46:00 -0400 Subject: [PATCH 6/8] fix example --- examples/PeerDiscovery.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/PeerDiscovery.js b/examples/PeerDiscovery.js index 6440718..2662698 100644 --- a/examples/PeerDiscovery.js +++ b/examples/PeerDiscovery.js @@ -5,7 +5,7 @@ peerman.discoverPeers(function(err, peers) { // we can use this array of peers to add to the manager // but let's limit it to 6 connections for this example var p = 0; - do { peerman.addPeer(peers[p]); p++; } while (p <= 6); + do { peerman.addPeer(peers[p]); p++; } while (p < 6); // then we can start the manager peerman.start(); }); From def036b79f27aba66d6fc2517b181cc3bb3ab481 Mon Sep 17 00:00:00 2001 From: Gordon Hall Date: Sun, 20 Apr 2014 14:04:52 -0400 Subject: [PATCH 7/8] setup connection pool and auto handling of bad seeds --- examples/PeerDiscovery.js | 9 +------ lib/PeerManager.js | 52 ++++++++++++++++++++++++++++----------- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/examples/PeerDiscovery.js b/examples/PeerDiscovery.js index 2662698..dfbec65 100644 --- a/examples/PeerDiscovery.js +++ b/examples/PeerDiscovery.js @@ -1,11 +1,4 @@ var PeerManager = require('../lib/PeerManager'); var peerman = new PeerManager(); -peerman.discoverPeers(function(err, peers) { - // we can use this array of peers to add to the manager - // but let's limit it to 6 connections for this example - var p = 0; - do { peerman.addPeer(peers[p]); p++; } while (p < 6); - // then we can start the manager - peerman.start(); -}); +peerman.discover({ limit: 6 }).start(); diff --git a/lib/PeerManager.js b/lib/PeerManager.js index d18bd98..85cbb22 100644 --- a/lib/PeerManager.js +++ b/lib/PeerManager.js @@ -3,7 +3,6 @@ var extend = imports.extend || require('extend'); var log = imports.log || require('../util/log'); var bitcoreDefaults = imports.config || require('../config'); var Connection = imports.Connection || require ('./Connection'); - var Peer = imports.Peer || require('./Peer'); GetAdjustedTime = imports.GetAdjustedTime || function () { @@ -18,6 +17,7 @@ function PeerManager(config) { this.timer = null; this.peers = []; + this.pool = []; this.connections = []; this.isConnected = false; this.peerDiscovery = false; @@ -30,8 +30,7 @@ function PeerManager(config) { // keep track of tried seeds and results this.seeds = { resolved: [], - failed: [], - results: {} + failed: [] }; } @@ -68,6 +67,13 @@ PeerManager.prototype.addPeer = function(peer, port) { } }; +PeerManager.prototype.removePeer = function(peer) { + var index = this.peers.indexOf(peer); + var exists = !!~index; + if (exists) this.peers.splice(index, 1); + return exists; +}; + PeerManager.prototype.checkStatus = function checkStatus() { // Make sure we are connected to all forcePeers if(this.peers.length) { @@ -91,7 +97,7 @@ PeerManager.prototype.checkStatus = function checkStatus() { }; PeerManager.prototype.connectTo = function(peer) { - log.info('connecting to '+peer); + log.info('connecting to ' + peer); try { return this.addConnection(peer.createConnection(), peer); } catch (e) { @@ -181,10 +187,16 @@ PeerManager.prototype.handleError = function(e) { }; PeerManager.prototype.handleDisconnect = function(e) { - log.info('disconnected from peer '+e.peer); + log.info('disconnected from peer ' + e.peer); var i = this.connections.indexOf(e.conn); if(i != -1) this.connections.splice(i, 1); + this.removePeer(e.peer); + log.info('replacing peer with one from the pool of: ' + this.pool.length); + if (this.pool.length) { + this.addPeer(this.pool.pop()); + } + if(!this.connections.length) { this.emit('netDisconnected'); this.isConnected = false; @@ -219,13 +231,15 @@ PeerManager.prototype.getActiveConnections = function () { return this.connections.slice(0); }; -PeerManager.prototype.discoverPeers = function(callback) { +PeerManager.prototype.discover = function(options, callback) { var self = this; var async = imports.async || require('async'); var dns = imports.dns || require('dns'); var networks = imports.networks || require('../networks'); var seeds = networks[self.config.network].dnsSeeds; + self.limit = options.limit || 12; + var dnsExecutor = seeds.map(function(seed) { return function(done) { // have we already resolved this seed? @@ -240,17 +254,30 @@ PeerManager.prototype.discoverPeers = function(callback) { return done(null, []); } - // otherwise resolve the dns seed to get some peers log.info('resolving dns seed '+ seed); + dns.resolve(seed, function(err, peers) { if (err) { log.err('failed to resolve dns seed '+ seed, err); self.seeds.failed.push(seed); return done(null, []); } + log.info('found '+ peers.length + ' peers from ' + seed); self.seeds.resolved.push(seed); - self.seeds.results[seed] = peers; + + // transform that list into a list of Peer instances + peers = peers.map(function(ip) { + return new Peer(ip, networks.defaultClientPort); + }); + + peers.forEach(function(p) { + if (self.peers.length < self.limit) self.addPeer(p); + else self.pool.push(p); + }); + + self.emit('peers', peers); + return done(null, peers); }); @@ -266,13 +293,10 @@ PeerManager.prototype.discoverPeers = function(callback) { peers = peers.concat(peerlist); }); - // transform that list into a list of Peer instances - peers = peers.map(function(ip) { - return new Peer(ip, networks.defaultClientPort); - }); - - callback(null, peers); + if (typeof callback === 'function') callback(null, peers); }); + + return self; }; module.exports = require('soop')(PeerManager); From 9934b29effef28461beb80bda3697676c3ebc7c0 Mon Sep 17 00:00:00 2001 From: Gordon Hall Date: Sun, 20 Apr 2014 14:31:07 -0400 Subject: [PATCH 8/8] update example, added log statements to status check in peer manager for showing how many good peers there are --- examples/PeerDiscovery.js | 2 +- lib/PeerManager.js | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/examples/PeerDiscovery.js b/examples/PeerDiscovery.js index dfbec65..b070c29 100644 --- a/examples/PeerDiscovery.js +++ b/examples/PeerDiscovery.js @@ -1,4 +1,4 @@ var PeerManager = require('../lib/PeerManager'); var peerman = new PeerManager(); -peerman.discover({ limit: 6 }).start(); +peerman.discover({ limit: 12 }).start(); diff --git a/lib/PeerManager.js b/lib/PeerManager.js index 85cbb22..c9f6924 100644 --- a/lib/PeerManager.js +++ b/lib/PeerManager.js @@ -90,6 +90,13 @@ PeerManager.prototype.checkStatus = function checkStatus() { } }); + // for debug purposes, print how many of our peers are actually connected + var connected = 0 + this.peers.forEach(function(p) { + if (p.connection && !p.connection._connecting) connected++ + }); + log.info(connected + ' of ' + this.peers.length + ' peers connected'); + Object.keys(peerIndex).forEach(function(i) { this.connectTo(peerIndex[i]); }.bind(this)); @@ -192,8 +199,8 @@ PeerManager.prototype.handleDisconnect = function(e) { if(i != -1) this.connections.splice(i, 1); this.removePeer(e.peer); - log.info('replacing peer with one from the pool of: ' + this.pool.length); if (this.pool.length) { + log.info('replacing peer using the pool of ' + this.pool.length + ' seeds'); this.addPeer(this.pool.pop()); }