diff --git a/Gruntfile.js b/Gruntfile.js index 743d9845..ab534a16 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -39,6 +39,13 @@ module.exports = function(grunt) { } } }, + mochaTest: { + options: { + reporter: 'spec', + }, + src: ['test/*.js'] + }, + nodemon: { dev: { options: { @@ -61,13 +68,6 @@ module.exports = function(grunt) { logConcurrentOutput: true } }, - mochaTest: { - options: { - reporter: 'spec', - require: 'server.js' - }, - src: ['test/*.js'] - }, env: { test: { NODE_ENV: 'test' @@ -87,7 +87,7 @@ module.exports = function(grunt) { grunt.option('force', true); //Default task(s). - grunt.registerTask('default', ['jshint', 'concurrent']); + grunt.registerTask('default', ['jshint','concurrent']); //Test task. grunt.registerTask('test', ['env:test', 'mochaTest']); diff --git a/Sync.js b/Sync.js index df300608..7269c8ff 100644 --- a/Sync.js +++ b/Sync.js @@ -6,9 +6,11 @@ function spec(b) { var RpcClient = require('bitcore/RpcClient').class(); var networks = require('bitcore/networks'); + var async = require('async'); var config = require('./config/config'); var Block = require('./app/models/Block'); + var Transaction=require('./app/models/Transaction'); function Sync(config) { this.network = config.networkName == 'testnet' ? networks.testnet : networks.livenet; @@ -39,8 +41,13 @@ function spec(b) { return cb(err); } - return that.getNextBlock(blockInfo.result.nextblockhash, cb); - + if (inBlock) { + inBlock.explodeTransactions(function (err) { + return that.getNextBlock(blockInfo.result.nextblockhash, cb); + }); + } + else + return that.getNextBlock(blockInfo.result.nextblockhash, cb); }); }); } @@ -70,7 +77,7 @@ function spec(b) { } - Sync.prototype.start = function (reindex, cb) { + Sync.prototype.start = function (opts, next) { mongoose.connect(config.db); @@ -81,15 +88,35 @@ function spec(b) { db.on('error', console.error.bind(console, 'connection error:')); - db.once('open', function callback () { + db.once('open', function (){ - that.syncBlocks(reindex, function(err) { - if (err) { - return cb(err); + async.series([ + function(cb){ + if (opts.destroy) { + console.log("Deleting Blocks..."); + Block.remove().exec(cb); + } + }, + function(cb){ + if (opts.destroy) { + console.log("Deleting TXs..."); + Transaction.remove().exec(cb); + } + }, + function(cb) { + that.syncBlocks(opts.reindex, function(err) { + if (err) { + return cb(err); + } + db.close(); + return cb(); + }); } - mongoose.connection.close(); - return cb(); + ], + function(err) { + if (err) return next(er); + return next(); }); }); } diff --git a/app/models/Block.js b/app/models/Block.js index 53272b52..511cd7cc 100644 --- a/app/models/Block.js +++ b/app/models/Block.js @@ -3,9 +3,11 @@ /** * Module dependencies. */ -var mongoose = require('mongoose'), - Schema = mongoose.Schema; +var mongoose = require('mongoose'), + Schema = mongoose.Schema; +var async = require('async'); +var Transaction = require('./Transaction'); /** * Block Schema @@ -38,6 +40,32 @@ var BlockSchema = new Schema({ }, }); +BlockSchema.methods.explodeTransactions = function(next) { + + // console.log('exploding %s', this.hash, typeof this.tx); + + async.forEach( this.tx, + function(tx, callback) { + // console.log('procesing TX %s', tx); + Transaction.create({ hash: tx }, function(err) { + if (err && ! err.toString().match(/E11000/)) { + return callback(); + } + if (err) { + + return callback(err); + } + return callback(); + + }); + }, + function(err) { + if (err) return next(err); + return next(); + } + ); +}; + /** * Validations */ @@ -65,4 +93,5 @@ BlockSchema.statics.fromHash = function(hash, cb) { }).exec(cb); }; + module.exports = mongoose.model('Block', BlockSchema); diff --git a/app/models/Transaction.js b/app/models/Transaction.js new file mode 100644 index 00000000..ba8678ec --- /dev/null +++ b/app/models/Transaction.js @@ -0,0 +1,41 @@ +'use strict'; + +/** + * Module dependencies. + */ +var mongoose = require('mongoose'), + Schema = mongoose.Schema; + + +/** + */ +var TransactionSchema = new Schema({ + hash: { + type: String, + index: true, + unique: true, + }, + parsed: { + type: Boolean, + default: false, + }, +}); + +/** + * Statics + */ + +TransactionSchema.statics.load = function(id, cb) { + this.findOne({ + _id: id + }).exec(cb); +}; + + +TransactionSchema.statics.fromHash = function(hash, cb) { + this.findOne({ + hash: hash, + }).exec(cb); +}; + +module.exports = mongoose.model('Transaction', TransactionSchema); diff --git a/package.json b/package.json index 51a566e7..1dd908e4 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "postinstall": "node node_modules/bower/bin/bower install" }, "dependencies": { + "async": "*", "classtool": "*", "commander": "*", "express": "~3.4.7", diff --git a/util/sync.js b/util/sync.js index f1be0a85..9a2bbbf8 100755 --- a/util/sync.js +++ b/util/sync.js @@ -6,97 +6,30 @@ require('buffertools').extend(); var SYNC_VERSION = '0.1'; var program = require('commander'); -var util = require('util'); -var RpcClient = require('../node_modules/bitcore/RpcClient').class(); -var networks = require('../node_modules/bitcore/networks'); - -var Block = require('../app/models/Block'); -var config = require('../config/config'); -var mongoose = require('mongoose'); +var Sync = require('../Sync').class(); program .version(SYNC_VERSION) - .option('-N --network [livenet]', 'Set bitcoin network [livenet]', 'livenet') + .option('-N --network [livenet]', 'Set bitcoin network [testnet]', 'testnet') .option('-R --reindex', 'Force reindexing', '0') + .option('-D --destroy', 'Remove current DB', '0') .parse(process.argv); -var networkName = program.network; -var network = networkName == 'testnet' ? networks.testnet : networks.livenet; +var sync = new Sync({ networkName: program.network }); +if (program.remove) { +} -mongoose.connect(config.db); -var db = mongoose.connection; -var rpc = new RpcClient(config.bitcoind); - - -db.on('error', console.error.bind(console, 'connection error:')); - -db.once('open', function callback () { - - syncBlocks(network, program.reindex, function(err) { - if (err) { - console.log(err); - } - mongoose.connection.close(); - }); +sync.start({ + reindex: program.reindex, + destroy: program.destroy, +}, function(err){ + if (err) { + console.log(err); + } + else { + console.log('Done!'); + } }); - - - -function getNextBlock(blockHash,cb) { - - if ( !blockHash ) { - console.log("done"); - return cb(); - } - - rpc.getBlock(blockHash, function(err, blockInfo) { - if (err) { - return cb(err); - } - - if ( ! ( blockInfo.result.height % 1000) ) { - var h = blockInfo.result.height, - d = blockInfo.result.confirmations; - console.log( util.format("Height: %d/%d [%d%%]", h, d, 100*h/(h+d))); - } - - Block.create( blockInfo.result, function(err, inBlock) { - - // E11000 => already exists - if (err && ! err.toString().match(/E11000/)) { - return cb(err); - } - - return getNextBlock(blockInfo.result.nextblockhash, cb); - - }); - }); - -} - -function syncBlocks(network, reindex, cb) { - - var genesisHash = network.genesisBlock.hash.reverse().toString('hex'); - - if (reindex) - return getNextBlock(genesisHash, cb); - - - Block.findOne({}, {}, { sort: { 'confirmations' : 1 } }, function(err, block) { - if (err) return cb(err); - - var nextHash = - block && block.hash - ? block.hash - : genesisHash - ; - - - console.log('Starting at hash: ' + nextHash); - return getNextBlock(nextHash, cb); - }); -} -