diff --git a/dev-util/read_block.js b/dev-util/read_block.js new file mode 100755 index 0000000..99fd0d8 --- /dev/null +++ b/dev-util/read_block.js @@ -0,0 +1,25 @@ +#!/usr/bin/env node +'use strict'; + +process.env.NODE_ENV = process.env.NODE_ENV || 'development'; + +var assert = require('assert'), + config = require('../config/config'), + BlockExtractor = require('../lib/BlockExtractor').class(), + networks = require('bitcore/networks'), + util = require('bitcore/util/util'); + + var be = new BlockExtractor(config.bitcoind.dataDir, config.network); + var network = config.network === 'testnet' ? networks.testnet: networks.livenet; +// console.log('[read_block.js.13]', be.nextFile() ); + + var c=0; + while (c++ < 100) { + be.getNextBlock(function(err, b) { + console.log('[read_block.js.14]',err, c, b?util.formatHashAlt(b.hash):''); //TODO + }); + } + + + + diff --git a/lib/BlockExtractor.js b/lib/BlockExtractor.js new file mode 100644 index 0000000..5f1b7d3 --- /dev/null +++ b/lib/BlockExtractor.js @@ -0,0 +1,158 @@ +'use strict'; + +require('classtool'); + +function spec() { + var Block = require('bitcore/block').class(), + networks = require('bitcore/networks'), + Parser = require('bitcore/util/BinaryParser').class(), + fs = require('fs'), + Buffer = require('buffer').Buffer, + glob = require('glob'), + async = require('async'); + + function BlockExtractor(dataDir, network) { + + var self = this; + var path = dataDir + '/blocks/blk*.dat'; + + self.dataDir = dataDir; + self.files = glob.sync(path); + self.nfiles = self.files.length; + + if (self.nfiles === 0) + throw new Error('Could not find block files at: ' + path); + + self.currentFileIndex = 0; + self.isCurrentRead = false; + self.currentBuffer = null; + self.currentParser = null; + self.network = network === 'testnet' ? networks.testnet: networks.livenet; + self.magic = self.network.magic.toString('hex'); + } + + BlockExtractor.prototype.currentFile = function() { + var self = this; + + return self.files[self.currentFileIndex]; + }; + + + BlockExtractor.prototype.nextFile = function() { + var self = this; + + if (self.currentFileIndex < 0) return false; + + var ret = true; + + self.isCurrentRead = false; + self.currentBuffer = null; + self.currentParser = null; + + if (self.currentFileIndex < self.nfiles - 1) { + self.currentFileIndex++; + } + else { + self.currentFileIndex=-1; + ret = false; + } + return ret; + }; + + BlockExtractor.prototype.readCurrentFileSync = function() { + var self = this; + + if (self.currentFileIndex < 0 || self.isCurrentRead) return; + + + self.isCurrentRead = true; + + var fname = self.currentFile(); + if (!fname) return; + + + var stats = fs.statSync(fname); + + var size = stats.size; + + console.log('Reading Blockfile %s [%d MB]', + fname, parseInt(size/1024/1024)); + + var fd = fs.openSync(fname, 'r'); + +// if (status) return cb(new Error(status.message)); + + var buffer = new Buffer(size); + + var num = fs.readSync(fd, buffer, 0, size, 0); + + self.currentBuffer = buffer; + self.currentParser = new Parser(buffer); + }; + + + + BlockExtractor.prototype.getNextBlock = function(cb) { + var self = this; + + var b; + var magic; + async.series([ + function (a_cb) { + + async.whilst( + function() { + return (!magic); + }, + function(w_cb) { + + self.readCurrentFileSync(); + + if (self.currentFileIndex < 0) return cb(); + + + magic = self.currentParser ? self.currentParser.buffer(4).toString('hex') + : null ; + + if (!self.currentParser || self.currentParser.eof()) { + magic = null; + if (self.nextFile()) { + console.log('Moving forward to file:' + self.currentFile() ); + return w_cb(); + } + else { + console.log('Finished all files'); + return cb(); + } + } + else { + return w_cb(); + } + }, a_cb); + }, + function (a_cb) { + if (magic !== self.magic) { + var e = new Error('CRITICAL ERROR: Magic number mismatch: ' + + magic + '!=' + self.magic); + return a_cb(e); + } + + // spacer? + self.currentParser.word32le(); + return a_cb(); + }, + function (a_cb) { + b = new Block(); + b.parse(self.currentParser); + b.getHash(); + return a_cb(); + }, + ], function(err) { + return cb(err,b); + }); + }; + + return BlockExtractor; +} +module.defineClass(spec); + diff --git a/test/integration/blockExtractor.js b/test/integration/blockExtractor.js new file mode 100644 index 0000000..e7ff291 --- /dev/null +++ b/test/integration/blockExtractor.js @@ -0,0 +1,72 @@ +#!/usr/bin/env node +'use strict'; + +process.env.NODE_ENV = process.env.NODE_ENV || 'development'; + + + +var assert = require('assert'), + config = require('../../config/config'), + BlockExtractor = require('../../lib/BlockExtractor').class(), + networks = require('bitcore/networks'), + util = require('bitcore/util/util'); + +//var txItemsValid = JSON.parse(fs.readFileSync('test/model/txitems.json')); + +describe('TransactionOut', function(){ + + var be = new BlockExtractor(config.bitcoind.dataDir, config.network); + + var network = config.network === 'testnet' ? networks.testnet: networks.livenet; + + it('should glob block files ', function(done) { + assert(be.files.length>0); + done(); + }); + + var lastTs; + + it('should read genesis block ', function(done) { + be.getNextBlock(function(err,b) { + assert(!err); + var genesisHashReversed = new Buffer(32); + network.genesisBlock.hash.copy(genesisHashReversed); + var genesis = util.formatHashFull(network.genesisBlock.hash); + + assert.equal(util.formatHashFull(b.hash),genesis); + assert.equal(b.nounce,network.genesisBlock.nounce); + assert.equal(b.timestamp,network.genesisBlock.timestamp); + assert.equal(b.merkle_root.toString('hex'),network.genesisBlock.merkle_root.toString('hex')); + + lastTs = b.timestamp; + done(); + }); + }); + + it('should read next testnet block ', function(done) { + be.getNextBlock(function(err,b) { + assert(!err); + assert(b.timestamp > lastTs, 'timestamp > genesis_ts'); + done(); + }); + }); + + it('should read 100000 blocks with no error ', function(done) { + + var i=0; + while(i++<100000) { + be.getNextBlock(function(err,b) { + assert(!err,err); + assert(lastTs < b.timestamp, 'genesisTS < b.timestamp: ' + lastTs + '<' + b.timestamp + ":" + i); + if(i % 1000 === 1) process.stdout.write('.'); + if(i === 100000) done(); + }); + } + }); + + + +}); + + +