blockExtractor class to read blocks from bitcoind .dat files

dev-util/read_block.js

@@ -0,0 +1,25 @@
+#!/usr/bin/env node
+'use strict';
+
+process.env.NODE_ENV = process.env.NODE_ENV || 'development';
+
+var assert        = require('assert'),
+  config          = require('../config/config'),
+  BlockExtractor  = require('../lib/BlockExtractor').class(),
+  networks        = require('bitcore/networks'),
+  util            = require('bitcore/util/util');
+
+  var be = new BlockExtractor(config.bitcoind.dataDir, config.network);
+  var network = config.network === 'testnet' ? networks.testnet: networks.livenet;
+//  console.log('[read_block.js.13]', be.nextFile() );
+
+  var c=0;
+  while (c++ < 100) {
+    be.getNextBlock(function(err, b) {
+      console.log('[read_block.js.14]',err, c,  b?util.formatHashAlt(b.hash):''); //TODO
+    });
+  }
+
+
+
+

lib/BlockExtractor.js

@@ -0,0 +1,158 @@
+'use strict';
+
+require('classtool');
+
+function spec() {
+  var Block = require('bitcore/block').class(),
+   networks = require('bitcore/networks'),
+   Parser   = require('bitcore/util/BinaryParser').class(),
+   fs       = require('fs'),
+   Buffer   = require('buffer').Buffer,
+   glob     = require('glob'),
+   async    = require('async');
+
+  function BlockExtractor(dataDir, network) {
+
+    var self = this;
+    var path = dataDir + '/blocks/blk*.dat';
+
+    self.dataDir = dataDir;
+    self.files   = glob.sync(path);
+    self.nfiles  = self.files.length;
+
+    if (self.nfiles === 0)
+      throw new Error('Could not find block files at: ' + path);
+
+    self.currentFileIndex = 0;
+    self.isCurrentRead    = false;
+    self.currentBuffer    = null;
+    self.currentParser    = null;
+    self.network = network === 'testnet' ? networks.testnet: networks.livenet;
+    self.magic   = self.network.magic.toString('hex');
+  }
+
+  BlockExtractor.prototype.currentFile = function() {
+    var self = this;
+
+    return self.files[self.currentFileIndex];
+  };
+
+
+  BlockExtractor.prototype.nextFile = function() {
+    var self = this;
+
+    if (self.currentFileIndex < 0) return false;
+
+    var ret  = true;
+
+    self.isCurrentRead = false;
+    self.currentBuffer = null;
+    self.currentParser = null;
+
+    if (self.currentFileIndex < self.nfiles - 1) {
+      self.currentFileIndex++;
+    }
+    else {
+      self.currentFileIndex=-1;
+      ret = false;
+    }
+    return ret;
+  };
+
+  BlockExtractor.prototype.readCurrentFileSync = function() {
+    var self = this;
+
+    if (self.currentFileIndex < 0 || self.isCurrentRead) return;
+
+
+    self.isCurrentRead = true;
+
+    var fname = self.currentFile();
+    if (!fname) return;
+
+
+    var stats = fs.statSync(fname);
+
+    var size = stats.size;
+
+    console.log('Reading Blockfile %s [%d MB]',
+              fname, parseInt(size/1024/1024));
+
+    var fd = fs.openSync(fname, 'r');
+
+//    if (status) return cb(new Error(status.message));
+
+    var buffer = new Buffer(size);
+
+    var num = fs.readSync(fd, buffer, 0, size, 0);
+
+    self.currentBuffer = buffer;
+    self.currentParser = new Parser(buffer);
+  };
+
+
+
+  BlockExtractor.prototype.getNextBlock = function(cb) {
+    var self = this;
+
+    var b;
+    var magic;
+    async.series([
+      function (a_cb) {
+  
+        async.whilst(
+          function() {
+            return (!magic);
+          },
+          function(w_cb) {
+
+            self.readCurrentFileSync();
+
+            if (self.currentFileIndex < 0) return cb();
+
+
+            magic = self.currentParser ? self.currentParser.buffer(4).toString('hex')
+                        : null ;
+
+            if (!self.currentParser || self.currentParser.eof()) {
+              magic = null;
+              if (self.nextFile()) {
+                console.log('Moving forward to file:' + self.currentFile() );
+                return w_cb();
+              }
+              else {
+                console.log('Finished all files');
+                return cb();
+              }
+            }
+            else {
+              return w_cb();
+            }
+          }, a_cb);
+      },
+      function (a_cb) {
+        if (magic !== self.magic) {
+          var e = new Error('CRITICAL ERROR: Magic number mismatch: ' +
+                    magic + '!=' + self.magic);
+          return a_cb(e);
+        }
+
+        // spacer?
+        self.currentParser.word32le();
+        return a_cb();
+      },
+      function (a_cb) {
+        b = new Block();
+        b.parse(self.currentParser);
+        b.getHash();
+        return a_cb();
+      },
+    ], function(err) {
+      return cb(err,b);
+    });
+  };
+
+  return BlockExtractor;
+}
+module.defineClass(spec);
+

test/integration/blockExtractor.js

@@ -0,0 +1,72 @@
+#!/usr/bin/env node
+'use strict';
+
+process.env.NODE_ENV = process.env.NODE_ENV || 'development';
+
+
+
+var assert        = require('assert'),
+  config          = require('../../config/config'),
+  BlockExtractor  = require('../../lib/BlockExtractor').class(),
+  networks        = require('bitcore/networks'),
+  util            =  require('bitcore/util/util');
+
+//var txItemsValid = JSON.parse(fs.readFileSync('test/model/txitems.json'));
+
+describe('TransactionOut', function(){
+
+  var be = new BlockExtractor(config.bitcoind.dataDir, config.network);
+
+  var network = config.network === 'testnet' ? networks.testnet: networks.livenet;
+
+  it('should glob block files ', function(done) {
+    assert(be.files.length>0);
+    done();
+  });
+
+  var lastTs;
+
+  it('should read genesis block ', function(done) {
+    be.getNextBlock(function(err,b) {
+      assert(!err);
+      var genesisHashReversed = new Buffer(32);
+      network.genesisBlock.hash.copy(genesisHashReversed);
+      var genesis = util.formatHashFull(network.genesisBlock.hash);
+
+      assert.equal(util.formatHashFull(b.hash),genesis);
+      assert.equal(b.nounce,network.genesisBlock.nounce);
+      assert.equal(b.timestamp,network.genesisBlock.timestamp);
+      assert.equal(b.merkle_root.toString('hex'),network.genesisBlock.merkle_root.toString('hex'));
+
+      lastTs = b.timestamp;
+      done();
+    });
+  });
+
+  it('should read next testnet block ', function(done) {
+    be.getNextBlock(function(err,b) {
+      assert(!err);
+      assert(b.timestamp > lastTs, 'timestamp > genesis_ts');
+      done();
+    });
+  });
+
+  it('should read 100000 blocks with no error ', function(done) {
+
+    var i=0;
+    while(i++<100000) {
+      be.getNextBlock(function(err,b) {
+        assert(!err,err);
+        assert(lastTs < b.timestamp, 'genesisTS < b.timestamp: ' + lastTs + '<' + b.timestamp + ":" + i);
+        if(i % 1000 === 1)  process.stdout.write('.');
+        if(i === 100000) done();
+      });
+    }
+  });
+
+
+
+});
+
+
+