optimize fields in mongo for better storage

app/controllers/blocks.js

@@ -37,13 +37,13 @@ exports.show = function(req, res) {
  * Show block by Height
  */
 exports.blockindex = function(req, res, next, height) {
-  Block.fromHeight(height, function(err, hash) {
+  Block.blockIndex(height, function(err, hashStr) {
     if (err) {
       console.log(err);
       res.status(400).send('Bad Request'); // TODO
     }
     else {
-      res.jsonp(hash);
+      res.jsonp(hashStr);
     }
   });
 };

app/models/Address.js

@@ -60,7 +60,7 @@ function spec() {
       // TODO TXout!
       //T
       function (cb) {
-        TransactionItem.find({addr:that.addrStr}).sort({ts:-1}).exec(function(err,txItems){
+        TransactionItem.find({addr:that.addrStr}).exec(function(err,txItems){
           if (err) return cb(err);
 
           txItems.forEach(function(txItem){

app/models/Block.js

@@ -8,7 +8,6 @@ var mongoose    = require('mongoose'),
     RpcClient   = require('bitcore/RpcClient').class(),
     util        = require('bitcore/util/util'),
     BitcoreBlock= require('bitcore/Block').class(),
-    Transaction = require('./Transaction').class(),
     TransactionItem = require('./TransactionItem'),
     config      = require('../../config/config')
     ;
@@ -21,19 +20,53 @@ var BlockSchema = new Schema({
   // For now we keep this as short as possible
   // More fields will be propably added as we move
   // forward with the UX
-  hash: {
+  _id: {
-    type: String,
+    type: Buffer,
     index: true,
     unique: true,
+    required: true,
   },
   time: Number,
-  nextBlockHash: String,
+  nextBlockHash: Buffer,
   isOrphan: Boolean,
 });
 
-/**
+BlockSchema.virtual('hash').get(function () {
- * Validations
+  return this._id;
- */
+});
+
+
+BlockSchema.virtual('hash').set(function (hash) {
+    this._id = hash;
+});
+
+
+
+
+BlockSchema.virtual('hashStr').get(function () {
+  return this._id.toString('hex');
+});
+
+
+BlockSchema.virtual('hashStr').set(function (hashStr) {
+  if (hashStr)
+    this._id = new Buffer(hashStr,'hex');
+  else
+    this._id = null;
+});
+
+
+
+BlockSchema.virtual('nextBlockHashStr').get(function () {
+  return this.nextBlockHash.toString('hex');
+});
+
+BlockSchema.virtual('nextBlockHashStr').set(function (hashStr) {
+  if (hashStr)
+    this.nextBlockHash = new Buffer(hashStr,'hex');
+  else
+    this.nextBlockHash = null;
+});
 
 /*
 BlockSchema.path('title').validate(function(title) {
@@ -54,9 +87,8 @@ BlockSchema.statics.customCreate = function(block, cb) {
   var newBlock = new That();
 
   newBlock.time = block.time ? block.time : Math.round(new Date().getTime() / 1000);
-  newBlock.hash = block.hash;
+  newBlock.hashStr = block.hash;
-  newBlock.nextBlockHash = block.nextBlockHash;
+  newBlock.nextBlockHashStr =  block.nextBlockHash;
-
 
   TransactionItem.createFromArray(block.tx, function(err, inserted_txs) {
     if (err) return cb(err);
@@ -67,40 +99,38 @@ BlockSchema.statics.customCreate = function(block, cb) {
   });
 };
 
-BlockSchema.statics.load = function(id, cb) {
-  this.findOne({
-    _id: id
-  }).exec(cb);
-};
 
-BlockSchema.statics.fromHeight = function(height, cb) {
+BlockSchema.statics.blockIndex = function(height, cb) {
   var rpc  = new RpcClient(config.bitcoind);
-  var hash = {};
+  var hashStr = {};
   rpc.getBlockHash(height, function(err, bh){
     if (err) return cb(err);
-    hash.blockHash = bh.result;
+    hashStr.blockHash = bh.result;
-    cb(null, hash);
+    cb(null, hashStr);
   });
 };
 
-BlockSchema.statics.fromHash = function(hash, cb) {
+BlockSchema.statics.fromHash = function(hashStr, cb) {
+
+  var hash = new Buffer(hashStr, 'hex');
+
   this.findOne({
     hash: hash,
   }).exec(cb);
 };
 
 
-BlockSchema.statics.fromHashWithInfo = function(hash, cb) {
+BlockSchema.statics.fromHashWithInfo = function(hashStr, cb) {
   var That = this;
 
-  this.fromHash(hash, function(err, block) {
+  That.fromHash(hashStr, function(err, block) {
     if (err) return cb(err);
 
     if (!block) {
       // No in mongo...but maybe in bitcoind... lets query it
       block = new That();
 
-      block.hash = hash;
+      block.hashStr = hashStr;
       block.getInfo(function(err, blockInfo) {
         if (err) return cb(err);
         if (!blockInfo) return cb();
@@ -121,10 +151,10 @@ BlockSchema.statics.fromHashWithInfo = function(hash, cb) {
 // TODO: Can we store the rpc instance in the Block object?
 BlockSchema.methods.getInfo = function (next) {
 
-  var that = this;
+  var self = this;
   var rpc  = new RpcClient(config.bitcoind);
 
-  rpc.getBlock(this.hash, function(err, blockInfo) {
+  rpc.getBlock(self.hashStr, function(err, blockInfo) {
     // Not found?
     if (err && err.code === -5) return next();
 
@@ -135,12 +165,10 @@ BlockSchema.methods.getInfo = function (next) {
      * Any other way to lazy load a property in a mongoose object?
      */
 
-    that.info = blockInfo.result;
+    self.info = blockInfo.result;
+    self.info.reward =  BitcoreBlock.getBlockValue(self.info.height) / util.COIN ;
 
-    that.info.reward =  BitcoreBlock.getBlockValue(that.info.height) / util.COIN ;
+    return next(null, self.info);
-
-    //console.log("THAT", that);
-    return next(null, that.info);
   });
 };
 

app/models/TransactionItem.js

@@ -10,9 +10,11 @@ var mongoose    = require('mongoose'),
     Schema      = mongoose.Schema;
 
 var CONCURRENCY = 15;
+// TODO: use bitcore networks module
+var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b';
 
 var TransactionItemSchema = new Schema({
-  txid: String,
+  txidBuf: Buffer,
   index: Number,
   addr: {
     type: String,
@@ -25,12 +27,24 @@ var TransactionItemSchema = new Schema({
 });
 
 
-// TODO: use bitcore networks module
-var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b';
-
-
 // Compound index
-TransactionItemSchema.index({txid: 1, index: 1, value_sat: 1}, {unique: true, dropDups: true});
+TransactionItemSchema.index({txidBuf: 1, index: 1, value_sat: 1}, {unique: true, dropDups: true});
+
+TransactionItemSchema.virtual('txid').get(function () {
+  return this.txidBuf.toString('hex');
+});
+
+TransactionItemSchema.virtual('txid').set(function (txidStr) {
+  if (txidStr)
+    this.txidBuf = new Buffer(txidStr,'hex');
+  else
+    this.txidBuf = null;
+});
+
+
+
+
+
 
 
 TransactionItemSchema.statics.load = function(id, cb) {
@@ -87,7 +101,6 @@ TransactionItemSchema.statics.explodeTransactionItems = function(txid, cb) {
             value_sat : -1 * i.valueSat,
             addr  : i.addr,
             index : i.n,
-            ts : info.time,
         }, next_in);
         if (addrs.indexOf(i.addr) === -1) {
           addrs.push(i.addr);
@@ -124,7 +137,6 @@ TransactionItemSchema.statics.explodeTransactionItems = function(txid, cb) {
               value_sat : o.valueSat,
               addr  : o.scriptPubKey.addresses[0], // TODO: only address 0?
               index : o.n,
-              ts : info.time,
           }, next_out);
           if (addrs.indexOf(o.scriptPubKey.addresses[0]) === -1) {
             addrs.push(o.scriptPubKey.addresses[0]);
@@ -139,7 +151,7 @@ TransactionItemSchema.statics.explodeTransactionItems = function(txid, cb) {
         if (err) {
           if (err.message.match(/E11000/)) {
             is_new = false;
-          } 
+          }
           else {
             console.log('ERR at TX %s: %s', txid,  err);
             return cb(err);

lib/Sync.js

@@ -62,7 +62,6 @@ function spec() {
     var self = this;
     async.series([
       function(b) { return self.db.collections.blocks.drop(b);},
-      function(b) { return self.db.collections.transactions.drop(b);},
       function(b) { return self.db.collections.transactionitems.drop(b);},
     ], next);
   };

test/model/block.js

@@ -1,11 +1,12 @@
 #!/usr/bin/env node
+'use strict';
 
 process.env.NODE_ENV = process.env.NODE_ENV || 'development';
 
 
 var TESTING_BLOCK = '000000000185678d3d7ecc9962c96418174431f93fe20bf216d5565272423f74';
 
-var 
+var
   mongoose= require('mongoose'),
   assert  = require('assert'),
   config      = require('../../config/config'),
@@ -28,21 +29,46 @@ describe('Block fromHashWithInfo', function(){
 
 
   it('should poll block\'s info from mongoose', function(done) {
-    var block2 = Block.fromHashWithInfo(TESTING_BLOCK, function(err, b2) {
+    Block.fromHashWithInfo(TESTING_BLOCK, function(err, b2) {
         if (err) done(err);
 
-        assert.equal(b2.hash, TESTING_BLOCK);
+
+        var h = new Buffer(TESTING_BLOCK,'hex');
+        assert(b2.hashStr === TESTING_BLOCK);
+        assert.equal(b2.hashStr, TESTING_BLOCK);
         done();
       });
   });
 
   it('should poll block\'s info from bitcoind', function(done) {
-    var block2 = Block.fromHashWithInfo(TESTING_BLOCK, function(err, b2) {
+    Block.fromHashWithInfo(TESTING_BLOCK, function(err, b2) {
         if (err) done(err);
         assert.equal(b2.info.hash, TESTING_BLOCK);
         assert.equal(b2.info.chainwork, '000000000000000000000000000000000000000000000000001b6dc969ffe847');
         done();
     });
   });
+
+
+  it('hash Virtuals SET', function(done) {
+    var b = new Block();
+    b.hashStr = 'a1a2';
+    assert.equal(b.hash.toString('hex'),'a1a2');
+    b.nextBlockHashStr = 'a1a3';
+    assert.equal(b.nextBlockHash.toString('hex'),'a1a3');
+    done();
+  });
+
+
+  it('hash Virtuals GET', function(done) {
+    var b = new Block();
+    b.hash = new Buffer('a1a2','hex');
+    assert.equal(b.hashStr,'a1a2');
+
+
+    b.nextBlockHash = new Buffer('b2b1','hex');
+    assert.equal(b.nextBlockHashStr,'b2b1');
+    done();
+  });
 });