sync with TX items!!!

app/models/Address.js

@@ -4,13 +4,11 @@
  * Module dependencies.
  */
 var mongoose    = require('mongoose'),
-    Schema      = mongoose.Schema,
+    Schema      = mongoose.Schema
-    RpcClient   = require('bitcore/RpcClient').class(),
-    config      = require('../../config/config')
     ;
 
 /**
- * Block Schema
+ * Addr Schema
  */
 var AddressSchema = new Schema({
 
@@ -22,26 +20,17 @@ var AddressSchema = new Schema({
     index: true,
     unique: true,
   },
-  balance: Number,
+  inputs: [{
-  totalReceived: Number,
+    type: mongoose.Schema.Types.ObjectId,
-  totalSent: Number,
+    ref: 'TransactionItem' //Edit: I'd put the schema. Silly me.
-  inTransactions: [String],
+  }],
+  output: [{
+    type: mongoose.Schema.Types.ObjectId,
+    ref: 'TransactionItem' //Edit: I'd put the schema. Silly me.
+  }],
 });
 
 
-/**
- * Validations
- */
-
-/*
-AddressSchema.path('title').validate(function(title) {
-    return title.length;
-},'Title cannot be blank');
-*/
-
-/**
- * Statics
- */
 
 AddressSchema.statics.load = function(id, cb) {
   this.findOne({
@@ -58,37 +47,13 @@ AddressSchema.statics.fromAddr = function(hash, cb) {
 
 
 AddressSchema.statics.fromAddrWithInfo = function(hash, cb) {
-  this.fromHash(hash, function(err, block) {
+  this.fromHash(hash, function(err, addr) {
     if (err) return cb(err);
-    if (!block) { return cb(new Error('Block not found')); }
+    if (!addr) { return cb(new Error('Addr not found')); }
-
+// TODO
-    block.getInfo(function(err) { return cb(err,block); } );
+//    addr.getInfo(function(err) { return cb(err,addr); } );
   });
 };
 
 
-
-// TODO: Can we store the rpc instance in the Block object?
-AddressSchema.methods.getInfo = function (next) {
-
-  var that = this;
-  var rpc  = new RpcClient(config.bitcoind);
-
-  rpc.getBlock(this.hash, function(err, blockInfo) {
-    if (err) return next(err);
-
-    /*
-     * Not sure this is the right way to do it.
-     * Any other way to lazy load a property in a mongoose object?
-     */
-
-    that.info = blockInfo.result;
-
-    //console.log("THAT", that);
-    return next(null, that.info);
-  });
-};
-
-
-
 module.exports = mongoose.model('Address', AddressSchema);

app/models/Transaction.js

@@ -27,6 +27,14 @@ var TransactionSchema = new Schema({
     index: true,
     unique: true,
   },
+  processed: {
+    type: Boolean,
+    default: false,
+  },
+  orphaned: {
+    type: Boolean,
+    default: false,
+  },
 });
 
 /**
@@ -154,22 +162,26 @@ TransactionSchema.methods.queryInfo = function (next) {
       }
       else {
         tx.ins.forEach(function(i) {
+  
+          if (i.value) {
+            that.info.vin[c].value = util.formatValue(i.value);
+            var n = util.valueToBigInt(i.value).toNumber();
+            valueIn           = valueIn.add( n );
 
-          that.info.vin[c].value = util.formatValue(i.value);
+            var scriptSig     = i.getScript();
-          var n = util.valueToBigInt(i.value).toNumber();
+            var pubKey        = scriptSig.simpleInPubKey();
-          valueIn           = valueIn.add( n );
 
-          var scriptSig     = i.getScript();
+            // We check for pubKey in case a broken / strange TX.
-          var pubKey        = scriptSig.simpleInPubKey();
+            if (pubKey) {
-
+              var pubKeyHash    = util.sha256ripe160(pubKey);
-          // We check for pubKey in case a broken / strange TX.
+              var addr          = new Address(network.addressPubkey, pubKeyHash);
-          if (pubKey) {
+              var addrStr       = addr.toString();
-            var pubKeyHash    = util.sha256ripe160(pubKey);
+              that.info.vin[c].addr  = addrStr;
-            var addr          = new Address(network.addressPubkey, pubKeyHash);
+            }
-            var addrStr       = addr.toString();
+          }
-            that.info.vin[c].addr  = addrStr;
+          else {
+            console.log("TX could not be parsed: %s,%d",txInfo.result.txid, c); 
           }
-
           c++;
         });
       }

app/models/TransactionItem.js

@@ -0,0 +1,35 @@
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+var mongoose    = require('mongoose'),
+    Schema      = mongoose.Schema;
+
+var TransactionItemSchema = new Schema({
+  txid: String,
+  index: Number,
+  addr: {
+    type: String,
+    index: true,
+  },
+  // >0 is Input <0 is Output
+  value: Number,
+});
+
+
+
+TransactionItemSchema.statics.load = function(id, cb) {
+  this.findOne({
+    _id: id
+  }).exec(cb);
+};
+
+
+TransactionItemSchema.statics.fromAddr = function(addr, cb) {
+  this.find({
+    addr: addr,
+  }).exec(cb);
+};
+
+module.exports = mongoose.model('TransactionItem', TransactionItemSchema);

lib/Sync.js

@@ -2,21 +2,17 @@
 
 require('classtool');
 
-/* We dont sync any contents from TXs, only their IDs are stored */
-
-var isSyncTxEnabled = 0;
 
 function spec() {
-  var mongoose = require('mongoose');
+  var mongoose        = require('mongoose');
-  var util = require('util');
+  var util            = require('util');
-
+  var RpcClient       = require('bitcore/RpcClient').class();
-  var RpcClient = require('bitcore/RpcClient').class();
+  var networks        = require('bitcore/networks');
-  var networks = require('bitcore/networks');
+  var async           = require('async');
-  var async = require('async');
+  var config          = require('../config/config');
-
+  var Block           = require('../app/models/Block');
-  var config = require('../config/config');
+  var Transaction     = require('../app/models/Transaction');
-  var Block = require('../app/models/Block');
+  var TransactionItem = require('../app/models/TransactionItem');
-  var Transaction = require('../app/models/Transaction');
 
   function Sync(config) {
     this.network = config.networkName === 'testnet' ? networks.testnet: networks.livenet;
@@ -151,6 +147,105 @@ function spec() {
     });
   };
 
+
+  Sync.prototype.processTXs = function(reindex, cb) {
+
+    var that = this;
+
+    console.log('Syncing TXs...');
+
+    var filter = reindex ? {} : { processed: false } ;
+
+    Transaction.find(filter,
+      function(err, txs) {
+        if (err) return cb(err);
+
+        var read = 0,
+          pull   = 0,
+          write  = 0,
+          total  = txs.length;
+
+        console.log('\tneed to pull %d txs', total);
+
+        if (!total) return cb();
+
+        async.each(txs, function(tx, next) {
+          if (read++ % 1000 === 0) progress_bar('read', read, total);
+
+          if (!tx.txid) {
+            console.log('NO TXID skipping...', tx);
+            return next();
+          }
+
+
+          // This will trigger an RPC call
+          Transaction.fromIdWithInfo( tx.txid, function(err,t) {
+            if (pull++ % 1000 === 0) progress_bar('\tpull', pull, total);
+
+            if (!err && t) {
+              var index = 0;
+
+              async.each(t.info.vin, function(i, next_in) {
+
+                /*
+                 * TODO Support multisigs???
+                 * how??
+                 */
+
+                if (i.addr && i.value) {
+                  TransactionItem.create({
+                    txid  : t.txid,
+                    value : -1 * i.value,
+                    addr  : i.addr,
+                    index : i.n,
+                  }, next_in);
+                }
+                else {
+                  if ( !i.coinbase )
+                    console.log ("TX: %s seems to be multisig IN. Skipping... ", t.txid);
+                  return next_in(); 
+                }
+              },
+              function (err) {
+                if (err) console.log (err);
+                  index = 0;
+                  async.each(t.info.vout, function(o, next_out) {
+
+                    /*
+                    * TODO Support multisigs
+                    */
+                    if (o.value &&  o.scriptPubKey 
+                        && o.scriptPubKey.addresses 
+                        && o.scriptPubKey.addresses[0]
+                       ) {
+                      TransactionItem.create({
+                        txid  : t.txid,
+                        value : o.value,
+                        addr  :  o.scriptPubKey.addresses[0],
+                        index : o.n,
+                      }, next_out);
+                    }
+                    else {
+                      console.log ("TX: %s,%d seems to be multisig OUT. Skipping... ", t.txid, o.n);
+                      return next_out(); 
+                    }
+                  },
+                  function (err) {
+                    if (err) console.log (err);
+                    if (write++ % 1000 === 0) progress_bar('\t\twrite', write, total);
+                    return next();
+                  });
+              });
+            }
+            else return next();
+          });
+        },
+        function(err) {
+          return cb(err);
+        });
+      });
+  };
+
   Sync.prototype.init = function(opts) {
     if (!(opts && opts.skip_db_connection)) {
       mongoose.connect(config.db);
@@ -191,13 +286,24 @@ function spec() {
         }
       },
       function(cb) {
-        if (isSyncTxEnabled && ! opts.skip_txs) {
+        if (! opts.skip_txs) {
+          that.processTXs(opts.reindex, cb);
+        }
+        else {
+          return cb();
+        }
+      }
+/* We dont sync any contents from TXs, only their IDs are stored 
+      function(cb) {
+        if (! opts.skip_txs) {
           that.syncTXs(opts.reindex, cb);
         }
         else {
           return cb();
         }
-      }], function(err) {
+      }
+*/
+      ], function(err) {
         return next(err);
       });
     });