2014-01-10 11:02:33 -08:00
|
|
|
'use strict';
|
2014-01-07 10:21:59 -08:00
|
|
|
|
2014-01-10 11:02:33 -08:00
|
|
|
require('classtool');
|
2014-01-08 09:50:37 -08:00
|
|
|
|
|
|
|
|
2014-01-10 11:02:33 -08:00
|
|
|
function spec() {
|
2014-01-10 16:42:39 -08:00
|
|
|
var mongoose = require('mongoose');
|
|
|
|
var util = require('util');
|
|
|
|
var RpcClient = require('bitcore/RpcClient').class();
|
|
|
|
var networks = require('bitcore/networks');
|
|
|
|
var async = require('async');
|
|
|
|
var config = require('../config/config');
|
|
|
|
var Block = require('../app/models/Block');
|
|
|
|
var Transaction = require('../app/models/Transaction');
|
|
|
|
var TransactionItem = require('../app/models/TransactionItem');
|
2014-01-10 11:57:16 -08:00
|
|
|
|
|
|
|
function Sync(config) {
|
|
|
|
this.network = config.networkName === 'testnet' ? networks.testnet: networks.livenet;
|
|
|
|
}
|
|
|
|
|
|
|
|
var progress_bar = function(string, current, total) {
|
|
|
|
console.log(util.format('\t%s %d/%d [%d%%]', string, current, total, parseInt(100 * current / total)));
|
|
|
|
};
|
|
|
|
|
|
|
|
Sync.prototype.getNextBlock = function(blockHash, cb) {
|
|
|
|
var that = this;
|
|
|
|
if (!blockHash) {
|
|
|
|
return cb();
|
|
|
|
}
|
|
|
|
this.rpc.getBlock(blockHash, function(err, blockInfo) {
|
|
|
|
if (err) return cb(err);
|
|
|
|
if (blockInfo.result.height % 1000 === 0) {
|
|
|
|
var h = blockInfo.result.height,
|
|
|
|
d = blockInfo.result.confirmations;
|
|
|
|
progress_bar('height', h, h + d);
|
|
|
|
}
|
|
|
|
|
|
|
|
that.storeBlock(blockInfo.result, function(err) {
|
|
|
|
if (!err) {
|
|
|
|
var txs = blockInfo.result.tx;
|
|
|
|
that.storeTxs(txs, function(err) {
|
|
|
|
if (!err) {
|
|
|
|
return that.getNextBlock(blockInfo.result.nextblockhash, cb);
|
|
|
|
}
|
|
|
|
});
|
|
|
|
}
|
|
|
|
});
|
|
|
|
});
|
|
|
|
};
|
|
|
|
|
|
|
|
Sync.prototype.storeBlock = function(block, cb) {
|
|
|
|
Block.create(block, function(err, inBlock) {
|
|
|
|
// E11000 => already exists
|
|
|
|
if (err && ! err.toString().match(/E11000/)) {
|
|
|
|
return cb(err);
|
|
|
|
}
|
|
|
|
cb();
|
|
|
|
});
|
|
|
|
};
|
|
|
|
|
|
|
|
Sync.prototype.storeTxs = function(txs, cb) {
|
|
|
|
Transaction.createFromArray(txs, cb);
|
|
|
|
};
|
|
|
|
|
|
|
|
Sync.prototype.syncBlocks = function(reindex, cb) {
|
|
|
|
var that = this;
|
|
|
|
var genesisHash = this.network.genesisBlock.hash.reverse().toString('hex');
|
|
|
|
|
|
|
|
console.log('Syncing Blocks... ' + reindex);
|
|
|
|
if (reindex) {
|
|
|
|
return this.getNextBlock(genesisHash, cb);
|
|
|
|
}
|
|
|
|
|
|
|
|
Block.findOne({},
|
|
|
|
{},
|
|
|
|
{
|
|
|
|
sort: {
|
|
|
|
'time': - 1
|
|
|
|
}
|
|
|
|
},
|
|
|
|
function(err, block) {
|
|
|
|
if (err) return cb(err);
|
|
|
|
|
|
|
|
var nextHash = block && block.hash ? block.hash: genesisHash;
|
|
|
|
|
|
|
|
console.log('\tStarting at hash: ' + nextHash);
|
|
|
|
return that.getNextBlock(nextHash, cb);
|
|
|
|
});
|
|
|
|
};
|
|
|
|
|
|
|
|
// This is not currently used. Transactions are represented by txid only
|
|
|
|
// in mongodb
|
|
|
|
Sync.prototype.syncTXs = function(reindex, cb) {
|
|
|
|
|
|
|
|
var that = this;
|
|
|
|
|
|
|
|
console.log('Syncing TXs...');
|
|
|
|
if (reindex) {
|
|
|
|
// TODO?
|
|
|
|
}
|
|
|
|
|
|
|
|
Transaction.find({
|
|
|
|
blockhash: null
|
|
|
|
},
|
|
|
|
function(err, txs) {
|
|
|
|
if (err) return cb(err);
|
|
|
|
|
|
|
|
var read = 0;
|
|
|
|
var pull = 0;
|
|
|
|
var write = 0;
|
|
|
|
var total = txs.length;
|
|
|
|
console.log('\tneed to pull %d txs', total);
|
|
|
|
|
|
|
|
if (!total) return cb();
|
|
|
|
|
|
|
|
async.each(txs, function(tx, next) {
|
|
|
|
if (!tx.txid) {
|
|
|
|
console.log('NO TXID skipping...', tx);
|
|
|
|
return next();
|
|
|
|
}
|
|
|
|
|
|
|
|
if (read++ % 1000 === 0) progress_bar('read', read, total);
|
|
|
|
|
|
|
|
that.rpc.getRawTransaction(tx.txid, 1, function(err, txInfo) {
|
|
|
|
|
|
|
|
if (pull++ % 1000 === 0) progress_bar('\tpull', pull, total);
|
|
|
|
|
|
|
|
if (!err && txInfo) {
|
|
|
|
Transaction.update({
|
|
|
|
txid: tx.txid
|
|
|
|
},
|
|
|
|
txInfo.result, function(err) {
|
|
|
|
if (err) return next(err);
|
|
|
|
|
|
|
|
if (write++ % 1000 === 0) progress_bar('\t\twrite', write, total);
|
|
|
|
|
|
|
|
return next();
|
|
|
|
});
|
|
|
|
}
|
|
|
|
else return next();
|
|
|
|
});
|
|
|
|
},
|
|
|
|
function(err) {
|
|
|
|
if (err) return cb(err);
|
|
|
|
return cb(err);
|
|
|
|
});
|
|
|
|
});
|
|
|
|
};
|
|
|
|
|
2014-01-10 16:42:39 -08:00
|
|
|
|
|
|
|
Sync.prototype.processTXs = function(reindex, cb) {
|
|
|
|
|
|
|
|
var that = this;
|
|
|
|
|
|
|
|
console.log('Syncing TXs...');
|
|
|
|
|
|
|
|
var filter = reindex ? {} : { processed: false } ;
|
|
|
|
|
|
|
|
Transaction.find(filter,
|
|
|
|
function(err, txs) {
|
|
|
|
if (err) return cb(err);
|
|
|
|
|
|
|
|
var read = 0,
|
|
|
|
pull = 0,
|
|
|
|
write = 0,
|
|
|
|
total = txs.length;
|
|
|
|
|
|
|
|
console.log('\tneed to pull %d txs', total);
|
|
|
|
|
|
|
|
if (!total) return cb();
|
|
|
|
|
|
|
|
async.each(txs, function(tx, next) {
|
|
|
|
if (read++ % 1000 === 0) progress_bar('read', read, total);
|
|
|
|
|
|
|
|
if (!tx.txid) {
|
|
|
|
console.log('NO TXID skipping...', tx);
|
|
|
|
return next();
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// This will trigger an RPC call
|
|
|
|
Transaction.fromIdWithInfo( tx.txid, function(err,t) {
|
|
|
|
if (pull++ % 1000 === 0) progress_bar('\tpull', pull, total);
|
|
|
|
|
|
|
|
if (!err && t) {
|
|
|
|
var index = 0;
|
|
|
|
|
|
|
|
async.each(t.info.vin, function(i, next_in) {
|
|
|
|
|
|
|
|
/*
|
|
|
|
* TODO Support multisigs???
|
|
|
|
* how??
|
|
|
|
*/
|
|
|
|
|
|
|
|
if (i.addr && i.value) {
|
|
|
|
TransactionItem.create({
|
|
|
|
txid : t.txid,
|
|
|
|
value : -1 * i.value,
|
|
|
|
addr : i.addr,
|
|
|
|
index : i.n,
|
|
|
|
}, next_in);
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
if ( !i.coinbase )
|
|
|
|
console.log ("TX: %s seems to be multisig IN. Skipping... ", t.txid);
|
|
|
|
return next_in();
|
|
|
|
}
|
|
|
|
},
|
|
|
|
function (err) {
|
|
|
|
if (err) console.log (err);
|
|
|
|
index = 0;
|
|
|
|
async.each(t.info.vout, function(o, next_out) {
|
|
|
|
|
|
|
|
/*
|
|
|
|
* TODO Support multisigs
|
|
|
|
*/
|
|
|
|
if (o.value && o.scriptPubKey
|
|
|
|
&& o.scriptPubKey.addresses
|
|
|
|
&& o.scriptPubKey.addresses[0]
|
|
|
|
) {
|
|
|
|
TransactionItem.create({
|
|
|
|
txid : t.txid,
|
|
|
|
value : o.value,
|
|
|
|
addr : o.scriptPubKey.addresses[0],
|
|
|
|
index : o.n,
|
|
|
|
}, next_out);
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
console.log ("TX: %s,%d seems to be multisig OUT. Skipping... ", t.txid, o.n);
|
|
|
|
return next_out();
|
|
|
|
}
|
|
|
|
},
|
|
|
|
function (err) {
|
|
|
|
if (err) console.log (err);
|
|
|
|
if (write++ % 1000 === 0) progress_bar('\t\twrite', write, total);
|
|
|
|
return next();
|
|
|
|
});
|
|
|
|
});
|
|
|
|
}
|
|
|
|
else return next();
|
|
|
|
});
|
|
|
|
},
|
|
|
|
function(err) {
|
|
|
|
return cb(err);
|
|
|
|
});
|
|
|
|
});
|
|
|
|
};
|
|
|
|
|
2014-01-10 11:57:16 -08:00
|
|
|
Sync.prototype.init = function(opts) {
|
2014-01-13 10:17:34 -08:00
|
|
|
if (!(opts && opts.skip_db_connection)) {
|
|
|
|
mongoose.connect(config.db);
|
|
|
|
}
|
2014-01-10 11:57:16 -08:00
|
|
|
this.db = mongoose.connection;
|
|
|
|
this.rpc = new RpcClient(config.bitcoind);
|
|
|
|
|
|
|
|
this.db.on('error', console.error.bind(console, 'connection error:'));
|
|
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
Sync.prototype.import_history = function(opts, next) {
|
|
|
|
|
|
|
|
var that = this;
|
|
|
|
this.db.once('open', function() {
|
|
|
|
async.series([
|
|
|
|
function(cb) {
|
|
|
|
if (opts.destroy) {
|
|
|
|
console.log('Deleting Blocks...');
|
|
|
|
that.db.collections.blocks.drop(cb);
|
|
|
|
} else {
|
|
|
|
cb();
|
|
|
|
}
|
|
|
|
},
|
|
|
|
function(cb) {
|
|
|
|
if (opts.destroy) {
|
|
|
|
console.log('Deleting TXs...');
|
|
|
|
that.db.collections.transactions.drop(cb);
|
|
|
|
} else {
|
|
|
|
cb();
|
|
|
|
}
|
|
|
|
},
|
|
|
|
function(cb) {
|
|
|
|
if (!opts.skip_blocks) {
|
|
|
|
that.syncBlocks(opts.reindex, cb);
|
|
|
|
} else {
|
|
|
|
cb();
|
|
|
|
}
|
|
|
|
},
|
|
|
|
function(cb) {
|
2014-01-10 16:42:39 -08:00
|
|
|
if (! opts.skip_txs) {
|
|
|
|
that.processTXs(opts.reindex, cb);
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
return cb();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
/* We dont sync any contents from TXs, only their IDs are stored
|
|
|
|
function(cb) {
|
|
|
|
if (! opts.skip_txs) {
|
2014-01-10 11:57:16 -08:00
|
|
|
that.syncTXs(opts.reindex, cb);
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
return cb();
|
|
|
|
}
|
2014-01-10 16:42:39 -08:00
|
|
|
}
|
|
|
|
*/
|
|
|
|
], function(err) {
|
2014-01-10 11:57:16 -08:00
|
|
|
return next(err);
|
|
|
|
});
|
|
|
|
});
|
|
|
|
};
|
|
|
|
|
|
|
|
Sync.prototype.close = function() {
|
|
|
|
console.log("closing connection");
|
|
|
|
this.db.close();
|
|
|
|
};
|
|
|
|
return Sync;
|
2014-01-10 11:02:33 -08:00
|
|
|
}
|
2014-01-07 10:21:59 -08:00
|
|
|
module.defineClass(spec);
|
2014-01-10 11:02:33 -08:00
|
|
|
|