zcash-hackworks
/
insight-ui-zcash
mirror of https://github.com/zcash-hackworks/insight-ui-zcash.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #183 from matiu/feature/leveldb 

Feature/leveldb
This commit is contained in:
Matias Alejo Garcia 2014-02-05 16:24:35 -02:00
parent 71949da2ff 3a3972616b
commit dbe65e6860
34 changed files with 1324 additions and 1246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
README.md
View File

@ -18,7 +18,6 @@ $ npm install -g bower
## Additional Packages
* Express - Defined as npm module in the [package.json](package.json) file.
* Mongoose - Defined as npm module in the [package.json](package.json) file.
* AngularJS - Defined as bower module in the [bower.json](bower.json) file.
* Twitter Bootstrap - Defined as bower module in the [bower.json](bower.json) file.
* UI Bootstrap - Defined as bower module in the [bower.json](bower.json) file.
@ -55,13 +54,15 @@ $ npm install -g bower
http://localhost:3000
If you get an error, please check the next section "Post-install"
## Syncing old blockchain data
Run sync from insight repository (to save old blocks and transactions in MongoDB):
Run sync from insight repository (to save old blocks and transactions in
LevelDB):
Create folders:
$ utils/sync.js
$ mkdir -p db/blocks
$ utils/sync.js -S
Check utils/sync.js --help for options.

66
app/controllers/blocks.js
View File

@ -3,17 +3,17 @@
/**
* Module dependencies.
*/
var mongoose = require('mongoose'),
Block = mongoose.model('Block'),
common = require('./common'),
async = require('async');
var common = require('./common'),
async = require('async'),
BlockDb = require('../../lib/BlockDb').class();
var bdb = new BlockDb();
/**
* Find block by hash ...
*/
exports.block = function(req, res, next, hash) {
Block.fromHashWithInfo(hash, function(err, block) {
bdb.fromHashWithInfo(hash, function(err, block) {
if (err || ! block)
return common.handleErrors(err, res, next);
else {
@ -37,7 +37,7 @@ exports.show = function(req, res) {
* Show block by Height
*/
exports.blockindex = function(req, res, next, height) {
Block.blockIndex(height, function(err, hashStr) {
bdb.blockIndex(height, function(err, hashStr) {
if (err) {
console.log(err);
res.status(400).send('Bad Request'); // TODO
@ -49,7 +49,7 @@ exports.blockindex = function(req, res, next, height) {
};
var getBlock = function(blockhash, cb) {
Block.fromHashWithInfo(blockhash, function(err, block) {
bdb.fromHashWithInfo(blockhash, function(err, block) {
if (err) {
console.log(err);
return cb(err);
@ -72,7 +72,7 @@ console.log('[blocks.js.60]: could not get %s from RPC. Orphan? Error?', blockha
* List of blocks by date
*/
exports.list = function(req, res) {
var limit = req.query.limit || 0;
var limit = req.query.limit || -1;
var isToday = false;
//helper to convert timestamps to yyyy-mm-dd format
@ -103,35 +103,27 @@ exports.list = function(req, res) {
var prev = formatTimestamp(new Date((gte - 86400) * 1000));
var next = formatTimestamp(new Date(lte * 1000));
Block
.find({
time: {
'$gte': gte,
'$lte': lte
bdb.getBlocksByDate(gte, lte, limit, function(err, blocks) {
if (err) {
res.status(500).send(err);
}
else {
var blockshashList = [];
for(var i=0;i<blocks.length;i++) {
blockshashList.unshift(blocks[i].hash);
}
})
.limit(limit)
.sort('-time')
.exec(function(err, blocks) {
if (err) {
res.status(500).send(err);
} else {
var blockshashList = [];
for(var i=0;i<blocks.length;i++) {
blockshashList.push(blocks[i].hash);
}
async.mapSeries(blockshashList, getBlock, function(err, allblocks) {
res.jsonp({
blocks: allblocks,
length: allblocks.length,
pagination: {
next: next,
prev: prev,
current: dateStr,
isToday: isToday
}
});
async.mapSeries(blockshashList, getBlock, function(err, allblocks) {
res.jsonp({
blocks: allblocks,
length: allblocks.length,
pagination: {
next: next,
prev: prev,
current: dateStr,
isToday: isToday
}
});
}
});
});
}
});
};

17
app/controllers/transactions.js
View File

@ -3,18 +3,23 @@
/**
* Module dependencies.
*/
var Transaction = require('../models/Transaction').class();
var Block = require('../models/Block');
var Address = require('../models/Address');
var async = require('async');
var common = require('./common');
var TransactionDb = require('../../lib/TransactionDb').class();
var BlockDb = require('../../lib/BlockDb').class();
var bdb = new BlockDb();
/**
* Find transaction by hash ...
*/
exports.transaction = function(req, res, next, txid) {
Transaction.fromIdWithInfo(txid, function(err, tx) {
var tDb = new TransactionDb();
tDb.fromIdWithInfo(txid, function(err, tx) {
if (err || ! tx)
return common.handleErrors(err, res);
else {
@ -37,7 +42,9 @@ exports.show = function(req, res) {
var getTransaction = function(txid, cb) {
Transaction.fromIdWithInfo(txid, function(err, tx) {
var tDb = new TransactionDb();
tDb.fromIdWithInfo(txid, function(err, tx) {
if (err) {
console.log(err);
}
@ -68,7 +75,7 @@ exports.list = function(req, res, next) {
var txs;
if (bId) {
Block.fromHashWithInfo(bId, function(err, block) {
bdb.fromHashWithInfo(bId, function(err, block) {
if (err) {
console.log(err);
return res.status(500).send('Internal Server Error');

81
app/models/Address.js
View File

@ -5,9 +5,9 @@ require('classtool');
function spec() {
var async = require('async');
var TransactionOut = require('./TransactionOut');
var BitcoreAddress = require('bitcore/Address').class();
var BitcoreUtil = require('bitcore/util/util');
var TransactionDb = require('../../lib/TransactionDb').class();
function Address(addrStr) {
this.balanceSat = 0;
@ -56,37 +56,25 @@ function spec() {
Address.prototype.update = function(next) {
var self = this;
var db = new TransactionDb();
async.series([
/* function (cb) {
TransactionIn.find({addr:self.addrStr}).exec(function(err,txIn){
if (err) return cb(err);
txIn.forEach(function(txItem){
self.balanceSat += txItem.value_sat;
self.totalReceivedSat += txItem.value_sat;
});
return cb();
});
},
*/
function (cb) {
TransactionOut.find({addr:self.addrStr}).exec(function(err,txOut){
db.fromAddr(self.addrStr, function(err,txOut){
if (err) return cb(err);
txOut.forEach(function(txItem){
var v = txItem.value_sat;
self.totalReceivedSat += txItem.value_sat;
self.totalReceivedSat += v;
self.transactions.push(txItem.txid);
if (! txItem.spendTxIdBuf) {
if (! txItem.spendTxId) {
// unspent
self.balanceSat += txItem.value_sat;
self.balanceSat += v;
self.txApperances +=1;
}
else {
// spent
self.totalSentSat += txItem.value_sat;
self.transactions.push(txItem.spendTxid);
self.totalSentSat += v;
self.transactions.push(txItem.spendTxId);
self.txApperances +=2;
}
});
@ -102,54 +90,3 @@ function spec() {
}
module.defineClass(spec);
/**
* Addr Schema Idea for moogose. Not used now.
*
var AddressSchema = new Schema({
// For now we keep this as short as possible
// More fields will be propably added as we move
// forward with the UX
addr: {
type: String,
index: true,
unique: true,
},
inputs: [{
type: mongoose.Schema.Types.ObjectId,
ref: 'TransactionItem' //Edit: I'd put the schema. Silly me.
}],
output: [{
type: mongoose.Schema.Types.ObjectId,
ref: 'TransactionItem' //Edit: I'd put the schema. Silly me.
}],
});
AddressSchema.statics.load = function(id, cb) {
this.findOne({
_id: id
}).exec(cb);
};
AddressSchema.statics.fromAddr = function(hash, cb) {
this.findOne({
hash: hash,
}).exec(cb);
};
AddressSchema.statics.fromAddrWithInfo = function(hash, cb) {
this.fromHash(hash, function(err, addr) {
if (err) return cb(err);
if (!addr) { return cb(new Error('Addr not found')); }
// TODO
// addr.getInfo(function(err) { return cb(err,addr); } );
});
};
module.exports = mongoose.model('Address', AddressSchema);
*/

186
app/models/Block.js
View File

@ -1,186 +0,0 @@
'use strict';
/**
* Module dependencies.
*/
var mongoose = require('mongoose'),
Schema = mongoose.Schema,
RpcClient = require('bitcore/RpcClient').class(),
util = require('bitcore/util/util'),
async = require('async'),
BitcoreBlock= require('bitcore/Block').class(),
TransactionOut = require('./TransactionOut'),
config = require('../../config/config')
;
/**
* Block Schema
*/
var BlockSchema = new Schema({
// For now we keep this as short as possible
// More fields will be propably added as we move
// forward with the UX
_id: {
type: Buffer,
index: true,
unique: true,
required: true,
},
time: Number,
nextBlockHash: Buffer,
isOrphan: Boolean,
});
BlockSchema.virtual('hash').get(function () {
return this._id;
});
BlockSchema.virtual('hash').set(function (hash) {
this._id = hash;
});
BlockSchema.virtual('hashStr').get(function () {
return this._id.toString('hex');
});
BlockSchema.virtual('hashStr').set(function (hashStr) {
if (hashStr)
this._id = new Buffer(hashStr,'hex');
else
this._id = null;
});
BlockSchema.virtual('nextBlockHashStr').get(function () {
return this.nextBlockHash.toString('hex');
});
BlockSchema.virtual('nextBlockHashStr').set(function (hashStr) {
if (hashStr)
this.nextBlockHash = new Buffer(hashStr,'hex');
else
this.nextBlockHash = null;
});
/*
BlockSchema.path('title').validate(function(title) {
return title.length;
},'Title cannot be blank');
*/
/**
* Statics
*/
BlockSchema.statics.customCreate = function(block, cb) {
var Self= this;
var BlockSchema = mongoose.model('Block', BlockSchema);
var newBlock = new Self();
newBlock.time = block.time ? block.time : Math.round(new Date().getTime() / 1000);
newBlock.hashStr = block.hash;
newBlock.isOrphan = block.isOrphan;
newBlock.nextBlockHashStr = block.nextBlockHash;
var insertedTxs, updateAddrs;
async.series([
function(a_cb) {
TransactionOut.createFromTxs(block.tx, block.isOrphan,
function(err, inInsertedTxs, inUpdateAddrs) {
insertedTxs = inInsertedTxs;
updateAddrs = inUpdateAddrs;
return a_cb(err);
});
}, function(a_cb) {
newBlock.save(function(err) {
return a_cb(err);
});
}],
function (err) {
return cb(err, newBlock, insertedTxs, updateAddrs);
});
};
BlockSchema.statics.blockIndex = function(height, cb) {
var rpc = new RpcClient(config.bitcoind);
var hashStr = {};
rpc.getBlockHash(height, function(err, bh){
if (err) return cb(err);
hashStr.blockHash = bh.result;
cb(null, hashStr);
});
};
BlockSchema.statics.fromHash = function(hashStr, cb) {
var hash = new Buffer(hashStr, 'hex');
this.findOne({
_id: hash,
}).exec(cb);
};
BlockSchema.statics.fromHashWithInfo = function(hashStr, cb) {
var That = this;
That.fromHash(hashStr, function(err, block) {
if (err) return cb(err);
if (!block) {
// No in mongo...but maybe in bitcoind... lets query it
block = new That();
block.hashStr = hashStr;
block.getInfo(function(err, blockInfo) {
if (err) return cb(err);
if (!blockInfo) return cb();
block.save(function(err) {
return cb(err,block);
});
});
}
else {
block.getInfo(function(err) {
return cb(err,block);
});
}
});
};
// TODO: Can we store the rpc instance in the Block object?
BlockSchema.methods.getInfo = function (next) {
var self = this;
var rpc = new RpcClient(config.bitcoind);
rpc.getBlock(self.hashStr, function(err, blockInfo) {
// Not found?
if (err && err.code === -5) return next();
if (err) return next(err);
/*
* Not sure this is the right way to do it.
* Any other way to lazy load a property in a mongoose object?
*/
self.info = blockInfo.result;
self.info.reward = BitcoreBlock.getBlockValue(self.info.height) / util.COIN ;
return next(null, self.info);
});
};
module.exports = mongoose.model('Block', BlockSchema);

86
app/models/Transaction.js
View File

@ -1,86 +0,0 @@
'use strict';
require('classtool');
function spec() {
var util = require('bitcore/util/util'),
TransactionRpc = require('../../lib/TransactionRpc').class(),
TransactionOut = require('./TransactionOut'),
async = require('async');
var CONCURRENCY = 20;
function Transaction() {
this.txid = null;
}
Transaction.fromIdWithInfo = function (txid,cb) {
var tx = new Transaction();
tx.txid = txid;
tx._fillInfo(function(err) {
if (err) return cb(err);
if (! tx.info ) return cb();
return cb(err,tx);
});
};
Transaction.prototype._fillInfo = function(next) {
var self = this;
TransactionRpc.getRpcInfo(self.txid, function(err, info) {
if (err) return next(err);
Transaction._fillOutpoints(info, function() {
self.info = info;
return next();
});
});
};
Transaction._fillOutpoints = function(info, cb) {
if (!info || info.isCoinBase) return cb();
var valueIn = 0;
var incompleteInputs = 0;
async.eachLimit(info.vin, CONCURRENCY, function(i, c_in) {
TransactionOut.fromTxIdN(i.txid, i.vout, function(err, out) {
if (err || !out || ! out.addr) {
console.log('Could not get TXouts in %s,%d from %s ', i.txid, i.vout, info.txid);
incompleteInputs = 1;
return c_in(); // error not scaled
}
i.addr = out.addr;
i.valueSat = out.value_sat;
i.value = out.value_sat / util.COIN;
valueIn += i.valueSat;
return c_in();
});
},
function () {
if (! incompleteInputs ) {
info.valueIn = valueIn / util.COIN;
info.fees = (valueIn - parseInt(info.valueOut * util.COIN)) / util.COIN ;
}
else {
info.incompleteInputs = 1;
}
return cb();
});
};
return Transaction;
}
module.defineClass(spec);

270
app/models/TransactionOut.js
View File

@ -1,270 +0,0 @@
'use strict';
/**
* Module dependencies.
*/
var mongoose = require('mongoose'),
async = require('async'),
util = require('bitcore/util/util'),
TransactionRpc = require('../../lib/TransactionRpc').class(),
Schema = mongoose.Schema;
var CONCURRENCY = 15;
// TODO: use bitcore networks module
var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b';
var TransactionOutSchema = new Schema({
txidBuf: {
type: Buffer,
index: true,
},
index: Number,
addr: {
type: String,
index: true,
},
value_sat: Number,
fromOrphan: Boolean,
spendTxIdBuf: Buffer,
spendIndex: Number,
spendFromOrphan: Boolean,
});
// Compound index
TransactionOutSchema.index({txidBuf: 1, index: 1}, {unique: true, sparse: true});
TransactionOutSchema.index({spendTxIdBuf: 1, spendIndex: 1}, {unique: true, sparse: true});
TransactionOutSchema.virtual('txid').get(function () {
return this.txidBuf.toString('hex');
});
TransactionOutSchema.virtual('spendTxid').get(function () {
if (!this.spendTxIdBuf) return (null);
return this.spendTxIdBuf.toString('hex');
});
TransactionOutSchema.virtual('txid').set(function (txidStr) {
if (txidStr)
this.txidBuf = new Buffer(txidStr,'hex');
else
this.txidBuf = null;
});
TransactionOutSchema.statics.fromTxId = function(txid, cb) {
var txidBuf = new Buffer(txid, 'hex');
this.find({
txidBuf: txidBuf,
}).exec(function (err,items) {
// sort by index
return cb(err,items.sort(function(a,b){
return a.index - b.index;
}));
});
};
TransactionOutSchema.statics.fromTxIdOne = function(txid, cb) {
var txidBuf = new Buffer(txid, 'hex');
this.find({
txidBuf: txidBuf,
}).exec(function (err,item) {
return cb(err, item[0]);
});
};
TransactionOutSchema.statics.fromTxIdN = function(txid, n, cb) {
var txidBuf = new Buffer(txid, 'hex');
this.findOne({
txidBuf: txidBuf, index: n
}).exec(cb);
};
TransactionOutSchema.statics.removeFromTxId = function(txid, cb) {
var txidBuf = new Buffer(txid, 'hex');
this.remove({ txidBuf: txidBuf }).exec(cb);
};
TransactionOutSchema.statics.storeTransactionOuts = function(txInfo, fromOrphan, cb) {
var Self = this;
var addrs = [];
var is_new = true;
if (txInfo.hash) {
// adapt bitcore TX object to bitcoind JSON response
txInfo.txid = txInfo.hash;
var count = 0;
txInfo.vin = txInfo.in.map(function (txin) {
var i = {};
if (txin.coinbase) {
txInfo.isCoinBase = true;
}
else {
i.txid= txin.prev_out.hash;
i.vout= txin.prev_out.n;
};
i.n = count++;
return i;
});
count = 0;
txInfo.vout = txInfo.out.map(function (txout) {
var o = {};
o.value = txout.value;
o.n = count++;
if (txout.addrStr){
o.scriptPubKey = {};
o.scriptPubKey.addresses = [txout.addrStr];
}
return o;
});
}
var bTxId = new Buffer(txInfo.txid,'hex');
async.series([
// Input Outpoints (mark them as spended)
function(p_c) {
if (txInfo.isCoinBase) return p_c();
async.forEachLimit(txInfo.vin, CONCURRENCY,
function(i, next_out) {
var b = new Buffer(i.txid,'hex');
var data = {
txidBuf: b,
index: i.vout,
spendTxIdBuf: bTxId,
spendIndex: i.n,
};
if (fromOrphan) data.spendFromOrphan = true;
Self.update({txidBuf: b, index: i.vout}, data, {upsert: true}, next_out);
},
function (err) {
if (err) {
if (!err.message.match(/E11000/)) {
console.log('ERR at TX %s: %s', txInfo.txid, err);
return cb(err);
}
}
return p_c();
});
},
// Parse Outputs
function(p_c) {
async.forEachLimit(txInfo.vout, CONCURRENCY,
function(o, next_out) {
if (o.value && o.scriptPubKey &&
o.scriptPubKey.addresses &&
o.scriptPubKey.addresses[0] &&
! o.scriptPubKey.addresses[1] // TODO : not supported
){
// This is only to broadcast (WIP)
// if (addrs.indexOf(o.scriptPubKey.addresses[0]) === -1) {
// addrs.push(o.scriptPubKey.addresses[0]);
// }
var data = {
txidBuf: bTxId,
index : o.n,
value_sat : o.value * util.COIN,
addr : o.scriptPubKey.addresses[0],
};
if (fromOrphan) data.fromOrphan = true;
Self.update({txidBuf: bTxId, index: o.n}, data, {upsert: true}, next_out);
}
else {
console.log ('WARN in TX: %s could not parse OUTPUT %d', txInfo.txid, o.n);
return next_out();
}
},
function (err) {
if (err) {
if (err.message.match(/E11000/)) {
is_new = false;
}
else {
console.log('ERR at TX %s: %s', txInfo.txid, err);
return cb(err);
}
}
return p_c();
});
}], function(err) {
return cb(err, addrs, is_new);
});
};
// txs can be a [hashes] or [txObjects]
TransactionOutSchema.statics.createFromTxs = function(txs, fromOrphan, next) {
var Self = this;
if (typeof fromOrphan === 'function') {
next = fromOrphan;
fromOrphan = false;
}
if (!txs) return next();
var inserted_txs = [];
var updated_addrs = {};
async.forEachLimit(txs, CONCURRENCY, function(t, each_cb) {
var txInfo;
async.series([
function(a_cb) {
if (typeof t !== 'string') {
txInfo = t;
return a_cb();
}
// Is it from genesis block? (testnet==livenet)
// TODO: parse it from networks.genesisTX?
if (t === genesisTXID) return a_cb();
TransactionRpc.getRpcInfo(t, function(err, inInfo) {
txInfo =inInfo;
return a_cb(err);
});
},
function(a_cb) {
if (!txInfo) return a_cb();
Self.storeTransactionOuts(txInfo, fromOrphan, function(err, addrs) {
if (err) return a_cb(err);
return a_cb();
});
}],
function(err) {
return each_cb(err);
});
},
function(err) {
return next(err, inserted_txs, updated_addrs);
});
};
module.exports = mongoose.model('TransactionOut', TransactionOutSchema);

2
config/config.js
View File

@ -23,7 +23,7 @@ module.exports = {
root: rootPath,
appName: 'Insight ' + env,
port: process.env.PORT || 3000,
db: 'mongodb://localhost/insight-' + env,
leveldb: './db',
bitcoind: {
protocol: process.env.BITCOIND_PROTO || 'http',
user: process.env.BITCOIND_USER || 'user',

25
dev-util/block-level.js Executable file
View File

@ -0,0 +1,25 @@
#!/usr/bin/env node
var
config = require('../config/config'),
levelup = require('levelup');
db = levelup(config.leveldb + '/blocks');
db.createReadStream({start: 'b-'})
.on('data', function (data) {
console.log('[block-level.js.11:data:]',data); //TODO
if (data==false) c++;
})
.on('error', function (err) {
return cb(err);
})
.on('close', function () {
return cb(null);
})
.on('end', function () {
return cb(null);
});

22
dev-util/get_tx.js
View File

@ -2,9 +2,7 @@
'use strict';
var util = require('util');
var T = require('../app/models/Transaction').class();
var mongoose= require('mongoose'),
config = require('../config/config');
var T = require('../lib/TransactionDb').class();
process.env.NODE_ENV = process.env.NODE_ENV || 'development';
@ -14,21 +12,15 @@ var hash = process.argv[2] || 'e2253359458db3e732c82a43fc62f56979ff59928f25a2df3
var t = new T();
t.fromIdWithInfo(hash, function(err, ret) {
mongoose.connect(config.db);
mongoose.connection.on('error', function(err) { console.log('error!', err); });
mongoose.connection.on('open', function() {
T.fromIdWithInfo(hash, function(err, ret) {
console.log('Err:');
console.log(err);
console.log('Err:');
console.log(err);
console.log('Ret:');
console.log(util.inspect(ret,{depth:null}));
mongoose.connection.close();
});
console.log('Ret:');
console.log(util.inspect(ret,{depth:null}));
});

17
dev-util/sync-level.js Normal file
View File

@ -0,0 +1,17 @@
#!/usr/bin/env node
'use strict';
var Sync = require('../lib/Sync').class();
var s = new Sync();
s.setOrphan(
'0000000000c2b1e8dab92a72741289e5ef0d4f375fd1b26f729da2ba979c028a',
'000000000228f9d02654459e09998c7557afa9082784c11226853f5feb805df9',
function (err) {
console.log('[sync-level.js.15]',err); //TODO
});

19
dev-util/tx-level.js Executable file
View File

@ -0,0 +1,19 @@
#!/usr/bin/env node
var
config = require('../config/config'),
levelup = require('levelup');
db = levelup(config.leveldb + '/txs');
var s = 'txouts-addr-mgqvRGJMwR9JU5VhJ3x9uX9MTkzTsmmDgQ';
db.createReadStream({start: s, end: s+'~'})
.on('data', function (data) {
console.log('[block-level.js.11:data:]',data); //TODO
if (data==false) c++;
})
.on('end', function () {
});

24
insight.js
View File

@ -9,9 +9,7 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development';
var express = require('express'),
fs = require('fs'),
PeerSync = require('./lib/PeerSync').class(),
HistoricSync = require('./lib/HistoricSync').class(),
mongoose = require('mongoose');
HistoricSync = require('./lib/HistoricSync').class();
//Initializing system variables
var config = require('./config/config');
@ -21,22 +19,6 @@ var config = require('./config/config');
*/
var expressApp = express();
/**
* Bootstrap db connection
*/
// If mongod is running
mongoose.connection.on('open', function() {
console.log('Connected to mongo server.');
});
// If mongod is not running
mongoose.connection.on('error', function(err) {
console.log('Could not connect to mongo server!');
console.log(err);
});
mongoose.connect(config.db);
/**
* Bootstrap models
*/
@ -67,7 +49,6 @@ if (!config.disableHistoricSync) {
historicSync = new HistoricSync();
historicSync.init({
skipDbConnection: true,
shouldBroadcast: true,
networkName: config.network
}, function(err) {
@ -76,7 +57,7 @@ if (!config.disableHistoricSync) {
console.log('[historic_sync] ' + txt);
}
else {
historicSync.smartImport(function(err){
historicSync.smartImport({}, function(err){
var txt = 'ended.';
if (err) txt = 'ABORTED with error: ' + err.message;
console.log('[historic_sync] ' + txt, historicSync.info());
@ -91,7 +72,6 @@ if (!config.disableHistoricSync) {
if (!config.disableP2pSync) {
var ps = new PeerSync();
ps.init({
skipDbConnection: true,
broadcast_txs: true,
broadcast_address_tx: true,
broadcast_blocks: true,

165
lib/BlockDb.js Normal file
View File

@ -0,0 +1,165 @@
'use strict';
require('classtool');
function spec(b) {
var TIMESTAMP_ROOT = 'b-ts-'; // b-ts-<ts> => <hash>
var PREV_ROOT = 'b-prev-'; // b-prev-<hash> => <prev_hash> (0 if orphan)
/**
* Module dependencies.
*/
var RpcClient = require('bitcore/RpcClient').class(),
util = require('bitcore/util/util'),
levelup = require('levelup'),
BitcoreBlock= require('bitcore/Block').class(),
config = require('../config/config');
var db = b.db || levelup(config.leveldb + '/blocks');
var rpc = b.rpc || new RpcClient(config.bitcoind);
var BlockDb = function() {
};
BlockDb.prototype.close = function(cb) {
db.close(cb);
};
BlockDb.prototype.drop = function(cb) {
var path = config.leveldb + '/blocks';
db.close(function() {
require('leveldown').destroy(path, function () {
db = levelup(path);
return cb();
});
});
};
BlockDb.prototype.add = function(b, cb) {
if (!b.hash) return cb(new Error('no Hash at Block.save'));
var time_key = TIMESTAMP_ROOT +
( b.time || Math.round(new Date().getTime() / 1000) );
db.batch()
.put(time_key, b.hash)
.put(PREV_ROOT + b.hash, b.previousblockhash)
.write(cb);
};
BlockDb.prototype.setOrphan = function(hash, cb) {
var k = PREV_ROOT + hash;
db.get(k, function (err,oldPrevHash) {
if (err || !oldPrevHash) return cb(err);
db.put(PREV_ROOT + hash, 0, function() {
return cb(err, oldPrevHash);
});
});
// We keep the block in TIMESTAMP_ROOT
};
//mainly for testing
BlockDb.prototype.setPrev = function(hash, prevHash, cb) {
db.put(PREV_ROOT + hash, prevHash, function(err) {
return cb(err);
});
};
//mainly for testing
BlockDb.prototype.getPrev = function(hash, cb) {
db.get(PREV_ROOT + hash, function(err,val) {
return cb(err,val);
});
};
BlockDb.prototype.countNotOrphan = function(cb) {
var c = 0;
console.log('Counting connected blocks. This could take some minutes');
db.createReadStream({start: PREV_ROOT, end: PREV_ROOT + '~' })
.on('data', function (data) {
if (data.value !== 0) c++;
})
.on('error', function (err) {
return cb(err);
})
.on('end', function () {
return cb(null, c);
});
};
BlockDb.prototype.has = function(hash, cb) {
var k = PREV_ROOT + hash;
db.get(k, function (err,val) {
var ret;
if (err && err.notFound) {
err = null;
ret = false;
}
if (typeof val !== 'undefined') {
ret = true;
}
return cb(err, ret);
});
};
BlockDb.prototype.fromHashWithInfo = function(hash, cb) {
rpc.getBlock(hash, function(err, info) {
// Not found?
if (err && err.code === -5) return cb();
if (err) return cb(err);
info.reward = BitcoreBlock.getBlockValue(info.height) / util.COIN ;
return cb(null, {
hash: hash,
info: info.result,
});
});
};
BlockDb.prototype.getBlocksByDate = function(start_ts, end_ts, limit, cb) {
var list = [];
db.createReadStream({
start: TIMESTAMP_ROOT + start_ts,
end: TIMESTAMP_ROOT + end_ts,
fillCache: true,
limit: parseInt(limit) // force to int
})
.on('data', function (data) {
list.push({
ts: data.key.replace(TIMESTAMP_ROOT, ''),
hash: data.value,
});
})
.on('error', function (err) {
return cb(err);
})
.on('end', function () {
return cb(null, list);
});
};
BlockDb.prototype.blockIndex = function(height, cb) {
var rpc = new RpcClient(config.bitcoind);
rpc.getBlockHash(height, function(err, bh){
if (err) return cb(err);
cb(null, { blockHash: bh.result });
});
};
return BlockDb;
}
module.defineClass(spec);

9
lib/BlockExtractor.js
View File

@ -113,7 +113,7 @@ function spec() {
magic = self.currentParser ? self.currentParser.buffer(4).toString('hex')
: null ;
if (!self.currentParser || self.currentParser.eof()) {
if (!self.currentParser || self.currentParser.eof() || magic === '00000000') {
magic = null;
if (self.nextFile()) {
console.log('Moving forward to file:' + self.currentFile() );
@ -121,7 +121,8 @@ function spec() {
}
else {
console.log('Finished all files');
return cb();
magic = null;
return w_cb();
}
}
else {
@ -130,6 +131,8 @@ function spec() {
}, a_cb);
},
function (a_cb) {
if (!magic) return a_cb();
if (magic !== self.magic) {
var e = new Error('CRITICAL ERROR: Magic number mismatch: ' +
magic + '!=' + self.magic);
@ -141,6 +144,8 @@ function spec() {
return a_cb();
},
function (a_cb) {
if (!magic) return a_cb();
b = new Block();
b.parse(self.currentParser);
b.getHash();

299
lib/HistoricSync.js
View File

@ -13,10 +13,10 @@ function spec() {
var networks = require('bitcore/networks');
var async = require('async');
var config = require('../config/config');
var Block = require('../app/models/Block');
var Sync = require('./Sync').class();
var sockets = require('../app/controllers/socket.js');
var BlockExtractor = require('./BlockExtractor.js').class();
//var Deserialize = require('bitcore/Deserialize');
var BAD_GEN_ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:';
@ -36,7 +36,8 @@ function spec() {
this.syncPercentage = 0;
this.syncedBlocks = 0;
this.skippedBlocks = 0;
this.orphanBlocks = 0;
this.type ='';
}
function p() {
@ -95,11 +96,13 @@ function spec() {
syncPercentage: this.syncPercentage,
skippedBlocks: this.skippedBlocks,
syncedBlocks: this.syncedBlocks,
orphanBlocks: this.orphanBlocks,
error: this.error,
type: this.type,
};
};
HistoricSync.prototype.showProgress = function(height) {
HistoricSync.prototype.showProgress = function() {
var self = this;
if (self.error) {
@ -109,7 +112,7 @@ function spec() {
self.syncPercentage = parseFloat(100 * (self.syncedBlocks + self.skippedBlocks) / self.blockChainHeight).toFixed(3);
if (self.syncPercentage > 100) self.syncPercentage = 100;
p(util.format('status: [%d%%] skipped: %d ', self.syncPercentage, self.skippedBlocks, height));
p(util.format('status: [%d%%] skipped: %d ', self.syncPercentage, self.skippedBlocks));
}
if (self.opts.shouldBroadcast) {
sockets.broadcastSyncInfo(self.info());
@ -128,14 +131,13 @@ function spec() {
async.series([
// Already got it?
function(c) {
Block.fromHash(blockHash, function(err, block) {
self.sync.bDb.has(blockHash, function(err, ret) {
if (err) {
p(err);
return c(err);
}
if (block) {
existed = true;
}
if (ret) existed = true;
return c();
});
},
@ -161,11 +163,7 @@ function spec() {
if (existed) return c();
self.sync.storeBlock(blockInfo, function(err) {
existed = err && err.toString().match(/E11000/);
if (err && ! existed) return c(err);
return c();
return c(err);
});
},
/* TODO: Should Start to sync backwards? (this is for partial syncs)
@ -181,14 +179,10 @@ function spec() {
], function(err) {
if (err) {
self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncedBlocks);
self.status = 'aborted';
self.showProgress();
p(self.err);
self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncedBlocks));
return cb(err);
}
else {
self.err = null;
self.status = 'syncing';
}
@ -244,9 +238,7 @@ function spec() {
addrStrs = [ addr.toString() ];
break;
case Script.TX_MULTISIG:
var addrs = [];
var chunks = s.capture();
chunks.forEach(function(chunk) {
var a = new Address(self.network.addressPubkey, bitutil.sha256ripe160(chunk));
addrStrs.push(a.toString());
@ -259,44 +251,40 @@ function spec() {
return addrStrs;
};
HistoricSync.prototype.getBlockFromFile = function(height, scanOpts, cb) {
HistoricSync.prototype.getBlockFromFile = function(scanOpts, cb) {
var self = this;
var nextHash;
var blockInfo;
var isMainChain;
var existed;
async.series([
// Is it in mainchain?
function(c) {
self.rpc.getBlockHash(height, function(err, res) {
if (err) return cb(err);
nextHash = res.result;
return c();
});
},
//show some (inacurate) status
function(c) {
if ( ( self.syncedBlocks + self.skippedBlocks) % self.step === 1) {
self.showProgress(height);
self.showProgress();
}
return c();
},
//get Info
function(c) {
self.blockExtractor.getNextBlock(function(err, b) {
if (err || ! b) return c(err);
blockInfo = b.getStandardizedObject(b.txs, self.network);
// blockInfo.curWork = Deserialize.intFromCompact(b.bits);
// We keep the RPC field names
blockInfo.previousblockhash = blockInfo.prev_block;
var ti=0;
// Get TX Address
b.txs.forEach(function(t) {
var objTx = blockInfo.tx[ti++];
//add time from block
objTx.time = blockInfo.time;
var to=0;
t.outs.forEach( function(o) {
@ -315,66 +303,83 @@ function spec() {
return c();
});
},
//store it
//check prev
function(c) {
if (blockInfo && self.prevHash && blockInfo.previousblockhash !== self.prevHash) {
isMainChain = blockInfo.hash === nextHash;
console.log('Hole found @%s', blockInfo.hash);
console.log('From: %s To: %s', self.prevHash, blockInfo.previousblockhash);
self.sync.checkOrphan(self.prevHash, blockInfo.previousblockhash, c);
}
else return c();
},
//check it
function(c) {
if (!blockInfo) return c();
blockInfo.isOrphan = !isMainChain;
/*
* In file sync, orphan blocks are just ignored.
* This is to simplify our schema and the
* sync process
*/
if (blockInfo.isOrphan) return c();
self.sync.storeBlock(blockInfo, function(err) {
existed = err && err.toString().match(/E11000/);
if (err && ! existed) return c(err);
return c();
self.sync.bDb.has(blockInfo.hash, function(err, had) {
existed = had;
return c(err);
});
},
], function(err) {
if (err) {
self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockInfo.hash, err, self.syncedBlocks);
self.status = 'aborted';
self.showProgress();
p(err);
return cb(err);
}
else {
// Continue
if (blockInfo) {
// mainchain
if (isMainChain) height++;
//store it
function(c) {
if (!blockInfo || existed) return c();
self.sync.storeBlock(blockInfo, function(err) {
return c(err);
});
},
function(c) {
if (blockInfo && blockInfo.hash) {
self.prevHash = blockInfo.hash;
if (existed)
self.skippedBlocks++;
else
self.syncedBlocks++;
self.err = null;
self.status = 'syncing';
} else
self.status = 'finished';
return self.getBlockFromFile(height, scanOpts, cb);
}
else {
self.err = null;
self.status = 'finished';
}
return c();
},
], function(err) {
if (err) {
self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockInfo ? blockInfo.hash : '-', err, self.syncedBlocks));
}
return cb(err);
});
};
HistoricSync.prototype.countNotOrphan = function(cb) {
var self = this;
if (self.notOrphanCount) return cb(null, self.notOrphanCount);
self.sync.bDb.countNotOrphan(function(err, count) {
if (err) return cb(err);
self.notOrphanCount = count;
return cb(null, self.notOrphanCount);
});
};
HistoricSync.prototype.getBlockCount = function(cb) {
var self = this;
if (self.blockChainHeight) return cb();
self.rpc.getBlockCount(function(err, res) {
if (err) return cb(err);
self.blockChainHeight = res.result;
return cb();
});
};
HistoricSync.prototype.importHistory = function(scanOpts, next) {
var self = this;
var retry_secs = 2;
var lastBlock;
async.series([
@ -386,13 +391,7 @@ function spec() {
return cb();
},
// We are not using getBestBlockHash, because is not available in all clients
function(cb) {
self.rpc.getBlockCount(function(err, res) {
if (err) return cb(err);
self.blockChainHeight = res.result;
return cb();
});
},
function (cb) { return self.getBlockCount(cb); },
function(cb) {
if (!scanOpts.reverse) return cb();
@ -406,10 +405,7 @@ function spec() {
function(cb) {
if (scanOpts.upToExisting) {
// should be isOrphan = true or null to be more accurate.
Block.count({
isOrphan: null
},
function(err, count) {
self.countNotOrphan(function(err, count) {
if (err) return cb(err);
self.syncedBlocks = count || 0;
@ -421,44 +417,16 @@ function spec() {
}
},
], function(err) {
var start, end;
function sync() {
if (scanOpts.reverse) {
start = lastBlock;
end = self.genesis;
scanOpts.prev = true;
}
else {
start = self.genesis;
end = null;
scanOpts.next = true;
}
p('Starting from: ', start);
p(' to : ', end);
p(' scanOpts: ', JSON.stringify(scanOpts));
if (scanOpts.fromFiles) {
self.getBlockFromFile(0, scanOpts, function(err) {
return next(err);
});
}
else {
self.getPrevNextBlock(start, end, scanOpts, function(err) {
if (err && err.message.match(/ECONNREFUSED/)) {
setTimeout(function() {
p('Retrying in %d secs', retry_secs);
sync();
},
retry_secs * 1000);
}
else return next(err);
});
}
if (err) {
self.setError(err);
return next(err, 0);
}
if (!self.step) {
// SETUP Sync params
var start, end;
if (!self.step) {
var step = parseInt( (self.blockChainHeight - self.syncedBlocks) / 1000);
if (self.opts.progressStep) {
@ -469,13 +437,42 @@ function spec() {
self.step = step;
}
if (err) {
self.setError(err);
return next(err, 0);
if (scanOpts.reverse) {
start = lastBlock;
end = self.genesis;
scanOpts.prev = true;
}
else {
sync();
start = self.genesis;
end = null;
scanOpts.next = true;
}
p('Starting from: ', start);
p(' to : ', end);
p(' scanOpts: ', JSON.stringify(scanOpts));
if (scanOpts.fromFiles) {
self.status = 'syncing';
self.type = 'from .dat Files';
async.whilst(function() {
return self.status === 'syncing';
}, function (w_cb) {
self.getBlockFromFile(scanOpts, function(err) {
setImmediate(function(){
return w_cb(err);
});
});
}, function(err) {
return next(err);
});
}
else {
self.type = 'from RPC calls';
self.getPrevNextBlock(start, end, scanOpts, function(err) {
return next(err);
});
}
});
};
@ -483,29 +480,39 @@ function spec() {
HistoricSync.prototype.smartImport = function(scanOpts, next) {
var self = this;
Block.fromHash(self.genesis, function(err, b) {
self.sync.bDb.has(self.genesis, function(err, b) {
if (err) return next(err);
self.countNotOrphan(function(err, count) {
if (err) return next(err);
self.getBlockCount(function(err) {
if (err) return next(err);
if (!b || scanOpts.destroy || count < self.blockChainHeight * 0.8 ) {
if (!b || scanOpts.destroy) {
p('Could not find Genesis block. Running FULL SYNC');
if (config.bitcoind.dataDir) {
p('bitcoind dataDir configured...importing blocks from .dat files');
scanOpts.fromFiles = true;
self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network);
}
else {
scanOpts.reverse = true;
}
}
else {
p('Genesis block found. Syncing upto known blocks.');
scanOpts.reverse = true;
scanOpts.upToExisting = true;
}
if (!b)
p('Could not find Genesis block. Running FULL SYNC');
else
p('Less that 80% of current blockchain is stored. Running FULL SYNC',
parseInt(count/self.blockChainHeight*100));
return self.importHistory(scanOpts, next);
if (config.bitcoind.dataDir) {
p('bitcoind dataDir configured...importing blocks from .dat files');
scanOpts.fromFiles = true;
self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network);
}
else {
scanOpts.reverse = true;
}
}
else {
p('Genesis block found. Syncing upto known blocks.');
p('Got %d out of %d blocks', count, self.blockChainHeight);
scanOpts.reverse = true;
scanOpts.upToExisting = true;
}
return self.importHistory(scanOpts, next);
});
});
});
};

114
lib/Sync.js
View File

@ -4,15 +4,16 @@ require('classtool');
function spec() {
var mongoose = require('mongoose');
var config = require('../config/config');
var Block = require('../app/models/Block');
var TransactionOut = require('../app/models/TransactionOut');
var sockets = require('../app/controllers/socket.js');
var BlockDb = require('./BlockDb').class();
var TransactionDb = require('./TransactionDb').class();
var async = require('async');
function Sync() {
this.bDb = new BlockDb();
this.txDb = new TransactionDb();
}
Sync.prototype.init = function(opts, cb) {
@ -20,70 +21,88 @@ function spec() {
self.opts = opts;
if (!(opts && opts.skipDbConnection)) {
if (mongoose.connection.readyState !== 1) {
mongoose.connect(config.db, function(err) {
if (err) {
console.log('CRITICAL ERROR: connecting to mongoDB:',err);
return (err);
}
});
}
self.db = mongoose.connection;
self.db.on('error', function(err) {
console.log('MongoDB ERROR:' + err);
return cb(err);
});
self.db.on('disconnect', function(err) {
console.log('MongoDB disconnect:' + err);
return cb(err);
});
return self.db.once('open', function(err) {
return cb(err);
});
}
else return cb();
return cb();
};
Sync.prototype.close = function() {
if ( this.db && this.db.readyState ) {
this.db.close();
}
Sync.prototype.close = function(cb) {
var self = this;
self.txDb.close(function() {
self.bDb.close(cb);
});
};
Sync.prototype.destroy = function(next) {
var self = this;
async.series([
function(b) { try {self.db.collections.blocks.drop(b);} catch (e) { return b(); } },
function(b) { try {self.db.collections.transactionitems.drop(b);} catch (e) { return b(); } },
function(b) { try {self.db.collections.transactionouts.drop(b);} catch (e) { return b(); } },
function(b) { self.bDb.drop(b); },
function(b) { self.txDb.drop(b); },
], next);
};
Sync.prototype.storeBlock = function(block, cb) {
var self = this;
Block.customCreate(block, function(err, block, inserted_txs, updated_addrs){
self.txDb.createFromBlock(block, function(err, insertedTxs, updateAddrs) {
if (err) return cb(err);
self._handleBroadcast(block, inserted_txs, updated_addrs);
return cb();
self.bDb.add(block, function(err){
if (err) return cb(err);
self._handleBroadcast(block, insertedTxs, updateAddrs);
return cb();
});
});
};
Sync.prototype._handleBroadcast = function(block, inserted_txs, updated_addrs) {
Sync.prototype.checkOrphan = function(fromBlock, toBlock, c) {
var self = this;
if (block && self.opts.broadcast_blocks) {
sockets.broadcast_block(block);
var hash = fromBlock;
var co = 0;
var limit = 10;
var cont = 1;
async.whilst(
function () {
if (++co > limit) {
console.log('[Sync.js.109] WARN: Reach reog depth limit'); //TODO
}
return cont && hash && hash !== toBlock && co < limit;
},
function (w_c) {
//check with RPC if the block is mainchain
self.bDb.fromHashWithInfo(hash, function (err, info) {
if (!info) {
console.log('[Sync.js.107:hash:ORPHAN]',hash); //TODO
self.txDb.setOrphan(hash, function(err) {
if (err) return w_c(err);
self.bDb.setOrphan(hash, function(err, prevHash){
hash = prevHash;
return w_c(err);
});
});
}
else {
console.log('[Sync.js.107:hash:NOT ORPHAN]',hash); //TODO
cont = 0;
return w_c();
}
});
},
function (err) {
return c(err);
}
);
};
Sync.prototype._handleBroadcast = function(hash, inserted_txs, updated_addrs) {
var self = this;
if (hash && self.opts.broadcast_blocks) {
sockets.broadcast_block({hash: hash});
}
if (inserted_txs && self.opts.broadcast_txs) {
@ -107,7 +126,8 @@ function spec() {
Sync.prototype.storeTxs = function(txs, cb) {
var self = this;
TransactionOut.createFromTxs(txs, function(err, inserted_txs, updated_addrs) {
// TODO
self.txDb.createFromTxs(txs, function(err, inserted_txs, updated_addrs) {
if (err) return cb(err);
self._handleBroadcast(null, inserted_txs, updated_addrs);

444
lib/TransactionDb.js Normal file
View File

@ -0,0 +1,444 @@
'use strict';
require('classtool');
function spec(b) {
// blockHash -> txid mapping (to reorgs)
var ROOT = 'tx-b-'; //tx-b-<txid>-<block> => 1/0 (connected or not)
// to show tx outs
var OUTS_ROOT = 'txouts-'; //txouts-<txid>-<n> => [addr, btc_sat]
// to sum up addr balance
var ADDR_ROOT = 'txouts-addr-'; //txouts-addr-<addr>-<ts>-<txid>-<n> => + btc_sat
var SPEND_ROOT = 'txouts-spend-';//txouts-spend-<txid(out)>-<n(out)> => [txid(in),n(in),ts]
// TODO: use bitcore networks module
var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b';
var CONCURRENCY = 100;
/**
* Module dependencies.
*/
var TransactionRpc = require('./TransactionRpc').class(),
util = require('bitcore/util/util'),
levelup = require('levelup'),
async = require('async'),
config = require('../config/config'),
assert = require('assert');
var db = b.db || levelup(config.leveldb + '/txs');
var TransactionDb = function() {
};
TransactionDb.prototype.close = function(cb) {
db.close(cb);
};
TransactionDb.prototype.drop = function(cb) {
var self = this;
var path = config.leveldb + '/txs';
db.close(function() {
require('leveldown').destroy(path, function () {
db = levelup(path);
return cb();
});
});
};
// TransactionDb.prototype.fromTxIdOne = function(txid, cb) { TODO
TransactionDb.prototype.has = function(txid, cb) {
var self = this;
var k = OUTS_ROOT + txid;
db.get(k, function (err,val) {
var ret;
if (err && err.notFound) {
err = null;
ret = false;
}
if (typeof val !== undefined) {
ret = true;
}
return cb(err, ret);
});
};
TransactionDb.prototype.fromTxId = function(txid, cb) {
var self = this;
var k = OUTS_ROOT + txid;
var ret=[];
// outs.
db.createReadStream({start: k, end: k + '~'})
.on('data', function (data) {
var k = data.key.split('-');
var v = data.value.split(':');
ret.push({
addr: v[0],
value_sat: parseInt(v[1]),
index: parseInt(k[2]),
});
})
.on('error', function (err) {
return cb(err);
})
.on('end', function () {
var k = SPEND_ROOT + txid;
var l = ret.length;
db.createReadStream({start: k, end: k + '~'})
.on('data', function (data) {
var k = data.key.split('-');
var v = data.value.split(':');
var set=0;
for(var i=0; i<l; i++) {
if (ret[i].index === parseInt(k[3])) {
ret[i].spendTxId = v[0];
ret[i].spendIndex = parseInt(v[1]);
set=1;
}
}
assert(set,'Spent could not be stored: tx ' + txid +
'spend in TX:' + k[2] + ',' + k[3]);
})
.on('error', function (err) {
return cb(err);
})
.on('end', function (err) {
return cb(err, ret);
});
});
};
TransactionDb.prototype._fillOutpoints = function(info, cb) {
var self = this;
if (!info || info.isCoinBase) return cb();
var valueIn = 0;
var incompleteInputs = 0;
async.eachLimit(info.vin, CONCURRENCY, function(i, c_in) {
self.fromTxIdN(i.txid, i.vout, function(err, addr, valueSat) {
if (err || !addr || !valueSat ) {
console.log('Could not get TXouts in %s,%d from %s ', i.txid, i.vout, info.txid);
incompleteInputs = 1;
return c_in(); // error not scaled
}
i.addr = addr;
i.valueSat = valueSat;
i.value = valueSat / util.COIN;
valueIn += i.valueSat;
return c_in();
});
},
function () {
if (! incompleteInputs ) {
info.valueIn = valueIn / util.COIN;
info.fees = (valueIn - parseInt(info.valueOut * util.COIN)) / util.COIN ;
}
else {
info.incompleteInputs = 1;
}
return cb();
});
};
TransactionDb.prototype._getInfo = function(txid, next) {
var self = this;
TransactionRpc.getRpcInfo(txid, function(err, info) {
if (err) return next(err);
self._fillOutpoints(info, function() {
return next(null, info);
});
});
};
TransactionDb.prototype.fromIdWithInfo = function (txid, cb) {
var self = this;
self._getInfo(txid, function(err, info) {
if (err) return cb(err);
if (!info ) return cb();
return cb(err, {txid: txid, info: info} );
});
};
TransactionDb.prototype.fromTxIdN = function(txid, n, cb) {
var self = this;
var k = OUTS_ROOT + txid + '-' + n;
db.get(k, function (err,val) {
if (err && err.notFound) {
err = null;
}
var a = val.split(':');
return cb(err, a[0], parseInt(a[1]));
});
};
TransactionDb.prototype.fromAddr = function(addr, cb) {
var self = this;
var k = ADDR_ROOT + addr;
var ret=[];
//
db.createReadStream({start: k, end: k + '~'})
.on('data', function (data) {
var k = data.key.split('-');
var v = data.value.split(':');
ret.push({
value_sat: parseInt(v[0]),
ts: parseInt(k[3]),
txid: k[4],
index: parseInt(k[5]),
});
})
.on('error', function (err) {
return cb(err);
})
.on('end', function () {
async.each(ret, function(o, e_c) {
var k = SPEND_ROOT + o.txid + '-' + o.index;
db.get(k, function(err, val) {
if (err && err.notFound) err=null;
if (err || !val) return e_c(err);
var v = val.split(':');
o.spendTxId= v[0];
o.spendIndex=parseInt(v[1]);
o.spendTs=parseInt(v[2]);
return e_c();
});
},
function(err) {
return cb(err,ret);
});
});
};
TransactionDb.prototype.removeFromTxId = function(txid, cb) {
var self = this;
async.series([
function(c) {
db.createReadStream({
start: OUTS_ROOT + txid,
end: OUTS_ROOT + txid + '~',
}).pipe(
db.createWriteStream({type:'del'})
).on('close', c);
},
function(c) {
db.createReadStream({
start: SPEND_ROOT + txid,
end: SPEND_ROOT + txid + '~'
})
.pipe(
db.createWriteStream({type:'del'})
).on('close', c);
}],
function(err) {
cb(err);
});
};
TransactionDb.prototype.adaptTxObject = function(txInfo) {
// adapt bitcore TX object to bitcoind JSON response
txInfo.txid = txInfo.hash;
var count = 0;
txInfo.vin = txInfo.in.map(function (txin) {
var i = {};
if (txin.coinbase) {
txInfo.isCoinBase = true;
}
else {
i.txid= txin.prev_out.hash;
i.vout= txin.prev_out.n;
}
i.n = count++;
return i;
});
count = 0;
txInfo.vout = txInfo.out.map(function (txout) {
var o = {};
o.value = txout.value;
o.n = count++;
if (txout.addrStr){
o.scriptPubKey = {};
o.scriptPubKey.addresses = [txout.addrStr];
}
return o;
});
};
TransactionDb.prototype.add = function(tx, cb) {
var self = this;
var addrs = [];
var is_new = true;
if (tx.hash) self.adaptTxObject(tx);
var ts = tx.time;
async.series([
// Input Outpoints (mark them as spended)
function(p_c) {
if (tx.isCoinBase) return p_c();
async.forEachLimit(tx.vin, CONCURRENCY,
function(i, next_out) {
db.batch()
.put( SPEND_ROOT + i.txid + '-' + i.vout ,
tx.txid + ':' + i.n + ':' + ts)
.write(next_out);
},
function (err) {
if (err) {
if (!err.message.match(/E11000/)) {
console.log('ERR at TX %s: %s', tx.txid, err);
return cb(err);
}
}
return p_c();
});
},
// Parse Outputs
function(p_c) {
async.forEachLimit(tx.vout, CONCURRENCY,
function(o, next_out) {
if (o.value && o.scriptPubKey &&
o.scriptPubKey.addresses &&
o.scriptPubKey.addresses[0] &&
! o.scriptPubKey.addresses[1] // TODO : not supported
){
// This is only to broadcast (WIP)
// if (addrs.indexOf(o.scriptPubKey.addresses[0]) === -1) {
// addrs.push(o.scriptPubKey.addresses[0]);
// }
var addr = o.scriptPubKey.addresses[0];
var sat = Math.round(o.value * util.COIN);
db.batch()
.put( OUTS_ROOT + tx.txid + '-' + o.n, addr + ':' + sat)
.put( ADDR_ROOT + addr + '-' + ts + '-' + tx.txid +
'-' + o.n, sat)
.write(next_out);
}
else {
console.log ('WARN in TX: %s could not parse OUTPUT %d', tx.txid, o.n);
return next_out();
}
},
function (err) {
if (err) {
if (err.message.match(/E11000/)) {
is_new = false;
}
else {
console.log('ERR at TX %s: %s', tx.txid, err);
return cb(err);
}
}
return p_c();
});
}], function(err) {
return cb(err, addrs, is_new);
});
};
TransactionDb.prototype.createFromArray = function(txs, blockHash, next) {
var self = this;
if (!txs) return next();
// TODO
var insertedTxs = [];
var updatedAddrs = {};
async.forEachLimit(txs, CONCURRENCY, function(t, each_cb) {
if (typeof t === 'string') {
// Is it from genesis block? (testnet==livenet)
// TODO: parse it from networks.genesisTX?
if (t === genesisTXID) return each_cb();
TransactionRpc.getRpcInfo(t, function(err, inInfo) {
if (!inInfo) return each_cb(err);
self.add(inInfo, function(err) {
if (err || !blockHash) return each_cb(err);
db.put(ROOT + t + '-' + blockHash, 1, function(err) {
return each_cb(err);
});
});
});
}
else {
self.add(t, function(err) {
if (err) return each_cb(err);
db.put(ROOT + t.txid + '-' + blockHash, 1, function(err) {
return each_cb(err);
});
});
}
},
function(err) {
return next(err, insertedTxs, updatedAddrs);
});
};
// txs can be a [hashes] or [txObjects]
TransactionDb.prototype.createFromBlock = function(b, next) {
var self = this;
if (!b || !b.tx) return next();
return self.createFromArray(b.tx, b.hash, next);
};
TransactionDb.prototype.setOrphan = function(blockHash, next) {
// var self = this;
//Get Txs
// TODO
//Mark Tx's output as fromOrphan
//Mark Tx's outpoiunt as fromOrphan. Undo spents
return next();
};
return TransactionDb;
}
module.defineClass(spec);

11
lib/TransactionRpc.js
View File

@ -3,7 +3,7 @@
require('classtool');
function spec() {
function spec(b) {
var RpcClient = require('bitcore/RpcClient').class(),
// networks = require('bitcore/network'),
BitcoreTransaction = require('bitcore/Transaction').class(),
@ -11,8 +11,9 @@ function spec() {
util = require('bitcore/util/util'),
config = require('../config/config');
var rpc = b.rpc || new RpcClient(config.bitcoind);
function TransactionRpc() {
this.dummy = null;
}
TransactionRpc._parseRpcResult = function(info) {
@ -37,9 +38,9 @@ function spec() {
// Outputs
var valueOut = 0;
info.vout.forEach( function(o) {
valueOut += o.value * util.COIN;
valueOut += o.value;
});
info.valueOut = valueOut / util.COIN;
info.valueOut = valueOut;
info.size = b.length;
return info;
@ -48,8 +49,6 @@ function spec() {
TransactionRpc.getRpcInfo = function(txid, cb) {
var Self = this;
var rpc = new RpcClient(config.bitcoind);
rpc.getRawTransaction(txid, 1, function(err, txInfo) {
// Not found?

3
package.json
View File

@ -52,13 +52,14 @@
},
"dependencies": {
"async": "*",
"leveldown": "*",
"levelup": "*",
"glob": "*",
"classtool": "*",
"commander": "*",
"bignum": "*",
"express": "~3.4.7",
"jade": "~1.0.2",
"mongoose": "~3.8.3",
"lodash": "~2.4.1",
"bower": "~1.2.8",
"buffertools": "*",

4
public/js/config.js
View File

@ -20,11 +20,11 @@ angular.module('insight').config(function($routeProvider) {
title: 'Home'
}).
when('/blocks', {
templateUrl: '/views/blocks_list.html',
templateUrl: '/views/block_list.html',
title: 'Bitcoin Blocks solved Today'
}).
when('/blocks-date/:blockDate', {
templateUrl: '/views/blocks_list.html',
templateUrl: '/views/block_list.html',
title: 'Bitcoin Blocks solved '
}).
when('/address/:addrStr', {

176
test/integration/01-transactionouts.js
View File

@ -5,60 +5,161 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development';
var mongoose = require('mongoose'),
assert = require('assert'),
var assert = require('assert'),
fs = require('fs'),
util = require('util'),
config = require('../../config/config'),
TransactionOut = require('../../app/models/TransactionOut');
TransactionDb = require('../../lib/TransactionDb').class();
var txItemsValid = JSON.parse(fs.readFileSync('test/model/txitems.json'));
var txItemsValid = JSON.parse(fs.readFileSync('test/integration/txitems.json'));
var txDb;
mongoose.connection.on('error', function(err) { console.log(err); });
describe('TransactionDb fromIdWithInfo', function(){
describe('TransactionOut', function(){
before(function(done) {
mongoose.connect(config.db);
done();
before(function(c) {
txDb = new TransactionDb();
return c();
});
after(function(done) {
mongoose.connection.close();
done();
var txid = '7e621eeb02874ab039a8566fd36f4591e65eca65313875221842c53de6907d6c';
it('tx info ' + txid, function(done) {
txDb.fromIdWithInfo(txid, function(err, tx) {
if (err) done(err);
assert.equal(tx.txid, txid);
assert(!tx.info.isCoinBase);
for(var i=0; i<20; i++)
assert(parseFloat(tx.info.vin[i].value) === parseFloat(50), 'input '+i);
assert(tx.info.vin[0].addr === 'msGKGCy2i8wbKS5Fo1LbWUTJnf1GoFFG59', 'addr 0');
assert(tx.info.vin[1].addr === 'mfye7oHsdrHbydtj4coPXCasKad2eYSv5P', 'addr 1');
done();
});
});
it('tx info', function(done) {
var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237';
txDb.fromIdWithInfo(txid, function(err, tx) {
if (err) done(err);
assert.equal(tx.txid, txid);
assert(!tx.info.isCoinBase);
done();
});
});
it('should pool tx\'s info from bitcoind', function(done) {
var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237';
txDb.fromIdWithInfo(txid, function(err, tx) {
if (err) done(err);
assert.equal(tx.info.txid, txid);
assert.equal(tx.info.blockhash, '000000000185678d3d7ecc9962c96418174431f93fe20bf216d5565272423f74');
assert.equal(tx.info.valueOut, 1.66174);
assert.equal(tx.info.fees, 0.0005 );
assert.equal(tx.info.size, 226 );
assert(!tx.info.isCoinBase);
done();
});
});
var txid1 = '2a104bab1782e9b6445583296d4a0ecc8af304e4769ceb64b890e8219c562399';
it('test a coinbase TX ' + txid1, function(done) {
txDb.fromIdWithInfo(txid1, function(err, tx) {
if (err) done(err);
assert(tx.info.isCoinBase);
assert.equal(tx.info.txid, txid1);
assert(!tx.info.feeds);
done();
});
});
var txid22 = '666';
it('test invalid TX ' + txid22, function(done) {
txDb.fromIdWithInfo(txid22, function(err, tx) {
if (err && err.message.match(/must.be.hexadecimal/)) {
return done();
}
else {
return done(err);
}
});
});
var txid23 = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca227';
it('test unexisting TX ' + txid23, function(done) {
txDb.fromIdWithInfo(txid23, function(err, tx) {
assert(!err);
assert(!tx);
return done();
});
});
var txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608';
it('create TX on the fly ' + txid2, function(done) {
txDb.fromIdWithInfo(txid2, function(err, tx) {
if (err) return done(err);
assert.equal(tx.info.txid, txid2);
done();
});
});
txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608';
it('test a broken TX ' + txid2, function(done) {
txDb.fromIdWithInfo(txid2, function(err, tx) {
if (err) return done(err);
assert.equal(tx.info.txid, txid2);
assert.equal(tx.info.vin[0].addr, 'n1JagbRWBDi6VMvG7HfZmXX74dB9eiHJzU');
done();
});
});
});
describe('TransactionDb Outs', function(){
before(function(c) {
txDb = new TransactionDb();
return c();
});
txItemsValid.forEach( function(v) {
if (v.disabled) return;
it('test a exploding tx ' + v.txid, function(done) {
it('test a processing tx ' + v.txid, function(done) {
this.timeout(60000);
// Remove first
TransactionOut.removeFromTxId(v.txid, function(err) {
TransactionOut._explodeTransactionOuts(v.txid, function(err, tx) {
if (err) done(err);
txDb.removeFromTxId(v.txid, function() {
TransactionOut
.fromTxId( v.txid, function(err, readItems) {
txDb.fromTxId( v.txid, function(err, readItems) {
assert.equal(readItems.length,0);
var unmatch={};
var unmatch=[];
txDb.createFromArray([v.txid], null, function(err) {
if (err) return done(err);
txDb.fromTxId( v.txid, function(err, readItems) {
v.items.forEach(function(validItem){
unmatch[validItem.addr] =1;
});
assert.equal(readItems.length,v.items.length);
v.items.forEach(function(validItem){
var readItem = readItems.shift();
assert.equal(readItem.addr,validItem.addr);
assert.equal(readItem.value_sat,validItem.value_sat);
assert.equal(readItem.index,validItem.index);
assert.equal(readItem.spendIndex, null);
assert.equal(readItem.spendTxIdBuf, null);
delete unmatch[validItem.addr];
});
var valid = util.inspect(v.items, { depth: null });
assert(!Object.keys(unmatch).length,'\n\tUnmatchs:' + Object.keys(unmatch) + "\n\n" +valid + '\nvs.\n' + readItems);
return done();
v.items.forEach(function(validItem){
unmatch[validItem.addr] =1;
});
v.items.forEach(function(validItem){
var readItem = readItems.shift();
assert.equal(readItem.addr,validItem.addr);
assert.equal(readItem.value_sat,validItem.value_sat);
assert.equal(readItem.index,validItem.index);
assert.equal(readItem.spendIndex, null);
assert.equal(readItem.spendTxIdBuf, null);
delete unmatch[validItem.addr];
});
var valid = util.inspect(v.items, { depth: null });
assert(!Object.keys(unmatch).length,'\n\tUnmatchs:' + Object.keys(unmatch) + "\n\n" +valid + '\nvs.\n' + readItems);
done();
});
});
});
@ -67,4 +168,3 @@ describe('TransactionOut', function(){
});

31
test/integration/02-transactionouts.js
View File

@ -5,59 +5,50 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development';
var mongoose = require('mongoose'),
var
assert = require('assert'),
fs = require('fs'),
util = require('util'),
async = require('async'),
config = require('../../config/config'),
TransactionOut = require('../../app/models/TransactionOut');
TransactionDb = require('../../lib/TransactionDb').class();
var spentValid = JSON.parse(fs.readFileSync('test/model/spent.json'));
var spentValid = JSON.parse(fs.readFileSync('test/integration/spent.json'));
mongoose.connection.on('error', function(err) { console.log(err); });
var txDb;
describe('TransactionDb Expenses', function(){
describe('TransactionOut Expenses', function(){
before(function(done) {
mongoose.connect(config.db);
before(function(c) {
txDb = new TransactionDb();
// lets spend!
async.each(Object.keys(spentValid),
function(txid,c_out) {
async.each(spentValid[txid],
function(i,c_in) {
TransactionOut._explodeTransactionOuts(i.txid, function(err) {
txDb.createFromArray([i.txid], null, function(err) {
return c_in();
});
},
function(err) {
console.log('Done spending ', txid); //TODO
return c_out();
}
);
},
function(err) {
console.log('[transactionouts.js.88]'); //TODO
return done();
return c();
}
);
});
after(function(done) {
mongoose.connection.close();
done();
});
Object.keys(spentValid).forEach( function(txid) {
it('test result of spending tx ' + txid, function(done) {
var s = spentValid[txid];
var c=0;
TransactionOut.fromTxId( txid, function(err, readItems) {
txDb.fromTxId( txid, function(err, readItems) {
s.forEach( function(v) {
assert.equal(readItems[c].spendTxIdBuf.toString('hex'),v.txid);
assert.equal(readItems[c].spendTxId,v.txid);
assert.equal(readItems[c].spendIndex,v.n);
c++;
});

35
test/integration/addr.js
View File

@ -1,25 +1,20 @@
#!/usr/bin/env node
'use strict';
process.env.NODE_ENV = process.env.NODE_ENV || 'development';
var
assert = require('assert'),
var assert = require('assert'),
fs = require('fs'),
mongoose= require('mongoose'),
config = require('../../config/config'),
Address = require('../../app/models/Address').class();
addrValid = JSON.parse(fs.readFileSync('test/model/addr.json'));
Address = require('../../app/models/Address').class(),
TransactionDb = require('../../lib/TransactionDb').class(),
addrValid = JSON.parse(fs.readFileSync('test/integration/addr.json'));
var txDb;
describe('Address balances', function(){
before(function(done) {
mongoose.connect(config.db);
done();
});
after(function(done) {
mongoose.connection.close();
done();
before(function(c) {
txDb = new TransactionDb();
return c();
});
addrValid.forEach( function(v) {
@ -30,24 +25,22 @@ describe('Address balances', function(){
it('Info for: ' + v.addr, function(done) {
this.timeout(5000);
var a = new Address(v.addr);
var a = new Address(v.addr, txDb);
a.update(function(err) {
if (err) done(err);
assert.equal(v.addr, a.addrStr);
console.log("TX count:" + a.transactions.length);
if (v.balance) assert.equal(v.balance, a.balance, 'balance: ' + a.balance);
if (v.txApperances)
assert.equal(v.txApperances, a.txApperances, 'txApperances: ' + a.txApperances );
if (v.totalReceived) assert.equal(v.totalReceived, a.totalReceived, 'received: ' + a.totalReceived );
if (v.totalSent) assert.equal(v.totalSent, a.totalSent, 'send: ' + a.totalSent);
if (v.txApperances)
assert.equal(v.txApperances, a.txApperances, 'txApperances: ' + a.txApperances );
if (v.balance) assert.equal(v.balance, a.balance, 'balance: ' + a.balance);
if (v.transactions) {
v.transactions.forEach( function(tx) {
assert(a.transactions.indexOf(tx)>-1);
assert(a.transactions.indexOf(tx)>-1,'have tx '+tx);
});
}
done();

25
test/integration/addr.json
View File

@ -1,9 +1,5 @@
[
{
"addr": "mgqvRGJMwR9JU5VhJ3x9uX9MTkzTsmmDgQ",
"balance": 43.1,
"txApperances": 19
},
{
"addr": "mp3Rzxx9s1A21SY3sjJ3CQoa2Xjph7e5eS",
"balance": 0,
@ -20,10 +16,10 @@
},
{
"addr": "mhPEfAmeKVwT7arwMYbhwnL2TfwuWbP4r4",
"balance": 1065,
"totalReceived": 1069,
"totalSent": 4,
"txApperances": 13
"txApperances": 13,
"balance": 1065,
"totalSent": 4
},
{
"addr": "n47CfqnKWdNwqY1UWxTmNJAqYutFxdH3zY",
@ -39,11 +35,16 @@
"totalSent": 3.9775,
"txApperances": 2
},
{
"addr": "mgqvRGJMwR9JU5VhJ3x9uX9MTkzTsmmDgQ",
"txApperances": 27,
"balance": 5.1
},
{
"addr": "mzW2hdZN2um7WBvTDerdahKqRgj3md9C29",
"txApperances": 2034,
"balance": 1049.69744099,
"totalReceived": 1049.69744099,
"txApperances": 6033,
"balance": 1049.69744101,
"totalReceived": 1049.69744101,
"totalSent": 0
},
{
@ -62,11 +63,11 @@
"transactions": [ "91800d80bb4c69b238c9bfd94eb5155ab821e6b25cae5c79903d12853bbb4ed5" ]
},
{
"disabled":1,
"addr": "mmvP3mTe53qxHdPqXEvdu8WdC7GfQ2vmx5",
"balance": 10580.50027254,
"totalReceived": 12157.65075053,
"totalSent": 1577.15047799,
"txApperances": 459,
"transactions": [
"91800d80bb4c69b238c9bfd94eb5155ab821e6b25cae5c79903d12853bbb4ed5",
"f6e80d4fd1a2377406856c67d0cee5ac7e5120993ff97e617ca9aac33b4c6b1e",

77
test/integration/block.js
View File

@ -7,68 +7,51 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development';
var TESTING_BLOCK = '000000000185678d3d7ecc9962c96418174431f93fe20bf216d5565272423f74';
var
mongoose= require('mongoose'),
assert = require('assert'),
config = require('../../config/config'),
Block = require('../../app/models/Block');
BlockDb = require('../../lib/BlockDb').class();
var bDb;
describe('BlockDb fromHashWithInfo', function(){
mongoose.connection.on('error', function(err) { console.log(err); });
describe('Block fromHashWithInfo', function(){
before(function(done) {
mongoose.connect(config.db);
done();
});
after(function(done) {
mongoose.connection.close();
done();
});
it('should poll block\'s info from mongoose', function(done) {
Block.fromHashWithInfo(TESTING_BLOCK, function(err, b2) {
if (err) done(err);
var h = new Buffer(TESTING_BLOCK,'hex');
assert(b2.hashStr === TESTING_BLOCK);
assert.equal(b2.hashStr, TESTING_BLOCK);
done();
});
before(function(c) {
bDb = new BlockDb();
return c();
});
it('should poll block\'s info from bitcoind', function(done) {
Block.fromHashWithInfo(TESTING_BLOCK, function(err, b2) {
bDb.fromHashWithInfo(TESTING_BLOCK, function(err, b2) {
if (err) done(err);
assert.equal(b2.hash, TESTING_BLOCK);
assert.equal(b2.info.hash, TESTING_BLOCK);
assert.equal(b2.info.chainwork, '000000000000000000000000000000000000000000000000001b6dc969ffe847');
done();
});
});
it('hash Virtuals SET', function(done) {
var b = new Block();
b.hashStr = 'a1a2';
assert.equal(b.hash.toString('hex'),'a1a2');
b.nextBlockHashStr = 'a1a3';
assert.equal(b.nextBlockHash.toString('hex'),'a1a3');
done();
it('return true in has', function(done) {
bDb.has(TESTING_BLOCK, function(err, has) {
assert.equal(has, true);
done();
});
});
it('setOrphan', function(done) {
var b16 = '00000000c4cbd75af741f3a2b2ff72d9ed4d83a048462c1efe331be31ccf006b';
var b17 = '00000000fe198cce4c8abf9dca0fee1182cb130df966cc428ad2a230df8da743';
it('hash Virtuals GET', function(done) {
var b = new Block();
b.hash = new Buffer('a1a2','hex');
assert.equal(b.hashStr,'a1a2');
b.nextBlockHash = new Buffer('b2b1','hex');
assert.equal(b.nextBlockHashStr,'b2b1');
done();
bDb.has(b17, function(err, has) {
assert(has);
bDb.setOrphan(b17, function(err, oldPrev) {
assert.equal(oldPrev, b16);
bDb.setPrev(b17, b16, function(err, oldPrev) {
bDb.getPrev(b17, function(err, p) {
assert.equal(p, b16);
done();
});
});
});
});
});
});

4
test/integration/blockExtractor.js
View File

@ -13,7 +13,7 @@ var assert = require('assert'),
//var txItemsValid = JSON.parse(fs.readFileSync('test/model/txitems.json'));
describe('TransactionOut', function(){
describe('BlockExtractor', function(){
var be = new BlockExtractor(config.bitcoind.dataDir, config.network);
@ -51,7 +51,7 @@ describe('TransactionOut', function(){
});
});
it('should read 100000 blocks with no error ', function(done) {
it.skip('should read 100000 blocks with no error ', function(done) {
var i=0;
while(i++<100000) {

37
test/integration/blocklist.js Normal file
View File

@ -0,0 +1,37 @@
#!/usr/bin/env node
'use strict';
process.env.NODE_ENV = process.env.NODE_ENV || 'development';
var TESTING_BLOCK0 = '000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943';
var TESTING_BLOCK1 = '00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206';
var START_TS = 1;
var END_TS = '1296688928~'; // 2/2/2011 23:23PM
var LIMIT = 2;
var assert = require('assert'),
BlockDb = require('../../lib/BlockDb').class();
var bDb;
describe('BlockDb getBlocksByDate', function(){
before(function(c) {
bDb = new BlockDb();
return c();
});
it('Get Hash by Date', function(done) {
bDb.getBlocksByDate(START_TS, END_TS, LIMIT, function(err, list) {
if (err) done(err);
assert(list, 'returns list');
assert.equal(list.length,2, 'list has 2 items');
assert.equal(list[0].hash, TESTING_BLOCK0);
assert.equal(list[1].hash, TESTING_BLOCK1);
done();
});
});
});

18
test/integration/status.js
View File

@ -1,25 +1,13 @@
#!/usr/bin/env node
'use strict';
process.env.NODE_ENV = process.env.NODE_ENV || 'development';
var
assert = require('assert'),
config = require('../../config/config'),
Status = require('../../app/models/Status').class(),
mongoose= require('mongoose');
var assert = require('assert'),
Status = require('../../app/models/Status').class();
describe('Status', function(){
before(function(done) {
mongoose.connect(config.db);
done();
});
after(function(done) {
mongoose.connection.close();
done();
});
it('getInfo', function(done) {
var d = new Status();

92
test/integration/sync.js Normal file
View File

@ -0,0 +1,92 @@
#!/usr/bin/env node
'use strict';
process.env.NODE_ENV = process.env.NODE_ENV || 'development';
var
assert = require('assert'),
async = require('async'),
Sync = require('../../lib/Sync').class();
var b = [
'00000000c4cbd75af741f3a2b2ff72d9ed4d83a048462c1efe331be31ccf006b', //B#16
'00000000fe198cce4c8abf9dca0fee1182cb130df966cc428ad2a230df8da743',
'000000008d55c3e978639f70af1d2bf1fe6f09cb3143e104405a599215c89a48',
'000000009b3bca4909f38313f2746120129cce4a699a1f552390955da470c5a9',
'00000000ede57f31cc598dc241d129ccb4d8168ef112afbdc870dc60a85f5dd3', //B#20
];
var fix = function(s,cb) {
async.each([1,2,3,4], function(i,c) {
s.bDb.setPrev(b[i],b[i-1], function() {
return c();
});
}, cb);
};
var test = function(s,cb) {
async.each([2,3,4], function(i,c) {
s.bDb.getPrev(b[i], function(err, p) {
assert.equal(p,0);
return c();
});
}, function() {
s.bDb.getPrev(b[1], function(err, p) {
assert.equal(p,b[0]);
return cb();
});
});
};
var testNo = function(s,cb) {
async.each([2,3,4], function(i,c) {
s.bDb.getPrev(b[i], function(err, p) {
assert.equal(p,b[i-1]);
return c();
});
}, function() {
s.bDb.getPrev(b[1], function(err, p) {
assert.equal(p,b[0]);
return cb();
});
});
};
var s;
describe('Sync checkOrphan', function(){
before(function(done) {
s = new Sync();
fix(s,done);
});
after(function(done) {
fix(s,function() {
s.close(done);
});
});
it('checkOrphan', function(done) {
this.timeout(100000);
s.bDb.has(b[0], function(err, has) {
assert(has);
s.bDb.has(b[1], function(err, has) {
assert(has);
s.checkOrphan(b[4],b[1], function() {
testNo(s,done);
});
});
});
});
});

120
test/integration/transaction.js
View File

@ -1,120 +0,0 @@
#!/usr/bin/env node
'use strict';
process.env.NODE_ENV = process.env.NODE_ENV || 'development';
var mongoose= require('mongoose'),
assert = require('assert'),
config = require('../../config/config'),
Transaction = require('../../app/models/Transaction').class();
mongoose.connection.on('error', function(err) { console.log(err); });
describe('Transaction', function(){
before(function(done) {
mongoose.connect(config.db);
done();
});
after(function(done) {
mongoose.connection.close();
done();
});
var txid = '7e621eeb02874ab039a8566fd36f4591e65eca65313875221842c53de6907d6c';
it('txid ' + txid, function(done) {
Transaction.fromIdWithInfo(txid, function(err, tx) {
if (err) done(err);
assert.equal(tx.txid, txid);
assert(!tx.info.isCoinBase);
for(var i=0; i<20; i++)
assert(parseFloat(tx.info.vin[i].value) === parseFloat(50), 'input '+i);
assert(tx.info.vin[0].addr === 'msGKGCy2i8wbKS5Fo1LbWUTJnf1GoFFG59', 'addr 0');
assert(tx.info.vin[1].addr === 'mfye7oHsdrHbydtj4coPXCasKad2eYSv5P', 'addr 1');
done();
});
});
it('should pool tx\'s object from mongoose', function(done) {
var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237';
Transaction.fromIdWithInfo(txid, function(err, tx) {
if (err) done(err);
assert.equal(tx.txid, txid);
assert(!tx.info.isCoinBase);
done();
});
});
it('should pool tx\'s info from bitcoind', function(done) {
var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237';
Transaction.fromIdWithInfo(txid, function(err, tx) {
if (err) done(err);
assert.equal(tx.info.txid, txid);
assert.equal(tx.info.blockhash, '000000000185678d3d7ecc9962c96418174431f93fe20bf216d5565272423f74');
assert.equal(tx.info.valueOut, 1.66174);
assert.equal(tx.info.fees, 0.0005 );
assert.equal(tx.info.size, 226 );
assert(!tx.info.isCoinBase);
done();
});
});
var txid1 = '2a104bab1782e9b6445583296d4a0ecc8af304e4769ceb64b890e8219c562399';
it('test a coinbase TX ' + txid1, function(done) {
Transaction.fromIdWithInfo(txid1, function(err, tx) {
if (err) done(err);
assert(tx.info.isCoinBase);
assert.equal(tx.info.txid, txid1);
assert(!tx.info.feeds);
done();
});
});
var txid22 = '666';
it('test invalid TX ' + txid22, function(done) {
Transaction.fromIdWithInfo(txid22, function(err, tx) {
if (err && err.message.match(/must.be.hexadecimal/)) {
return done();
}
else {
return done(err);
}
});
});
var txid23 = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca227';
it('test unexisting TX ' + txid23, function(done) {
Transaction.fromIdWithInfo(txid23, function(err, tx) {
assert(!err);
assert(!tx);
return done();
});
});
var txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608';
it('create TX on the fly ' + txid2, function(done) {
Transaction.fromIdWithInfo(txid2, function(err, tx) {
if (err) return done(err);
assert.equal(tx.info.txid, txid2);
done();
});
});
var txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608';
it('test a broken TX ' + txid2, function(done) {
Transaction.fromIdWithInfo(txid2, function(err, tx) {
if (err) return done(err);
assert.equal(tx.info.txid, txid2);
assert.equal(tx.info.vin[0].addr, 'n1JagbRWBDi6VMvG7HfZmXX74dB9eiHJzU');
done();
});
});
});

67
test/integration/transactionouts-expenses.js
View File

@ -1,67 +0,0 @@
#!/usr/bin/env node
'use strict';
process.env.NODE_ENV = process.env.NODE_ENV || 'development';
var mongoose = require('mongoose'),
assert = require('assert'),
fs = require('fs'),
util = require('util'),
async = require('async'),
config = require('../../config/config'),
TransactionOut = require('../../app/models/TransactionOut');
var spentValid = JSON.parse(fs.readFileSync('test/model/spent.json'));
mongoose.connection.on('error', function(err) { console.log(err); });
describe('TransactionOut Expenses', function(){
before(function(done) {
mongoose.connect(config.db);
// lets spend!
async.each(Object.keys(spentValid),
function(txid,c_out) {
async.each(spentValid[txid],
function(i,c_in) {
TransactionOut._explodeTransactionOuts(i.txid, function(err) {
return c_in();
});
},
function(err) {
return c_out();
}
);
},
function(err) {
console.log('[transactionouts.js.88]'); //TODO
return done();
}
);
});
after(function(done) {
mongoose.connection.close();
done();
});
Object.keys(spentValid).forEach( function(txid) {
it('test result of spending tx ' + txid, function(done) {
var s = spentValid[txid];
var c=0;
TransactionOut.fromTxId( txid, function(err, readItems) {
s.forEach( function(v) {
assert.equal(readItems[c].spendTxIdBuf.toString('hex'),v.txid);
assert.equal(readItems[c].spendIndex,v.n);
c++;
});
done();
});
});
});
});

10
test/integration/txitems.json
View File

@ -5,6 +5,10 @@
},
{
"txid": "21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237",
"toRm": [
"txouts-spend-86a03cac7d87f596008c6d5a8d3fd8b88842932ea6f0337673eda16f6b472f7f-0",
"txouts-spend-bcd8da8ee847da377f8aaca92502c05e5f914c6a2452753146013b0e642a25a0-0"
],
"items": [
{
"addr": "mzjLe62faUqCSjkwQkwPAL5nYyR8K132fA",
@ -20,6 +24,9 @@
},
{
"txid": "b633a6249d4a2bc123e7f8a151cae2d4afd17aa94840009f8697270c7818ceee",
"toRm": [
"txouts-spend-01621403689cb4a95699a3dbae029d7031c5667678ef14e2054793954fb27917-0"
],
"items": [
{
"addr": "mhfQJUSissP6nLM5pz6DxHfctukrrLct2T",
@ -35,6 +42,9 @@
},
{
"txid": "ca2f42e44455b8a84434de139efea1fe2c7d71414a8939e0a20f518849085c3b",
"toRm": [
"txouts-spend-2d7b680fb06e4d7eeb65ca49ac7522276586e0090b7fe662fc708129429c5e6a-0"
],
"items": [
{
"addr": "mhqyL1nDQDo1WLH9qH8sjRjx2WwrnmAaXE",

3
util/sync.js
View File

@ -1,4 +1,5 @@
#! /usr/bin/env node
#!/usr/bin/env node
'use strict';