diff --git a/README.md b/README.md index 4ee92d8f..9b8d6916 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,6 @@ $ npm install -g bower ## Additional Packages * Express - Defined as npm module in the [package.json](package.json) file. -* Mongoose - Defined as npm module in the [package.json](package.json) file. * AngularJS - Defined as bower module in the [bower.json](bower.json) file. * Twitter Bootstrap - Defined as bower module in the [bower.json](bower.json) file. * UI Bootstrap - Defined as bower module in the [bower.json](bower.json) file. @@ -55,13 +54,15 @@ $ npm install -g bower http://localhost:3000 - If you get an error, please check the next section "Post-install" - ## Syncing old blockchain data - Run sync from insight repository (to save old blocks and transactions in MongoDB): + Run sync from insight repository (to save old blocks and transactions in + LevelDB): + + Create folders: - $ utils/sync.js + $ mkdir -p db/blocks + $ utils/sync.js -S Check utils/sync.js --help for options. diff --git a/app/controllers/blocks.js b/app/controllers/blocks.js index 726a0864..468bc13a 100644 --- a/app/controllers/blocks.js +++ b/app/controllers/blocks.js @@ -3,17 +3,17 @@ /** * Module dependencies. */ -var mongoose = require('mongoose'), - Block = mongoose.model('Block'), - common = require('./common'), - async = require('async'); +var common = require('./common'), + async = require('async'), + BlockDb = require('../../lib/BlockDb').class(); +var bdb = new BlockDb(); /** * Find block by hash ... */ exports.block = function(req, res, next, hash) { - Block.fromHashWithInfo(hash, function(err, block) { + bdb.fromHashWithInfo(hash, function(err, block) { if (err || ! block) return common.handleErrors(err, res, next); else { @@ -37,7 +37,7 @@ exports.show = function(req, res) { * Show block by Height */ exports.blockindex = function(req, res, next, height) { - Block.blockIndex(height, function(err, hashStr) { + bdb.blockIndex(height, function(err, hashStr) { if (err) { console.log(err); res.status(400).send('Bad Request'); // TODO @@ -49,7 +49,7 @@ exports.blockindex = function(req, res, next, height) { }; var getBlock = function(blockhash, cb) { - Block.fromHashWithInfo(blockhash, function(err, block) { + bdb.fromHashWithInfo(blockhash, function(err, block) { if (err) { console.log(err); return cb(err); @@ -72,7 +72,7 @@ console.log('[blocks.js.60]: could not get %s from RPC. Orphan? Error?', blockha * List of blocks by date */ exports.list = function(req, res) { - var limit = req.query.limit || 0; + var limit = req.query.limit || -1; var isToday = false; //helper to convert timestamps to yyyy-mm-dd format @@ -103,35 +103,27 @@ exports.list = function(req, res) { var prev = formatTimestamp(new Date((gte - 86400) * 1000)); var next = formatTimestamp(new Date(lte * 1000)); - Block - .find({ - time: { - '$gte': gte, - '$lte': lte + bdb.getBlocksByDate(gte, lte, limit, function(err, blocks) { + if (err) { + res.status(500).send(err); + } + else { + var blockshashList = []; + for(var i=0;i => + var PREV_ROOT = 'b-prev-'; // b-prev- => (0 if orphan) + + + /** + * Module dependencies. + */ + var RpcClient = require('bitcore/RpcClient').class(), + util = require('bitcore/util/util'), + levelup = require('levelup'), + BitcoreBlock= require('bitcore/Block').class(), + config = require('../config/config'); + var db = b.db || levelup(config.leveldb + '/blocks'); + var rpc = b.rpc || new RpcClient(config.bitcoind); + + + var BlockDb = function() { + }; + + BlockDb.prototype.close = function(cb) { + db.close(cb); + }; + + BlockDb.prototype.drop = function(cb) { + var path = config.leveldb + '/blocks'; + db.close(function() { + require('leveldown').destroy(path, function () { + db = levelup(path); + return cb(); + }); + }); + }; + + BlockDb.prototype.add = function(b, cb) { + if (!b.hash) return cb(new Error('no Hash at Block.save')); + + + var time_key = TIMESTAMP_ROOT + + ( b.time || Math.round(new Date().getTime() / 1000) ); + + db.batch() + .put(time_key, b.hash) + .put(PREV_ROOT + b.hash, b.previousblockhash) + .write(cb); + }; + + + + BlockDb.prototype.setOrphan = function(hash, cb) { + var k = PREV_ROOT + hash; + + db.get(k, function (err,oldPrevHash) { + if (err || !oldPrevHash) return cb(err); + db.put(PREV_ROOT + hash, 0, function() { + return cb(err, oldPrevHash); + }); + }); + // We keep the block in TIMESTAMP_ROOT + }; + + //mainly for testing + BlockDb.prototype.setPrev = function(hash, prevHash, cb) { + db.put(PREV_ROOT + hash, prevHash, function(err) { + return cb(err); + }); + }; + + //mainly for testing + BlockDb.prototype.getPrev = function(hash, cb) { + db.get(PREV_ROOT + hash, function(err,val) { + return cb(err,val); + }); + }; + + + + BlockDb.prototype.countNotOrphan = function(cb) { + var c = 0; + console.log('Counting connected blocks. This could take some minutes'); + db.createReadStream({start: PREV_ROOT, end: PREV_ROOT + '~' }) + .on('data', function (data) { + if (data.value !== 0) c++; + }) + .on('error', function (err) { + return cb(err); + }) + .on('end', function () { + return cb(null, c); + }); + }; + + BlockDb.prototype.has = function(hash, cb) { + var k = PREV_ROOT + hash; + db.get(k, function (err,val) { + var ret; + if (err && err.notFound) { + err = null; + ret = false; + } + if (typeof val !== 'undefined') { + ret = true; + } + return cb(err, ret); + }); + }; + + + BlockDb.prototype.fromHashWithInfo = function(hash, cb) { + rpc.getBlock(hash, function(err, info) { + // Not found? + if (err && err.code === -5) return cb(); + if (err) return cb(err); + + info.reward = BitcoreBlock.getBlockValue(info.height) / util.COIN ; + + return cb(null, { + hash: hash, + info: info.result, + }); + }); + }; + + BlockDb.prototype.getBlocksByDate = function(start_ts, end_ts, limit, cb) { + var list = []; + db.createReadStream({ + start: TIMESTAMP_ROOT + start_ts, + end: TIMESTAMP_ROOT + end_ts, + fillCache: true, + limit: parseInt(limit) // force to int + }) + .on('data', function (data) { + list.push({ + ts: data.key.replace(TIMESTAMP_ROOT, ''), + hash: data.value, + }); + }) + .on('error', function (err) { + return cb(err); + }) + .on('end', function () { + return cb(null, list); + }); + }; + + BlockDb.prototype.blockIndex = function(height, cb) { + var rpc = new RpcClient(config.bitcoind); + rpc.getBlockHash(height, function(err, bh){ + if (err) return cb(err); + + cb(null, { blockHash: bh.result }); + }); + }; + + return BlockDb; +} +module.defineClass(spec); + + diff --git a/lib/BlockExtractor.js b/lib/BlockExtractor.js index 857ed71b..92c6740e 100644 --- a/lib/BlockExtractor.js +++ b/lib/BlockExtractor.js @@ -113,7 +113,7 @@ function spec() { magic = self.currentParser ? self.currentParser.buffer(4).toString('hex') : null ; - if (!self.currentParser || self.currentParser.eof()) { + if (!self.currentParser || self.currentParser.eof() || magic === '00000000') { magic = null; if (self.nextFile()) { console.log('Moving forward to file:' + self.currentFile() ); @@ -121,7 +121,8 @@ function spec() { } else { console.log('Finished all files'); - return cb(); + magic = null; + return w_cb(); } } else { @@ -130,6 +131,8 @@ function spec() { }, a_cb); }, function (a_cb) { + if (!magic) return a_cb(); + if (magic !== self.magic) { var e = new Error('CRITICAL ERROR: Magic number mismatch: ' + magic + '!=' + self.magic); @@ -141,6 +144,8 @@ function spec() { return a_cb(); }, function (a_cb) { + if (!magic) return a_cb(); + b = new Block(); b.parse(self.currentParser); b.getHash(); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 3ab8f60d..6b547bc3 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -13,10 +13,10 @@ function spec() { var networks = require('bitcore/networks'); var async = require('async'); var config = require('../config/config'); - var Block = require('../app/models/Block'); var Sync = require('./Sync').class(); var sockets = require('../app/controllers/socket.js'); var BlockExtractor = require('./BlockExtractor.js').class(); + //var Deserialize = require('bitcore/Deserialize'); var BAD_GEN_ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:'; @@ -36,7 +36,8 @@ function spec() { this.syncPercentage = 0; this.syncedBlocks = 0; this.skippedBlocks = 0; - + this.orphanBlocks = 0; + this.type =''; } function p() { @@ -95,11 +96,13 @@ function spec() { syncPercentage: this.syncPercentage, skippedBlocks: this.skippedBlocks, syncedBlocks: this.syncedBlocks, + orphanBlocks: this.orphanBlocks, error: this.error, + type: this.type, }; }; - HistoricSync.prototype.showProgress = function(height) { + HistoricSync.prototype.showProgress = function() { var self = this; if (self.error) { @@ -109,7 +112,7 @@ function spec() { self.syncPercentage = parseFloat(100 * (self.syncedBlocks + self.skippedBlocks) / self.blockChainHeight).toFixed(3); if (self.syncPercentage > 100) self.syncPercentage = 100; - p(util.format('status: [%d%%] skipped: %d ', self.syncPercentage, self.skippedBlocks, height)); + p(util.format('status: [%d%%] skipped: %d ', self.syncPercentage, self.skippedBlocks)); } if (self.opts.shouldBroadcast) { sockets.broadcastSyncInfo(self.info()); @@ -128,14 +131,13 @@ function spec() { async.series([ // Already got it? function(c) { - Block.fromHash(blockHash, function(err, block) { + self.sync.bDb.has(blockHash, function(err, ret) { if (err) { p(err); return c(err); } - if (block) { - existed = true; - } + + if (ret) existed = true; return c(); }); }, @@ -161,11 +163,7 @@ function spec() { if (existed) return c(); self.sync.storeBlock(blockInfo, function(err) { - - existed = err && err.toString().match(/E11000/); - - if (err && ! existed) return c(err); - return c(); + return c(err); }); }, /* TODO: Should Start to sync backwards? (this is for partial syncs) @@ -181,14 +179,10 @@ function spec() { ], function(err) { if (err) { - self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncedBlocks); - self.status = 'aborted'; - self.showProgress(); - p(self.err); + self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncedBlocks)); return cb(err); } else { - self.err = null; self.status = 'syncing'; } @@ -244,9 +238,7 @@ function spec() { addrStrs = [ addr.toString() ]; break; case Script.TX_MULTISIG: - var addrs = []; var chunks = s.capture(); - chunks.forEach(function(chunk) { var a = new Address(self.network.addressPubkey, bitutil.sha256ripe160(chunk)); addrStrs.push(a.toString()); @@ -259,44 +251,40 @@ function spec() { return addrStrs; }; - - HistoricSync.prototype.getBlockFromFile = function(height, scanOpts, cb) { + HistoricSync.prototype.getBlockFromFile = function(scanOpts, cb) { var self = this; - - var nextHash; var blockInfo; - var isMainChain; var existed; - async.series([ - // Is it in mainchain? - function(c) { - self.rpc.getBlockHash(height, function(err, res) { - if (err) return cb(err); - - nextHash = res.result; - return c(); - }); - }, //show some (inacurate) status function(c) { if ( ( self.syncedBlocks + self.skippedBlocks) % self.step === 1) { - self.showProgress(height); + self.showProgress(); } return c(); }, //get Info function(c) { + self.blockExtractor.getNextBlock(function(err, b) { if (err || ! b) return c(err); blockInfo = b.getStandardizedObject(b.txs, self.network); + // blockInfo.curWork = Deserialize.intFromCompact(b.bits); + // We keep the RPC field names + blockInfo.previousblockhash = blockInfo.prev_block; var ti=0; // Get TX Address b.txs.forEach(function(t) { + + var objTx = blockInfo.tx[ti++]; + + //add time from block + objTx.time = blockInfo.time; + var to=0; t.outs.forEach( function(o) { @@ -315,66 +303,83 @@ function spec() { return c(); }); }, - //store it + //check prev function(c) { + if (blockInfo && self.prevHash && blockInfo.previousblockhash !== self.prevHash) { - isMainChain = blockInfo.hash === nextHash; + console.log('Hole found @%s', blockInfo.hash); + console.log('From: %s To: %s', self.prevHash, blockInfo.previousblockhash); + self.sync.checkOrphan(self.prevHash, blockInfo.previousblockhash, c); + } + else return c(); + }, + //check it + function(c) { + if (!blockInfo) return c(); - blockInfo.isOrphan = !isMainChain; - - /* - * In file sync, orphan blocks are just ignored. - * This is to simplify our schema and the - * sync process - */ - if (blockInfo.isOrphan) return c(); - - self.sync.storeBlock(blockInfo, function(err) { - existed = err && err.toString().match(/E11000/); - - if (err && ! existed) return c(err); - return c(); + self.sync.bDb.has(blockInfo.hash, function(err, had) { + existed = had; + return c(err); }); }, - ], function(err) { - - if (err) { - self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockInfo.hash, err, self.syncedBlocks); - self.status = 'aborted'; - self.showProgress(); - p(err); - return cb(err); - } - else { - - // Continue - if (blockInfo) { - - // mainchain - if (isMainChain) height++; + //store it + function(c) { + if (!blockInfo || existed) return c(); + self.sync.storeBlock(blockInfo, function(err) { + return c(err); + }); + }, + function(c) { + if (blockInfo && blockInfo.hash) { + self.prevHash = blockInfo.hash; + if (existed) + self.skippedBlocks++; + else self.syncedBlocks++; - self.err = null; - self.status = 'syncing'; + } else + self.status = 'finished'; - return self.getBlockFromFile(height, scanOpts, cb); - } - else { - self.err = null; - self.status = 'finished'; - } + return c(); + }, + ], function(err) { + if (err) { + self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockInfo ? blockInfo.hash : '-', err, self.syncedBlocks)); } return cb(err); }); }; + HistoricSync.prototype.countNotOrphan = function(cb) { + var self = this; + + if (self.notOrphanCount) return cb(null, self.notOrphanCount); + + + self.sync.bDb.countNotOrphan(function(err, count) { + if (err) return cb(err); + self.notOrphanCount = count; + return cb(null, self.notOrphanCount); + }); + }; + + + HistoricSync.prototype.getBlockCount = function(cb) { + var self = this; + + if (self.blockChainHeight) return cb(); + + self.rpc.getBlockCount(function(err, res) { + if (err) return cb(err); + self.blockChainHeight = res.result; + return cb(); + }); + }; HistoricSync.prototype.importHistory = function(scanOpts, next) { var self = this; - var retry_secs = 2; - var lastBlock; async.series([ @@ -386,13 +391,7 @@ function spec() { return cb(); }, // We are not using getBestBlockHash, because is not available in all clients - function(cb) { - self.rpc.getBlockCount(function(err, res) { - if (err) return cb(err); - self.blockChainHeight = res.result; - return cb(); - }); - }, + function (cb) { return self.getBlockCount(cb); }, function(cb) { if (!scanOpts.reverse) return cb(); @@ -406,10 +405,7 @@ function spec() { function(cb) { if (scanOpts.upToExisting) { // should be isOrphan = true or null to be more accurate. - Block.count({ - isOrphan: null - }, - function(err, count) { + self.countNotOrphan(function(err, count) { if (err) return cb(err); self.syncedBlocks = count || 0; @@ -421,44 +417,16 @@ function spec() { } }, ], function(err) { - var start, end; - function sync() { - if (scanOpts.reverse) { - start = lastBlock; - end = self.genesis; - scanOpts.prev = true; - } - else { - start = self.genesis; - end = null; - scanOpts.next = true; - } - p('Starting from: ', start); - p(' to : ', end); - p(' scanOpts: ', JSON.stringify(scanOpts)); - - if (scanOpts.fromFiles) { - self.getBlockFromFile(0, scanOpts, function(err) { - return next(err); - }); - } - else { - self.getPrevNextBlock(start, end, scanOpts, function(err) { - if (err && err.message.match(/ECONNREFUSED/)) { - setTimeout(function() { - p('Retrying in %d secs', retry_secs); - sync(); - }, - retry_secs * 1000); - } - else return next(err); - }); - } + if (err) { + self.setError(err); + return next(err, 0); } - if (!self.step) { + // SETUP Sync params + var start, end; + if (!self.step) { var step = parseInt( (self.blockChainHeight - self.syncedBlocks) / 1000); if (self.opts.progressStep) { @@ -469,13 +437,42 @@ function spec() { self.step = step; } - if (err) { - self.setError(err); - return next(err, 0); + if (scanOpts.reverse) { + start = lastBlock; + end = self.genesis; + scanOpts.prev = true; } else { - sync(); + start = self.genesis; + end = null; + scanOpts.next = true; } + p('Starting from: ', start); + p(' to : ', end); + p(' scanOpts: ', JSON.stringify(scanOpts)); + + if (scanOpts.fromFiles) { + self.status = 'syncing'; + self.type = 'from .dat Files'; + async.whilst(function() { + return self.status === 'syncing'; + }, function (w_cb) { + self.getBlockFromFile(scanOpts, function(err) { + setImmediate(function(){ + return w_cb(err); + }); + }); + }, function(err) { + return next(err); + }); + } + else { + self.type = 'from RPC calls'; + self.getPrevNextBlock(start, end, scanOpts, function(err) { + return next(err); + }); + } + }); }; @@ -483,29 +480,39 @@ function spec() { HistoricSync.prototype.smartImport = function(scanOpts, next) { var self = this; - Block.fromHash(self.genesis, function(err, b) { - + self.sync.bDb.has(self.genesis, function(err, b) { if (err) return next(err); + self.countNotOrphan(function(err, count) { + if (err) return next(err); + self.getBlockCount(function(err) { + if (err) return next(err); + if (!b || scanOpts.destroy || count < self.blockChainHeight * 0.8 ) { - if (!b || scanOpts.destroy) { - p('Could not find Genesis block. Running FULL SYNC'); - if (config.bitcoind.dataDir) { - p('bitcoind dataDir configured...importing blocks from .dat files'); - scanOpts.fromFiles = true; - self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); - } - else { - scanOpts.reverse = true; - } - } - else { - p('Genesis block found. Syncing upto known blocks.'); - scanOpts.reverse = true; - scanOpts.upToExisting = true; - } + if (!b) + p('Could not find Genesis block. Running FULL SYNC'); + else + p('Less that 80% of current blockchain is stored. Running FULL SYNC', + parseInt(count/self.blockChainHeight*100)); - return self.importHistory(scanOpts, next); + if (config.bitcoind.dataDir) { + p('bitcoind dataDir configured...importing blocks from .dat files'); + scanOpts.fromFiles = true; + self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); + } + else { + scanOpts.reverse = true; + } + } + else { + p('Genesis block found. Syncing upto known blocks.'); + p('Got %d out of %d blocks', count, self.blockChainHeight); + scanOpts.reverse = true; + scanOpts.upToExisting = true; + } + return self.importHistory(scanOpts, next); + }); + }); }); }; diff --git a/lib/Sync.js b/lib/Sync.js index bb445fad..d46e818c 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -4,15 +4,15 @@ require('classtool'); function spec() { - var mongoose = require('mongoose'); - var config = require('../config/config'); - var Block = require('../app/models/Block'); - var TransactionOut = require('../app/models/TransactionOut'); var sockets = require('../app/controllers/socket.js'); + var BlockDb = require('./BlockDb').class(); + var TransactionDb = require('./TransactionDb').class(); var async = require('async'); function Sync() { + this.bDb = new BlockDb(); + this.txDb = new TransactionDb(); } Sync.prototype.init = function(opts, cb) { @@ -20,70 +20,88 @@ function spec() { self.opts = opts; - if (!(opts && opts.skipDbConnection)) { - - if (mongoose.connection.readyState !== 1) { - mongoose.connect(config.db, function(err) { - if (err) { - console.log('CRITICAL ERROR: connecting to mongoDB:',err); - return (err); - } - }); - } - - self.db = mongoose.connection; - - self.db.on('error', function(err) { - console.log('MongoDB ERROR:' + err); - return cb(err); - }); - - self.db.on('disconnect', function(err) { - console.log('MongoDB disconnect:' + err); - return cb(err); - }); - - return self.db.once('open', function(err) { - return cb(err); - }); - } - else return cb(); + return cb(); }; - Sync.prototype.close = function() { - if ( this.db && this.db.readyState ) { - this.db.close(); - } + Sync.prototype.close = function(cb) { + var self = this; + self.txDb.close(function() { + self.bDb.close(cb); + }); }; Sync.prototype.destroy = function(next) { var self = this; async.series([ - function(b) { try {self.db.collections.blocks.drop(b);} catch (e) { return b(); } }, - function(b) { try {self.db.collections.transactionitems.drop(b);} catch (e) { return b(); } }, - function(b) { try {self.db.collections.transactionouts.drop(b);} catch (e) { return b(); } }, + function(b) { self.bDb.drop(b); }, + function(b) { self.txDb.drop(b); }, ], next); }; + Sync.prototype.storeBlock = function(block, cb) { var self = this; - Block.customCreate(block, function(err, block, inserted_txs, updated_addrs){ + self.txDb.createFromBlock(block, function(err, insertedTxs, updateAddrs) { if (err) return cb(err); - self._handleBroadcast(block, inserted_txs, updated_addrs); - - return cb(); + self.bDb.add(block, function(err){ + if (err) return cb(err); + self._handleBroadcast(block, insertedTxs, updateAddrs); + return cb(); + }); }); }; - - Sync.prototype._handleBroadcast = function(block, inserted_txs, updated_addrs) { + Sync.prototype.checkOrphan = function(fromBlock, toBlock, c) { var self = this; - if (block && self.opts.broadcast_blocks) { - sockets.broadcast_block(block); + var hash = fromBlock; + + var co = 0; + var limit = 10; + var cont = 1; + + async.whilst( + function () { + if (++co > limit) { +console.log('[Sync.js.109] WARN: Reach reog depth limit'); //TODO + } + return cont && hash && hash !== toBlock && co < limit; + }, + function (w_c) { + //check with RPC if the block is mainchain + self.bDb.fromHashWithInfo(hash, function (err, info) { + + if (!info) { + console.log('[Sync.js.107:hash:ORPHAN]',hash); //TODO + self.txDb.setOrphan(hash, function(err) { + if (err) return w_c(err); + self.bDb.setOrphan(hash, function(err, prevHash){ + hash = prevHash; + return w_c(err); + }); + }); + } + else { + console.log('[Sync.js.107:hash:NOT ORPHAN]',hash); //TODO + cont = 0; + return w_c(); + } + }); + }, + function (err) { + return c(err); + } + ); + }; + + Sync.prototype._handleBroadcast = function(hash, inserted_txs, updated_addrs) { + var self = this; + + if (hash && self.opts.broadcast_blocks) { + sockets.broadcast_block({hash: hash}); } if (inserted_txs && self.opts.broadcast_txs) { @@ -107,7 +125,8 @@ function spec() { Sync.prototype.storeTxs = function(txs, cb) { var self = this; - TransactionOut.createFromTxs(txs, function(err, inserted_txs, updated_addrs) { + // TODO + self.txDb.createFromTxs(txs, function(err, inserted_txs, updated_addrs) { if (err) return cb(err); self._handleBroadcast(null, inserted_txs, updated_addrs); diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js new file mode 100644 index 00000000..1774bae6 --- /dev/null +++ b/lib/TransactionDb.js @@ -0,0 +1,438 @@ +'use strict'; + +require('classtool'); + + +function spec(b) { + + // blockHash -> txid mapping (to reorgs) + var ROOT = 'tx-b-'; //tx-b-- => 1/0 (connected or not) + + // to show tx outs + var OUTS_ROOT = 'txouts-'; //txouts-- => [addr, btc_sat] + + // to sum up addr balance + var ADDR_ROOT = 'txouts-addr-'; //txouts-addr---- => + btc_sat + var SPEND_ROOT = 'txouts-spend-';//txouts-spend-- => [txid(in),n(in),ts] + + // TODO: use bitcore networks module + var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b'; + var CONCURRENCY = 100; + + /** + * Module dependencies. + */ + var TransactionRpc = require('./TransactionRpc').class(), + util = require('bitcore/util/util'), + levelup = require('levelup'), + async = require('async'), + config = require('../config/config'), + assert = require('assert'); + var db = b.db || levelup(config.leveldb + '/txs'); + + var TransactionDb = function() { + }; + + TransactionDb.prototype.close = function(cb) { + db.close(cb); + }; + + TransactionDb.prototype.drop = function(cb) { + var path = config.leveldb + '/txs'; + db.close(function() { + require('leveldown').destroy(path, function () { + db = levelup(path); + return cb(); + }); + }); + }; + + +// TransactionDb.prototype.fromTxIdOne = function(txid, cb) { TODO + TransactionDb.prototype.has = function(txid, cb) { + + var k = OUTS_ROOT + txid; + db.get(k, function (err,val) { + + var ret; + + if (err && err.notFound) { + err = null; + ret = false; + } + if (typeof val !== undefined) { + ret = true; + } + return cb(err, ret); + }); + }; + + TransactionDb.prototype.fromTxId = function(txid, cb) { + + var k = OUTS_ROOT + txid; + var ret=[]; + + // outs. + db.createReadStream({start: k, end: k + '~'}) + .on('data', function (data) { + var k = data.key.split('-'); + var v = data.value.split(':'); + ret.push({ + addr: v[0], + value_sat: parseInt(v[1]), + index: parseInt(k[2]), + }); + }) + .on('error', function (err) { + return cb(err); + }) + .on('end', function () { + var k = SPEND_ROOT + txid; + var l = ret.length; + db.createReadStream({start: k, end: k + '~'}) + .on('data', function (data) { + var k = data.key.split('-'); + var v = data.value.split(':'); + var set=0; + for(var i=0; i -
-
-

- Developers API Documentation -

-
- - - - -

Overview

-

The Insight REST API provides you with a convenient, powerful and simple way to read data from the bitcoin network and build your own services with it.

-

The REST API currently only supports JSON for the response formats.

-

The API is free to use.

- - - -

API Reference

- -

Blocks

- -

Get Block

- -

URI:

- - 
/api/block/00000000009890591fbacf147ffc6c246fa7bad0f2c75a68e3e1ba48b1636d23
- -

Result:

- - 
-{
-  hash: "00000000009890591fbacf147ffc6c246fa7bad0f2c75a68e3e1ba48b1636d23",
-  confirmations: 37,
-  size: 1287,
-  height: 177951,
-  version: 2,
-  merkleroot: "4d549d51e0bca64030b7f3ab74fcc9a82f3de7bba5d6c3f9a8cb2028399fc590",
-  tx: [
-    "1b8354de2102f0f3f72f7e59fd3fdc0a406fa92675a78d7bb1f599067685385a",
-    "babe2d8b24649df7951e696a4b1686e28764d2d7bbf94d45f612ff598a14017d",
-    "0351d523fec44a04990b8f9513c44b358eac8b88d5df1074ed36d3ffef5f1582",
-    "09f1a8dd15a989377753690f2293e5509fa0763a26af661cf7b56eebc6999c59",
-    "04d972767c5500805ccd41961e2b538b5334af3e41635b2a2ec3750a3c96565a",
-    "df11424536f618ba511436702f97e886b83a1a0e7d1578a26160177274ca0a39"
-  ],
-  time: 1391192318,
-  nonce: 2374664549,
-  bits: "1c00ffff",
-  difficulty: 256,
-  chainwork: "0000000000000000000000000000000000000000000000000413951687599d9f",
-  previousblockhash: "000000000034a41747ec564a890d8192fbd2ec0bfa71667b81db145575cde08e",
-  nextblockhash: "0000000000868ef72603e45325fc0464a5424b7539c51f5905e0ffd2bafc7115",
-  reward: 50
-}
- -

Get Block information by date

- -

URI:

- - 
/api/blocks/?blockDate=yyyy-mm-dd
- -

Result:

- - 
-{
-  blocks: [ Array information of blocks ],
-  length: 2636,
-  pagination: {
-    next: "2014-01-30",
-    prev: "2014-01-28",
-    current: "2014-01-29",
-    isToday: false
-  }
-}
- -

Get hash of block by height

- -

URI:

- - 
/api/block-index/0
- -

Result:

- - 
-{
-  blockHash: "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943"
-}
- -

Transactions

- -

Get Transaction

- -

URI:

- - 
/api/tx/0099d9342edab8a59fa4d84cb807cf599ce2210e5240bc39500d8f6b6f4779c0
- -

Result:

- - 
-{
-  hex: "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0d0358b702017e062f503253482fffffffff01108a082a010000002321029f58ed5b58f1941664d67bcc2ccb5b25c50cc2324ccc9643ff7f8271118c4609ac00000000",
-  txid: "0099d9342edab8a59fa4d84cb807cf599ce2210e5240bc39500d8f6b6f4779c0",
-  version: 1,
-  locktime: 0,
-  vin: [
-    {
-      coinbase: "0358b702017e062f503253482f",
-      sequence: 4294967295,
-      reward: 50,
-      n: 0
-    }
-  ],
-  vout: [
-    {
-      value: 50.0017,
-      n: 0,
-      scriptPubKey: {
-        asm: "029f58ed5b58f1941664d67bcc2ccb5b25c50cc2324ccc9643ff7f8271118c4609 OP_CHECKSIG",
-        hex: "21029f58ed5b58f1941664d67bcc2ccb5b25c50cc2324ccc9643ff7f8271118c4609ac",
-        reqSigs: 1,
-        type: "pubkey",
-        addresses: [
-          "mt3r2JMiWqGkHUiMqduo2HoHw5iAHMT7P9"
-        ]
-      }
-    }
-  ],
-  blockhash: "0000000000478e746e78012066a14db61ee9a447b35607a42b54d15060eb23d6",
-  confirmations: 10,
-  time: 1391201940,
-  blocktime: 1391201940,
-  isCoinBase: true,
-  valueIn: 50,
-  valueOut: 50.0017,
-  size: 108
-}
-
- -

Get Transactions by BlockHash

- -

URI:

- - 
/api/txs/?block=0000000001a6b96db5e1af9f01458c5f0dd880d9cce6e9ed200afb9f0941991b&pageNum=1
- -

Result:

- - 
-{
-  pagesTotal: 2,
-  txs: [ Array of Transactions ]
-}
-
- -

Get Transactions by String Address

- -

URI:

- - 
/api/txs/?address=mqxfgc12qXdm4ekLrCKPDYqamxUEtK78Vg&pageNum=0
- -

Result:

- - 
-{
-  pagesTotal: 4,
-  txs: [ Array of Transactions ]
-}
-
- -

Address

- -

Get Address

- -

URI:

- - 
/api/addr/mqxfgc12qXdm4ekLrCKPDYqamxUEtK78Vg
- -

Result:

- - 
-{
-  balanceSat: 5031766000,
-  totalReceivedSat: 5853132000,
-  totalSentSat: 821366000,
-  txApperances: 66,
-  transactions: [ Array of Transaction IDs ],
-  addrStr: "mqxfgc12qXdm4ekLrCKPDYqamxUEtK78Vg",
-  totalSent: 8.21366,
-  balance: 50.31766,
-  totalReceived: 58.53132
-}
-
- -

Application Status

- -

Get Information

- -

URI:

- - 
/api/status?q=getInfo
- -

Result:

- - 
-{
-  info: {
-    version: 89900,
-    protocolversion: 70002,
-    walletversion: 60000,
-    balance: 0,
-    blocks: 178019,
-    timeoffset: -345,
-    connections: 9,
-    proxy: "",
-    difficulty: 1,
-    testnet: true,
-    keypoololdest: 1389273625,
-    keypoolsize: 101,
-    paytxfee: 0,
-    errors: "This is a pre-release test build - use at your own risk - do not use for mining or merchant applications"
-  }
-}
-
- -

Get Difficulty

- -

URI:

- - 
/api/status?q=getDifficulty
- -

Result:

- - 
-{
-  difficulty: 1
-}
-
- -

Get TxOutSetInfo

- -

URI:

- - 
/api/status?q=getTxOutSetInfo
- -

Result:

- - 
-{
-  txoutsetinfo: {
-    height: 178020,
-    bestblock: "00000000fc182ffd65fa397dbe0c60b1932bee4ccef407b36b7d6d9d5fdb5962",
-    transactions: 242191,
-    txouts: 421180,
-    bytes_serialized: 25490639,
-    hash_serialized: "68decf93d8c883a31c061fcd393966d36fbed07c05b4df739da85585cb70e077",
-    total_amount: 8900499.25420614
-  }
-}
-
- -

Get Last Block Hash

- -

URI:

- - 
/api/status?q=getLastBlockHash
- -

Result:

- - 
-{
-  lastblockhash: "00000000fc182ffd65fa397dbe0c60b1932bee4ccef407b36b7d6d9d5fdb5962"
-}
-
- -

Server Information

- -

Get information of Bitcoin synchronization

- -

URI:

- - 
/api/sync
- -

Result:

- - 
-{
-  status: "finished",
-  blockChainHeight: 178009,
-  syncPercentage: 100,
-  skippedBlocks: 0,
-  syncedBlocks: 178011,
-  error: null
-}
-
- -

Get API Version

- -

URI:

- - 
/api/version
- -

Result:

- - 
-{
-  version: "0.0.1"
-}
-
- - - - -

Sandbox for test API

- -

API live documentation

- -
- diff --git a/test/integration/01-transactionouts.js b/test/integration/01-transactionouts.js index 57a7f2d4..0fc7e0ff 100644 --- a/test/integration/01-transactionouts.js +++ b/test/integration/01-transactionouts.js @@ -5,60 +5,161 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; -var mongoose = require('mongoose'), - assert = require('assert'), +var assert = require('assert'), fs = require('fs'), util = require('util'), - config = require('../../config/config'), - TransactionOut = require('../../app/models/TransactionOut'); + TransactionDb = require('../../lib/TransactionDb').class(); -var txItemsValid = JSON.parse(fs.readFileSync('test/model/txitems.json')); +var txItemsValid = JSON.parse(fs.readFileSync('test/integration/txitems.json')); +var txDb; -mongoose.connection.on('error', function(err) { console.log(err); }); +describe('TransactionDb fromIdWithInfo', function(){ -describe('TransactionOut', function(){ - - before(function(done) { - mongoose.connect(config.db); - done(); + before(function(c) { + txDb = new TransactionDb(); + return c(); }); - after(function(done) { - mongoose.connection.close(); - done(); + + var txid = '7e621eeb02874ab039a8566fd36f4591e65eca65313875221842c53de6907d6c'; + it('tx info ' + txid, function(done) { + txDb.fromIdWithInfo(txid, function(err, tx) { + + if (err) done(err); + assert.equal(tx.txid, txid); + assert(!tx.info.isCoinBase); + + for(var i=0; i<20; i++) + assert(parseFloat(tx.info.vin[i].value) === parseFloat(50), 'input '+i); + assert(tx.info.vin[0].addr === 'msGKGCy2i8wbKS5Fo1LbWUTJnf1GoFFG59', 'addr 0'); + assert(tx.info.vin[1].addr === 'mfye7oHsdrHbydtj4coPXCasKad2eYSv5P', 'addr 1'); + done(); + }); + }); + + it('tx info', function(done) { + var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237'; + txDb.fromIdWithInfo(txid, function(err, tx) { + if (err) done(err); + assert.equal(tx.txid, txid); + assert(!tx.info.isCoinBase); + done(); + }); + }); + + it('should pool tx\'s info from bitcoind', function(done) { + var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237'; + txDb.fromIdWithInfo(txid, function(err, tx) { + if (err) done(err); + assert.equal(tx.info.txid, txid); + assert.equal(tx.info.blockhash, '000000000185678d3d7ecc9962c96418174431f93fe20bf216d5565272423f74'); + assert.equal(tx.info.valueOut, 1.66174); + assert.equal(tx.info.fees, 0.0005 ); + assert.equal(tx.info.size, 226 ); + assert(!tx.info.isCoinBase); + done(); + }); + }); + + var txid1 = '2a104bab1782e9b6445583296d4a0ecc8af304e4769ceb64b890e8219c562399'; + it('test a coinbase TX ' + txid1, function(done) { + txDb.fromIdWithInfo(txid1, function(err, tx) { + if (err) done(err); + assert(tx.info.isCoinBase); + assert.equal(tx.info.txid, txid1); + assert(!tx.info.feeds); + done(); + }); + }); + var txid22 = '666'; + it('test invalid TX ' + txid22, function(done) { + txDb.fromIdWithInfo(txid22, function(err, tx) { + if (err && err.message.match(/must.be.hexadecimal/)) { + return done(); + } + else { + return done(err); + } + }); + }); + + var txid23 = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca227'; + it('test unexisting TX ' + txid23, function(done) { + + txDb.fromIdWithInfo(txid23, function(err, tx) { + assert(!err); + assert(!tx); + return done(); + }); + }); + + + + var txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608'; + it('create TX on the fly ' + txid2, function(done) { + txDb.fromIdWithInfo(txid2, function(err, tx) { + if (err) return done(err); + assert.equal(tx.info.txid, txid2); + done(); + }); + }); + + txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608'; + it('test a broken TX ' + txid2, function(done) { + txDb.fromIdWithInfo(txid2, function(err, tx) { + if (err) return done(err); + assert.equal(tx.info.txid, txid2); + assert.equal(tx.info.vin[0].addr, 'n1JagbRWBDi6VMvG7HfZmXX74dB9eiHJzU'); + done(); + }); + }); +}); + +describe('TransactionDb Outs', function(){ + + before(function(c) { + txDb = new TransactionDb(); + return c(); }); txItemsValid.forEach( function(v) { if (v.disabled) return; - it('test a exploding tx ' + v.txid, function(done) { + it('test a processing tx ' + v.txid, function(done) { + this.timeout(60000); // Remove first - TransactionOut.removeFromTxId(v.txid, function(err) { - TransactionOut._explodeTransactionOuts(v.txid, function(err, tx) { - if (err) done(err); + txDb.removeFromTxId(v.txid, function() { - TransactionOut - .fromTxId( v.txid, function(err, readItems) { + txDb.fromTxId( v.txid, function(err, readItems) { + assert.equal(readItems.length,0); - var unmatch={}; + var unmatch=[]; + txDb.createFromArray([v.txid], null, function(err) { + if (err) return done(err); + + txDb.fromTxId( v.txid, function(err, readItems) { + + v.items.forEach(function(validItem){ + unmatch[validItem.addr] =1; + }); + assert.equal(readItems.length,v.items.length); + + v.items.forEach(function(validItem){ + var readItem = readItems.shift(); + + assert.equal(readItem.addr,validItem.addr); + assert.equal(readItem.value_sat,validItem.value_sat); + assert.equal(readItem.index,validItem.index); + assert.equal(readItem.spendIndex, null); + assert.equal(readItem.spendTxIdBuf, null); + delete unmatch[validItem.addr]; + }); + + var valid = util.inspect(v.items, { depth: null }); + assert(!Object.keys(unmatch).length,'\n\tUnmatchs:' + Object.keys(unmatch) + "\n\n" +valid + '\nvs.\n' + readItems); + return done(); - v.items.forEach(function(validItem){ - unmatch[validItem.addr] =1; }); - v.items.forEach(function(validItem){ - var readItem = readItems.shift(); - assert.equal(readItem.addr,validItem.addr); - assert.equal(readItem.value_sat,validItem.value_sat); - assert.equal(readItem.index,validItem.index); - assert.equal(readItem.spendIndex, null); - assert.equal(readItem.spendTxIdBuf, null); - delete unmatch[validItem.addr]; - }); - - var valid = util.inspect(v.items, { depth: null }); - assert(!Object.keys(unmatch).length,'\n\tUnmatchs:' + Object.keys(unmatch) + "\n\n" +valid + '\nvs.\n' + readItems); - done(); - }); }); }); @@ -67,4 +168,3 @@ describe('TransactionOut', function(){ }); - diff --git a/test/integration/02-transactionouts.js b/test/integration/02-transactionouts.js index 8097f41c..112a761d 100644 --- a/test/integration/02-transactionouts.js +++ b/test/integration/02-transactionouts.js @@ -5,59 +5,50 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; -var mongoose = require('mongoose'), +var assert = require('assert'), fs = require('fs'), util = require('util'), async = require('async'), config = require('../../config/config'), - TransactionOut = require('../../app/models/TransactionOut'); + TransactionDb = require('../../lib/TransactionDb').class(); -var spentValid = JSON.parse(fs.readFileSync('test/model/spent.json')); +var spentValid = JSON.parse(fs.readFileSync('test/integration/spent.json')); -mongoose.connection.on('error', function(err) { console.log(err); }); +var txDb; +describe('TransactionDb Expenses', function(){ - -describe('TransactionOut Expenses', function(){ - - before(function(done) { - mongoose.connect(config.db); + before(function(c) { + txDb = new TransactionDb(); // lets spend! async.each(Object.keys(spentValid), function(txid,c_out) { async.each(spentValid[txid], function(i,c_in) { - TransactionOut._explodeTransactionOuts(i.txid, function(err) { + txDb.createFromArray([i.txid], null, function(err) { return c_in(); }); }, function(err) { - console.log('Done spending ', txid); //TODO return c_out(); } ); }, function(err) { - console.log('[transactionouts.js.88]'); //TODO - return done(); + return c(); } ); }); - after(function(done) { - mongoose.connection.close(); - done(); - }); - Object.keys(spentValid).forEach( function(txid) { it('test result of spending tx ' + txid, function(done) { var s = spentValid[txid]; var c=0; - TransactionOut.fromTxId( txid, function(err, readItems) { + txDb.fromTxId( txid, function(err, readItems) { s.forEach( function(v) { - assert.equal(readItems[c].spendTxIdBuf.toString('hex'),v.txid); + assert.equal(readItems[c].spendTxId,v.txid); assert.equal(readItems[c].spendIndex,v.n); c++; }); diff --git a/test/integration/addr.js b/test/integration/addr.js index 362bcac6..8f0d23e6 100644 --- a/test/integration/addr.js +++ b/test/integration/addr.js @@ -1,25 +1,20 @@ #!/usr/bin/env node +'use strict'; process.env.NODE_ENV = process.env.NODE_ENV || 'development'; -var - assert = require('assert'), +var assert = require('assert'), fs = require('fs'), - mongoose= require('mongoose'), - config = require('../../config/config'), - Address = require('../../app/models/Address').class(); - addrValid = JSON.parse(fs.readFileSync('test/model/addr.json')); + Address = require('../../app/models/Address').class(), + TransactionDb = require('../../lib/TransactionDb').class(), + addrValid = JSON.parse(fs.readFileSync('test/integration/addr.json')); + var txDb; describe('Address balances', function(){ - before(function(done) { - mongoose.connect(config.db); - done(); - }); - - after(function(done) { - mongoose.connection.close(); - done(); + before(function(c) { + txDb = new TransactionDb(); + return c(); }); addrValid.forEach( function(v) { @@ -30,24 +25,22 @@ describe('Address balances', function(){ it('Info for: ' + v.addr, function(done) { this.timeout(5000); - var a = new Address(v.addr); + var a = new Address(v.addr, txDb); a.update(function(err) { if (err) done(err); assert.equal(v.addr, a.addrStr); - console.log("TX count:" + a.transactions.length); - - if (v.balance) assert.equal(v.balance, a.balance, 'balance: ' + a.balance); + if (v.txApperances) + assert.equal(v.txApperances, a.txApperances, 'txApperances: ' + a.txApperances ); if (v.totalReceived) assert.equal(v.totalReceived, a.totalReceived, 'received: ' + a.totalReceived ); if (v.totalSent) assert.equal(v.totalSent, a.totalSent, 'send: ' + a.totalSent); - if (v.txApperances) - assert.equal(v.txApperances, a.txApperances, 'txApperances: ' + a.txApperances ); + if (v.balance) assert.equal(v.balance, a.balance, 'balance: ' + a.balance); if (v.transactions) { v.transactions.forEach( function(tx) { - assert(a.transactions.indexOf(tx)>-1); + assert(a.transactions.indexOf(tx)>-1,'have tx '+tx); }); } done(); diff --git a/test/integration/addr.json b/test/integration/addr.json index 1812aaba..090c1c61 100644 --- a/test/integration/addr.json +++ b/test/integration/addr.json @@ -1,9 +1,5 @@ [ - { - "addr": "mgqvRGJMwR9JU5VhJ3x9uX9MTkzTsmmDgQ", - "balance": 43.1, - "txApperances": 19 - }, + { "addr": "mp3Rzxx9s1A21SY3sjJ3CQoa2Xjph7e5eS", "balance": 0, @@ -20,10 +16,10 @@ }, { "addr": "mhPEfAmeKVwT7arwMYbhwnL2TfwuWbP4r4", - "balance": 1065, "totalReceived": 1069, - "totalSent": 4, - "txApperances": 13 + "txApperances": 13, + "balance": 1065, + "totalSent": 4 }, { "addr": "n47CfqnKWdNwqY1UWxTmNJAqYutFxdH3zY", @@ -39,11 +35,16 @@ "totalSent": 3.9775, "txApperances": 2 }, + { + "addr": "mgqvRGJMwR9JU5VhJ3x9uX9MTkzTsmmDgQ", + "txApperances": 27, + "balance": 5.1 + }, { "addr": "mzW2hdZN2um7WBvTDerdahKqRgj3md9C29", - "txApperances": 2034, - "balance": 1049.69744099, - "totalReceived": 1049.69744099, + "txApperances": 6033, + "balance": 1049.69744101, + "totalReceived": 1049.69744101, "totalSent": 0 }, { @@ -62,11 +63,11 @@ "transactions": [ "91800d80bb4c69b238c9bfd94eb5155ab821e6b25cae5c79903d12853bbb4ed5" ] }, { - "disabled":1, "addr": "mmvP3mTe53qxHdPqXEvdu8WdC7GfQ2vmx5", "balance": 10580.50027254, "totalReceived": 12157.65075053, "totalSent": 1577.15047799, + "txApperances": 459, "transactions": [ "91800d80bb4c69b238c9bfd94eb5155ab821e6b25cae5c79903d12853bbb4ed5", "f6e80d4fd1a2377406856c67d0cee5ac7e5120993ff97e617ca9aac33b4c6b1e", diff --git a/test/integration/block.js b/test/integration/block.js index 9ca09c09..e61655b6 100644 --- a/test/integration/block.js +++ b/test/integration/block.js @@ -7,68 +7,51 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; var TESTING_BLOCK = '000000000185678d3d7ecc9962c96418174431f93fe20bf216d5565272423f74'; var - mongoose= require('mongoose'), assert = require('assert'), config = require('../../config/config'), - Block = require('../../app/models/Block'); + BlockDb = require('../../lib/BlockDb').class(); + +var bDb; + +describe('BlockDb fromHashWithInfo', function(){ -mongoose.connection.on('error', function(err) { console.log(err); }); - -describe('Block fromHashWithInfo', function(){ - - before(function(done) { - mongoose.connect(config.db); - done(); - }); - - after(function(done) { - mongoose.connection.close(); - done(); - }); - - - it('should poll block\'s info from mongoose', function(done) { - Block.fromHashWithInfo(TESTING_BLOCK, function(err, b2) { - if (err) done(err); - - - var h = new Buffer(TESTING_BLOCK,'hex'); - assert(b2.hashStr === TESTING_BLOCK); - assert.equal(b2.hashStr, TESTING_BLOCK); - done(); - }); + before(function(c) { + bDb = new BlockDb(); + return c(); }); it('should poll block\'s info from bitcoind', function(done) { - Block.fromHashWithInfo(TESTING_BLOCK, function(err, b2) { + bDb.fromHashWithInfo(TESTING_BLOCK, function(err, b2) { if (err) done(err); + assert.equal(b2.hash, TESTING_BLOCK); assert.equal(b2.info.hash, TESTING_BLOCK); assert.equal(b2.info.chainwork, '000000000000000000000000000000000000000000000000001b6dc969ffe847'); done(); }); }); - - - it('hash Virtuals SET', function(done) { - var b = new Block(); - b.hashStr = 'a1a2'; - assert.equal(b.hash.toString('hex'),'a1a2'); - b.nextBlockHashStr = 'a1a3'; - assert.equal(b.nextBlockHash.toString('hex'),'a1a3'); - done(); + it('return true in has', function(done) { + bDb.has(TESTING_BLOCK, function(err, has) { + assert.equal(has, true); + done(); + }); }); + it('setOrphan', function(done) { + var b16 = '00000000c4cbd75af741f3a2b2ff72d9ed4d83a048462c1efe331be31ccf006b'; + var b17 = '00000000fe198cce4c8abf9dca0fee1182cb130df966cc428ad2a230df8da743'; - - it('hash Virtuals GET', function(done) { - var b = new Block(); - b.hash = new Buffer('a1a2','hex'); - assert.equal(b.hashStr,'a1a2'); - - - b.nextBlockHash = new Buffer('b2b1','hex'); - assert.equal(b.nextBlockHashStr,'b2b1'); - done(); + bDb.has(b17, function(err, has) { + assert(has); + bDb.setOrphan(b17, function(err, oldPrev) { + assert.equal(oldPrev, b16); + bDb.setPrev(b17, b16, function(err, oldPrev) { + bDb.getPrev(b17, function(err, p) { + assert.equal(p, b16); + done(); + }); + }); + }); + }); }); }); diff --git a/test/integration/blockExtractor.js b/test/integration/blockExtractor.js index e7ff291a..13c7e8d2 100644 --- a/test/integration/blockExtractor.js +++ b/test/integration/blockExtractor.js @@ -13,7 +13,7 @@ var assert = require('assert'), //var txItemsValid = JSON.parse(fs.readFileSync('test/model/txitems.json')); -describe('TransactionOut', function(){ +describe('BlockExtractor', function(){ var be = new BlockExtractor(config.bitcoind.dataDir, config.network); @@ -51,7 +51,7 @@ describe('TransactionOut', function(){ }); }); - it('should read 100000 blocks with no error ', function(done) { + it.skip('should read 100000 blocks with no error ', function(done) { var i=0; while(i++<100000) { diff --git a/test/integration/blocklist.js b/test/integration/blocklist.js new file mode 100644 index 00000000..d1ed33dc --- /dev/null +++ b/test/integration/blocklist.js @@ -0,0 +1,37 @@ +#!/usr/bin/env node +'use strict'; + +process.env.NODE_ENV = process.env.NODE_ENV || 'development'; + +var TESTING_BLOCK0 = '000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943'; +var TESTING_BLOCK1 = '00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206'; +var START_TS = 1; +var END_TS = '1296688928~'; // 2/2/2011 23:23PM +var LIMIT = 2; + +var assert = require('assert'), + BlockDb = require('../../lib/BlockDb').class(); + +var bDb; + +describe('BlockDb getBlocksByDate', function(){ + + + before(function(c) { + bDb = new BlockDb(); + return c(); + }); + + it('Get Hash by Date', function(done) { + + bDb.getBlocksByDate(START_TS, END_TS, LIMIT, function(err, list) { + if (err) done(err); + assert(list, 'returns list'); + assert.equal(list.length,2, 'list has 2 items'); + assert.equal(list[0].hash, TESTING_BLOCK0); + assert.equal(list[1].hash, TESTING_BLOCK1); + done(); + }); + }); +}); + diff --git a/test/integration/status.js b/test/integration/status.js index e7f078af..663e4730 100644 --- a/test/integration/status.js +++ b/test/integration/status.js @@ -1,25 +1,13 @@ #!/usr/bin/env node +'use strict'; process.env.NODE_ENV = process.env.NODE_ENV || 'development'; -var - assert = require('assert'), - config = require('../../config/config'), - Status = require('../../app/models/Status').class(), - mongoose= require('mongoose'); +var assert = require('assert'), + Status = require('../../app/models/Status').class(); describe('Status', function(){ - before(function(done) { - mongoose.connect(config.db); - done(); - }); - - after(function(done) { - mongoose.connection.close(); - done(); - }); - it('getInfo', function(done) { var d = new Status(); diff --git a/test/integration/sync.js b/test/integration/sync.js new file mode 100644 index 00000000..82b87c4e --- /dev/null +++ b/test/integration/sync.js @@ -0,0 +1,92 @@ +#!/usr/bin/env node +'use strict'; + +process.env.NODE_ENV = process.env.NODE_ENV || 'development'; + + +var + assert = require('assert'), + async = require('async'), + Sync = require('../../lib/Sync').class(); + + +var b = [ + '00000000c4cbd75af741f3a2b2ff72d9ed4d83a048462c1efe331be31ccf006b', //B#16 + '00000000fe198cce4c8abf9dca0fee1182cb130df966cc428ad2a230df8da743', + '000000008d55c3e978639f70af1d2bf1fe6f09cb3143e104405a599215c89a48', + '000000009b3bca4909f38313f2746120129cce4a699a1f552390955da470c5a9', + '00000000ede57f31cc598dc241d129ccb4d8168ef112afbdc870dc60a85f5dd3', //B#20 +]; + +var fix = function(s,cb) { + async.each([1,2,3,4], function(i,c) { + s.bDb.setPrev(b[i],b[i-1], function() { + return c(); + }); + }, cb); +}; + +var test = function(s,cb) { + async.each([2,3,4], function(i,c) { + s.bDb.getPrev(b[i], function(err, p) { + assert.equal(p,0); + return c(); + }); + }, function() { + s.bDb.getPrev(b[1], function(err, p) { + assert.equal(p,b[0]); + return cb(); + }); + }); +}; + + + +var testNo = function(s,cb) { + async.each([2,3,4], function(i,c) { + s.bDb.getPrev(b[i], function(err, p) { + assert.equal(p,b[i-1]); + return c(); + }); + }, function() { + s.bDb.getPrev(b[1], function(err, p) { + assert.equal(p,b[0]); + return cb(); + }); + }); +}; + + + + +var s; +describe('Sync checkOrphan', function(){ + + before(function(done) { + s = new Sync(); + fix(s,done); + }); + + after(function(done) { + + fix(s,function() { + s.close(done); + }); + + }); + + it('checkOrphan', function(done) { + this.timeout(100000); + + s.bDb.has(b[0], function(err, has) { + assert(has); + s.bDb.has(b[1], function(err, has) { + assert(has); + s.checkOrphan(b[4],b[1], function() { + testNo(s,done); + }); + }); + }); + }); +}); + diff --git a/test/integration/transaction.js b/test/integration/transaction.js deleted file mode 100644 index d5a87c06..00000000 --- a/test/integration/transaction.js +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env node -'use strict'; - -process.env.NODE_ENV = process.env.NODE_ENV || 'development'; - - - -var mongoose= require('mongoose'), - assert = require('assert'), - config = require('../../config/config'), - Transaction = require('../../app/models/Transaction').class(); - - -mongoose.connection.on('error', function(err) { console.log(err); }); - -describe('Transaction', function(){ - - before(function(done) { - mongoose.connect(config.db); - done(); - }); - - after(function(done) { - mongoose.connection.close(); - done(); - }); - var txid = '7e621eeb02874ab039a8566fd36f4591e65eca65313875221842c53de6907d6c'; - it('txid ' + txid, function(done) { - Transaction.fromIdWithInfo(txid, function(err, tx) { - if (err) done(err); - assert.equal(tx.txid, txid); - assert(!tx.info.isCoinBase); - - for(var i=0; i<20; i++) - assert(parseFloat(tx.info.vin[i].value) === parseFloat(50), 'input '+i); - assert(tx.info.vin[0].addr === 'msGKGCy2i8wbKS5Fo1LbWUTJnf1GoFFG59', 'addr 0'); - assert(tx.info.vin[1].addr === 'mfye7oHsdrHbydtj4coPXCasKad2eYSv5P', 'addr 1'); - done(); - }); - }); - - it('should pool tx\'s object from mongoose', function(done) { - var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237'; - Transaction.fromIdWithInfo(txid, function(err, tx) { - if (err) done(err); - assert.equal(tx.txid, txid); - assert(!tx.info.isCoinBase); - done(); - }); - }); - - it('should pool tx\'s info from bitcoind', function(done) { - var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237'; - Transaction.fromIdWithInfo(txid, function(err, tx) { - if (err) done(err); - assert.equal(tx.info.txid, txid); - assert.equal(tx.info.blockhash, '000000000185678d3d7ecc9962c96418174431f93fe20bf216d5565272423f74'); - assert.equal(tx.info.valueOut, 1.66174); - assert.equal(tx.info.fees, 0.0005 ); - assert.equal(tx.info.size, 226 ); - assert(!tx.info.isCoinBase); - done(); - }); - }); - - var txid1 = '2a104bab1782e9b6445583296d4a0ecc8af304e4769ceb64b890e8219c562399'; - it('test a coinbase TX ' + txid1, function(done) { - Transaction.fromIdWithInfo(txid1, function(err, tx) { - if (err) done(err); - assert(tx.info.isCoinBase); - assert.equal(tx.info.txid, txid1); - assert(!tx.info.feeds); - done(); - }); - }); - var txid22 = '666'; - it('test invalid TX ' + txid22, function(done) { - Transaction.fromIdWithInfo(txid22, function(err, tx) { - if (err && err.message.match(/must.be.hexadecimal/)) { - return done(); - } - else { - return done(err); - } - }); - }); - - var txid23 = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca227'; - it('test unexisting TX ' + txid23, function(done) { - - Transaction.fromIdWithInfo(txid23, function(err, tx) { - assert(!err); - assert(!tx); - return done(); - }); - }); - - - - var txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608'; - it('create TX on the fly ' + txid2, function(done) { - Transaction.fromIdWithInfo(txid2, function(err, tx) { - if (err) return done(err); - assert.equal(tx.info.txid, txid2); - done(); - }); - }); - - var txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608'; - it('test a broken TX ' + txid2, function(done) { - Transaction.fromIdWithInfo(txid2, function(err, tx) { - if (err) return done(err); - assert.equal(tx.info.txid, txid2); - assert.equal(tx.info.vin[0].addr, 'n1JagbRWBDi6VMvG7HfZmXX74dB9eiHJzU'); - done(); - }); - }); - -}); - diff --git a/test/integration/transactionouts-expenses.js b/test/integration/transactionouts-expenses.js deleted file mode 100644 index b8b04beb..00000000 --- a/test/integration/transactionouts-expenses.js +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env node -'use strict'; - -process.env.NODE_ENV = process.env.NODE_ENV || 'development'; - - - -var mongoose = require('mongoose'), - assert = require('assert'), - fs = require('fs'), - util = require('util'), - async = require('async'), - config = require('../../config/config'), - TransactionOut = require('../../app/models/TransactionOut'); - -var spentValid = JSON.parse(fs.readFileSync('test/model/spent.json')); - -mongoose.connection.on('error', function(err) { console.log(err); }); - - - -describe('TransactionOut Expenses', function(){ - - before(function(done) { - mongoose.connect(config.db); - - // lets spend! - async.each(Object.keys(spentValid), - function(txid,c_out) { - async.each(spentValid[txid], - function(i,c_in) { - TransactionOut._explodeTransactionOuts(i.txid, function(err) { - return c_in(); - }); - }, - function(err) { - return c_out(); - } - ); - }, - function(err) { - console.log('[transactionouts.js.88]'); //TODO - return done(); - } - ); - }); - - after(function(done) { - mongoose.connection.close(); - done(); - }); - - Object.keys(spentValid).forEach( function(txid) { - it('test result of spending tx ' + txid, function(done) { - var s = spentValid[txid]; - var c=0; - TransactionOut.fromTxId( txid, function(err, readItems) { - s.forEach( function(v) { - assert.equal(readItems[c].spendTxIdBuf.toString('hex'),v.txid); - assert.equal(readItems[c].spendIndex,v.n); - c++; - }); - done(); - }); - }); - }); -}); diff --git a/test/integration/txitems.json b/test/integration/txitems.json index 72ff9699..bee3b43f 100644 --- a/test/integration/txitems.json +++ b/test/integration/txitems.json @@ -5,6 +5,10 @@ }, { "txid": "21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237", + "toRm": [ + "txouts-spend-86a03cac7d87f596008c6d5a8d3fd8b88842932ea6f0337673eda16f6b472f7f-0", + "txouts-spend-bcd8da8ee847da377f8aaca92502c05e5f914c6a2452753146013b0e642a25a0-0" + ], "items": [ { "addr": "mzjLe62faUqCSjkwQkwPAL5nYyR8K132fA", @@ -20,6 +24,9 @@ }, { "txid": "b633a6249d4a2bc123e7f8a151cae2d4afd17aa94840009f8697270c7818ceee", + "toRm": [ + "txouts-spend-01621403689cb4a95699a3dbae029d7031c5667678ef14e2054793954fb27917-0" + ], "items": [ { "addr": "mhfQJUSissP6nLM5pz6DxHfctukrrLct2T", @@ -35,6 +42,9 @@ }, { "txid": "ca2f42e44455b8a84434de139efea1fe2c7d71414a8939e0a20f518849085c3b", + "toRm": [ + "txouts-spend-2d7b680fb06e4d7eeb65ca49ac7522276586e0090b7fe662fc708129429c5e6a-0" + ], "items": [ { "addr": "mhqyL1nDQDo1WLH9qH8sjRjx2WwrnmAaXE", diff --git a/util/sync.js b/util/sync.js index 3c1af7d9..7a9a0946 100755 --- a/util/sync.js +++ b/util/sync.js @@ -1,4 +1,5 @@ -#! /usr/bin/env node +#!/usr/bin/env node + 'use strict';