diff --git a/app/models/Address.js b/app/models/Address.js index 370a4b12..7ec5ad1f 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -151,6 +151,7 @@ Address.prototype._addTxItem = function(txItem, notxlist) { this.unconfirmedBalanceSat += v; this.unconfirmedTxApperances += add; } + return txs; }; diff --git a/config/config.js b/config/config.js index c89fc3a9..17f8134c 100644 --- a/config/config.js +++ b/config/config.js @@ -9,6 +9,9 @@ var path = require('path'), b_port, p2p_port; +var packageStr = fs.readFileSync('package.json'); +var version = JSON.parse(packageStr).version; + function getUserHome() { return process.env[(process.platform == 'win32') ? 'USERPROFILE' : 'HOME']; @@ -57,6 +60,59 @@ if (!dataDir) { } dataDir += network === 'testnet' ? 'testnet3' : ''; +var safeConfirmations = process.env.SAFE_CONFIRMATIONS || 6; + + +var bitcoindConf = { + protocol: process.env.BITCOIND_PROTO || 'http', + user: process.env.BITCOIND_USER || 'user', + pass: process.env.BITCOIND_PASS || 'pass', + host: process.env.BITCOIND_HOST || '127.0.0.1', + port: process.env.BITCOIND_PORT || b_port, + p2pPort: process.env.BITCOIND_P2P_PORT || p2p_port, + dataDir: dataDir, + // DO NOT CHANGE THIS! + disableAgent: true +}; + +/*jshint multistr: true */ +console.log( +'\n\ + ____ _ __ __ ___ _ \n\ + / _/___ _____(_)___ _/ /_ / /_ / | ____ (_)\n\ + / // __ \\/ ___/ / __ `/ __ \\/ __/ / /\| \| / __ \\/ / \n\ + _/ // / / (__ ) / /_/ / / / / /_ / ___ |/ /_/ / / \n\ +/___/_/ /_/____/_/\\__, /_/ /_/\\__/ /_/ |_/ .___/_/ \n\ + /____/ /_/ \n\ +\n\t\t\t\t\t\tv%s\n\ + # Configuration:\n\ +\t\tNetwork: %s\tINSIGHT_NETWORK\n\ +\t\tDatabase Path: %s\tINSIGHT_DB\n\ +\t\tSafe Confirmations: %s\tSAFE_CONFIRMATIONS\n\ + # Bicoind Connection configuration:\n\ +\t\tRPC Username: %s\tBITCOIND_USER\n\ +\t\tRPC Password: %s\tBITCOIND_PASS\n\ +\t\tRPC Protocol: %s\tBITCOIND_PROTO\n\ +\t\tRPC Host: %s\tBITCOIND_HOST\n\ +\t\tRPC Port: %s\tBITCOIND_PORT\n\ +\t\tP2P Port: %s\tBITCOIND_P2P_PORT\n\ +\t\tData Dir: %s\tBITCOIND_DATADIR\n\ +\t\t%s\n\ +\nChange setting by assigning the enviroment variables in the last column. Example:\n\ + $ INSIGHT_NETWORK="testnet" BITCOIND_HOST="123.123.123.123" ./insight.js\ +\n\n', +version, +network, home, safeConfirmations, +bitcoindConf.user, +bitcoindConf.pass?'Yes(hidden)':'No', +bitcoindConf.protocol, +bitcoindConf.host, +bitcoindConf.port, +bitcoindConf.p2p_port, +dataDir+(network==='testnet'?'*':''), +(network==='testnet'?'* (/testnet3 is added automatically)':'') +); + if (! fs.existsSync(db)){ @@ -71,17 +127,7 @@ module.exports = { apiPrefix: '/api', port: port, leveldb: db, - bitcoind: { - protocol: process.env.BITCOIND_PROTO || 'http', - user: process.env.BITCOIND_USER || 'user', - pass: process.env.BITCOIND_PASS || 'pass', - host: process.env.BITCOIND_HOST || '127.0.0.1', - port: process.env.BITCOIND_PORT || b_port, - p2pPort: process.env.BITCOIND_P2P_PORT || p2p_port, - dataDir: dataDir, - // DO NOT CHANGE THIS! - disableAgent: true - }, + bitcoind: bitcoindConf, network: network, disableP2pSync: false, disableHistoricSync: false, @@ -92,5 +138,5 @@ module.exports = { keys: { segmentio: process.env.INSIGHT_SEGMENTIO_KEY }, - safeConfirmations: 6, // PLEASE NOTE THAT *FULL RESYNC* IS NEEDED TO CHANGE safeConfirmations + safeConfirmations: safeConfirmations, // PLEASE NOTE THAT *FULL RESYNC* IS NEEDED TO CHANGE safeConfirmations }; diff --git a/insight.js b/insight.js index 1cb74c08..215118db 100755 --- a/insight.js +++ b/insight.js @@ -1,6 +1,6 @@ +#!/usr/bin/env node + 'use strict'; - - //Set the node enviornment variable if not set before process.env.NODE_ENV = process.env.NODE_ENV || 'development'; diff --git a/lib/BlockExtractor.js b/lib/BlockExtractor.js index a242a6f4..2c8a6aa2 100644 --- a/lib/BlockExtractor.js +++ b/lib/BlockExtractor.js @@ -16,7 +16,6 @@ function BlockExtractor(dataDir, network) { self.dataDir = dataDir; self.files = glob.sync(path); self.nfiles = self.files.length; - self.errorCount =0; if (self.nfiles === 0) throw new Error('Could not find block files at: ' + path); @@ -98,40 +97,29 @@ BlockExtractor.prototype.getNextBlock = function(cb) { async.whilst( function() { - return (!magic || magic === '00000000'); + return (!magic); }, function(w_cb) { - magic = null; self.readCurrentFileSync(); + if (self.currentFileIndex < 0) return cb(); - var byte0 = self.currentParser ? self.currentParser.buffer(1).toString('hex') : null; - if (byte0) { - // Grab 3 bytes from block without removing them - var p = self.currentParser.pos; - var bytes123 = self.currentParser.subject.toString('hex',p,p+3); - magic = byte0 + bytes123; - - if (magic !=='00000000' && magic !== self.magic) { - - if (self.errorCount++ > 4) - return cb(new Error('CRITICAL ERROR: Magic number mismatch: ' + - magic + '!=' + self.magic)); - - magic=null; - } - } - - if (!self.currentParser || self.currentParser.eof() ) { - if (self.nextFile()) - console.log('Moving forward to file:' + self.currentFile() ); - else - console.log('Finished all files'); + magic = self.currentParser ? self.currentParser.buffer(4).toString('hex') + : null ; + if (!self.currentParser || self.currentParser.eof() || magic === '00000000') { magic = null; - return w_cb(); + if (self.nextFile()) { + console.log('Moving forward to file:' + self.currentFile() ); + return w_cb(); + } + else { + console.log('Finished all files'); + magic = null; + return w_cb(); + } } else { return w_cb(); @@ -140,8 +128,14 @@ BlockExtractor.prototype.getNextBlock = function(cb) { }, function (a_cb) { if (!magic) return a_cb(); - // Remove 3 bytes from magic and spacer - self.currentParser.buffer(3+4); + if (magic !== self.magic) { + var e = new Error('CRITICAL ERROR: Magic number mismatch: ' + + magic + '!=' + self.magic); + return a_cb(e); + } + + // spacer? + self.currentParser.word32le(); return a_cb(); }, function (a_cb) { @@ -150,7 +144,6 @@ BlockExtractor.prototype.getNextBlock = function(cb) { b = new Block(); b.parse(self.currentParser); b.getHash(); - self.errorCount=0; return a_cb(); }, ], function(err) { diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 4140a4f7..b0b65b52 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -36,13 +36,14 @@ function HistoricSync(opts) { this.rpc = new RpcClient(config.bitcoind); this.sync = new Sync(opts); + this.height =0; } HistoricSync.prototype.showProgress = function() { var self = this; if ( self.status ==='syncing' && - ( self.syncedBlocks ) % self.step !== 1) return; + ( self.height ) % self.step !== 1) return; if (self.error) error(self.error); @@ -82,7 +83,7 @@ HistoricSync.prototype.info = function() { status: this.status, blockChainHeight: this.blockChainHeight, syncPercentage: this.syncPercentage, - syncedBlocks: this.syncedBlocks, + height: this.height, syncTipHash: this.sync.tip, error: this.error, type: this.type, @@ -92,7 +93,7 @@ HistoricSync.prototype.info = function() { }; HistoricSync.prototype.updatePercentage = function() { - var r = this.syncedBlocks / this.blockChainHeight; + var r = this.height / this.blockChainHeight; this.syncPercentage = parseFloat(100 * r).toFixed(3); if (this.syncPercentage > 100) this.syncPercentage = 100; }; @@ -153,16 +154,6 @@ HistoricSync.prototype.getBlockFromFile = function(cb) { }); }; -HistoricSync.prototype.updateConnectedCountDB = function(cb) { - var self = this; - self.sync.bDb.countConnected(function(err, count) { - self.connectedCountDB = count || 0; - self.syncedBlocks = count || 0; - return cb(err); - }); -}; - - HistoricSync.prototype.updateBlockChainHeight = function(cb) { var self = this; @@ -199,7 +190,7 @@ HistoricSync.prototype.updateStartBlock = function(next) { self.startBlock = self.genesis; - self.sync.bDb.getTip(function(err,tip) { + self.sync.bDb.getTip(function(err,tip, height) { if (!tip) return next(); var blockInfo; @@ -219,6 +210,12 @@ HistoricSync.prototype.updateStartBlock = function(next) { function(err) { if (err) return next(err); var ret = false; + + var d = Math.abs(height-blockInfo.height); + if (d>6) { + error('Previous Tip block tip height differs by %d. Please delete and resync (-D)',d); + process.exit(1); + } if ( self.blockChainHeight === blockInfo.height || blockInfo.confirmations > 0) { ret = false; @@ -227,7 +224,6 @@ HistoricSync.prototype.updateStartBlock = function(next) { oldtip = tip; if (!tip) throw new Error('Previous blockchain tip was not found on bitcoind. Please reset Insight DB. Tip was:'+tip) - tip = blockInfo.previousblockhash; info('Previous TIP is now orphan. Back to:' + tip); ret = true; @@ -236,7 +232,8 @@ HistoricSync.prototype.updateStartBlock = function(next) { }, function(err) { self.startBlock = tip; - info('Resuming sync from block:'+tip); + self.height = height; + info('Resuming sync from block: %s #%d',tip,height); return next(err); } ); @@ -247,7 +244,7 @@ HistoricSync.prototype.prepareFileSync = function(opts, next) { var self = this; if ( opts.forceRPC || !config.bitcoind.dataDir || - self.connectedCountDB > self.blockChainHeight * 0.9) return next(); + self.height > self.blockChainHeight * 0.9) return next(); try { @@ -260,6 +257,7 @@ HistoricSync.prototype.prepareFileSync = function(opts, next) { self.getFn = self.getBlockFromFile; self.allowReorgs = true; self.sync.bDb.getLastFileIndex(function(err, idx) { + if (opts.forceStartFile) self.blockExtractor.currentFileIndex = opts.forceStartFile; else if (idx) self.blockExtractor.currentFileIndex = idx; @@ -280,7 +278,9 @@ HistoricSync.prototype.prepareFileSync = function(opts, next) { return w_cb(err); }); }); - }, next); + }, function(err){ + return next(err); + }); }); }; @@ -298,7 +298,7 @@ HistoricSync.prototype.prepareRpcSync = function(opts, next) { HistoricSync.prototype.showSyncStartMessage = function() { var self = this; - info('Got ' + self.connectedCountDB + + info('Got ' + self.height + ' blocks in current DB, out of ' + self.blockChainHeight + ' block at bitcoind'); if (self.blockExtractor) { @@ -317,7 +317,7 @@ HistoricSync.prototype.showSyncStartMessage = function() { HistoricSync.prototype.setupSyncStatus = function() { var self = this; - var step = parseInt( (self.blockChainHeight - self.syncedBlocks) / 1000); + var step = parseInt( (self.blockChainHeight - self.height) / 1000); if (step < 10) step = 10; self.step = step; @@ -335,10 +335,10 @@ HistoricSync.prototype.prepareToSync = function(opts, next) { self.status = 'starting'; async.series([ function(s_c) { - self.checkNetworkSettings(s_c); + self.sync.txDb.checkVersion02(s_c); }, function(s_c) { - self.updateConnectedCountDB(s_c); + self.checkNetworkSettings(s_c); }, function(s_c) { self.updateBlockChainHeight(s_c); @@ -382,12 +382,10 @@ HistoricSync.prototype.start = function(opts, next) { function (w_cb) { self.getFn(function(err,blockInfo) { if (err) return w_cb(self.setError(err)); - if (blockInfo && blockInfo.hash - && (!opts.stopAt || opts.stopAt !== blockInfo.hash) - ) { - self.syncedBlocks++; - self.sync.storeTipBlock(blockInfo, self.allowReorgs, function(err) { + if (blockInfo && blockInfo.hash && (!opts.stopAt || opts.stopAt !== blockInfo.hash)) { + self.sync.storeTipBlock(blockInfo, self.allowReorgs, function(err, height) { if (err) return w_cb(self.setError(err)); + self.height=height; setImmediate(function(){ return w_cb(err); }); diff --git a/lib/Sync.js b/lib/Sync.js index bc39f659..7809f4f6 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -164,7 +164,7 @@ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { if (err && err.toString().match(/WARN/)) { err = null; } - return cb(err); + return cb(err, height); }); }; diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 8cc37682..e58df2e0 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -8,7 +8,8 @@ var OUTS_PREFIX = 'txo-'; //txo-- => [addr, btc_sat] var SPENT_PREFIX = 'txs-'; //txs---- = ts // to sum up addr balance (only outs, spents are gotten later) -var ADDR_PREFIX = 'txa-'; //txa--- => + btc_sat:ts [:-](spent) +var ADDR_PREFIX = 'txa-'; //txa--- + // => + btc_sat:ts [:isConfirmed:[scriptPubKey|isSpendConfirmed:SpentTxid:SpentVout:SpentTs] // TODO: use bitcore networks module var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b'; @@ -178,7 +179,7 @@ TransactionDb.prototype._fillOutpoints = function(txInfo, cb) { var incompleteInputs = 0; async.eachLimit(txInfo.vin, CONCURRENCY, function(i, c_in) { - self.fromTxIdN(i.txid, i.vout, txInfo.confirmations, function(err, ret) { + self.fromTxIdN(i.txid, i.vout, function(err, ret) { if (!ret || !ret.addr || !ret.valueSat) { info('Could not get TXouts in %s,%d from %s ', i.txid, i.vout, txInfo.txid); if (ret) i.unconfirmedInput = ret.unconfirmedInput; @@ -272,7 +273,7 @@ TransactionDb.prototype.fromIdWithInfo = function(txid, cb) { }); }; -TransactionDb.prototype.fromTxIdN = function(txid, n, confirmations, cb) { +TransactionDb.prototype.fromTxIdN = function(txid, n, cb) { var self = this; var k = OUTS_PREFIX + txid + '-' + n; @@ -289,18 +290,6 @@ TransactionDb.prototype.fromTxIdN = function(txid, n, confirmations, cb) { valueSat: parseInt(a[1]), }; - /* - * If this TxID comes from an RPC request - * the .confirmations value from bitcoind is available - * so we could avoid checking if the input was double spent - * - * This speed up address calculations by ~30% - * - if (confirmations >= CONFIRMATION_NR_TO_NOT_CHECK) { - return cb(null, ret); - } - */ - // spent? var k = SPENT_PREFIX + txid + '-' + n + '-'; db.createReadStream({ @@ -671,4 +660,17 @@ TransactionDb.prototype.getPoolInfo = function(txid, cb) { }); }; + +TransactionDb.prototype.checkVersion02 = function(cb) { + var k = 'txb-f0315ffc38709d70ad5647e22048358dd3745f3ce3874223c80a7c92fab0c8ba-00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206'; + db.get(k, function(err, val) { + if (val) { + console.log('\n#############################\n\n ## Insight API DB is older that v0.2. Please resync using:\n $ util/sync.js -D\n More information at Insight API\'s Readme.md'); + process.exit(1); + } + return cb(); + }); +}; + + module.exports = require('soop')(TransactionDb); diff --git a/test/integration/addr.js b/test/integration/addr.js index 0e726824..b5fc5399 100644 --- a/test/integration/addr.js +++ b/test/integration/addr.js @@ -31,8 +31,11 @@ describe('Address balances', function() { a.update(function(err) { if (err) done(err); v.addr.should.equal(a.addrStr); - a.unconfirmedTxApperances.should.equal(v.unconfirmedTxApperances || 0, 'unconfirmedTxApperances'); - a.unconfirmedBalanceSat.should.equal(v.unconfirmedBalanceSat || 0, 'unconfirmedBalanceSat'); + + if (v.unconfirmedTxApperances) + a.unconfirmedTxApperances.should.equal(v.unconfirmedTxApperances || 0, 'unconfirmedTxApperances'); + if (v.unconfirmedBalanceSat) + a.unconfirmedBalanceSat.should.equal(v.unconfirmedBalanceSat || 0, 'unconfirmedBalanceSat'); if (v.txApperances) a.txApperances.should.equal(v.txApperances, 'txApperances');