From cab25cf397f67f8da60c3af3e2f120cff7fd5729 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 28 Dec 2015 15:43:20 -0500 Subject: [PATCH 01/20] Address Service: Start to use streams for memory optimization with large queries --- lib/services/address/constants.js | 41 + lib/services/address/encoding.js | 211 +++++ lib/services/address/index.js | 800 +++++++----------- .../address/streams/inputs-transform.js | 40 + .../address/streams/outputs-transform.js | 42 + 5 files changed, 663 insertions(+), 471 deletions(-) create mode 100644 lib/services/address/constants.js create mode 100644 lib/services/address/encoding.js create mode 100644 lib/services/address/streams/inputs-transform.js create mode 100644 lib/services/address/streams/outputs-transform.js diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js new file mode 100644 index 00000000..7a0cf8dd --- /dev/null +++ b/lib/services/address/constants.js @@ -0,0 +1,41 @@ +'use strict'; + +var exports = {}; + +exports.PREFIXES = { + OUTPUTS: new Buffer('02', 'hex'), // Query outputs by address and/or height + SPENTS: new Buffer('03', 'hex'), // Query inputs by address and/or height + SPENTSMAP: new Buffer('05', 'hex') // Get the input that spends an output +}; + +exports.MEMPREFIXES = { + OUTPUTS: new Buffer('01', 'hex'), // Query mempool outputs by address + SPENTS: new Buffer('02', 'hex'), // Query mempool inputs by address + SPENTSMAP: new Buffer('03', 'hex') // Query mempool for the input that spends an output +}; + +// To save space, we're only storing the PubKeyHash or ScriptHash in our index. +// To avoid intentional unspendable collisions, which have been seen on the blockchain, +// we must store the hash type (PK or Script) as well. +exports.HASH_TYPES = { + PUBKEY: new Buffer('01', 'hex'), + REDEEMSCRIPT: new Buffer('02', 'hex') +}; + +// Translates from our enum type back into the hash types returned by +// bitcore-lib/address. +exports.HASH_TYPES_READABLE = { + '01': 'pubkeyhash', + '02': 'scripthash' +}; + +exports.HASH_TYPES_MAP = { + 'pubkeyhash': exports.HASH_TYPES.PUBKEY, + 'scripthash': exports.HASH_TYPES.REDEEMSCRIPT +}; + +exports.SPACER_MIN = new Buffer('00', 'hex'); +exports.SPACER_MAX = new Buffer('ff', 'hex'); + +module.exports = exports; + diff --git a/lib/services/address/encoding.js b/lib/services/address/encoding.js new file mode 100644 index 00000000..68313754 --- /dev/null +++ b/lib/services/address/encoding.js @@ -0,0 +1,211 @@ +'use strict'; + +var bitcore = require('bitcore-lib'); +var BufferReader = bitcore.encoding.BufferReader; +var Address = bitcore.Address; +var PublicKey = bitcore.PublicKey; +var constants = require('./constants'); +var $ = bitcore.util.preconditions; + +var exports = {}; + +exports.encodeSpentIndexSyncKey = function(txidBuffer, outputIndex) { + var outputIndexBuffer = new Buffer(4); + outputIndexBuffer.writeUInt32BE(outputIndex); + var key = Buffer.concat([ + txidBuffer, + outputIndexBuffer + ]); + return key.toString('binary'); +}; + +exports.encodeOutputKey = function(hashBuffer, hashTypeBuffer, height, txidBuffer, outputIndex) { + var heightBuffer = new Buffer(4); + heightBuffer.writeUInt32BE(height); + var outputIndexBuffer = new Buffer(4); + outputIndexBuffer.writeUInt32BE(outputIndex); + var key = Buffer.concat([ + constants.PREFIXES.OUTPUTS, + hashBuffer, + hashTypeBuffer, + constants.SPACER_MIN, + heightBuffer, + txidBuffer, + outputIndexBuffer + ]); + return key; +}; + +exports.decodeOutputKey = function(buffer) { + var reader = new BufferReader(buffer); + var prefix = reader.read(1); + var hashBuffer = reader.read(20); + var hashTypeBuffer = reader.read(1); + var spacer = reader.read(1); + var height = reader.readUInt32BE(); + var txid = reader.read(32); + var outputIndex = reader.readUInt32BE(); + return { + prefix: prefix, + hashBuffer: hashBuffer, + hashTypeBuffer: hashTypeBuffer, + height: height, + txid: txid, + outputIndex: outputIndex + }; +}; + +exports.encodeOutputValue = function(satoshis, scriptBuffer) { + var satoshisBuffer = new Buffer(8); + satoshisBuffer.writeDoubleBE(satoshis); + return Buffer.concat([satoshisBuffer, scriptBuffer]); +}; + +exports.decodeOutputValue = function(buffer) { + var satoshis = buffer.readDoubleBE(0); + var scriptBuffer = buffer.slice(8, buffer.length); + return { + satoshis: satoshis, + scriptBuffer: scriptBuffer + }; +}; + +exports.encodeInputKey = function(hashBuffer, hashTypeBuffer, height, prevTxIdBuffer, outputIndex) { + var heightBuffer = new Buffer(4); + heightBuffer.writeUInt32BE(height); + var outputIndexBuffer = new Buffer(4); + outputIndexBuffer.writeUInt32BE(outputIndex); + return Buffer.concat([ + constants.PREFIXES.SPENTS, + hashBuffer, + hashTypeBuffer, + constants.SPACER_MIN, + heightBuffer, + prevTxIdBuffer, + outputIndexBuffer + ]); +}; + +exports.decodeInputKey = function(buffer) { + var reader = new BufferReader(buffer); + var prefix = reader.read(1); + var hashBuffer = reader.read(20); + var hashTypeBuffer = reader.read(1); + var spacer = reader.read(1); + var height = reader.readUInt32BE(); + var prevTxId = reader.read(32); + var outputIndex = reader.readUInt32BE(); + return { + prefix: prefix, + hashBuffer: hashBuffer, + hashTypeBuffer: hashTypeBuffer, + height: height, + prevTxId: prevTxId, + outputIndex: outputIndex + }; +}; + +exports.encodeInputValue = function(txidBuffer, inputIndex) { + var inputIndexBuffer = new Buffer(4); + inputIndexBuffer.writeUInt32BE(inputIndex); + return Buffer.concat([ + txidBuffer, + inputIndexBuffer + ]); +}; + +exports.decodeInputValue = function(buffer) { + var txid = buffer.slice(0, 32); + var inputIndex = buffer.readUInt32BE(32); + return { + txid: txid, + inputIndex: inputIndex + }; +}; + +exports.encodeInputKeyMap = function(outputTxIdBuffer, outputIndex) { + var outputIndexBuffer = new Buffer(4); + outputIndexBuffer.writeUInt32BE(outputIndex); + return Buffer.concat([ + constants.PREFIXES.SPENTSMAP, + outputTxIdBuffer, + outputIndexBuffer + ]); +}; + +exports.decodeInputKeyMap = function(buffer) { + var txid = buffer.slice(1, 33); + var outputIndex = buffer.readUInt32BE(33); + return { + outputTxId: txid, + outputIndex: outputIndex + }; +}; + +exports.encodeInputValueMap = function(inputTxIdBuffer, inputIndex) { + var inputIndexBuffer = new Buffer(4); + inputIndexBuffer.writeUInt32BE(inputIndex); + return Buffer.concat([ + inputTxIdBuffer, + inputIndexBuffer + ]); +}; + +exports.decodeInputValueMap = function(buffer) { + var txid = buffer.slice(0, 32); + var inputIndex = buffer.readUInt32BE(32); + return { + inputTxId: txid, + inputIndex: inputIndex + }; +}; + +exports.getAddressInfo = function(addressStr) { + var addrObj = bitcore.Address(addressStr); + var hashTypeBuffer = constants.HASH_TYPES_MAP[addrObj.type]; + + return { + hashBuffer: addrObj.hashBuffer, + hashTypeBuffer: hashTypeBuffer, + hashTypeReadable: addrObj.type + }; +}; + +/** + * This function is optimized to return address information about an output script + * without constructing a Bitcore Address instance. + * @param {Script} - An instance of a Bitcore Script + * @param {Network|String} - The network for the address + */ +exports.extractAddressInfoFromScript = function(script, network) { + $.checkArgument(network, 'Second argument is expected to be a network'); + var hashBuffer; + var addressType; + var hashTypeBuffer; + if (script.isPublicKeyHashOut()) { + hashBuffer = script.chunks[2].buf; + hashTypeBuffer = constants.HASH_TYPES.PUBKEY; + addressType = Address.PayToPublicKeyHash; + } else if (script.isScriptHashOut()) { + hashBuffer = script.chunks[1].buf; + hashTypeBuffer = constants.HASH_TYPES.REDEEMSCRIPT; + addressType = Address.PayToScriptHash; + } else if (script.isPublicKeyOut()) { + var pubkey = script.chunks[0].buf; + var address = Address.fromPublicKey(new PublicKey(pubkey), network); + hashBuffer = address.hashBuffer; + hashTypeBuffer = constants.HASH_TYPES.PUBKEY; + // pay-to-publickey doesn't have an address, however for compatibility + // purposes, we can create an address + addressType = Address.PayToPublicKeyHash; + } else { + return false; + } + return { + hashBuffer: hashBuffer, + hashTypeBuffer: hashTypeBuffer, + addressType: addressType + }; +}; + +module.exports = exports; diff --git a/lib/services/address/index.js b/lib/services/address/index.js index ef7e3e2c..cf37e7e2 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -16,11 +16,14 @@ var memdown = require('memdown'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; var Hash = bitcore.crypto.Hash; -var BufferReader = bitcore.encoding.BufferReader; var EventEmitter = require('events').EventEmitter; -var PublicKey = bitcore.PublicKey; var Address = bitcore.Address; var AddressHistory = require('./history'); +var constants = require('./constants'); +var encoding = require('./encoding'); +var InputsTransformStream = require('./streams/inputs-transform'); +var OutputsTransformStream = require('./streams/outputs-transform'); + /** * The Address Service builds upon the Database Service and the Bitcoin Service to add additional @@ -58,42 +61,6 @@ AddressService.dependencies = [ 'db' ]; -AddressService.PREFIXES = { - OUTPUTS: new Buffer('02', 'hex'), // Query outputs by address and/or height - SPENTS: new Buffer('03', 'hex'), // Query inputs by address and/or height - SPENTSMAP: new Buffer('05', 'hex') // Get the input that spends an output -}; - -AddressService.MEMPREFIXES = { - OUTPUTS: new Buffer('01', 'hex'), // Query mempool outputs by address - SPENTS: new Buffer('02', 'hex'), // Query mempool inputs by address - SPENTSMAP: new Buffer('03', 'hex') // Query mempool for the input that spends an output -}; - -// To save space, we're only storing the PubKeyHash or ScriptHash in our index. -// To avoid intentional unspendable collisions, which have been seen on the blockchain, -// we must store the hash type (PK or Script) as well. -AddressService.HASH_TYPES = { - PUBKEY: new Buffer('01', 'hex'), - REDEEMSCRIPT: new Buffer('02', 'hex') -}; - -// Translates from our enum type back into the hash types returned by -// bitcore-lib/address. -AddressService.HASH_TYPES_READABLE = { - '01': 'pubkeyhash', - '02': 'scripthash' -}; - -// Trnaslates from address types to our enum type. -AddressService.HASH_TYPES_MAP = { - 'pubkeyhash': AddressService.HASH_TYPES.PUBKEY, - 'scripthash': AddressService.HASH_TYPES.REDEEMSCRIPT -}; - -AddressService.SPACER_MIN = new Buffer('00', 'hex'); -AddressService.SPACER_MAX = new Buffer('ff', 'hex'); - AddressService.prototype.start = function(callback) { var self = this; @@ -205,7 +172,7 @@ AddressService.prototype.transactionOutputHandler = function(messages, tx, outpu return; } - var addressInfo = this._extractAddressInfoFromScript(script); + var addressInfo = encoding.extractAddressInfoFromScript(script, this.node.network); if (!addressInfo) { return; } @@ -312,7 +279,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { if (!output.script) { continue; } - var addressInfo = this._extractAddressInfoFromScript(output.script); + var addressInfo = encoding.extractAddressInfoFromScript(output.script, this.node.network); if (!addressInfo) { continue; } @@ -322,14 +289,14 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { outputIndexBuffer.writeUInt32BE(outputIndex); var outKey = Buffer.concat([ - AddressService.MEMPREFIXES.OUTPUTS, + constants.MEMPREFIXES.OUTPUTS, addressInfo.hashBuffer, addressInfo.hashTypeBuffer, txidBuffer, outputIndexBuffer ]); - var outValue = this._encodeOutputValue(output.satoshis, output._scriptBuffer); + var outValue = encoding.encodeOutputValue(output.satoshis, output._scriptBuffer); operations.push({ type: action, @@ -347,7 +314,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { inputOutputIndexBuffer.writeUInt32BE(input.outputIndex); // Add an additional small spent index for fast synchronous lookups - var spentIndexSyncKey = this._encodeSpentIndexSyncKey( + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( input.prevTxId, input.outputIndex ); @@ -359,7 +326,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { // Add a more detailed spent index with values var spentIndexKey = Buffer.concat([ - AddressService.MEMPREFIXES.SPENTSMAP, + constants.MEMPREFIXES.SPENTSMAP, input.prevTxId, inputOutputIndexBuffer ]); @@ -380,15 +347,15 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { var inputHashType; if (input.script.isPublicKeyHashIn()) { inputHashBuffer = Hash.sha256ripemd160(input.script.chunks[1].buf); - inputHashType = AddressService.HASH_TYPES.PUBKEY; + inputHashType = constants.HASH_TYPES.PUBKEY; } else if (input.script.isScriptHashIn()) { inputHashBuffer = Hash.sha256ripemd160(input.script.chunks[input.script.chunks.length - 1].buf); - inputHashType = AddressService.HASH_TYPES.REDEEMSCRIPT; + inputHashType = constants.HASH_TYPES.REDEEMSCRIPT; } else { continue; } var inputKey = Buffer.concat([ - AddressService.MEMPREFIXES.SPENTS, + constants.MEMPREFIXES.SPENTS, inputHashBuffer, inputHashType, input.prevTxId, @@ -417,42 +384,6 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { this.mempoolIndex.batch(operations, callback); }; -/** - * This function is optimized to return address information about an output script - * without constructing a Bitcore Address instance. - * @param {Script} - An instance of a Bitcore Script - * @private - */ -AddressService.prototype._extractAddressInfoFromScript = function(script) { - var hashBuffer; - var addressType; - var hashTypeBuffer; - if (script.isPublicKeyHashOut()) { - hashBuffer = script.chunks[2].buf; - hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; - addressType = Address.PayToPublicKeyHash; - } else if (script.isScriptHashOut()) { - hashBuffer = script.chunks[1].buf; - hashTypeBuffer = AddressService.HASH_TYPES.REDEEMSCRIPT; - addressType = Address.PayToScriptHash; - } else if (script.isPublicKeyOut()) { - var pubkey = script.chunks[0].buf; - var address = Address.fromPublicKey(new PublicKey(pubkey), this.node.network); - hashBuffer = address.hashBuffer; - hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; - // pay-to-publickey doesn't have an address, however for compatibility - // purposes, we can create an address - addressType = Address.PayToPublicKeyHash; - } else { - return false; - } - return { - hashBuffer: hashBuffer, - hashTypeBuffer: hashTypeBuffer, - addressType: addressType - }; -}; - /** * The Database Service will run this function when blocks are connected and * disconnected to the chain during syncing and reorganizations. @@ -494,7 +425,7 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { continue; } - var addressInfo = this._extractAddressInfoFromScript(script); + var addressInfo = encoding.extractAddressInfoFromScript(script, this.node.network); if (!addressInfo) { continue; } @@ -504,9 +435,9 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { // can have a time that is previous to the previous block (however not // less than the mean of the 11 previous blocks) and not greater than 2 // hours in the future. - var key = this._encodeOutputKey(addressInfo.hashBuffer, addressInfo.hashTypeBuffer, - height, txidBuffer, outputIndex); - var value = this._encodeOutputValue(output.satoshis, output._scriptBuffer); + var key = encoding.encodeOutputKey(addressInfo.hashBuffer, addressInfo.hashTypeBuffer, + height, txidBuffer, outputIndex); + var value = encoding.encodeOutputValue(output.satoshis, output._scriptBuffer); operations.push({ type: action, key: key, @@ -549,10 +480,10 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { if (input.script.isPublicKeyHashIn()) { inputHash = Hash.sha256ripemd160(input.script.chunks[1].buf); - inputHashType = AddressService.HASH_TYPES.PUBKEY; + inputHashType = constants.HASH_TYPES.PUBKEY; } else if (input.script.isScriptHashIn()) { inputHash = Hash.sha256ripemd160(input.script.chunks[input.script.chunks.length - 1].buf); - inputHashType = AddressService.HASH_TYPES.REDEEMSCRIPT; + inputHashType = constants.HASH_TYPES.REDEEMSCRIPT; } else { continue; } @@ -560,8 +491,8 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { var prevTxIdBuffer = new Buffer(input.prevTxId, 'hex'); // To be able to query inputs by address and spent height - var inputKey = this._encodeInputKey(inputHash, inputHashType, height, prevTxIdBuffer, input.outputIndex); - var inputValue = this._encodeInputValue(txidBuffer, inputIndex); + var inputKey = encoding.encodeInputKey(inputHash, inputHashType, height, prevTxIdBuffer, input.outputIndex); + var inputValue = encoding.encodeInputValue(txidBuffer, inputIndex); operations.push({ type: action, @@ -570,8 +501,8 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { }); // To be able to search for an input spending an output - var inputKeyMap = this._encodeInputKeyMap(prevTxIdBuffer, input.outputIndex); - var inputValueMap = this._encodeInputValueMap(txidBuffer, inputIndex); + var inputKeyMap = encoding.encodeInputKeyMap(prevTxIdBuffer, input.outputIndex); + var inputValueMap = encoding.encodeInputValueMap(txidBuffer, inputIndex); operations.push({ type: action, @@ -587,168 +518,6 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { }); }; -AddressService.prototype._encodeSpentIndexSyncKey = function(txidBuffer, outputIndex) { - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - var key = Buffer.concat([ - txidBuffer, - outputIndexBuffer - ]); - return key.toString('binary'); -}; - -AddressService.prototype._encodeOutputKey = function(hashBuffer, hashTypeBuffer, height, txidBuffer, outputIndex) { - var heightBuffer = new Buffer(4); - heightBuffer.writeUInt32BE(height); - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - var key = Buffer.concat([ - AddressService.PREFIXES.OUTPUTS, - hashBuffer, - hashTypeBuffer, - AddressService.SPACER_MIN, - heightBuffer, - txidBuffer, - outputIndexBuffer - ]); - return key; -}; - -AddressService.prototype._decodeOutputKey = function(buffer) { - var reader = new BufferReader(buffer); - var prefix = reader.read(1); - var hashBuffer = reader.read(20); - var hashTypeBuffer = reader.read(1); - var spacer = reader.read(1); - var height = reader.readUInt32BE(); - var txid = reader.read(32); - var outputIndex = reader.readUInt32BE(); - return { - prefix: prefix, - hashBuffer: hashBuffer, - hashTypeBuffer: hashTypeBuffer, - height: height, - txid: txid, - outputIndex: outputIndex - }; -}; - -AddressService.prototype._encodeOutputValue = function(satoshis, scriptBuffer) { - var satoshisBuffer = new Buffer(8); - satoshisBuffer.writeDoubleBE(satoshis); - return Buffer.concat([satoshisBuffer, scriptBuffer]); -}; - -AddressService.prototype._decodeOutputValue = function(buffer) { - var satoshis = buffer.readDoubleBE(0); - var scriptBuffer = buffer.slice(8, buffer.length); - return { - satoshis: satoshis, - scriptBuffer: scriptBuffer - }; -}; - -AddressService.prototype._encodeInputKey = function(hashBuffer, hashTypeBuffer, height, prevTxIdBuffer, outputIndex) { - var heightBuffer = new Buffer(4); - heightBuffer.writeUInt32BE(height); - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - return Buffer.concat([ - AddressService.PREFIXES.SPENTS, - hashBuffer, - hashTypeBuffer, - AddressService.SPACER_MIN, - heightBuffer, - prevTxIdBuffer, - outputIndexBuffer - ]); -}; - -AddressService.prototype._decodeInputKey = function(buffer) { - var reader = new BufferReader(buffer); - var prefix = reader.read(1); - var hashBuffer = reader.read(20); - var hashTypeBuffer = reader.read(1); - var spacer = reader.read(1); - var height = reader.readUInt32BE(); - var prevTxId = reader.read(32); - var outputIndex = reader.readUInt32BE(); - return { - prefix: prefix, - hashBuffer: hashBuffer, - hashTypeBuffer: hashTypeBuffer, - height: height, - prevTxId: prevTxId, - outputIndex: outputIndex - }; -}; - -AddressService.prototype._encodeInputValue = function(txidBuffer, inputIndex) { - var inputIndexBuffer = new Buffer(4); - inputIndexBuffer.writeUInt32BE(inputIndex); - return Buffer.concat([ - txidBuffer, - inputIndexBuffer - ]); -}; - -AddressService.prototype._decodeInputValue = function(buffer) { - var txid = buffer.slice(0, 32); - var inputIndex = buffer.readUInt32BE(32); - return { - txid: txid, - inputIndex: inputIndex - }; -}; - -AddressService.prototype._encodeInputKeyMap = function(outputTxIdBuffer, outputIndex) { - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - return Buffer.concat([ - AddressService.PREFIXES.SPENTSMAP, - outputTxIdBuffer, - outputIndexBuffer - ]); -}; - -AddressService.prototype._decodeInputKeyMap = function(buffer) { - var txid = buffer.slice(1, 33); - var outputIndex = buffer.readUInt32BE(33); - return { - outputTxId: txid, - outputIndex: outputIndex - }; -}; - -AddressService.prototype._encodeInputValueMap = function(inputTxIdBuffer, inputIndex) { - var inputIndexBuffer = new Buffer(4); - inputIndexBuffer.writeUInt32BE(inputIndex); - return Buffer.concat([ - inputTxIdBuffer, - inputIndexBuffer - ]); -}; - -AddressService.prototype._decodeInputValueMap = function(buffer) { - var txid = buffer.slice(0, 32); - var inputIndex = buffer.readUInt32BE(32); - return { - inputTxId: txid, - inputIndex: inputIndex - }; -}; - -AddressService.prototype._getAddressInfo = function(addressStr) { - var addrObj = bitcore.Address(addressStr); - var hashTypeBuffer = AddressService.HASH_TYPES_MAP[addrObj.type]; - - return { - hashBuffer: addrObj.hashBuffer, - hashTypeBuffer: hashTypeBuffer, - hashTypeReadable: addrObj.type - }; -}; - /** * This function is responsible for emitting events to any subscribers to the * `address/transaction` event. @@ -926,12 +695,12 @@ AddressService.prototype.getInputForOutput = function(txid, outputIndex, options txidBuffer = new Buffer(txid, 'hex'); } if (options.queryMempool) { - var spentIndexSyncKey = this._encodeSpentIndexSyncKey(txidBuffer, outputIndex); + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey(txidBuffer, outputIndex); if (this.mempoolSpentIndex[spentIndexSyncKey]) { return this._getSpentMempool(txidBuffer, outputIndex, callback); } } - var key = this._encodeInputKeyMap(txidBuffer, outputIndex); + var key = encoding.encodeInputKeyMap(txidBuffer, outputIndex); var dbOptions = { valueEncoding: 'binary', keyEncoding: 'binary' @@ -942,7 +711,7 @@ AddressService.prototype.getInputForOutput = function(txid, outputIndex, options } else if (err) { return callback(err); } - var value = self._decodeInputValueMap(buffer); + var value = encoding.decodeInputValueMap(buffer); callback(null, { inputTxId: value.inputTxId.toString('hex'), inputIndex: value.inputIndex @@ -950,10 +719,78 @@ AddressService.prototype.getInputForOutput = function(txid, outputIndex, options }); }; +/** + * A streaming equivalent to `getInputs`, and returns a transform stream with data + * emitted in the same format as `getInputs`. + * + * @param {String} addressStr - The relevant address + * @param {Object} options - Additional options for query the outputs + * @param {Number} [options.start] - The relevant start block height + * @param {Number} [options.end] - The relevant end block height + * @param {Function} callback + */ +AddressService.prototype.createInputsStream = function(addressStr, options, callback) { + + var inputStream = new InputsTransformStream({ + address: new Address(addressStr, this.node.network), + tipHeight: this.node.services.db.tip.__height + }); + + var stream = this.createInputsDBStream(addressStr, options).pipe(inputStream); + + return stream; + +}; + +AddressService.prototype.createInputsDBStream = function(addressStr, options) { + var stream; + var addrObj = encoding.getAddressInfo(addressStr); + var hashBuffer = addrObj.hashBuffer; + var hashTypeBuffer = addrObj.hashTypeBuffer; + + if (options.start && options.end) { + + var endBuffer = new Buffer(4); + endBuffer.writeUInt32BE(options.end); + + var startBuffer = new Buffer(4); + startBuffer.writeUInt32BE(options.start + 1); + + stream = this.node.services.db.store.createReadStream({ + gte: Buffer.concat([ + constants.PREFIXES.SPENTS, + hashBuffer, + hashTypeBuffer, + constants.SPACER_MIN, + endBuffer + ]), + lte: Buffer.concat([ + constants.PREFIXES.SPENTS, + hashBuffer, + hashTypeBuffer, + constants.SPACER_MIN, + startBuffer + ]), + valueEncoding: 'binary', + keyEncoding: 'binary' + }); + } else { + var allKey = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer]); + stream = this.node.services.db.store.createReadStream({ + gte: Buffer.concat([allKey, constants.SPACER_MIN]), + lte: Buffer.concat([allKey, constants.SPACER_MAX]), + valueEncoding: 'binary', + keyEncoding: 'binary' + }); + } + + return stream; +}; + /** * Will give inputs that spend previous outputs for an address as an object with: * address - The base58check encoded address - * hashType - The type of the address, e.g. 'pubkeyhash' or 'scripthash' + * hashtype - The type of the address, e.g. 'pubkeyhash' or 'scripthash' * txid - A string of the transaction hash * outputIndex - A number of corresponding transaction input * height - The height of the block the transaction was included, will be -1 for mempool transactions @@ -971,67 +808,15 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { var self = this; var inputs = []; - var stream; - var addrObj = this._getAddressInfo(addressStr); + var addrObj = encoding.getAddressInfo(addressStr); var hashBuffer = addrObj.hashBuffer; var hashTypeBuffer = addrObj.hashTypeBuffer; - if (!hashTypeBuffer) { - return callback(new Error('Unknown address type: ' + addrObj.hashTypeReadable + ' for address: ' + addressStr)); - } - if (options.start && options.end) { - - var endBuffer = new Buffer(4); - endBuffer.writeUInt32BE(options.end); - - var startBuffer = new Buffer(4); - startBuffer.writeUInt32BE(options.start + 1); - - stream = this.node.services.db.store.createReadStream({ - gte: Buffer.concat([ - AddressService.PREFIXES.SPENTS, - hashBuffer, - hashTypeBuffer, - AddressService.SPACER_MIN, - endBuffer - ]), - lte: Buffer.concat([ - AddressService.PREFIXES.SPENTS, - hashBuffer, - hashTypeBuffer, - AddressService.SPACER_MIN, - startBuffer - ]), - valueEncoding: 'binary', - keyEncoding: 'binary' - }); - } else { - var allKey = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer]); - stream = this.node.services.db.store.createReadStream({ - gte: Buffer.concat([allKey, AddressService.SPACER_MIN]), - lte: Buffer.concat([allKey, AddressService.SPACER_MAX]), - valueEncoding: 'binary', - keyEncoding: 'binary' - }); - } - - stream.on('data', function(data) { - - var key = self._decodeInputKey(data.key); - var value = self._decodeInputValue(data.value); - - var input = { - address: addressStr, - hashType: addrObj.hashTypeReadable, - txid: value.txid.toString('hex'), - inputIndex: value.inputIndex, - height: key.height, - confirmations: self.node.services.db.tip.__height - key.height + 1 - }; + var stream = this.createInputsStream(addressStr, options); + stream.on('data', function(input) { inputs.push(input); - }); var error; @@ -1042,7 +827,7 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { } }); - stream.on('close', function() { + stream.on('end', function() { if (error) { return callback(error); } @@ -1071,16 +856,16 @@ AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, ha var stream = self.mempoolIndex.createReadStream({ gte: Buffer.concat([ - AddressService.MEMPREFIXES.SPENTS, + constants.MEMPREFIXES.SPENTS, hashBuffer, hashTypeBuffer, - AddressService.SPACER_MIN + constants.SPACER_MIN ]), lte: Buffer.concat([ - AddressService.MEMPREFIXES.SPENTS, + constants.MEMPREFIXES.SPENTS, hashBuffer, hashTypeBuffer, - AddressService.SPACER_MAX + constants.SPACER_MAX ]), valueEncoding: 'binary', keyEncoding: 'binary' @@ -1091,7 +876,7 @@ AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, ha var inputIndex = data.value.readUInt32BE(32); var output = { address: addressStr, - hashType: AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], + hashType: constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], txid: txid.toString('hex'), //TODO use a buffer inputIndex: inputIndex, height: -1, @@ -1121,7 +906,7 @@ AddressService.prototype._getSpentMempool = function(txidBuffer, outputIndex, ca var outputIndexBuffer = new Buffer(4); outputIndexBuffer.writeUInt32BE(outputIndex); var spentIndexKey = Buffer.concat([ - AddressService.MEMPREFIXES.SPENTSMAP, + constants.MEMPREFIXES.SPENTSMAP, txidBuffer, outputIndexBuffer ]); @@ -1142,10 +927,69 @@ AddressService.prototype._getSpentMempool = function(txidBuffer, outputIndex, ca ); }; +AddressService.prototype.createOutputsStream = function(addressStr, options) { + + var outputStream = new OutputsTransformStream({ + address: new Address(addressStr, this.node.network), + tipHeight: this.node.services.db.tip.__height + }); + + var stream = this.createOutputsDBStream(addressStr, options).pipe(outputStream); + + return stream; + +}; + +AddressService.prototype.createOutputsDBStream = function(addressStr, options) { + + var addrObj = encoding.getAddressInfo(addressStr); + var hashBuffer = addrObj.hashBuffer; + var hashTypeBuffer = addrObj.hashTypeBuffer; + var stream; + + if (options.start && options.end) { + + var startBuffer = new Buffer(4); + startBuffer.writeUInt32BE(options.start + 1); + var endBuffer = new Buffer(4); + endBuffer.writeUInt32BE(options.end); + + stream = this.node.services.db.store.createReadStream({ + gte: Buffer.concat([ + constants.PREFIXES.OUTPUTS, + hashBuffer, + hashTypeBuffer, + constants.SPACER_MIN, + endBuffer + ]), + lte: Buffer.concat([ + constants.PREFIXES.OUTPUTS, + hashBuffer, + hashTypeBuffer, + constants.SPACER_MIN, + startBuffer + ]), + valueEncoding: 'binary', + keyEncoding: 'binary' + }); + } else { + var allKey = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer]); + stream = this.node.services.db.store.createReadStream({ + gte: Buffer.concat([allKey, constants.SPACER_MIN]), + lte: Buffer.concat([allKey, constants.SPACER_MAX]), + valueEncoding: 'binary', + keyEncoding: 'binary' + }); + } + + return stream; + +}; + /** * Will give outputs for an address as an object with: * address - The base58check encoded address - * hashType - The type of the address, e.g. 'pubkeyhash' or 'scripthash' + * hashtype - The type of the address, e.g. 'pubkeyhash' or 'scripthash' * txid - A string of the transaction hash * outputIndex - A number of corresponding transaction output * height - The height of the block the transaction was included, will be -1 for mempool transactions @@ -1165,7 +1009,7 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { $.checkArgument(_.isObject(options), 'Second argument is expected to be an options object.'); $.checkArgument(_.isFunction(callback), 'Third argument is expected to be a callback function.'); - var addrObj = this._getAddressInfo(addressStr); + var addrObj = encoding.getAddressInfo(addressStr); var hashBuffer = addrObj.hashBuffer; var hashTypeBuffer = addrObj.hashTypeBuffer; if (!hashTypeBuffer) { @@ -1173,61 +1017,10 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { } var outputs = []; - var stream; - - if (options.start && options.end) { - - var startBuffer = new Buffer(4); - startBuffer.writeUInt32BE(options.start + 1); - var endBuffer = new Buffer(4); - endBuffer.writeUInt32BE(options.end); - - stream = this.node.services.db.store.createReadStream({ - gte: Buffer.concat([ - AddressService.PREFIXES.OUTPUTS, - hashBuffer, - hashTypeBuffer, - AddressService.SPACER_MIN, - endBuffer - ]), - lte: Buffer.concat([ - AddressService.PREFIXES.OUTPUTS, - hashBuffer, - hashTypeBuffer, - AddressService.SPACER_MIN, - startBuffer - ]), - valueEncoding: 'binary', - keyEncoding: 'binary' - }); - } else { - var allKey = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer]); - stream = this.node.services.db.store.createReadStream({ - gte: Buffer.concat([allKey, AddressService.SPACER_MIN]), - lte: Buffer.concat([allKey, AddressService.SPACER_MAX]), - valueEncoding: 'binary', - keyEncoding: 'binary' - }); - } + var stream = this.createOutputsStream(addressStr, options); stream.on('data', function(data) { - - var key = self._decodeOutputKey(data.key); - var value = self._decodeOutputValue(data.value); - - var output = { - address: addressStr, - hashType: addrObj.hashTypeReadable, - txid: key.txid.toString('hex'), //TODO use a buffer - outputIndex: key.outputIndex, - height: key.height, - satoshis: value.satoshis, - script: value.scriptBuffer.toString('hex'), //TODO use a buffer - confirmations: self.node.services.db.tip.__height - key.height + 1 - }; - - outputs.push(output); - + outputs.push(data); }); var error; @@ -1238,7 +1031,7 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { } }); - stream.on('close', function() { + stream.on('end', function() { if (error) { return callback(error); } @@ -1266,16 +1059,16 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h var stream = self.mempoolIndex.createReadStream({ gte: Buffer.concat([ - AddressService.MEMPREFIXES.OUTPUTS, + constants.MEMPREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, - AddressService.SPACER_MIN + constants.SPACER_MIN ]), lte: Buffer.concat([ - AddressService.MEMPREFIXES.OUTPUTS, + constants.MEMPREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, - AddressService.SPACER_MAX + constants.SPACER_MAX ]), valueEncoding: 'binary', keyEncoding: 'binary' @@ -1285,10 +1078,10 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h // Format of data: prefix: 1, hashBuffer: 20, hashTypeBuffer: 1, txid: 32, outputIndex: 4 var txid = data.key.slice(22, 54); var outputIndex = data.key.readUInt32BE(54); - var value = self._decodeOutputValue(data.value); + var value = encoding.decodeOutputValue(data.value); var output = { address: addressStr, - hashType: AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], + hashType: constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], txid: txid.toString('hex'), //TODO use a buffer outputIndex: outputIndex, height: -1, @@ -1407,7 +1200,7 @@ AddressService.prototype.isSpent = function(output, options, callback) { var spent = self.node.services.bitcoind.isSpent(txid, output.outputIndex); if (!spent && queryMempool) { var txidBuffer = new Buffer(txid, 'hex'); - var spentIndexSyncKey = this._encodeSpentIndexSyncKey(txidBuffer, output.outputIndex); + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey(txidBuffer, output.outputIndex); spent = self.mempoolSpentIndex[spentIndexSyncKey] ? true : false; } setImmediate(function() { @@ -1474,111 +1267,176 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba * @param {Boolean} [options.noTxList] - if set, txid array will not be included * @param {Function} callback */ -AddressService.prototype.getAddressSummary = function(address, options, callback) { +AddressService.prototype.getAddressSummary = function(addressArg, options, callback) { var self = this; - var opt = { - queryMempool: true + var address = new Address(addressArg); + + async.waterfall([ + function(next) { + self._getAddressInputsSummary(address, options, next); + }, + function(result, next) { + self._getAddressOutputsSummary(address, options, result, next); + } + ], function(err, result) { + if (err) { + return callback(err); + } + + var confirmedTxids = Object.keys(result.appearanceIds); + var unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); + + var summary = { + totalReceived: result.totalReceived, + totalSpent: result.totalSpent, + balance: result.balance, + unconfirmedBalance: result.unconfirmedBalance, + appearances: confirmedTxids.length, + unconfirmedAppearances: unconfirmedTxids.length + }; + + if (!options.noTxList) { + var txids = confirmedTxids.concat(unconfirmedTxids); + + // sort by height + summary.txids = txids.sort(function(a, b) { + return a.height > b.height ? 1 : -1; + }).map(function(obj) { + return obj.txid; + }).filter(function(value, index, self) { + return self.indexOf(value) === index; + }); + } + + callback(null, summary); + + }); + +}; + +AddressService.prototype._getAddressInputsSummary = function(address, options, callback) { + $.checkArgument(address instanceof Address); + var self = this; + + var error = null; + var result = { + appearanceIds: {}, + unconfirmedAppearanceIds: {}, }; - var outputs; - var inputs; + var inputsStream = self.createInputsStream(address, options); + inputsStream.on('data', function(input) { + var txid = input.txid; + result.appearanceIds[txid] = true; + }); - async.parallel( - [ - function(next) { - self.getInputs(address, opt, function(err, ins) { - inputs = ins; - next(err); - }); - }, - function(next) { - self.getOutputs(address, opt, function(err, outs) { - outputs = outs; - next(err); - }); + inputsStream.on('error', function(err) { + error = err; + }); + + inputsStream.on('end', function() { + + var addressStr = address.toString(); + var hashBuffer = address.hashBuffer; + var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; + + self._getInputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolInputs) { + if (err) { + return callback(err); } - ], - function(err) { - if(err) { + for(var i = 0; i < mempoolInputs.length; i++) { + var input = mempoolInputs[i]; + result.unconfirmedAppearanceIds[input.txid] = true; + } + callback(error, result); + }); + }); +}; + +AddressService.prototype._getAddressOutputsSummary = function(address, options, result, callback) { + $.checkArgument(address instanceof Address); + $.checkArgument(!_.isUndefined(result) && + !_.isUndefined(result.appearanceIds) && + !_.isUndefined(result.unconfirmedAppearanceIds)); + + var self = this; + + var outputStream = self.createOutputsStream(address, options); + + result.totalReceived = 0; + result.totalSpent = 0; + result.balance = 0; + result.unconfirmedBalance = 0; + + outputStream.on('data', function(output) { + + var txid = output.txid; + var outputIndex = output.outputIndex; + + // Bitcoind's isSpent only works for confirmed transactions + var spentDB = self.node.services.bitcoind.isSpent(txid, outputIndex); + result.totalReceived += output.satoshis; + result.appearanceIds[txid] = true; + + if (spentDB) { + result.totalSpent += output.satoshis; + } else { + result.balance += output.satoshis; + } + + // Check to see if this output is spent in the mempool and if so + // we will subtract it from the unconfirmedBalance (a.k.a unconfirmedDelta) + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( + new Buffer(txid, 'hex'), // TODO: get buffer directly + outputIndex + ); + var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; + if (spentMempool) { + result.unconfirmedBalance -= output.satoshis; + } + + }); + + var error = null; + + outputStream.on('error', function(err) { + error = err; + }); + + outputStream.on('end', function() { + + var addressStr = address.toString(); + var hashBuffer = address.hashBuffer; + var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; + + self._getOutputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolOutputs) { + if (err) { return callback(err); } - var totalReceived = 0; - var totalSpent = 0; - var balance = 0; - var unconfirmedBalance = 0; - var appearanceIds = {}; - var unconfirmedAppearanceIds = {}; - var txids = []; + for(var i = 0; i < mempoolOutputs.length; i++) { + var output = mempoolOutputs[i]; - for(var i = 0; i < outputs.length; i++) { - // Bitcoind's isSpent only works for confirmed transactions - var spentDB = self.node.services.bitcoind.isSpent(outputs[i].txid, outputs[i].outputIndex); - var spentIndexSyncKey = self._encodeSpentIndexSyncKey( - new Buffer(outputs[i].txid, 'hex'), // TODO: get buffer directly - outputs[i].outputIndex + result.unconfirmedAppearanceIds[output.txid] = true; + + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( + new Buffer(output.txid, 'hex'), // TODO: get buffer directly + output.outputIndex ); var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; - - txids.push(outputs[i]); - - if(outputs[i].confirmations) { - totalReceived += outputs[i].satoshis; - balance += outputs[i].satoshis; - appearanceIds[outputs[i].txid] = true; - } else { - unconfirmedBalance += outputs[i].satoshis; - unconfirmedAppearanceIds[outputs[i].txid] = true; - } - - if(spentDB || spentMempool) { - if(spentDB) { - totalSpent += outputs[i].satoshis; - balance -= outputs[i].satoshis; - } else if(!outputs[i].confirmations) { - unconfirmedBalance -= outputs[i].satoshis; - } + // Only add this to the balance if it's not spent in the mempool already + if (!spentMempool) { + result.unconfirmedBalance += output.satoshis; } } - for(var j = 0; j < inputs.length; j++) { - if (inputs[j].confirmations) { - appearanceIds[inputs[j].txid] = true; - } else { - unconfirmedAppearanceIds[outputs[j].txid] = true; - } - } + callback(error, result); - var summary = { - totalReceived: totalReceived, - totalSpent: totalSpent, - balance: balance, - unconfirmedBalance: unconfirmedBalance, - appearances: Object.keys(appearanceIds).length, - unconfirmedAppearances: Object.keys(unconfirmedAppearanceIds).length - }; + }); - if(!options.noTxList) { - for(var i = 0; i < inputs.length; i++) { - txids.push(inputs[i]); - } + }); - // sort by height - txids = txids.sort(function(a, b) { - return a.height > b.height ? 1 : -1; - }).map(function(obj) { - return obj.txid; - }).filter(function(value, index, self) { - return self.indexOf(value) === index; - }); - - summary.txids = txids; - } - - callback(null, summary); - } - ); }; module.exports = AddressService; diff --git a/lib/services/address/streams/inputs-transform.js b/lib/services/address/streams/inputs-transform.js new file mode 100644 index 00000000..8b8f71d3 --- /dev/null +++ b/lib/services/address/streams/inputs-transform.js @@ -0,0 +1,40 @@ +'use strict'; + +var Transform = require('stream').Transform; +var inherits = require('util').inherits; +var bitcore = require('bitcore-lib'); +var encodingUtil = require('../encoding'); +var $ = bitcore.util.preconditions; + +function InputsTransformStream(options) { + $.checkArgument(options.address instanceof bitcore.Address); + Transform.call(this, { + objectMode: true + }); + this._address = options.address; + this._addressStr = this._address.toString(); + this._tipHeight = options.tipHeight; +} +inherits(InputsTransformStream, Transform); + +InputsTransformStream.prototype._transform = function(chunk, encoding, callback) { + var self = this; + + var key = encodingUtil.decodeInputKey(chunk.key); + var value = encodingUtil.decodeInputValue(chunk.value); + + var input = { + address: this._addressStr, + hashType: this._address.type, + txid: value.txid.toString('hex'), + inputIndex: value.inputIndex, + height: key.height, + confirmations: this._tipHeight - key.height + 1 + }; + + self.push(input); + callback(); + +}; + +module.exports = InputsTransformStream; diff --git a/lib/services/address/streams/outputs-transform.js b/lib/services/address/streams/outputs-transform.js new file mode 100644 index 00000000..b9c8e8d3 --- /dev/null +++ b/lib/services/address/streams/outputs-transform.js @@ -0,0 +1,42 @@ +'use strict'; + +var Transform = require('stream').Transform; +var inherits = require('util').inherits; +var bitcore = require('bitcore-lib'); +var encodingUtil = require('../encoding'); +var $ = bitcore.util.preconditions; + +function OutputsTransformStream(options) { + Transform.call(this, { + objectMode: true + }); + $.checkArgument(options.address instanceof bitcore.Address); + this._address = options.address; + this._addressStr = this._address.toString(); + this._tipHeight = options.tipHeight; +} +inherits(OutputsTransformStream, Transform); + +OutputsTransformStream.prototype._transform = function(chunk, encoding, callback) { + var self = this; + + var key = encodingUtil.decodeOutputKey(chunk.key); + var value = encodingUtil.decodeOutputValue(chunk.value); + + var output = { + address: this._addressStr, + hashType: this._address.type, + txid: key.txid.toString('hex'), //TODO use a buffer + outputIndex: key.outputIndex, + height: key.height, + satoshis: value.satoshis, + script: value.scriptBuffer.toString('hex'), //TODO use a buffer + confirmations: this._tipHeight - key.height + 1 + }; + + self.push(output); + callback(); + +}; + +module.exports = OutputsTransformStream; From 40eb4f50aec0d841daea783e3055ed16e570af7e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 30 Dec 2015 00:16:22 -0500 Subject: [PATCH 02/20] Address Service: Start to cache `getAddressSummary` based on range of block heights --- lib/services/address/constants.js | 4 + lib/services/address/encoding.js | 59 +++++++++++ lib/services/address/index.js | 167 +++++++++++++++++++++++------- lib/services/db.js | 6 +- 4 files changed, 197 insertions(+), 39 deletions(-) diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js index 7a0cf8dd..dc0b18b7 100644 --- a/lib/services/address/constants.js +++ b/lib/services/address/constants.js @@ -37,5 +37,9 @@ exports.HASH_TYPES_MAP = { exports.SPACER_MIN = new Buffer('00', 'hex'); exports.SPACER_MAX = new Buffer('ff', 'hex'); +// The total number of transactions that an address can receive before it will start +// to cache the summary to disk. +exports.SUMMARY_CACHE_THRESHOLD = 10000; + module.exports = exports; diff --git a/lib/services/address/encoding.js b/lib/services/address/encoding.js index 68313754..ca42fc7f 100644 --- a/lib/services/address/encoding.js +++ b/lib/services/address/encoding.js @@ -160,6 +160,65 @@ exports.decodeInputValueMap = function(buffer) { }; }; +exports.encodeSummaryCacheKey = function(address) { + return Buffer.concat([address.hashBuffer, constants.HASH_TYPES_BUFFER[address.type]]); +}; + +exports.decodeSummaryCacheKey = function(buffer, network) { + var hashBuffer = buffer.read(20); + var type = constants.HASH_TYPES_READABLE[buffer.read(20, 2).toString('hex')]; + var address = new Address({ + hashBuffer: hashBuffer, + type: type, + network: network + }); + return address; +}; + +exports.encodeSummaryCacheValue = function(cache, tipHeight) { + var buffer = new Buffer(new Array(20)); + buffer.writeUInt32BE(tipHeight); + buffer.writeDoubleBE(cache.result.totalReceived, 4); + buffer.writeDoubleBE(cache.result.balance, 12); + var txidBuffers = []; + for (var key in cache.result.appearanceIds) { + txidBuffers.push(new Buffer(key, 'hex')); + } + var txidsBuffer = Buffer.concat(txidBuffers); + var value = Buffer.concat([buffer, txidsBuffer]); + + return value; +}; + +exports.decodeSummaryCacheValue = function(buffer) { + + var height = buffer.readUInt32BE(); + var totalReceived = buffer.readDoubleBE(4); + var balance = buffer.readDoubleBE(12); + + // read 32 byte chunks until exhausted + var appearanceIds = {}; + var pos = 16; + while(pos < buffer.length) { + var txid = buffer.slice(pos, pos + 32).toString('hex'); + appearanceIds[txid] = true; + pos += 32; + } + + var cache = { + height: height, + result: { + appearanceIds: appearanceIds, + totalReceived: totalReceived, + balance: balance, + unconfirmedAppearanceIds: {}, // unconfirmed values are never stored in cache + unconfirmedBalance: 0 + } + }; + + return cache; +}; + exports.getAddressInfo = function(addressStr) { var addrObj = bitcore.Address(addressStr); var hashTypeBuffer = constants.HASH_TYPES_MAP[addrObj.type]; diff --git a/lib/services/address/index.js b/lib/services/address/index.js index cf37e7e2..665a32dc 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -44,7 +44,10 @@ var AddressService = function(options) { this.node.services.bitcoind.on('tx', this.transactionHandler.bind(this)); this.node.services.bitcoind.on('txleave', this.transactionLeaveHandler.bind(this)); + this.summaryCacheThreshold = options.summaryCacheThreshold || constants.SUMMARY_CACHE_THRESHOLD; + this._setMempoolIndexPath(); + this._setSummaryCachePath(); if (options.mempoolMemoryIndex) { this.levelupStore = memdown; } else { @@ -74,6 +77,7 @@ AddressService.prototype.start = function(callback) { } }, function(next) { + // Setup new mempool index if (!fs.existsSync(self.mempoolIndexPath)) { mkdirp(self.mempoolIndexPath, next); } else { @@ -87,7 +91,21 @@ AddressService.prototype.start = function(callback) { db: self.levelupStore, keyEncoding: 'binary', valueEncoding: 'binary', - fillCache: false + fillCache: false, + maxOpenFiles: 200 + }, + next + ); + }, + function(next) { + self.summaryCache = levelup( + self.summaryCachePath, + { + db: self.levelupStore, + keyEncoding: 'binary', + valueEncoding: 'binary', + fillCache: false, + maxOpenFiles: 200 }, next ); @@ -102,21 +120,35 @@ AddressService.prototype.stop = function(callback) { }; /** - * This function will set `this.dataPath` based on `this.node.network`. + * This function will set `this.summaryCachePath` based on `this.node.network`. + * @private + */ +AddressService.prototype._setSummaryCachePath = function() { + this.summaryCachePath = this._getDBPathFor('bitcore-addresssummary.db'); +}; + +/** + * This function will set `this.mempoolIndexPath` based on `this.node.network`. * @private */ AddressService.prototype._setMempoolIndexPath = function() { + this.mempoolIndexPath = this._getDBPathFor('bitcore-addressmempool.db'); +}; + +AddressService.prototype._getDBPathFor = function(dbname) { $.checkState(this.node.datadir, 'Node is expected to have a "datadir" property'); + var path; var regtest = Networks.get('regtest'); if (this.node.network === Networks.livenet) { - this.mempoolIndexPath = this.node.datadir + '/bitcore-addressmempool.db'; + path = this.node.datadir + '/' + dbname; } else if (this.node.network === Networks.testnet) { - this.mempoolIndexPath = this.node.datadir + '/testnet3/bitcore-addressmempool.db'; + path = this.node.datadir + '/testnet3/' + dbname; } else if (this.node.network === regtest) { - this.mempoolIndexPath = this.node.datadir + '/regtest/bitcore-addressmempool.db'; + path = this.node.datadir + '/regtest/' + dbname; } else { throw new Error('Unknown network: ' + this.network); } + return path; }; /** @@ -1270,32 +1302,49 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba AddressService.prototype.getAddressSummary = function(addressArg, options, callback) { var self = this; + var startTime = new Date(); + var address = new Address(addressArg); + var tipHeight = this.node.services.db.tip.__height; async.waterfall([ function(next) { - self._getAddressInputsSummary(address, options, next); + self._getAddressSummaryCache(address, next); }, - function(result, next) { - self._getAddressOutputsSummary(address, options, result, next); + function(cache, next) { + self._getAddressInputsSummary(address, cache, tipHeight, next); + }, + function(cache, next) { + self._getAddressOutputsSummary(address, cache, tipHeight, next); + }, + function(cache, next) { + self._saveAddressSummaryCache(address, cache, tipHeight, next); } - ], function(err, result) { + ], function(err, cache) { if (err) { return callback(err); } + var result = cache.result; var confirmedTxids = Object.keys(result.appearanceIds); var unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); var summary = { totalReceived: result.totalReceived, - totalSpent: result.totalSpent, + totalSpent: result.totalReceived - result.balance, balance: result.balance, - unconfirmedBalance: result.unconfirmedBalance, appearances: confirmedTxids.length, + unconfirmedBalance: result.unconfirmedBalance, unconfirmedAppearances: unconfirmedTxids.length }; + var timeDelta = new Date() - startTime; + if (timeDelta > 5000) { + var seconds = Math.round(timeDelta / 1000); + log.warn('Slow (' + seconds + 's) getAddressSummary request for address: ' + address.toString()); + log.warn('Address Summary:', summary); + } + if (!options.noTxList) { var txids = confirmedTxids.concat(unconfirmedTxids); @@ -1315,20 +1364,64 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb }; -AddressService.prototype._getAddressInputsSummary = function(address, options, callback) { +AddressService.prototype._saveAddressSummaryCache = function(address, cache, tipHeight, callback) { + var transactionLength = Object.keys(cache.result.appearanceIds).length; + var exceedsCacheThreshold = (transactionLength > this.summaryCacheThreshold); + if (exceedsCacheThreshold) { + log.info('Saving address summary cache for: ' + address.toString() + 'at height: ' + tipHeight); + var key = encoding.encodeSummaryCacheKey(address); + var value = encoding.encodeSummaryCacheValue(cache, tipHeight); + this.summaryCache.put(key, value, function(err) { + if (err) { + return callback(err); + } + callback(null, cache); + }); + } else { + callback(null, cache); + } +}; + +AddressService.prototype._getAddressSummaryCache = function(address, callback) { + var baseCache = { + result: { + appearanceIds: {}, + totalReceived: 0, + balance: 0, + unconfirmedAppearanceIds: {}, + unconfirmedBalance: 0 + } + }; + var key = encoding.encodeSummaryCacheKey(address); + this.summaryCache.get(key, { + valueEncoding: 'binary', + keyEncoding: 'binary' + }, function(err, buffer) { + if (err instanceof levelup.errors.NotFoundError) { + return callback(null, baseCache); + } else if (err) { + return callback(err); + } + var cache = encoding.decodeSummaryCacheValue(buffer); + callback(null, cache); + }); +}; + +AddressService.prototype._getAddressInputsSummary = function(address, cache, tipHeight, callback) { $.checkArgument(address instanceof Address); var self = this; var error = null; - var result = { - appearanceIds: {}, - unconfirmedAppearanceIds: {}, + + var opts = { + start: _.isUndefined(cache.height) ? 0 : cache.height + 1, + end: tipHeight }; - var inputsStream = self.createInputsStream(address, options); + var inputsStream = self.createInputsStream(address, opts); inputsStream.on('data', function(input) { var txid = input.txid; - result.appearanceIds[txid] = true; + cache.result.appearanceIds[txid] = true; }); inputsStream.on('error', function(err) { @@ -1347,27 +1440,27 @@ AddressService.prototype._getAddressInputsSummary = function(address, options, c } for(var i = 0; i < mempoolInputs.length; i++) { var input = mempoolInputs[i]; - result.unconfirmedAppearanceIds[input.txid] = true; + cache.result.unconfirmedAppearanceIds[input.txid] = true; } - callback(error, result); + callback(error, cache); }); }); }; -AddressService.prototype._getAddressOutputsSummary = function(address, options, result, callback) { +AddressService.prototype._getAddressOutputsSummary = function(address, cache, tipHeight, callback) { $.checkArgument(address instanceof Address); - $.checkArgument(!_.isUndefined(result) && - !_.isUndefined(result.appearanceIds) && - !_.isUndefined(result.unconfirmedAppearanceIds)); + $.checkArgument(!_.isUndefined(cache.result) && + !_.isUndefined(cache.result.appearanceIds) && + !_.isUndefined(cache.result.unconfirmedAppearanceIds)); var self = this; - var outputStream = self.createOutputsStream(address, options); + var opts = { + start: _.isUndefined(cache.height) ? 0 : cache.height + 1, + end: tipHeight + }; - result.totalReceived = 0; - result.totalSpent = 0; - result.balance = 0; - result.unconfirmedBalance = 0; + var outputStream = self.createOutputsStream(address, opts); outputStream.on('data', function(output) { @@ -1376,13 +1469,11 @@ AddressService.prototype._getAddressOutputsSummary = function(address, options, // Bitcoind's isSpent only works for confirmed transactions var spentDB = self.node.services.bitcoind.isSpent(txid, outputIndex); - result.totalReceived += output.satoshis; - result.appearanceIds[txid] = true; + cache.result.totalReceived += output.satoshis; + cache.result.appearanceIds[txid] = true; - if (spentDB) { - result.totalSpent += output.satoshis; - } else { - result.balance += output.satoshis; + if (!spentDB) { + cache.result.balance += output.satoshis; } // Check to see if this output is spent in the mempool and if so @@ -1393,7 +1484,7 @@ AddressService.prototype._getAddressOutputsSummary = function(address, options, ); var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; if (spentMempool) { - result.unconfirmedBalance -= output.satoshis; + cache.result.unconfirmedBalance -= output.satoshis; } }); @@ -1418,7 +1509,7 @@ AddressService.prototype._getAddressOutputsSummary = function(address, options, for(var i = 0; i < mempoolOutputs.length; i++) { var output = mempoolOutputs[i]; - result.unconfirmedAppearanceIds[output.txid] = true; + cache.result.unconfirmedAppearanceIds[output.txid] = true; var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( new Buffer(output.txid, 'hex'), // TODO: get buffer directly @@ -1427,11 +1518,11 @@ AddressService.prototype._getAddressOutputsSummary = function(address, options, var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; // Only add this to the balance if it's not spent in the mempool already if (!spentMempool) { - result.unconfirmedBalance += output.satoshis; + cache.result.unconfirmedBalance += output.satoshis; } } - callback(error, result); + callback(error, cache); }); diff --git a/lib/services/db.js b/lib/services/db.js index 0c655051..c3c3dfd4 100644 --- a/lib/services/db.js +++ b/lib/services/db.js @@ -46,6 +46,8 @@ function DB(options) { this._setDataPath(); + this.maxOpenFiles = options.maxOpenFiles || DB.DEFAULT_MAX_OPEN_FILES; + this.levelupStore = leveldown; if (options.store) { this.levelupStore = options.store; @@ -68,6 +70,8 @@ DB.PREFIXES = { TIP: new Buffer('04', 'hex') }; +DB.DEFAULT_MAX_OPEN_FILES = 200; + /** * This function will set `this.dataPath` based on `this.node.network`. * @private @@ -98,7 +102,7 @@ DB.prototype.start = function(callback) { } this.genesis = Block.fromBuffer(this.node.services.bitcoind.genesisBuffer); - this.store = levelup(this.dataPath, { db: this.levelupStore }); + this.store = levelup(this.dataPath, { db: this.levelupStore, maxOpenFiles: this.maxOpenFiles }); this.node.services.bitcoind.on('tx', this.transactionHandler.bind(this)); this.once('ready', function() { From cef2f7686de33b7bbda126b853cf65498b0df882 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 5 Jan 2016 16:47:29 -0500 Subject: [PATCH 03/20] Address Service: Limit the length of outputs that can be queried at a time --- lib/services/address/constants.js | 6 ++++++ lib/services/address/index.js | 18 ++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js index dc0b18b7..f11b3fe7 100644 --- a/lib/services/address/constants.js +++ b/lib/services/address/constants.js @@ -41,5 +41,11 @@ exports.SPACER_MAX = new Buffer('ff', 'hex'); // to cache the summary to disk. exports.SUMMARY_CACHE_THRESHOLD = 10000; + +// The default maximum length queries +exports.MAX_INPUTS_QUERY_LENGTH = 50000; +exports.MAX_OUTPUTS_QUERY_LENGTH = 50000; + + module.exports = exports; diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 665a32dc..1eeb0404 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -45,6 +45,8 @@ var AddressService = function(options) { this.node.services.bitcoind.on('txleave', this.transactionLeaveHandler.bind(this)); this.summaryCacheThreshold = options.summaryCacheThreshold || constants.SUMMARY_CACHE_THRESHOLD; + this.maxInputsQueryLength = options.maxInputsQueryLength || constants.MAX_INPUTS_QUERY_LENGTH; + this.maxOutputsQueryLength = options.maxOutputsQueryLength || constants.MAX_OUTPUTS_QUERY_LENGTH; this._setMempoolIndexPath(); this._setSummaryCachePath(); @@ -849,6 +851,12 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { stream.on('data', function(input) { inputs.push(input); + if (inputs.length > self.maxInputsQueryLength) { + log.warn('Tried to query too many inputs (' + self.maxInputsQueryLength + ') for address '+ addressStr); + error = new Error('Maximum number of inputs (' + self.maxInputsQueryLength + ') per query reached'); + stream.pause(); + stream.end(); + } }); var error; @@ -859,7 +867,7 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { } }); - stream.on('end', function() { + stream.on('finish', function() { if (error) { return callback(error); } @@ -1053,6 +1061,12 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { stream.on('data', function(data) { outputs.push(data); + if (outputs.length > self.maxOutputsQueryLength) { + log.warn('Tried to query too many outputs (' + self.maxOutputsQueryLength + ') for address ' + addressStr); + error = new Error('Maximum number of outputs (' + self.maxOutputsQueryLength + ') per query reached'); + stream.pause(); + stream.end(); + } }); var error; @@ -1063,7 +1077,7 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { } }); - stream.on('end', function() { + stream.on('finish', function() { if (error) { return callback(error); } From 8298e380ed195699bb9e2f1a9cd98224d92563a0 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 6 Jan 2016 19:14:45 -0500 Subject: [PATCH 04/20] Address Service: Use streams to combine inputs and outputs --- lib/services/address/history.js | 164 ++++++---------------- lib/services/address/streams/combined.js | 166 +++++++++++++++++++++++ 2 files changed, 207 insertions(+), 123 deletions(-) create mode 100644 lib/services/address/streams/combined.js diff --git a/lib/services/address/history.js b/lib/services/address/history.js index 61e77158..1795fede 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -2,6 +2,7 @@ var bitcore = require('bitcore-lib'); var async = require('async'); +var CombinedStream = require('./streams/combined'); var _ = bitcore.deps._; /** @@ -19,7 +20,6 @@ function AddressHistory(args) { } else { this.addresses = [args.addresses]; } - this.transactionInfo = []; this.combinedArray = []; this.detailedArray = []; } @@ -35,129 +35,47 @@ AddressHistory.prototype.get = function(callback) { var self = this; var totalCount; - async.eachLimit( - self.addresses, - AddressHistory.MAX_ADDRESS_QUERIES, - function(address, next) { - self.getTransactionInfo(address, next); - }, - function(err) { - if (err) { - return callback(err); - } - - self.combineTransactionInfo(); - totalCount = Number(self.combinedArray.length); - self.sortAndPaginateCombinedArray(); - - async.eachSeries( - self.combinedArray, - function(txInfo, next) { - self.getDetailedInfo(txInfo, next); - }, - function(err) { - if (err) { - return callback(err); - } - callback(null, { - totalCount: totalCount, - items: self.detailedArray - }); - } - ); - } - ); -}; - -/** - * This function will retrieve input and output information for an address - * and set the property `this.transactionInfo`. - * @param {String} address - A base58check encoded address - * @param {Function} next - */ -AddressHistory.prototype.getTransactionInfo = function(address, next) { - var self = this; - - var args = { - start: self.options.start, - end: self.options.end, - queryMempool: _.isUndefined(self.options.queryMempool) ? true : self.options.queryMempool - }; - - var outputs; - var inputs; - - async.parallel([ - function(done) { - self.node.services.address.getOutputs(address, args, function(err, result) { - if (err) { - return done(err); - } - outputs = result; - done(); - }); - }, - function(done) { - self.node.services.address.getInputs(address, args, function(err, result) { - if (err) { - return done(err); - } - inputs = result; - done(); - }); - } - ], function(err) { - if (err) { - return next(err); - } - self.transactionInfo = self.transactionInfo.concat(outputs, inputs); - next(); - }); -}; - -/** - * This function combines results from getInputs and getOutputs at - * `this.transactionInfo` to be "txid" unique at `this.combinedArray`. - */ -AddressHistory.prototype.combineTransactionInfo = function() { - var combinedArrayMap = {}; - this.combinedArray = []; - var l = this.transactionInfo.length; - for(var i = 0; i < l; i++) { - var item = this.transactionInfo[i]; - var mapKey = item.txid; - if (combinedArrayMap[mapKey] >= 0) { - var combined = this.combinedArray[combinedArrayMap[mapKey]]; - if (!combined.addresses[item.address]) { - combined.addresses[item.address] = { - outputIndexes: [], - inputIndexes: [] - }; - } - if (item.outputIndex >= 0) { - combined.satoshis += item.satoshis; - combined.addresses[item.address].outputIndexes.push(item.outputIndex); - } else if (item.inputIndex >= 0) { - combined.addresses[item.address].inputIndexes.push(item.inputIndex); - } - } else { - item.addresses = {}; - item.addresses[item.address] = { - outputIndexes: [], - inputIndexes: [] - }; - if (item.outputIndex >= 0) { - item.addresses[item.address].outputIndexes.push(item.outputIndex); - } else if (item.inputIndex >= 0) { - item.addresses[item.address].inputIndexes.push(item.inputIndex); - } - delete item.outputIndex; - delete item.inputIndex; - delete item.address; - this.combinedArray.push(item); - combinedArrayMap[mapKey] = this.combinedArray.length - 1; - } + // TODO: handle multiple addresses (restore previous functionality) + if (self.addresses.length > 1) { + return callback('Only single address queries supported currently'); } + + var address = self.addresses[0]; + + var combinedStream = new CombinedStream({ + inputStream: this.node.services.address.createInputsStream(address, this.options), + outputStream: this.node.services.address.createOutputsStream(address, this.options) + }); + + // Results from the transaction info stream are grouped into + // sets based on block height + combinedStream.on('data', function(block) { + self.combinedArray = self.combinedArray.concat(block); + }); + + combinedStream.on('end', function() { + totalCount = Number(self.combinedArray.length); + + self.sortAndPaginateCombinedArray(); + + // TODO: Add the mempool transactions + + async.eachSeries( + self.combinedArray, + function(txInfo, next) { + self.getDetailedInfo(txInfo, next); + }, + function(err) { + if (err) { + return callback(err); + } + callback(null, { + totalCount: totalCount, + items: self.detailedArray + }); + } + ); + }); }; /** diff --git a/lib/services/address/streams/combined.js b/lib/services/address/streams/combined.js new file mode 100644 index 00000000..050d8474 --- /dev/null +++ b/lib/services/address/streams/combined.js @@ -0,0 +1,166 @@ +'use strict'; + +var ReadableStream = require('stream').Readable; +var inherits = require('util').inherits; + +function TransactionInfoStream(options) { + ReadableStream.call(this, { + objectMode: true + }); + + // TODO: Be able to specify multiple input and output streams + // so that it's possible to query multiple addresses at the same time. + this._inputStream = options.inputStream; + this._outputStream = options.outputStream; + + // This holds a collection of combined inputs and outputs + // grouped into the matching block heights. + this._blocks = {}; + + this._inputCurrentHeight = 0; + this._outputCurrentHeight = 0; + this._inputFinishedHeights = []; + this._outputFinishedHeights = []; + this._inputEnded = false; + this._outputEnded = false; + + this._listenStreamEvents(); +} + +inherits(TransactionInfoStream, ReadableStream); + +TransactionInfoStream.prototype._listenStreamEvents = function() { + var self = this; + + self._inputStream.on('data', function(input) { + self._addToBlock(input); + if (input.height > self._inputCurrentHeight) { + self._inputFinishedHeights.push(input.height); + } + self._inputCurrentHeight = input.height; + self._maybePushBlock(); + }); + + self._outputStream.on('data', function(output) { + self._addToBlock(output); + if (output.height > self._outputCurrentHeight) { + self._outputFinishedHeights.push(output.height); + } + self._outputCurrentHeight = output.height; + self._maybePushBlock(); + }); + + self._inputStream.on('end', function() { + self._inputFinishedHeights.push(self._inputCurrentHeight); + self._inputEnded = true; + self._maybeEndStream(); + }); + + self._outputStream.on('end', function() { + self._outputFinishedHeights.push(self._outputCurrentHeight); + self._outputEnded = true; + self._maybeEndStream(); + }); + +}; + +TransactionInfoStream.prototype._read = function() { + this._inputStream.resume(); + this._outputStream.resume(); +}; + +TransactionInfoStream.prototype._addToBlock = function(data) { + if (!this._blocks[data.height]) { + this._blocks[data.height] = []; + } + this._blocks[data.height].push(data); +}; + +TransactionInfoStream.prototype._maybeEndStream = function() { + if (this._inputEnded && this._outputEnded) { + this._pushRemainingBlocks(); + this.push(null); + } +}; + +TransactionInfoStream.prototype._pushRemainingBlocks = function() { + var keys = Object.keys(this._blocks); + for (var i = 0; i < keys.length; i++) { + this.push(this._blocks[keys[i]]); + delete this._blocks[keys[i]]; + } +}; + +TransactionInfoStream.prototype._combineTransactionInfo = function(transactionInfo) { + var combinedArrayMap = {}; + var combinedArray = []; + var l = transactionInfo.length; + for(var i = 0; i < l; i++) { + var item = transactionInfo[i]; + var mapKey = item.txid; + if (combinedArrayMap[mapKey] >= 0) { + var combined = combinedArray[combinedArrayMap[mapKey]]; + if (!combined.addresses[item.address]) { + combined.addresses[item.address] = { + outputIndexes: [], + inputIndexes: [] + }; + } + if (item.outputIndex >= 0) { + combined.satoshis += item.satoshis; + combined.addresses[item.address].outputIndexes.push(item.outputIndex); + } else if (item.inputIndex >= 0) { + combined.addresses[item.address].inputIndexes.push(item.inputIndex); + } + } else { + item.addresses = {}; + item.addresses[item.address] = { + outputIndexes: [], + inputIndexes: [] + }; + if (item.outputIndex >= 0) { + item.addresses[item.address].outputIndexes.push(item.outputIndex); + } else if (item.inputIndex >= 0) { + item.addresses[item.address].inputIndexes.push(item.inputIndex); + } + delete item.outputIndex; + delete item.inputIndex; + delete item.address; + combinedArray.push(item); + combinedArrayMap[mapKey] = combinedArray.length - 1; + } + } + return combinedArray; +}; + +TransactionInfoStream.prototype._maybePushBlock = function() { + if (!this._inputFinishedHeights[0] && !this._outputFinishedHeights[0]) { + return; + } + + var inputFinished = this._inputFinishedHeights[0]; + var outputFinished = this._outputFinishedHeights[0]; + var bothFinished; + + if (inputFinished === outputFinished) { + bothFinished = inputFinished; + this._inputFinishedHeights.shift(); + this._outputFinishedHeights.shift(); + } else if (inputFinished <= outputFinished) { + bothFinished = inputFinished; + this._inputFinishedHeights.shift(); + } else if (outputFinished <= inputFinished) { + bothFinished = outputFinished; + this._outputFinishedHeights.shift(); + } + + if (bothFinished) { + var block = this._combineTransactionInfo(this._blocks[bothFinished]); + this.push(block); + delete this._blocks[bothFinished]; + //this._inputStream.pause(); + //this._outputStream.pause(); + } +}; + +module.exports = TransactionInfoStream; From 5c4f3c4453f89b4e7c7ce160c88460c980b2498f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 6 Jan 2016 21:20:10 -0500 Subject: [PATCH 05/20] Address Service: Use address summary cache for pagination --- lib/services/address/constants.js | 2 +- lib/services/address/encoding.js | 18 ++- lib/services/address/history.js | 125 ++++++++++++----- lib/services/address/index.js | 44 ++++-- lib/services/address/streams/combined.js | 166 ----------------------- 5 files changed, 133 insertions(+), 222 deletions(-) delete mode 100644 lib/services/address/streams/combined.js diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js index f11b3fe7..6ab3aba1 100644 --- a/lib/services/address/constants.js +++ b/lib/services/address/constants.js @@ -45,7 +45,7 @@ exports.SUMMARY_CACHE_THRESHOLD = 10000; // The default maximum length queries exports.MAX_INPUTS_QUERY_LENGTH = 50000; exports.MAX_OUTPUTS_QUERY_LENGTH = 50000; - +exports.MAX_HISTORY_QUERY_LENGTH = 1000; module.exports = exports; diff --git a/lib/services/address/encoding.js b/lib/services/address/encoding.js index ca42fc7f..2f83faab 100644 --- a/lib/services/address/encoding.js +++ b/lib/services/address/encoding.js @@ -181,8 +181,12 @@ exports.encodeSummaryCacheValue = function(cache, tipHeight) { buffer.writeDoubleBE(cache.result.totalReceived, 4); buffer.writeDoubleBE(cache.result.balance, 12); var txidBuffers = []; - for (var key in cache.result.appearanceIds) { - txidBuffers.push(new Buffer(key, 'hex')); + for (var i = 0; i < cache.result.txids.length; i++) { + var buf = new Buffer(new Array(36)); + var txid = cache.result.txids[i]; + buf.write(txid, 'hex'); + buf.writeUInt32BE(cache.result.appearanceIds[txid], 32); + txidBuffers.push(buf); } var txidsBuffer = Buffer.concat(txidBuffers); var value = Buffer.concat([buffer, txidsBuffer]); @@ -198,17 +202,21 @@ exports.decodeSummaryCacheValue = function(buffer) { // read 32 byte chunks until exhausted var appearanceIds = {}; - var pos = 16; + var txids = []; + var pos = 20; while(pos < buffer.length) { var txid = buffer.slice(pos, pos + 32).toString('hex'); - appearanceIds[txid] = true; - pos += 32; + var txidHeight = buffer.readUInt32BE(pos + 32); + txids.push(txid); + appearanceIds[txid] = txidHeight; + pos += 36; } var cache = { height: height, result: { appearanceIds: appearanceIds, + txids: txids, totalReceived: totalReceived, balance: balance, unconfirmedAppearanceIds: {}, // unconfirmed values are never stored in cache diff --git a/lib/services/address/history.js b/lib/services/address/history.js index 1795fede..dc15766c 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -2,9 +2,10 @@ var bitcore = require('bitcore-lib'); var async = require('async'); -var CombinedStream = require('./streams/combined'); var _ = bitcore.deps._; +var constants = require('./constants'); + /** * This represents an instance that keeps track of data over a series of * asynchronous I/O calls to get the transaction history for a group of @@ -20,7 +21,21 @@ function AddressHistory(args) { } else { this.addresses = [args.addresses]; } - this.combinedArray = []; + + this.maxHistoryQueryLength = constants.MAX_HISTORY_QUERY_LENGTH; + + this.addressStrings = []; + for (var i = 0; i < this.addresses.length; i++) { + var address = this.addresses[i]; + if (address instanceof bitcore.Address) { + this.addressStrings.push(address.toString()); + } else if (_.isString(address)) { + this.addressStrings.push(address); + } else { + throw new TypeError('Addresses are expected to be strings'); + } + } + this.detailedArray = []; } @@ -42,28 +57,33 @@ AddressHistory.prototype.get = function(callback) { var address = self.addresses[0]; - var combinedStream = new CombinedStream({ - inputStream: this.node.services.address.createInputsStream(address, this.options), - outputStream: this.node.services.address.createOutputsStream(address, this.options) - }); + this.node.services.address.getAddressSummary(address, this.options, function(err, summary) { + if (err) { + return callback(err); + } - // Results from the transaction info stream are grouped into - // sets based on block height - combinedStream.on('data', function(block) { - self.combinedArray = self.combinedArray.concat(block); - }); + totalCount = summary.txids.length; - combinedStream.on('end', function() { - totalCount = Number(self.combinedArray.length); + // TODO: Make sure txids are sorted by height and time + var fromOffset = summary.txids.length - self.options.from; + var toOffset = summary.txids.length - self.options.to; + var txids = summary.txids.slice(toOffset, fromOffset); - self.sortAndPaginateCombinedArray(); + // Verify that this query isn't too long + if (txids.length > self.maxHistoryQueryLength) { + return callback(new Error( + 'Maximum length query (' + self.maxAddressQueryLength + ') exceeded for addresses:' + + this.address.join(',') + )); + } - // TODO: Add the mempool transactions + // Reverse to include most recent at the top + txids.reverse(); async.eachSeries( - self.combinedArray, - function(txInfo, next) { - self.getDetailedInfo(txInfo, next); + txids, + function(txid, next) { + self.getDetailedInfo(txid, next); }, function(err) { if (err) { @@ -75,12 +95,15 @@ AddressHistory.prototype.get = function(callback) { }); } ); + }); + }; /** * A helper function to sort and slice/paginate the `combinedArray` */ +// TODO: Remove once txids summary results are verified to be sorted AddressHistory.prototype.sortAndPaginateCombinedArray = function() { this.combinedArray.sort(AddressHistory.sortByHeight); if (!_.isUndefined(this.options.from) && !_.isUndefined(this.options.to)) { @@ -94,6 +117,7 @@ AddressHistory.prototype.sortAndPaginateCombinedArray = function() { * @param {Object} a - An item from the `combinedArray` * @param {Object} b */ +// TODO: Remove once txids summary results are verified to be sorted AddressHistory.sortByHeight = function(a, b) { if (a.height < 0 && b.height < 0) { // Both are from the mempool, compare timestamps @@ -123,12 +147,12 @@ AddressHistory.sortByHeight = function(a, b) { * @param {Object} txInfo - An item from the `combinedArray` * @param {Function} next */ -AddressHistory.prototype.getDetailedInfo = function(txInfo, next) { +AddressHistory.prototype.getDetailedInfo = function(txid, next) { var self = this; var queryMempool = _.isUndefined(self.options.queryMempool) ? true : self.options.queryMempool; self.node.services.db.getTransactionWithBlockInfo( - txInfo.txid, + txid, queryMempool, function(err, transaction) { if (err) { @@ -136,13 +160,15 @@ AddressHistory.prototype.getDetailedInfo = function(txInfo, next) { } transaction.populateInputs(self.node.services.db, [], function(err) { - if(err) { + if (err) { return next(err); } + var addressDetails = self.getAddressDetailsForTransaction(transaction); + self.detailedArray.push({ - addresses: txInfo.addresses, - satoshis: self.getSatoshisDetail(transaction, txInfo), + addresses: addressDetails.addresses, + satoshis: addressDetails.satoshis, height: transaction.__height, confirmations: self.getConfirmationsDetail(transaction), timestamp: transaction.__timestamp, @@ -169,23 +195,52 @@ AddressHistory.prototype.getConfirmationsDetail = function(transaction) { return confirmations; }; -/** - * A helper function for `getDetailedInfo` for getting the satoshis. - * @param {Transaction} transaction - A transaction populated with previous outputs - * @param {Object} txInfo - An item from `combinedArray` - */ -AddressHistory.prototype.getSatoshisDetail = function(transaction, txInfo) { - var satoshis = txInfo.satoshis || 0; +AddressHistory.prototype.getAddressDetailsForTransaction = function(transaction) { + var result = { + addresses: {}, + satoshis: 0 + }; - for(var address in txInfo.addresses) { - if (txInfo.addresses[address].inputIndexes.length >= 0) { - for(var j = 0; j < txInfo.addresses[address].inputIndexes.length; j++) { - satoshis -= transaction.inputs[txInfo.addresses[address].inputIndexes[j]].output.satoshis; + for (var inputIndex = 0; inputIndex < transaction.inputs.length; inputIndex++) { + var input = transaction.inputs[inputIndex]; + if (!input.script) { + continue; + } + var inputAddress = input.script.toAddress(this.node.network); + if (inputAddress && this.addressStrings.indexOf(inputAddress.toString()) > 0) { + if (!result.addresses[inputAddress]) { + result.addresses[inputAddress] = { + inputIndexes: [], + outputIndexes: [] + }; + } else { + result.addresses[inputAddress].inputIndexes.push(inputIndex); } + result.satoshis -= input.output.satoshis; } } - return satoshis; + for (var outputIndex = 0; outputIndex < transaction.outputs.length; outputIndex++) { + var output = transaction.outputs[outputIndex]; + if (!output.script) { + continue; + } + var outputAddress = output.script.toAddress(this.node.network); + if (outputAddress && this.addressStrings.indexOf(outputAddress.toString()) > 0) { + if (!result.addresses[outputAddress]) { + result.addresses[outputAddress] = { + inputIndexes: [], + outputIndexes: [] + }; + } else { + result.addresses[outputAddress].inputIndexes.push(outputIndex); + } + result.satoshis += output.satoshis; + } + } + + return result; + }; module.exports = AddressHistory; diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 1eeb0404..0d3a5576 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -763,7 +763,7 @@ AddressService.prototype.getInputForOutput = function(txid, outputIndex, options * @param {Number} [options.end] - The relevant end block height * @param {Function} callback */ -AddressService.prototype.createInputsStream = function(addressStr, options, callback) { +AddressService.prototype.createInputsStream = function(addressStr, options) { var inputStream = new InputsTransformStream({ address: new Address(addressStr, this.node.network), @@ -1331,6 +1331,9 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb function(cache, next) { self._getAddressOutputsSummary(address, cache, tipHeight, next); }, + function(cache, next) { + self._sortTxids(cache, tipHeight, next); + }, function(cache, next) { self._saveAddressSummaryCache(address, cache, tipHeight, next); } @@ -1340,7 +1343,7 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb } var result = cache.result; - var confirmedTxids = Object.keys(result.appearanceIds); + var confirmedTxids = result.txids; var unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); var summary = { @@ -1360,16 +1363,7 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb } if (!options.noTxList) { - var txids = confirmedTxids.concat(unconfirmedTxids); - - // sort by height - summary.txids = txids.sort(function(a, b) { - return a.height > b.height ? 1 : -1; - }).map(function(obj) { - return obj.txid; - }).filter(function(value, index, self) { - return self.indexOf(value) === index; - }); + summary.txids = confirmedTxids.concat(unconfirmedTxids); } callback(null, summary); @@ -1378,8 +1372,22 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb }; +AddressService.prototype._sortTxids = function(cache, tipHeight, callback) { + if (cache.height === tipHeight) { + return callback(null, cache); + } + cache.result.txids = Object.keys(cache.result.appearanceIds); + cache.result.txids.sort(function(a, b) { + return cache.result.appearanceIds[a] - cache.result.appearanceIds[b]; + }); + callback(null, cache); +}; + AddressService.prototype._saveAddressSummaryCache = function(address, cache, tipHeight, callback) { - var transactionLength = Object.keys(cache.result.appearanceIds).length; + if (cache.height === tipHeight) { + return callback(null, cache); + } + var transactionLength = cache.result.txids.length; var exceedsCacheThreshold = (transactionLength > this.summaryCacheThreshold); if (exceedsCacheThreshold) { log.info('Saving address summary cache for: ' + address.toString() + 'at height: ' + tipHeight); @@ -1422,6 +1430,9 @@ AddressService.prototype._getAddressSummaryCache = function(address, callback) { }; AddressService.prototype._getAddressInputsSummary = function(address, cache, tipHeight, callback) { + if (cache.height === tipHeight) { + return callback(null, cache); + } $.checkArgument(address instanceof Address); var self = this; @@ -1435,7 +1446,7 @@ AddressService.prototype._getAddressInputsSummary = function(address, cache, tip var inputsStream = self.createInputsStream(address, opts); inputsStream.on('data', function(input) { var txid = input.txid; - cache.result.appearanceIds[txid] = true; + cache.result.appearanceIds[txid] = input.height; }); inputsStream.on('error', function(err) { @@ -1462,6 +1473,9 @@ AddressService.prototype._getAddressInputsSummary = function(address, cache, tip }; AddressService.prototype._getAddressOutputsSummary = function(address, cache, tipHeight, callback) { + if (cache.height === tipHeight) { + return callback(null, cache); + } $.checkArgument(address instanceof Address); $.checkArgument(!_.isUndefined(cache.result) && !_.isUndefined(cache.result.appearanceIds) && @@ -1484,7 +1498,7 @@ AddressService.prototype._getAddressOutputsSummary = function(address, cache, ti // Bitcoind's isSpent only works for confirmed transactions var spentDB = self.node.services.bitcoind.isSpent(txid, outputIndex); cache.result.totalReceived += output.satoshis; - cache.result.appearanceIds[txid] = true; + cache.result.appearanceIds[txid] = output.height; if (!spentDB) { cache.result.balance += output.satoshis; diff --git a/lib/services/address/streams/combined.js b/lib/services/address/streams/combined.js deleted file mode 100644 index 050d8474..00000000 --- a/lib/services/address/streams/combined.js +++ /dev/null @@ -1,166 +0,0 @@ -'use strict'; - -var ReadableStream = require('stream').Readable; -var inherits = require('util').inherits; - -function TransactionInfoStream(options) { - ReadableStream.call(this, { - objectMode: true - }); - - // TODO: Be able to specify multiple input and output streams - // so that it's possible to query multiple addresses at the same time. - this._inputStream = options.inputStream; - this._outputStream = options.outputStream; - - // This holds a collection of combined inputs and outputs - // grouped into the matching block heights. - this._blocks = {}; - - this._inputCurrentHeight = 0; - this._outputCurrentHeight = 0; - this._inputFinishedHeights = []; - this._outputFinishedHeights = []; - this._inputEnded = false; - this._outputEnded = false; - - this._listenStreamEvents(); -} - -inherits(TransactionInfoStream, ReadableStream); - -TransactionInfoStream.prototype._listenStreamEvents = function() { - var self = this; - - self._inputStream.on('data', function(input) { - self._addToBlock(input); - if (input.height > self._inputCurrentHeight) { - self._inputFinishedHeights.push(input.height); - } - self._inputCurrentHeight = input.height; - self._maybePushBlock(); - }); - - self._outputStream.on('data', function(output) { - self._addToBlock(output); - if (output.height > self._outputCurrentHeight) { - self._outputFinishedHeights.push(output.height); - } - self._outputCurrentHeight = output.height; - self._maybePushBlock(); - }); - - self._inputStream.on('end', function() { - self._inputFinishedHeights.push(self._inputCurrentHeight); - self._inputEnded = true; - self._maybeEndStream(); - }); - - self._outputStream.on('end', function() { - self._outputFinishedHeights.push(self._outputCurrentHeight); - self._outputEnded = true; - self._maybeEndStream(); - }); - -}; - -TransactionInfoStream.prototype._read = function() { - this._inputStream.resume(); - this._outputStream.resume(); -}; - -TransactionInfoStream.prototype._addToBlock = function(data) { - if (!this._blocks[data.height]) { - this._blocks[data.height] = []; - } - this._blocks[data.height].push(data); -}; - -TransactionInfoStream.prototype._maybeEndStream = function() { - if (this._inputEnded && this._outputEnded) { - this._pushRemainingBlocks(); - this.push(null); - } -}; - -TransactionInfoStream.prototype._pushRemainingBlocks = function() { - var keys = Object.keys(this._blocks); - for (var i = 0; i < keys.length; i++) { - this.push(this._blocks[keys[i]]); - delete this._blocks[keys[i]]; - } -}; - -TransactionInfoStream.prototype._combineTransactionInfo = function(transactionInfo) { - var combinedArrayMap = {}; - var combinedArray = []; - var l = transactionInfo.length; - for(var i = 0; i < l; i++) { - var item = transactionInfo[i]; - var mapKey = item.txid; - if (combinedArrayMap[mapKey] >= 0) { - var combined = combinedArray[combinedArrayMap[mapKey]]; - if (!combined.addresses[item.address]) { - combined.addresses[item.address] = { - outputIndexes: [], - inputIndexes: [] - }; - } - if (item.outputIndex >= 0) { - combined.satoshis += item.satoshis; - combined.addresses[item.address].outputIndexes.push(item.outputIndex); - } else if (item.inputIndex >= 0) { - combined.addresses[item.address].inputIndexes.push(item.inputIndex); - } - } else { - item.addresses = {}; - item.addresses[item.address] = { - outputIndexes: [], - inputIndexes: [] - }; - if (item.outputIndex >= 0) { - item.addresses[item.address].outputIndexes.push(item.outputIndex); - } else if (item.inputIndex >= 0) { - item.addresses[item.address].inputIndexes.push(item.inputIndex); - } - delete item.outputIndex; - delete item.inputIndex; - delete item.address; - combinedArray.push(item); - combinedArrayMap[mapKey] = combinedArray.length - 1; - } - } - return combinedArray; -}; - -TransactionInfoStream.prototype._maybePushBlock = function() { - if (!this._inputFinishedHeights[0] && !this._outputFinishedHeights[0]) { - return; - } - - var inputFinished = this._inputFinishedHeights[0]; - var outputFinished = this._outputFinishedHeights[0]; - var bothFinished; - - if (inputFinished === outputFinished) { - bothFinished = inputFinished; - this._inputFinishedHeights.shift(); - this._outputFinishedHeights.shift(); - } else if (inputFinished <= outputFinished) { - bothFinished = inputFinished; - this._inputFinishedHeights.shift(); - } else if (outputFinished <= inputFinished) { - bothFinished = outputFinished; - this._outputFinishedHeights.shift(); - } - - if (bothFinished) { - var block = this._combineTransactionInfo(this._blocks[bothFinished]); - this.push(block); - delete this._blocks[bothFinished]; - //this._inputStream.pause(); - //this._outputStream.pause(); - } -}; - -module.exports = TransactionInfoStream; From 8d2f69c5fddd0cbf3504e2833b8c771995c261dc Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 11 Jan 2016 14:34:04 -0500 Subject: [PATCH 06/20] Address Service: Restored multi-address history queries - Restored functionality to be able to query the history of multiple addresses in one query - Sorted mempool transactions by timestamp in txid lists --- lib/services/address/constants.js | 11 ++- lib/services/address/history.js | 121 ++++++++++++++++-------------- lib/services/address/index.js | 35 ++++++--- 3 files changed, 94 insertions(+), 73 deletions(-) diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js index 6ab3aba1..00fbafbe 100644 --- a/lib/services/address/constants.js +++ b/lib/services/address/constants.js @@ -36,16 +36,21 @@ exports.HASH_TYPES_MAP = { exports.SPACER_MIN = new Buffer('00', 'hex'); exports.SPACER_MAX = new Buffer('ff', 'hex'); +exports.TIMESTAMP_MIN = new Buffer('0000000000000000', 'hex'); +exports.TIMESTAMP_MAX = new Buffer('ffffffffffffffff', 'hex'); // The total number of transactions that an address can receive before it will start // to cache the summary to disk. exports.SUMMARY_CACHE_THRESHOLD = 10000; - -// The default maximum length queries +// The maximum number of inputs that can be queried at once exports.MAX_INPUTS_QUERY_LENGTH = 50000; +// The maximum number of outputs that can be queried at once exports.MAX_OUTPUTS_QUERY_LENGTH = 50000; -exports.MAX_HISTORY_QUERY_LENGTH = 1000; +// The maximum number of transactions that can be queried at once +exports.MAX_HISTORY_QUERY_LENGTH = 100; +// The maximum number of addresses that can be queried at once +exports.MAX_ADDRESSES_QUERY = 100; module.exports = exports; diff --git a/lib/services/address/history.js b/lib/services/address/history.js index dc15766c..f1b42cd5 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -22,7 +22,8 @@ function AddressHistory(args) { this.addresses = [args.addresses]; } - this.maxHistoryQueryLength = constants.MAX_HISTORY_QUERY_LENGTH; + this.maxHistoryQueryLength = args.options.maxHistoryQueryLength || constants.MAX_HISTORY_QUERY_LENGTH; + this.maxAddressesQuery = args.options.maxAddressesQuery || constants.MAX_ADDRESSES_QUERY; this.addressStrings = []; for (var i = 0; i < this.addresses.length; i++) { @@ -39,7 +40,33 @@ function AddressHistory(args) { this.detailedArray = []; } -AddressHistory.MAX_ADDRESS_QUERIES = 20; +AddressHistory.prototype._mergeAndSortTxids = function(summaries) { + var appearanceIds = {}; + var unconfirmedAppearanceIds = {}; + for (var i = 0; i < summaries.length; i++) { + var summary = summaries[i]; + for (var key in summary.appearanceIds) { + appearanceIds[key] = summary.appearanceIds[key]; + delete summary.appearanceIds[key]; + } + for (var unconfirmedKey in summary.unconfirmedAppearanceIds) { + unconfirmedAppearanceIds[unconfirmedKey] = summary.unconfirmedAppearanceIds[key]; + delete summary.unconfirmedAppearanceIds[key]; + } + } + var confirmedTxids = Object.keys(appearanceIds); + confirmedTxids.sort(function(a, b) { + // Confirmed are sorted by height + return appearanceIds[a] - appearanceIds[b]; + }); + var unconfirmedTxids = Object.keys(unconfirmedAppearanceIds); + unconfirmedTxids.sort(function(a, b) { + // Unconfirmed are sorted by timestamp + return unconfirmedAppearanceIds[a] - unconfirmedAppearanceIds[b]; + }); + var txids = confirmedTxids.concat(unconfirmedTxids); + return txids; +}; /** * This function will give detailed history for the configured @@ -50,30 +77,49 @@ AddressHistory.prototype.get = function(callback) { var self = this; var totalCount; - // TODO: handle multiple addresses (restore previous functionality) - if (self.addresses.length > 1) { - return callback('Only single address queries supported currently'); + if (this.addresses.length > this.maxAddressesQuery) { + return callback(new Error('Maximum number of addresses (' + this.maxAddressQuery + ') exceeded')); } - var address = self.addresses[0]; + if (this.addresses.length === 0) { + var address = this.addresses[0]; + self.node.services.address.getAddressSummary(address, this.options, function(err, summary) { + if (err) { + return callback(err); + } + return finish(summary.txids); + }); + } else { + var opts = _.clone(this.options); + opts.fullTxList = true; + async.map( + self.addresses, + function(address, next) { + self.node.services.address.getAddressSummary(address, opts, next); + }, + function(err, summaries) { + if (err) { + return callback(err); + } + var txids = self._mergeAndSortTxids(summaries); + return finish(txids); + } + ); + } - this.node.services.address.getAddressSummary(address, this.options, function(err, summary) { - if (err) { - return callback(err); - } + function finish(allTxids) { + totalCount = allTxids.length; - totalCount = summary.txids.length; - - // TODO: Make sure txids are sorted by height and time - var fromOffset = summary.txids.length - self.options.from; - var toOffset = summary.txids.length - self.options.to; - var txids = summary.txids.slice(toOffset, fromOffset); + // Slice the page starting with the most recent + var fromOffset = totalCount - self.options.from; + var toOffset = totalCount - self.options.to; + var txids = allTxids.slice(toOffset, fromOffset); // Verify that this query isn't too long if (txids.length > self.maxHistoryQueryLength) { return callback(new Error( 'Maximum length query (' + self.maxAddressQueryLength + ') exceeded for addresses:' + - this.address.join(',') + self.address.join(',') )); } @@ -96,49 +142,8 @@ AddressHistory.prototype.get = function(callback) { } ); - }); - -}; - -/** - * A helper function to sort and slice/paginate the `combinedArray` - */ -// TODO: Remove once txids summary results are verified to be sorted -AddressHistory.prototype.sortAndPaginateCombinedArray = function() { - this.combinedArray.sort(AddressHistory.sortByHeight); - if (!_.isUndefined(this.options.from) && !_.isUndefined(this.options.to)) { - this.combinedArray = this.combinedArray.slice(this.options.from, this.options.to); } -}; -/** - * A helper sort function to order by height and then by date - * for transactions that are in the mempool. - * @param {Object} a - An item from the `combinedArray` - * @param {Object} b - */ -// TODO: Remove once txids summary results are verified to be sorted -AddressHistory.sortByHeight = function(a, b) { - if (a.height < 0 && b.height < 0) { - // Both are from the mempool, compare timestamps - if (a.timestamp === b.timestamp) { - return 0; - } else { - return a.timestamp < b.timestamp ? 1 : -1; - } - } else if (a.height < 0 && b.height > 0) { - // A is from the mempool and B is in a block - return -1; - } else if (a.height > 0 && b.height < 0) { - // A is in a block and B is in the mempool - return 1; - } else if (a.height === b.height) { - // The heights are equal - return 0; - } else { - // Otherwise compare heights - return a.height < b.height ? 1 : -1; - } }; /** diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 0d3a5576..0a0a8bac 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -298,6 +298,8 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { /* jshint maxstatements: 100 */ var operations = []; + var timestampBuffer = new Buffer(new Array(8)); + timestampBuffer.writeDoubleBE(new Date().getTime()); var action = 'put'; if (!add) { @@ -326,6 +328,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { constants.MEMPREFIXES.OUTPUTS, addressInfo.hashBuffer, addressInfo.hashTypeBuffer, + timestampBuffer, txidBuffer, outputIndexBuffer ]); @@ -392,6 +395,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { constants.MEMPREFIXES.SPENTS, inputHashBuffer, inputHashType, + timestampBuffer, input.prevTxId, inputOutputIndexBuffer ]); @@ -899,22 +903,23 @@ AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, ha constants.MEMPREFIXES.SPENTS, hashBuffer, hashTypeBuffer, - constants.SPACER_MIN + constants.TIMESTAMP_MIN ]), lte: Buffer.concat([ constants.MEMPREFIXES.SPENTS, hashBuffer, hashTypeBuffer, - constants.SPACER_MAX + constants.TIMESTAMP_MAX ]), valueEncoding: 'binary', keyEncoding: 'binary' }); stream.on('data', function(data) { + var timestamp = data.key.readDoubleBE(22); var txid = data.value.slice(0, 32); var inputIndex = data.value.readUInt32BE(32); - var output = { + var input = { address: addressStr, hashType: constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], txid: txid.toString('hex'), //TODO use a buffer @@ -922,7 +927,7 @@ AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, ha height: -1, confirmations: 0 }; - mempoolInputs.push(output); + mempoolInputs.push(input); }); var error; @@ -1108,22 +1113,24 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h constants.MEMPREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, - constants.SPACER_MIN + constants.TIMESTAMP_MIN ]), lte: Buffer.concat([ constants.MEMPREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, - constants.SPACER_MAX + constants.TIMESTAMP_MAX ]), valueEncoding: 'binary', keyEncoding: 'binary' }); stream.on('data', function(data) { - // Format of data: prefix: 1, hashBuffer: 20, hashTypeBuffer: 1, txid: 32, outputIndex: 4 - var txid = data.key.slice(22, 54); - var outputIndex = data.key.readUInt32BE(54); + // Format of data: + // prefix: 1, hashBuffer: 20, hashTypeBuffer: 1, timestamp: 8, txid: 32, outputIndex: 4 + var timestamp = data.key.readDoubleBE(22); + var txid = data.key.slice(30, 62); + var outputIndex = data.key.readUInt32BE(62); var value = encoding.decodeOutputValue(data.value); var output = { address: addressStr, @@ -1131,6 +1138,7 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h txid: txid.toString('hex'), //TODO use a buffer outputIndex: outputIndex, height: -1, + timestamp: timestamp, satoshis: value.satoshis, script: value.scriptBuffer.toString('hex'), //TODO use a buffer confirmations: 0 @@ -1362,7 +1370,10 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb log.warn('Address Summary:', summary); } - if (!options.noTxList) { + if (options.fullTxList) { + summary.appearanceIds = result.appearanceIds; + summary.unconfirmedAppearanceIds = result.unconfirmedAppearanceIds; + } else if (!options.noTxList) { summary.txids = confirmedTxids.concat(unconfirmedTxids); } @@ -1465,7 +1476,7 @@ AddressService.prototype._getAddressInputsSummary = function(address, cache, tip } for(var i = 0; i < mempoolInputs.length; i++) { var input = mempoolInputs[i]; - cache.result.unconfirmedAppearanceIds[input.txid] = true; + cache.result.unconfirmedAppearanceIds[input.txid] = input.timestamp; } callback(error, cache); }); @@ -1537,7 +1548,7 @@ AddressService.prototype._getAddressOutputsSummary = function(address, cache, ti for(var i = 0; i < mempoolOutputs.length; i++) { var output = mempoolOutputs[i]; - cache.result.unconfirmedAppearanceIds[output.txid] = true; + cache.result.unconfirmedAppearanceIds[output.txid] = output.timestamp; var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( new Buffer(output.txid, 'hex'), // TODO: get buffer directly From 188ff28ec786660179f8f156cf511b2c056562f8 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 11 Jan 2016 16:51:00 -0500 Subject: [PATCH 07/20] Address Service: Fixed HASH_TYPES_MAP naming issue --- lib/services/address/encoding.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/address/encoding.js b/lib/services/address/encoding.js index 2f83faab..1f0933ad 100644 --- a/lib/services/address/encoding.js +++ b/lib/services/address/encoding.js @@ -161,7 +161,7 @@ exports.decodeInputValueMap = function(buffer) { }; exports.encodeSummaryCacheKey = function(address) { - return Buffer.concat([address.hashBuffer, constants.HASH_TYPES_BUFFER[address.type]]); + return Buffer.concat([address.hashBuffer, constants.HASH_TYPES_MAP[address.type]]); }; exports.decodeSummaryCacheKey = function(buffer, network) { From 4fcec8755c3007bdbf78d07486182034ec0bdb7c Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 11 Jan 2016 18:45:51 -0500 Subject: [PATCH 08/20] Address Service: Fixed many bugs from tests - Refactored getAddressSummary and added several tests - Fixed bugs revealed from the integration regtests - Updated many unit tests --- integration/regtest-node.js | 36 +- lib/services/address/constants.js | 2 + lib/services/address/encoding.js | 32 +- lib/services/address/history.js | 58 +- lib/services/address/index.js | 424 +++++--- package.json | 4 +- test/services/address/encoding.unit.js | 103 ++ test/services/address/history.unit.js | 309 +----- test/services/address/index.unit.js | 1346 +++++++++++++++++++----- 9 files changed, 1525 insertions(+), 789 deletions(-) create mode 100644 test/services/address/encoding.unit.js diff --git a/integration/regtest-node.js b/integration/regtest-node.js index fb7ff6bf..e05e3039 100644 --- a/integration/regtest-node.js +++ b/integration/regtest-node.js @@ -28,6 +28,7 @@ var Transaction = index.Transaction; var BitcoreNode = index.Node; var AddressService = index.services.Address; var BitcoinService = index.services.Bitcoin; +var encoding = require('../lib/services/address/encoding'); var DBService = index.services.DB; var testWIF = 'cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG'; var testKey; @@ -43,22 +44,6 @@ describe('Node Functionality', function() { before(function(done) { this.timeout(30000); - // Add the regtest network - bitcore.Networks.remove(bitcore.Networks.testnet); - bitcore.Networks.add({ - name: 'regtest', - alias: 'regtest', - pubkeyhash: 0x6f, - privatekey: 0xef, - scripthash: 0xc4, - xpubkey: 0x043587cf, - xprivkey: 0x04358394, - networkMagic: 0xfabfb5da, - port: 18444, - dnsSeeds: [ ] - }); - regtest = bitcore.Networks.get('regtest'); - var datadir = __dirname + '/data'; testKey = bitcore.PrivateKey(testWIF); @@ -93,6 +78,9 @@ describe('Node Functionality', function() { node = new BitcoreNode(configuration); + regtest = bitcore.Networks.get('regtest'); + should.exist(regtest); + node.on('error', function(err) { log.error(err); }); @@ -208,7 +196,7 @@ describe('Node Functionality', function() { // We need to add a transaction to the mempool so that the next block will // have a different hash as the hash has been invalidated. - client.sendToAddress(testKey.toAddress().toString(), 10, function(err) { + client.sendToAddress(testKey.toAddress(regtest).toString(), 10, function(err) { if (err) { throw err; } @@ -250,7 +238,7 @@ describe('Node Functionality', function() { var address; var unspentOutput; before(function() { - address = testKey.toAddress().toString(); + address = testKey.toAddress(regtest).toString(); }); it('should be able to get the balance of the test address', function(done) { node.services.address.getBalance(address, false, function(err, balance) { @@ -333,19 +321,19 @@ describe('Node Functionality', function() { /* jshint maxstatements: 50 */ testKey2 = bitcore.PrivateKey.fromWIF('cNfF4jXiLHQnFRsxaJyr2YSGcmtNYvxQYSakNhuDGxpkSzAwn95x'); - address2 = testKey2.toAddress().toString(); + address2 = testKey2.toAddress(regtest).toString(); testKey3 = bitcore.PrivateKey.fromWIF('cVTYQbaFNetiZcvxzXcVMin89uMLC43pEBMy2etgZHbPPxH5obYt'); - address3 = testKey3.toAddress().toString(); + address3 = testKey3.toAddress(regtest).toString(); testKey4 = bitcore.PrivateKey.fromWIF('cPNQmfE31H2oCUFqaHpfSqjDibkt7XoT2vydLJLDHNTvcddCesGw'); - address4 = testKey4.toAddress().toString(); + address4 = testKey4.toAddress(regtest).toString(); testKey5 = bitcore.PrivateKey.fromWIF('cVrzm9gCmnzwEVMGeCxY6xLVPdG3XWW97kwkFH3H3v722nb99QBF'); - address5 = testKey5.toAddress().toString(); + address5 = testKey5.toAddress(regtest).toString(); testKey6 = bitcore.PrivateKey.fromWIF('cPfMesNR2gsQEK69a6xe7qE44CZEZavgMUak5hQ74XDgsRmmGBYF'); - address6 = testKey6.toAddress().toString(); + address6 = testKey6.toAddress(regtest).toString(); var tx = new Transaction(); tx.from(unspentOutput); @@ -726,7 +714,7 @@ describe('Node Functionality', function() { node.services.bitcoind.sendTransaction(tx.serialize()); setImmediate(function() { - var addrObj = node.services.address._getAddressInfo(address); + var addrObj = encoding.getAddressInfo(address); node.services.address._getOutputsMempool(address, addrObj.hashBuffer, addrObj.hashTypeBuffer, function(err, outs) { if (err) { diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js index 00fbafbe..7cc8ef58 100644 --- a/lib/services/address/constants.js +++ b/lib/services/address/constants.js @@ -36,6 +36,8 @@ exports.HASH_TYPES_MAP = { exports.SPACER_MIN = new Buffer('00', 'hex'); exports.SPACER_MAX = new Buffer('ff', 'hex'); +exports.SPACER_HEIGHT_MIN = new Buffer('0000000000', 'hex'); +exports.SPACER_HEIGHT_MAX = new Buffer('ffffffffff', 'hex'); exports.TIMESTAMP_MIN = new Buffer('0000000000000000', 'hex'); exports.TIMESTAMP_MAX = new Buffer('ffffffffffffffff', 'hex'); diff --git a/lib/services/address/encoding.js b/lib/services/address/encoding.js index 1f0933ad..81421a18 100644 --- a/lib/services/address/encoding.js +++ b/lib/services/address/encoding.js @@ -61,6 +61,12 @@ exports.encodeOutputValue = function(satoshis, scriptBuffer) { return Buffer.concat([satoshisBuffer, scriptBuffer]); }; +exports.encodeOutputMempoolValue = function(satoshis, timestampBuffer, scriptBuffer) { + var satoshisBuffer = new Buffer(8); + satoshisBuffer.writeDoubleBE(satoshis); + return Buffer.concat([satoshisBuffer, timestampBuffer, scriptBuffer]); +}; + exports.decodeOutputValue = function(buffer) { var satoshis = buffer.readDoubleBE(0); var scriptBuffer = buffer.slice(8, buffer.length); @@ -70,6 +76,17 @@ exports.decodeOutputValue = function(buffer) { }; }; +exports.decodeOutputMempoolValue = function(buffer) { + var satoshis = buffer.readDoubleBE(0); + var timestamp = buffer.readDoubleBE(8); + var scriptBuffer = buffer.slice(16, buffer.length); + return { + satoshis: satoshis, + timestamp: timestamp, + scriptBuffer: scriptBuffer + }; +}; + exports.encodeInputKey = function(hashBuffer, hashTypeBuffer, height, prevTxIdBuffer, outputIndex) { var heightBuffer = new Buffer(4); heightBuffer.writeUInt32BE(height); @@ -175,7 +192,8 @@ exports.decodeSummaryCacheKey = function(buffer, network) { return address; }; -exports.encodeSummaryCacheValue = function(cache, tipHeight) { +exports.encodeSummaryCacheValue = function(cache, tipHeight, tipHash) { + var tipHashBuffer = new Buffer(tipHash, 'hex'); var buffer = new Buffer(new Array(20)); buffer.writeUInt32BE(tipHeight); buffer.writeDoubleBE(cache.result.totalReceived, 4); @@ -189,21 +207,22 @@ exports.encodeSummaryCacheValue = function(cache, tipHeight) { txidBuffers.push(buf); } var txidsBuffer = Buffer.concat(txidBuffers); - var value = Buffer.concat([buffer, txidsBuffer]); + var value = Buffer.concat([tipHashBuffer, buffer, txidsBuffer]); return value; }; exports.decodeSummaryCacheValue = function(buffer) { - var height = buffer.readUInt32BE(); - var totalReceived = buffer.readDoubleBE(4); - var balance = buffer.readDoubleBE(12); + var hash = buffer.slice(0, 32).toString('hex'); + var height = buffer.readUInt32BE(32); + var totalReceived = buffer.readDoubleBE(36); + var balance = buffer.readDoubleBE(44); // read 32 byte chunks until exhausted var appearanceIds = {}; var txids = []; - var pos = 20; + var pos = 52; while(pos < buffer.length) { var txid = buffer.slice(pos, pos + 32).toString('hex'); var txidHeight = buffer.readUInt32BE(pos + 32); @@ -214,6 +233,7 @@ exports.decodeSummaryCacheValue = function(buffer) { var cache = { height: height, + hash: hash, result: { appearanceIds: appearanceIds, txids: txids, diff --git a/lib/services/address/history.js b/lib/services/address/history.js index f1b42cd5..eb689828 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -64,8 +64,7 @@ AddressHistory.prototype._mergeAndSortTxids = function(summaries) { // Unconfirmed are sorted by timestamp return unconfirmedAppearanceIds[a] - unconfirmedAppearanceIds[b]; }); - var txids = confirmedTxids.concat(unconfirmedTxids); - return txids; + return confirmedTxids.concat(unconfirmedTxids); }; /** @@ -81,7 +80,7 @@ AddressHistory.prototype.get = function(callback) { return callback(new Error('Maximum number of addresses (' + this.maxAddressQuery + ') exceeded')); } - if (this.addresses.length === 0) { + if (this.addresses.length === 1) { var address = this.addresses[0]; self.node.services.address.getAddressSummary(address, this.options, function(err, summary) { if (err) { @@ -111,9 +110,14 @@ AddressHistory.prototype.get = function(callback) { totalCount = allTxids.length; // Slice the page starting with the most recent - var fromOffset = totalCount - self.options.from; - var toOffset = totalCount - self.options.to; - var txids = allTxids.slice(toOffset, fromOffset); + var txids; + if (self.options.from >= 0 && self.options.to >= 0) { + var fromOffset = totalCount - self.options.from; + var toOffset = totalCount - self.options.to; + txids = allTxids.slice(toOffset, fromOffset); + } else { + txids = allTxids; + } // Verify that this query isn't too long if (txids.length > self.maxHistoryQueryLength) { @@ -212,16 +216,19 @@ AddressHistory.prototype.getAddressDetailsForTransaction = function(transaction) continue; } var inputAddress = input.script.toAddress(this.node.network); - if (inputAddress && this.addressStrings.indexOf(inputAddress.toString()) > 0) { - if (!result.addresses[inputAddress]) { - result.addresses[inputAddress] = { - inputIndexes: [], - outputIndexes: [] - }; - } else { - result.addresses[inputAddress].inputIndexes.push(inputIndex); + if (inputAddress) { + var inputAddressString = inputAddress.toString(); + if (this.addressStrings.indexOf(inputAddressString) >= 0) { + if (!result.addresses[inputAddressString]) { + result.addresses[inputAddressString] = { + inputIndexes: [inputIndex], + outputIndexes: [] + }; + } else { + result.addresses[inputAddressString].inputIndexes.push(inputIndex); + } + result.satoshis -= input.output.satoshis; } - result.satoshis -= input.output.satoshis; } } @@ -231,16 +238,19 @@ AddressHistory.prototype.getAddressDetailsForTransaction = function(transaction) continue; } var outputAddress = output.script.toAddress(this.node.network); - if (outputAddress && this.addressStrings.indexOf(outputAddress.toString()) > 0) { - if (!result.addresses[outputAddress]) { - result.addresses[outputAddress] = { - inputIndexes: [], - outputIndexes: [] - }; - } else { - result.addresses[outputAddress].inputIndexes.push(outputIndex); + if (outputAddress) { + var outputAddressString = outputAddress.toString(); + if (this.addressStrings.indexOf(outputAddressString) >= 0) { + if (!result.addresses[outputAddressString]) { + result.addresses[outputAddressString] = { + inputIndexes: [], + outputIndexes: [outputIndex] + }; + } else { + result.addresses[outputAddressString].outputIndexes.push(outputIndex); + } + result.satoshis += output.satoshis; } - result.satoshis += output.satoshis; } } diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 0a0a8bac..22c2420f 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -328,12 +328,15 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { constants.MEMPREFIXES.OUTPUTS, addressInfo.hashBuffer, addressInfo.hashTypeBuffer, - timestampBuffer, txidBuffer, outputIndexBuffer ]); - var outValue = encoding.encodeOutputValue(output.satoshis, output._scriptBuffer); + var outValue = encoding.encodeOutputMempoolValue( + output.satoshis, + timestampBuffer, + output._scriptBuffer + ); operations.push({ type: action, @@ -395,13 +398,13 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { constants.MEMPREFIXES.SPENTS, inputHashBuffer, inputHashType, - timestampBuffer, input.prevTxId, inputOutputIndexBuffer ]); var inputValue = Buffer.concat([ txidBuffer, - inputIndexBuffer + inputIndexBuffer, + timestampBuffer ]); operations.push({ type: action, @@ -768,13 +771,17 @@ AddressService.prototype.getInputForOutput = function(txid, outputIndex, options * @param {Function} callback */ AddressService.prototype.createInputsStream = function(addressStr, options) { - var inputStream = new InputsTransformStream({ address: new Address(addressStr, this.node.network), tipHeight: this.node.services.db.tip.__height }); - var stream = this.createInputsDBStream(addressStr, options).pipe(inputStream); + var stream = this.createInputsDBStream(addressStr, options) + .on('error', function(err) { + // Forward the error + inputStream.emit('error', err); + inputStream.end(); + }).pipe(inputStream); return stream; @@ -786,23 +793,27 @@ AddressService.prototype.createInputsDBStream = function(addressStr, options) { var hashBuffer = addrObj.hashBuffer; var hashTypeBuffer = addrObj.hashTypeBuffer; - if (options.start && options.end) { + if (options.start >= 0 && options.end >= 0) { var endBuffer = new Buffer(4); - endBuffer.writeUInt32BE(options.end); + endBuffer.writeUInt32BE(options.end, 0); var startBuffer = new Buffer(4); - startBuffer.writeUInt32BE(options.start + 1); + // Because the key has additional data following it, we don't have an ability + // to use "gte" or "lte" we can only use "gt" and "lt", we therefore need to adjust the number + // to be one value larger to include it. + var adjustedStart = options.start + 1; + startBuffer.writeUInt32BE(adjustedStart, 0); stream = this.node.services.db.store.createReadStream({ - gte: Buffer.concat([ + gt: Buffer.concat([ constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, constants.SPACER_MIN, endBuffer ]), - lte: Buffer.concat([ + lt: Buffer.concat([ constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, @@ -815,8 +826,8 @@ AddressService.prototype.createInputsDBStream = function(addressStr, options) { } else { var allKey = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer]); stream = this.node.services.db.store.createReadStream({ - gte: Buffer.concat([allKey, constants.SPACER_MIN]), - lte: Buffer.concat([allKey, constants.SPACER_MAX]), + gt: Buffer.concat([allKey, constants.SPACER_HEIGHT_MIN]), + lt: Buffer.concat([allKey, constants.SPACER_HEIGHT_MAX]), valueEncoding: 'binary', keyEncoding: 'binary' }); @@ -903,27 +914,28 @@ AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, ha constants.MEMPREFIXES.SPENTS, hashBuffer, hashTypeBuffer, - constants.TIMESTAMP_MIN + constants.SPACER_MIN ]), lte: Buffer.concat([ constants.MEMPREFIXES.SPENTS, hashBuffer, hashTypeBuffer, - constants.TIMESTAMP_MAX + constants.SPACER_MAX ]), valueEncoding: 'binary', keyEncoding: 'binary' }); stream.on('data', function(data) { - var timestamp = data.key.readDoubleBE(22); var txid = data.value.slice(0, 32); var inputIndex = data.value.readUInt32BE(32); + var timestamp = data.value.readDoubleBE(36); var input = { address: addressStr, hashType: constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], txid: txid.toString('hex'), //TODO use a buffer inputIndex: inputIndex, + timestamp: timestamp, height: -1, confirmations: 0 }; @@ -979,7 +991,13 @@ AddressService.prototype.createOutputsStream = function(addressStr, options) { tipHeight: this.node.services.db.tip.__height }); - var stream = this.createOutputsDBStream(addressStr, options).pipe(outputStream); + var stream = this.createOutputsDBStream(addressStr, options) + .on('error', function(err) { + // Forward the error + outputStream.emit('error', err); + outputStream.end(); + }) + .pipe(outputStream); return stream; @@ -992,22 +1010,27 @@ AddressService.prototype.createOutputsDBStream = function(addressStr, options) { var hashTypeBuffer = addrObj.hashTypeBuffer; var stream; - if (options.start && options.end) { + if (options.start >= 0 && options.end >= 0) { + + var endBuffer = new Buffer(4); + endBuffer.writeUInt32BE(options.end, 0); var startBuffer = new Buffer(4); - startBuffer.writeUInt32BE(options.start + 1); - var endBuffer = new Buffer(4); - endBuffer.writeUInt32BE(options.end); + // Because the key has additional data following it, we don't have an ability + // to use "gte" or "lte" we can only use "gt" and "lt", we therefore need to adjust the number + // to be one value larger to include it. + var startAdjusted = options.start + 1; + startBuffer.writeUInt32BE(startAdjusted, 0); stream = this.node.services.db.store.createReadStream({ - gte: Buffer.concat([ + gt: Buffer.concat([ constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, constants.SPACER_MIN, endBuffer ]), - lte: Buffer.concat([ + lt: Buffer.concat([ constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, @@ -1020,8 +1043,8 @@ AddressService.prototype.createOutputsDBStream = function(addressStr, options) { } else { var allKey = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer]); stream = this.node.services.db.store.createReadStream({ - gte: Buffer.concat([allKey, constants.SPACER_MIN]), - lte: Buffer.concat([allKey, constants.SPACER_MAX]), + gt: Buffer.concat([allKey, constants.SPACER_HEIGHT_MIN]), + lt: Buffer.concat([allKey, constants.SPACER_HEIGHT_MAX]), valueEncoding: 'binary', keyEncoding: 'binary' }); @@ -1113,13 +1136,13 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h constants.MEMPREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, - constants.TIMESTAMP_MIN + constants.SPACER_MIN ]), lte: Buffer.concat([ constants.MEMPREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, - constants.TIMESTAMP_MAX + constants.SPACER_MAX ]), valueEncoding: 'binary', keyEncoding: 'binary' @@ -1127,18 +1150,17 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h stream.on('data', function(data) { // Format of data: - // prefix: 1, hashBuffer: 20, hashTypeBuffer: 1, timestamp: 8, txid: 32, outputIndex: 4 - var timestamp = data.key.readDoubleBE(22); - var txid = data.key.slice(30, 62); - var outputIndex = data.key.readUInt32BE(62); - var value = encoding.decodeOutputValue(data.value); + // prefix: 1, hashBuffer: 20, hashTypeBuffer: 1, txid: 32, outputIndex: 4 + var txid = data.key.slice(22, 54); + var outputIndex = data.key.readUInt32BE(54); + var value = encoding.decodeOutputMempoolValue(data.value); var output = { address: addressStr, hashType: constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], txid: txid.toString('hex'), //TODO use a buffer outputIndex: outputIndex, height: -1, - timestamp: timestamp, + timestamp: value.timestamp, satoshis: value.satoshis, script: value.scriptBuffer.toString('hex'), //TODO use a buffer confirmations: 0 @@ -1325,43 +1347,25 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb var self = this; var startTime = new Date(); - var address = new Address(addressArg); - var tipHeight = this.node.services.db.tip.__height; + + if (_.isUndefined(options.queryMempool)) { + options.queryMempool = true; + } async.waterfall([ function(next) { - self._getAddressSummaryCache(address, next); + self._getAddressConfirmedSummary(address, options, next); }, function(cache, next) { - self._getAddressInputsSummary(address, cache, tipHeight, next); - }, - function(cache, next) { - self._getAddressOutputsSummary(address, cache, tipHeight, next); - }, - function(cache, next) { - self._sortTxids(cache, tipHeight, next); - }, - function(cache, next) { - self._saveAddressSummaryCache(address, cache, tipHeight, next); + self._getAddressMempoolSummary(address, options, cache, next); } ], function(err, cache) { if (err) { return callback(err); } - var result = cache.result; - var confirmedTxids = result.txids; - var unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); - - var summary = { - totalReceived: result.totalReceived, - totalSpent: result.totalReceived - result.balance, - balance: result.balance, - appearances: confirmedTxids.length, - unconfirmedBalance: result.unconfirmedBalance, - unconfirmedAppearances: unconfirmedTxids.length - }; + var summary = self._transformAddressSummaryFromCache(cache, options); var timeDelta = new Date() - startTime; if (timeDelta > 5000) { @@ -1370,52 +1374,30 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb log.warn('Address Summary:', summary); } - if (options.fullTxList) { - summary.appearanceIds = result.appearanceIds; - summary.unconfirmedAppearanceIds = result.unconfirmedAppearanceIds; - } else if (!options.noTxList) { - summary.txids = confirmedTxids.concat(unconfirmedTxids); - } - callback(null, summary); }); }; -AddressService.prototype._sortTxids = function(cache, tipHeight, callback) { - if (cache.height === tipHeight) { - return callback(null, cache); - } - cache.result.txids = Object.keys(cache.result.appearanceIds); - cache.result.txids.sort(function(a, b) { - return cache.result.appearanceIds[a] - cache.result.appearanceIds[b]; +AddressService.prototype._getAddressConfirmedSummary = function(address, options, callback) { + var self = this; + var tipHeight = this.node.services.db.tip.__height; + + self._getAddressConfirmedSummaryCache(address, options, function(err, cache) { + if (err) { + return callback(err); + } + // Immediately give cache is already current, otherwise update + if (cache && cache.height === tipHeight) { + return callback(null, cache); + } + self._updateAddressConfirmedSummaryCache(address, options, cache, tipHeight, callback); }); - callback(null, cache); }; -AddressService.prototype._saveAddressSummaryCache = function(address, cache, tipHeight, callback) { - if (cache.height === tipHeight) { - return callback(null, cache); - } - var transactionLength = cache.result.txids.length; - var exceedsCacheThreshold = (transactionLength > this.summaryCacheThreshold); - if (exceedsCacheThreshold) { - log.info('Saving address summary cache for: ' + address.toString() + 'at height: ' + tipHeight); - var key = encoding.encodeSummaryCacheKey(address); - var value = encoding.encodeSummaryCacheValue(cache, tipHeight); - this.summaryCache.put(key, value, function(err) { - if (err) { - return callback(err); - } - callback(null, cache); - }); - } else { - callback(null, cache); - } -}; - -AddressService.prototype._getAddressSummaryCache = function(address, callback) { +AddressService.prototype._getAddressConfirmedSummaryCache = function(address, options, callback) { + var self = this; var baseCache = { result: { appearanceIds: {}, @@ -1425,6 +1407,11 @@ AddressService.prototype._getAddressSummaryCache = function(address, callback) { unconfirmedBalance: 0 } }; + // Use the base cache if the "start" and "end" options have been used + // We only save and retrieve a cache for the summary of all history + if (options.start >= 0 || options.end >= 0) { + return callback(null, baseCache); + } var key = encoding.encodeSummaryCacheKey(address); this.summaryCache.get(key, { valueEncoding: 'binary', @@ -1436,25 +1423,64 @@ AddressService.prototype._getAddressSummaryCache = function(address, callback) { return callback(err); } var cache = encoding.decodeSummaryCacheValue(buffer); + + // Use base cache if the cached tip/height doesn't match (e.g. there has been a reorg) + var blockIndex = self.node.services.bitcoind.getBlockIndex(cache.height); + if (cache.hash !== blockIndex.hash) { + return callback(null, baseCache); + } + callback(null, cache); }); }; -AddressService.prototype._getAddressInputsSummary = function(address, cache, tipHeight, callback) { - if (cache.height === tipHeight) { - return callback(null, cache); - } - $.checkArgument(address instanceof Address); +AddressService.prototype._updateAddressConfirmedSummaryCache = function(address, options, cache, tipHeight, callback) { var self = this; + var optionsPartial = _.clone(options); + var isHeightQuery = (options.start >= 0 || options.end >= 0); + if (!isHeightQuery) { + // We will pick up from the last point cached and query for all blocks + // proceeding the cache + var cacheHeight = _.isUndefined(cache.height) ? 0 : cache.height + 1; + optionsPartial.start = tipHeight; + optionsPartial.end = cacheHeight; + } else { + $.checkState(_.isUndefined(cache.height)); + } + + async.waterfall([ + function(next) { + self._getAddressConfirmedInputsSummary(address, cache, optionsPartial, next); + }, + function(cache, next) { + self._getAddressConfirmedOutputsSummary(address, cache, optionsPartial, next); + }, + function(cache, next) { + self._setAndSortTxidsFromAppearanceIds(cache, next); + } + ], function(err, cache) { + + // Skip saving the cache if the "start" or "end" options have been used, or + // if the transaction length does not exceed the caching threshold. + // We only want to cache full history results for addresses that have a large + // number of transactions. + var exceedsCacheThreshold = (cache.result.txids.length > self.summaryCacheThreshold); + if (exceedsCacheThreshold && !isHeightQuery) { + self._saveAddressConfirmedSummaryCache(address, cache, tipHeight, callback); + } else { + callback(null, cache); + } + + }); +}; + +AddressService.prototype._getAddressConfirmedInputsSummary = function(address, cache, options, callback) { + $.checkArgument(address instanceof Address); + var self = this; var error = null; - var opts = { - start: _.isUndefined(cache.height) ? 0 : cache.height + 1, - end: tipHeight - }; - - var inputsStream = self.createInputsStream(address, opts); + var inputsStream = self.createInputsStream(address, options); inputsStream.on('data', function(input) { var txid = input.txid; cache.result.appearanceIds[txid] = input.height; @@ -1465,28 +1491,14 @@ AddressService.prototype._getAddressInputsSummary = function(address, cache, tip }); inputsStream.on('end', function() { - - var addressStr = address.toString(); - var hashBuffer = address.hashBuffer; - var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; - - self._getInputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolInputs) { - if (err) { - return callback(err); - } - for(var i = 0; i < mempoolInputs.length; i++) { - var input = mempoolInputs[i]; - cache.result.unconfirmedAppearanceIds[input.txid] = input.timestamp; - } - callback(error, cache); - }); + if (error) { + return callback(error); + } + callback(null, cache); }); }; -AddressService.prototype._getAddressOutputsSummary = function(address, cache, tipHeight, callback) { - if (cache.height === tipHeight) { - return callback(null, cache); - } +AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, cache, options, callback) { $.checkArgument(address instanceof Address); $.checkArgument(!_.isUndefined(cache.result) && !_.isUndefined(cache.result.appearanceIds) && @@ -1494,12 +1506,7 @@ AddressService.prototype._getAddressOutputsSummary = function(address, cache, ti var self = this; - var opts = { - start: _.isUndefined(cache.height) ? 0 : cache.height + 1, - end: tipHeight - }; - - var outputStream = self.createOutputsStream(address, opts); + var outputStream = self.createOutputsStream(address, options); outputStream.on('data', function(output) { @@ -1514,16 +1521,19 @@ AddressService.prototype._getAddressOutputsSummary = function(address, cache, ti if (!spentDB) { cache.result.balance += output.satoshis; } + // TODO: subtract if spent (because of cache)? - // Check to see if this output is spent in the mempool and if so - // we will subtract it from the unconfirmedBalance (a.k.a unconfirmedDelta) - var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( - new Buffer(txid, 'hex'), // TODO: get buffer directly - outputIndex - ); - var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; - if (spentMempool) { - cache.result.unconfirmedBalance -= output.satoshis; + if (options.queryMempool) { + // Check to see if this output is spent in the mempool and if so + // we will subtract it from the unconfirmedBalance (a.k.a unconfirmedDelta) + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( + new Buffer(txid, 'hex'), // TODO: get buffer directly + outputIndex + ); + var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; + if (spentMempool) { + cache.result.unconfirmedBalance -= output.satoshis; + } } }); @@ -1535,38 +1545,112 @@ AddressService.prototype._getAddressOutputsSummary = function(address, cache, ti }); outputStream.on('end', function() { - - var addressStr = address.toString(); - var hashBuffer = address.hashBuffer; - var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; - - self._getOutputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolOutputs) { - if (err) { - return callback(err); - } - - for(var i = 0; i < mempoolOutputs.length; i++) { - var output = mempoolOutputs[i]; - - cache.result.unconfirmedAppearanceIds[output.txid] = output.timestamp; - - var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( - new Buffer(output.txid, 'hex'), // TODO: get buffer directly - output.outputIndex - ); - var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; - // Only add this to the balance if it's not spent in the mempool already - if (!spentMempool) { - cache.result.unconfirmedBalance += output.satoshis; - } - } - - callback(error, cache); - - }); - + if (error) { + return callback(error); + } + callback(null, cache); }); }; +AddressService.prototype._setAndSortTxidsFromAppearanceIds = function(cache, callback) { + cache.result.txids = Object.keys(cache.result.appearanceIds); + cache.result.txids.sort(function(a, b) { + return cache.result.appearanceIds[a] - cache.result.appearanceIds[b]; + }); + callback(null, cache); +}; + +AddressService.prototype._saveAddressConfirmedSummaryCache = function(address, cache, tipHeight, callback) { + + log.info('Saving address summary cache for: ' + address.toString() + 'at height: ' + tipHeight); + var key = encoding.encodeSummaryCacheKey(address); + var tipBlockIndex = this.node.services.bitcoind.getBlockIndex(tipHeight); + var value = encoding.encodeSummaryCacheValue(cache, tipHeight, tipBlockIndex.hash); + this.summaryCache.put(key, value, function(err) { + if (err) { + return callback(err); + } + callback(null, cache); + }); + +}; + +AddressService.prototype._getAddressMempoolSummary = function(address, options, cache, callback) { + var self = this; + + // Skip if the options do not want to include the mempool + if (!options.queryMempool) { + return callback(null, cache); + } + + var addressStr = address.toString(); + var hashBuffer = address.hashBuffer; + var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; + + async.waterfall([ + function(next) { + self._getInputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolInputs) { + if (err) { + return next(err); + } + for(var i = 0; i < mempoolInputs.length; i++) { + var input = mempoolInputs[i]; + cache.result.unconfirmedAppearanceIds[input.txid] = input.timestamp; + } + next(null, cache); + }); + + }, function(cache, next) { + self._getOutputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolOutputs) { + if (err) { + return next(err); + } + for(var i = 0; i < mempoolOutputs.length; i++) { + var output = mempoolOutputs[i]; + + cache.result.unconfirmedAppearanceIds[output.txid] = output.timestamp; + + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( + new Buffer(output.txid, 'hex'), // TODO: get buffer directly + output.outputIndex + ); + var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; + // Only add this to the balance if it's not spent in the mempool already + if (!spentMempool) { + cache.result.unconfirmedBalance += output.satoshis; + } + } + next(null, cache); + }); + } + ], callback); +}; + +AddressService.prototype._transformAddressSummaryFromCache = function(cache, options) { + + var result = cache.result; + var confirmedTxids = cache.result.txids; + var unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); + + var summary = { + totalReceived: result.totalReceived, + totalSpent: result.totalReceived - result.balance, + balance: result.balance, + appearances: confirmedTxids.length, + unconfirmedBalance: result.unconfirmedBalance, + unconfirmedAppearances: unconfirmedTxids.length + }; + + if (options.fullTxList) { + summary.appearanceIds = result.appearanceIds; + summary.unconfirmedAppearanceIds = result.unconfirmedAppearanceIds; + } else if (!options.noTxList) { + summary.txids = confirmedTxids.concat(unconfirmedTxids); + } + + return summary; + +}; + module.exports = AddressService; diff --git a/package.json b/package.json index 5e0a4478..deb421cf 100644 --- a/package.json +++ b/package.json @@ -54,8 +54,8 @@ "commander": "^2.8.1", "errno": "^0.1.4", "express": "^4.13.3", - "leveldown": "^1.4.2", - "levelup": "^1.2.1", + "leveldown": "^1.4.3", + "levelup": "^1.3.1", "liftoff": "^2.2.0", "memdown": "^1.0.0", "mkdirp": "0.5.0", diff --git a/test/services/address/encoding.unit.js b/test/services/address/encoding.unit.js new file mode 100644 index 00000000..e5ba7376 --- /dev/null +++ b/test/services/address/encoding.unit.js @@ -0,0 +1,103 @@ +'use strict'; + +var chai = require('chai'); +var should = chai.should(); +var sinon = require('sinon'); +var bitcorenode = require('../../../'); +var bitcore = require('bitcore-lib'); +var Address = bitcore.Address; +var Script = bitcore.Script; +var AddressService = bitcorenode.services.Address; +var Networks = bitcore.Networks; +var encoding = require('../../../lib/services/address/encoding'); + +var mockdb = { +}; + +var mocknode = { + network: Networks.testnet, + datadir: 'testdir', + db: mockdb, + services: { + bitcoind: { + on: sinon.stub() + } + } +}; + +describe('Address Service Encoding', function() { + + describe('#encodeSpentIndexSyncKey', function() { + it('will encode to 36 bytes (string)', function() { + var txidBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); + var key = encoding.encodeSpentIndexSyncKey(txidBuffer, 12); + key.length.should.equal(36); + }); + it('will be able to decode encoded value', function() { + var txid = '3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7'; + var txidBuffer = new Buffer(txid, 'hex'); + var key = encoding.encodeSpentIndexSyncKey(txidBuffer, 12); + var keyBuffer = new Buffer(key, 'binary'); + keyBuffer.slice(0, 32).toString('hex').should.equal(txid); + var outputIndex = keyBuffer.readUInt32BE(32); + outputIndex.should.equal(12); + }); + }); + + describe('#_encodeInputKeyMap/#_decodeInputKeyMap roundtrip', function() { + var encoded; + var outputTxIdBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); + it('encode key', function() { + encoded = encoding.encodeInputKeyMap(outputTxIdBuffer, 13); + }); + it('decode key', function() { + var key = encoding.decodeInputKeyMap(encoded); + key.outputTxId.toString('hex').should.equal(outputTxIdBuffer.toString('hex')); + key.outputIndex.should.equal(13); + }); + }); + + describe('#_encodeInputValueMap/#_decodeInputValueMap roundtrip', function() { + var encoded; + var inputTxIdBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); + it('encode key', function() { + encoded = encoding.encodeInputValueMap(inputTxIdBuffer, 7); + }); + it('decode key', function() { + var key = encoding.decodeInputValueMap(encoded); + key.inputTxId.toString('hex').should.equal(inputTxIdBuffer.toString('hex')); + key.inputIndex.should.equal(7); + }); + }); + + + describe('#extractAddressInfoFromScript', function() { + it('pay-to-publickey', function() { + var pubkey = new bitcore.PublicKey('022df8750480ad5b26950b25c7ba79d3e37d75f640f8e5d9bcd5b150a0f85014da'); + var script = Script.buildPublicKeyOut(pubkey); + var info = encoding.extractAddressInfoFromScript(script, Networks.livenet); + info.addressType.should.equal(Address.PayToPublicKeyHash); + info.hashBuffer.toString('hex').should.equal('9674af7395592ec5d91573aa8d6557de55f60147'); + }); + it('pay-to-publickeyhash', function() { + var script = Script('OP_DUP OP_HASH160 20 0x0000000000000000000000000000000000000000 OP_EQUALVERIFY OP_CHECKSIG'); + var info = encoding.extractAddressInfoFromScript(script, Networks.livenet); + info.addressType.should.equal(Address.PayToPublicKeyHash); + info.hashBuffer.toString('hex').should.equal('0000000000000000000000000000000000000000'); + }); + it('pay-to-scripthash', function() { + var script = Script('OP_HASH160 20 0x0000000000000000000000000000000000000000 OP_EQUAL'); + var info = encoding.extractAddressInfoFromScript(script, Networks.livenet); + info.addressType.should.equal(Address.PayToScriptHash); + info.hashBuffer.toString('hex').should.equal('0000000000000000000000000000000000000000'); + }); + it('non-address script type', function() { + var buf = new Buffer(40); + buf.fill(0); + var script = Script('OP_RETURN 40 0x' + buf.toString('hex')); + var info = encoding.extractAddressInfoFromScript(script, Networks.livenet); + info.should.equal(false); + }); + }); + +}); diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index 8092a2f0..4745c624 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -23,8 +23,6 @@ describe('Address Service History', function() { history.node.should.equal(node); history.options.should.equal(options); history.addresses.should.equal(addresses); - history.transactionInfo.should.deep.equal([]); - history.combinedArray.should.deep.equal([]); history.detailedArray.should.deep.equal([]); }); it('will set addresses an array if only sent a string', function() { @@ -40,27 +38,29 @@ describe('Address Service History', function() { describe('#get', function() { it('will complete the async each limit series', function(done) { var addresses = [address]; + var summary = { + txids: [] + }; var history = new AddressHistory({ - node: {}, + node: { + services: { + address: { + getAddressSummary: sinon.stub().callsArgWith(2, null, summary) + } + } + }, options: {}, addresses: addresses }); var expected = [{}]; history.detailedArray = expected; - history.combinedArray = [{}]; - history.getTransactionInfo = sinon.stub().callsArg(1); - history.combineTransactionInfo = sinon.stub(); - history.sortAndPaginateCombinedArray = sinon.stub(); history.getDetailedInfo = sinon.stub().callsArg(1); - history.sortTransactionsIntoArray = sinon.stub(); history.get(function(err, results) { if (err) { throw err; } - history.getTransactionInfo.callCount.should.equal(1); history.getDetailedInfo.callCount.should.equal(1); history.combineTransactionInfo.callCount.should.equal(1); - history.sortAndPaginateCombinedArray.callCount.should.equal(1); results.should.deep.equal({ totalCount: 1, items: expected @@ -78,149 +78,15 @@ describe('Address Service History', function() { var expected = [{}]; history.sortedArray = expected; history.transactionInfo = [{}]; - history.getTransactionInfo = sinon.stub().callsArg(1); - history.paginateSortedArray = sinon.stub(); history.getDetailedInfo = sinon.stub().callsArgWith(1, new Error('test')); history.get(function(err) { err.message.should.equal('test'); done(); }); }); - it('handle an error from getTransactionInfo', function(done) { - var addresses = [address]; - var history = new AddressHistory({ - node: {}, - options: {}, - addresses: addresses - }); - var expected = [{}]; - history.sortedArray = expected; - history.transactionInfo = [{}]; - history.getTransactionInfo = sinon.stub().callsArgWith(1, new Error('test')); - history.get(function(err) { - err.message.should.equal('test'); - done(); - }); - }); }); - describe('#getTransactionInfo', function() { - it('will handle an error from getInputs', function(done) { - var history = new AddressHistory({ - node: { - services: { - address: { - getOutputs: sinon.stub().callsArgWith(2, null, []), - getInputs: sinon.stub().callsArgWith(2, new Error('test')) - } - } - }, - options: {}, - addresses: [] - }); - history.getTransactionInfo(address, function(err) { - err.message.should.equal('test'); - done(); - }); - }); - it('will handle an error from getOutputs', function(done) { - var history = new AddressHistory({ - node: { - services: { - address: { - getOutputs: sinon.stub().callsArgWith(2, new Error('test')), - getInputs: sinon.stub().callsArgWith(2, null, []) - } - } - }, - options: {}, - addresses: [] - }); - history.getTransactionInfo(address, function(err) { - err.message.should.equal('test'); - done(); - }); - }); - it('will call getOutputs and getInputs with the correct options', function() { - var startTimestamp = 1438289011844; - var endTimestamp = 1438289012412; - var expectedArgs = { - start: new Date(startTimestamp * 1000), - end: new Date(endTimestamp * 1000), - queryMempool: true - }; - var history = new AddressHistory({ - node: { - services: { - address: { - getOutputs: sinon.stub().callsArgWith(2, null, []), - getInputs: sinon.stub().callsArgWith(2, null, []) - } - } - }, - options: { - start: new Date(startTimestamp * 1000), - end: new Date(endTimestamp * 1000), - queryMempool: true - }, - addresses: [] - }); - history.transactionInfo = [{}]; - history.getTransactionInfo(address, function(err) { - if (err) { - throw err; - } - history.node.services.address.getOutputs.args[0][1].should.deep.equal(expectedArgs); - history.node.services.address.getInputs.args[0][1].should.deep.equal(expectedArgs); - }); - }); - it('will handle empty results from getOutputs and getInputs', function() { - var history = new AddressHistory({ - node: { - services: { - address: { - getOutputs: sinon.stub().callsArgWith(2, null, []), - getInputs: sinon.stub().callsArgWith(2, null, []) - } - } - }, - options: {}, - addresses: [] - }); - history.transactionInfo = [{}]; - history.getTransactionInfo(address, function(err) { - if (err) { - throw err; - } - history.transactionInfo.length.should.equal(1); - history.node.services.address.getOutputs.args[0][0].should.equal(address); - }); - }); - it('will concatenate outputs and inputs', function() { - var history = new AddressHistory({ - node: { - services: { - address: { - getOutputs: sinon.stub().callsArgWith(2, null, [{}]), - getInputs: sinon.stub().callsArgWith(2, null, [{}]) - } - } - }, - options: {}, - addresses: [] - }); - history.transactionInfo = [{}]; - history.getTransactionInfo(address, function(err) { - if (err) { - throw err; - } - history.transactionInfo.length.should.equal(3); - history.node.services.address.getOutputs.args[0][0].should.equal(address); - }); - }); - }); - - describe('@sortByHeight', function() { + describe('#_mergeAndSortTxids', function() { it('will sort latest to oldest using height', function() { var transactionInfo = [ { @@ -386,131 +252,6 @@ describe('Address Service History', function() { }); }); - describe('#sortAndPaginateCombinedArray', function() { - it('from 0 to 2', function() { - var history = new AddressHistory({ - node: {}, - options: { - from: 0, - to: 2 - }, - addresses: [] - }); - history.combinedArray = [ - { - height: 13 - }, - { - height: 14, - }, - { - height: 12 - } - ]; - history.sortAndPaginateCombinedArray(); - history.combinedArray.length.should.equal(2); - history.combinedArray[0].height.should.equal(14); - history.combinedArray[1].height.should.equal(13); - }); - it('from 0 to 4 (exceeds length)', function() { - var history = new AddressHistory({ - node: {}, - options: { - from: 0, - to: 4 - }, - addresses: [] - }); - history.combinedArray = [ - { - height: 13 - }, - { - height: 14, - }, - { - height: 12 - } - ]; - history.sortAndPaginateCombinedArray(); - history.combinedArray.length.should.equal(3); - history.combinedArray[0].height.should.equal(14); - history.combinedArray[1].height.should.equal(13); - history.combinedArray[2].height.should.equal(12); - }); - it('from 0 to 1', function() { - var history = new AddressHistory({ - node: {}, - options: { - from: 0, - to: 1 - }, - addresses: [] - }); - history.combinedArray = [ - { - height: 13 - }, - { - height: 14, - }, - { - height: 12 - } - ]; - history.sortAndPaginateCombinedArray(); - history.combinedArray.length.should.equal(1); - history.combinedArray[0].height.should.equal(14); - }); - it('from 2 to 3', function() { - var history = new AddressHistory({ - node: {}, - options: { - from: 2, - to: 3 - }, - addresses: [] - }); - history.combinedArray = [ - { - height: 13 - }, - { - height: 14, - }, - { - height: 12 - } - ]; - history.sortAndPaginateCombinedArray(); - history.combinedArray.length.should.equal(1); - history.combinedArray[0].height.should.equal(12); - }); - it('from 10 to 20 (out of range)', function() { - var history = new AddressHistory({ - node: {}, - options: { - from: 10, - to: 20 - }, - addresses: [] - }); - history.combinedArray = [ - { - height: 13 - }, - { - height: 14, - }, - { - height: 12 - } - ]; - history.sortAndPaginateCombinedArray(); - history.combinedArray.length.should.equal(0); - }); - }); - describe('#getDetailedInfo', function() { it('will add additional information to existing this.transactions', function() { var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; @@ -602,7 +343,7 @@ describe('Address Service History', function() { } }, options: {}, - addresses: [] + addresses: [txAddress] }); var transactionInfo = { addresses: {}, @@ -614,7 +355,7 @@ describe('Address Service History', function() { transactionInfo.addresses[txAddress] = {}; transactionInfo.addresses[txAddress].outputIndexes = [1]; transactionInfo.addresses[txAddress].inputIndexes = []; - history.getDetailedInfo(transactionInfo, function(err) { + history.getDetailedInfo(txid, function(err) { if (err) { throw err; } @@ -653,28 +394,4 @@ describe('Address Service History', function() { history.getConfirmationsDetail(transaction).should.equal(1); }); }); - describe('#getSatoshisDetail', function() { - it('subtract inputIndexes satoshis without outputIndexes', function() { - var history = new AddressHistory({ - node: {}, - options: {}, - addresses: [] - }); - var transaction = { - inputs: [ - { - output: { - satoshis: 10000 - } - } - ] - }; - var txInfo = { - addresses: {} - }; - txInfo.addresses[address] = {}; - txInfo.addresses[address].inputIndexes = [0]; - history.getSatoshisDetail(transaction, txInfo).should.equal(-10000); - }); - }); }); diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index e7460a1d..193a9f8a 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -2,6 +2,8 @@ var should = require('chai').should(); var sinon = require('sinon'); +var stream = require('stream'); +var levelup = require('levelup'); var proxyquire = require('proxyquire'); var bitcorenode = require('../../../'); var AddressService = bitcorenode.services.Address; @@ -9,13 +11,15 @@ var blockData = require('../../data/livenet-345003.json'); var bitcore = require('bitcore-lib'); var memdown = require('memdown'); var leveldown = require('leveldown'); -var Script = bitcore.Script; -var Address = bitcore.Address; var Networks = bitcore.Networks; var EventEmitter = require('events').EventEmitter; var errors = bitcorenode.errors; var Transaction = require('../../../lib/transaction'); var txData = require('../../data/transaction.json'); +var index = require('../../../lib'); +var log = index.log; +var constants = require('../../../lib/services/address/constants'); +var encoding = require('../../../lib/services/address/encoding'); var mockdb = { }; @@ -96,7 +100,8 @@ describe('Address Service', function() { done(); }); }); - it('start levelup db for mempool index', function(done) { + it('start levelup db for mempool and summary index', function(done) { + var levelupStub = sinon.stub().callsArg(2); var TestAddressService = proxyquire('../../../lib/services/address', { 'fs': { existsSync: sinon.stub().returns(true) @@ -104,14 +109,7 @@ describe('Address Service', function() { 'leveldown': { destroy: sinon.stub().callsArgWith(1, null) }, - 'levelup': function(dbPath, options, callback) { - dbPath.should.equal('testdir/testnet3/bitcore-addressmempool.db'); - options.db.should.equal(memdown); - options.keyEncoding.should.equal('binary'); - options.valueEncoding.should.equal('binary'); - options.fillCache.should.equal(false); - setImmediate(callback); - }, + 'levelup': levelupStub, 'mkdirp': sinon.stub().callsArgWith(1, null) }); var am = new TestAddressService({ @@ -119,6 +117,16 @@ describe('Address Service', function() { node: mocknode }); am.start(function() { + levelupStub.callCount.should.equal(2); + var dbPath1 = levelupStub.args[0][0]; + dbPath1.should.equal('testdir/testnet3/bitcore-addressmempool.db'); + var options = levelupStub.args[0][1]; + options.db.should.equal(memdown); + options.keyEncoding.should.equal('binary'); + options.valueEncoding.should.equal('binary'); + options.fillCache.should.equal(false); + var dbPath2 = levelupStub.args[1][0]; + dbPath2.should.equal('testdir/testnet3/bitcore-addresssummary.db'); done(); }); }); @@ -253,7 +261,7 @@ describe('Address Service', function() { }); it('should load the db with regtest', function() { // Switch to use regtest - // Networks.remove(Networks.testnet); + Networks.remove(Networks.testnet); Networks.add({ name: 'regtest', alias: 'regtest', @@ -387,43 +395,6 @@ describe('Address Service', function() { }); }); - describe('#_extractAddressInfoFromScript', function() { - var am; - before(function() { - am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.node.network = Networks.livenet; - }); - it('pay-to-publickey', function() { - var pubkey = new bitcore.PublicKey('022df8750480ad5b26950b25c7ba79d3e37d75f640f8e5d9bcd5b150a0f85014da'); - var script = Script.buildPublicKeyOut(pubkey); - var info = am._extractAddressInfoFromScript(script); - info.addressType.should.equal(Address.PayToPublicKeyHash); - info.hashBuffer.toString('hex').should.equal('9674af7395592ec5d91573aa8d6557de55f60147'); - }); - it('pay-to-publickeyhash', function() { - var script = Script('OP_DUP OP_HASH160 20 0x0000000000000000000000000000000000000000 OP_EQUALVERIFY OP_CHECKSIG'); - var info = am._extractAddressInfoFromScript(script); - info.addressType.should.equal(Address.PayToPublicKeyHash); - info.hashBuffer.toString('hex').should.equal('0000000000000000000000000000000000000000'); - }); - it('pay-to-scripthash', function() { - var script = Script('OP_HASH160 20 0x0000000000000000000000000000000000000000 OP_EQUAL'); - var info = am._extractAddressInfoFromScript(script); - info.addressType.should.equal(Address.PayToScriptHash); - info.hashBuffer.toString('hex').should.equal('0000000000000000000000000000000000000000'); - }); - it('non-address script type', function() { - var buf = new Buffer(40); - buf.fill(0); - var script = Script('OP_RETURN 40 0x' + buf.toString('hex')); - var info = am._extractAddressInfoFromScript(script); - info.should.equal(false); - }); - }); - describe('#blockHandler', function() { var am; var testBlock = bitcore.Block.fromString(blockData); @@ -524,6 +495,7 @@ describe('Address Service', function() { var testnode = { datadir: 'testdir', db: db, + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -559,73 +531,6 @@ describe('Address Service', function() { }); }); - describe('#_encodeSpentIndexSyncKey', function() { - it('will encode to 36 bytes (string)', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var txidBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); - var key = am._encodeSpentIndexSyncKey(txidBuffer, 12); - key.length.should.equal(36); - }); - it('will be able to decode encoded value', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var txid = '3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7'; - var txidBuffer = new Buffer(txid, 'hex'); - var key = am._encodeSpentIndexSyncKey(txidBuffer, 12); - var keyBuffer = new Buffer(key, 'binary'); - keyBuffer.slice(0, 32).toString('hex').should.equal(txid); - var outputIndex = keyBuffer.readUInt32BE(32); - outputIndex.should.equal(12); - }); - }); - - describe('#_encodeInputKeyMap/#_decodeInputKeyMap roundtrip', function() { - var encoded; - var outputTxIdBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); - it('encode key', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - encoded = am._encodeInputKeyMap(outputTxIdBuffer, 13); - }); - it('decode key', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var key = am._decodeInputKeyMap(encoded); - key.outputTxId.toString('hex').should.equal(outputTxIdBuffer.toString('hex')); - key.outputIndex.should.equal(13); - }); - }); - - describe('#_encodeInputValueMap/#_decodeInputValueMap roundtrip', function() { - var encoded; - var inputTxIdBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); - it('encode key', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - encoded = am._encodeInputValueMap(inputTxIdBuffer, 7); - }); - it('decode key', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var key = am._decodeInputValueMap(encoded); - key.inputTxId.toString('hex').should.equal(inputTxIdBuffer.toString('hex')); - key.inputIndex.should.equal(7); - }); - }); - describe('#transactionEventHandler', function() { it('will emit a transaction if there is a subscriber', function(done) { var am = new AddressService({ @@ -817,18 +722,127 @@ describe('Address Service', function() { }); + describe('#createInputsStream', function() { + it('transform stream from buffer into object', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + tip: { + __height: 157 + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var streamStub = new stream.Readable(); + streamStub._read = function() { /* do nothing */ }; + addressService.createInputsDBStream = sinon.stub().returns(streamStub); + var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var testStream = addressService.createInputsStream(address, {}); + testStream.once('data', function(data) { + data.address.should.equal('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'); + data.hashType.should.equal('pubkeyhash'); + data.txid.should.equal('7b94e3c39386845ea383b8e726b20b5172ccd3ef9be008bbb133e3b63f07df72'); + data.inputIndex.should.equal(1); + data.height.should.equal(157); + data.confirmations.should.equal(1); + done(); + }); + streamStub.emit('data', { + key: new Buffer('030b2f0a0c31bfe0406b0ccc1381fdbe311946dadc01000000009d786cfeae288d74aaf9f51f215f9882e7bd7bc18af7a550683c4d7c6962f6372900000004', 'hex'), + value: new Buffer('7b94e3c39386845ea383b8e726b20b5172ccd3ef9be008bbb133e3b63f07df7200000001', 'hex') + }); + streamStub.emit('end'); + }); + }); + + describe('#createInputsDBStream', function() { + it('will stream all keys', function() { + var streamStub = sinon.stub().returns({}); + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + store: { + createReadStream: streamStub + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = {}; + var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var testStream = addressService.createInputsDBStream(address, options); + should.exist(testStream); + streamStub.callCount.should.equal(1); + var expectedGt = '03038a213afdfc551fc658e9a2a58a86e98d69b687010000000000'; + // The expected "lt" value should be one value above the start value, due + // to the keys having additional data following it and can't be "equal". + var expectedLt = '03038a213afdfc551fc658e9a2a58a86e98d69b68701ffffffffff'; + streamStub.args[0][0].gt.toString('hex').should.equal(expectedGt); + streamStub.args[0][0].lt.toString('hex').should.equal(expectedLt); + }); + it('will stream keys based on a range of block heights', function() { + var streamStub = sinon.stub().returns({}); + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + store: { + createReadStream: streamStub + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = { + start: 1, + end: 0 + }; + var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var testStream = addressService.createInputsDBStream(address, options); + should.exist(testStream); + streamStub.callCount.should.equal(1); + var expectedGt = '03038a213afdfc551fc658e9a2a58a86e98d69b687010000000000'; + // The expected "lt" value should be one value above the start value, due + // to the keys having additional data following it and can't be "equal". + var expectedLt = '03038a213afdfc551fc658e9a2a58a86e98d69b687010000000002'; + streamStub.args[0][0].gt.toString('hex').should.equal(expectedGt); + streamStub.args[0][0].lt.toString('hex').should.equal(expectedLt); + }); + }); + describe('#getInputs', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; + var hashTypeBuffer = constants.HASH_TYPES.PUBKEY; var db = { tip: { __height: 1 } }; var testnode = { - network: Networks.testnet, + network: Networks.livenet, datadir: 'testdir', services: { db: db, @@ -845,14 +859,18 @@ describe('Address Service', function() { }); it('will add mempool inputs on close', function(done) { - var testStream = new EventEmitter(); + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; var db = { store: { createReadStream: sinon.stub().returns(testStream) + }, + tip: { + __height: 10 } }; var testnode = { - network: Networks.testnet, + network: Networks.livenet, datadir: 'testdir', services: { db: db, @@ -883,10 +901,11 @@ describe('Address Service', function() { inputs[0].height.should.equal(-1); done(); }); - testStream.emit('close'); + testStream.push(null); }); it('will get inputs for an address and timestamp', function(done) { - var testStream = new EventEmitter(); + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; var args = { start: 15, end: 12, @@ -895,12 +914,12 @@ describe('Address Service', function() { var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - var gte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, + var gt = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); - ops.gte.toString('hex').should.equal(gte.toString('hex')); - var lte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, + ops.gt.toString('hex').should.equal(gt.toString('hex')); + var lt = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); - ops.lte.toString('hex').should.equal(lte.toString('hex')); + ops.lt.toString('hex').should.equal(lt.toString('hex')); createReadStreamCallCount++; return testStream; } @@ -924,20 +943,21 @@ describe('Address Service', function() { value: new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae700000000', 'hex') }; testStream.emit('data', data); - testStream.emit('close'); + testStream.push(null); }); it('should get inputs for address', function(done) { - var testStream = new EventEmitter(); + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; var args = { queryMempool: true }; var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - var gte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('00', 'hex')]); - ops.gte.toString('hex').should.equal(gte.toString('hex')); - var lte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('ff', 'hex')]); - ops.lte.toString('hex').should.equal(lte.toString('hex')); + var gt = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('0000000000', 'hex')]); + ops.gt.toString('hex').should.equal(gt.toString('hex')); + var lt = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('ffffffffff', 'hex')]); + ops.lt.toString('hex').should.equal(lt.toString('hex')); createReadStreamCallCount++; return testStream; } @@ -960,15 +980,16 @@ describe('Address Service', function() { value: new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae700000000', 'hex') }; testStream.emit('data', data); - testStream.emit('close'); + testStream.push(null); }); it('should give an error if the readstream has an error', function(done) { - var testStream = new EventEmitter(); + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; am.node.services.db.store = { createReadStream: sinon.stub().returns(testStream) }; - am.getOutputs(address, {}, function(err, outputs) { + am.getInputs(address, {}, function(err, outputs) { should.exist(err); err.message.should.equal('readstreamerror'); done(); @@ -976,7 +997,7 @@ describe('Address Service', function() { testStream.emit('error', new Error('readstreamerror')); setImmediate(function() { - testStream.emit('close'); + testStream.push(null); }); }); @@ -986,7 +1007,7 @@ describe('Address Service', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; + var hashTypeBuffer = constants.HASH_TYPES.PUBKEY; var db = { tip: { __height: 1 @@ -1025,39 +1046,44 @@ describe('Address Service', function() { }); }); it('it will parse data', function(done) { - var testStream = new EventEmitter(); + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; am.mempoolIndex = {}; am.mempoolIndex.createReadStream = sinon.stub().returns(testStream); - am._getInputsMempool(address, hashBuffer, hashTypeBuffer, function(err, outputs) { + var nowTime = new Date().getTime(); + + am._getInputsMempool(address, hashBuffer, hashTypeBuffer, function(err, inputs) { should.not.exist(err); - outputs.length.should.equal(1); - outputs[0].address.should.equal(address); - outputs[0].txid.should.equal(txid); - outputs[0].hashType.should.equal('pubkeyhash'); - outputs[0].hashType.should.equal(AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); - outputs[0].inputIndex.should.equal(5); - outputs[0].height.should.equal(-1); - outputs[0].confirmations.should.equal(0); + inputs.length.should.equal(1); + var input = inputs[0]; + input.address.should.equal(address); + input.txid.should.equal(txid); + input.hashType.should.equal('pubkeyhash'); + input.hashType.should.equal(constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); + input.inputIndex.should.equal(5); + input.height.should.equal(-1); + input.confirmations.should.equal(0); + input.timestamp.should.equal(nowTime); done(); }); var txid = '5d32f0fff6871c377e00c16f48ebb5e89c723d0b9dd25f68fdda70c3392bee61'; var inputIndex = 5; var inputIndexBuffer = new Buffer(4); + var timestampBuffer = new Buffer(new Array(8)); + timestampBuffer.writeDoubleBE(nowTime); inputIndexBuffer.writeUInt32BE(inputIndex); var valueData = Buffer.concat([ new Buffer(txid, 'hex'), - inputIndexBuffer + inputIndexBuffer, + timestampBuffer ]); - // Note: key is not used currently testStream.emit('data', { value: valueData }); - setImmediate(function() { - testStream.emit('close'); - }); + testStream.emit('close'); }); }); @@ -1105,18 +1131,129 @@ describe('Address Service', function() { }); }); + describe('#createOutputsStream', function() { + it('transform stream from buffer into object', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + tip: { + __height: 157 + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var streamStub = new stream.Readable(); + streamStub._read = function() { /* do nothing */ }; + addressService.createOutputsDBStream = sinon.stub().returns(streamStub); + var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var testStream = addressService.createOutputsStream(address, {}); + testStream.once('data', function(data) { + data.address.should.equal('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'); + data.hashType.should.equal('pubkeyhash'); + data.txid.should.equal('4078b72b09391f5146e2c564f5847d49b179f9946b253f780f65b140d46ef6f9'); + data.outputIndex.should.equal(2); + data.height.should.equal(157); + data.satoshis.should.equal(10000); + data.script.toString('hex').should.equal('76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac'); + data.confirmations.should.equal(1); + done(); + }); + streamStub.emit('data', { + key: new Buffer('020b2f0a0c31bfe0406b0ccc1381fdbe311946dadc01000000009d4078b72b09391f5146e2c564f5847d49b179f9946b253f780f65b140d46ef6f900000002', 'hex'), + value: new Buffer('40c388000000000076a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac', 'hex') + }); + streamStub.emit('end'); + }); + }); + + describe('#createOutputsDBStream', function() { + it('will stream all keys', function() { + var streamStub = sinon.stub().returns({}); + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + store: { + createReadStream: streamStub + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = {}; + var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var testStream = addressService.createOutputsDBStream(address, options); + should.exist(testStream); + streamStub.callCount.should.equal(1); + var expectedGt = '02038a213afdfc551fc658e9a2a58a86e98d69b687010000000000'; + // The expected "lt" value should be one value above the start value, due + // to the keys having additional data following it and can't be "equal". + var expectedLt = '02038a213afdfc551fc658e9a2a58a86e98d69b68701ffffffffff'; + streamStub.args[0][0].gt.toString('hex').should.equal(expectedGt); + streamStub.args[0][0].lt.toString('hex').should.equal(expectedLt); + }); + it('will stream keys based on a range of block heights', function() { + var streamStub = sinon.stub().returns({}); + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + store: { + createReadStream: streamStub + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = { + start: 1, + end: 0 + }; + var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var testStream = addressService.createOutputsDBStream(address, options); + should.exist(testStream); + streamStub.callCount.should.equal(1); + var expectedGt = '02038a213afdfc551fc658e9a2a58a86e98d69b687010000000000'; + // The expected "lt" value should be one value above the start value, due + // to the keys having additional data following it and can't be "equal". + var expectedLt = '02038a213afdfc551fc658e9a2a58a86e98d69b687010000000002'; + streamStub.args[0][0].gt.toString('hex').should.equal(expectedGt); + streamStub.args[0][0].lt.toString('hex').should.equal(expectedLt); + }); + }); + describe('#getOutputs', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; var hashBuffer = bitcore.Address('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W').hashBuffer; - var hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; + var hashTypeBuffer = constants.HASH_TYPES.PUBKEY; var db = { tip: { __height: 1 } }; var testnode = { - network: Networks.testnet, + network: Networks.livenet, datadir: 'testdir', services: { db: db, @@ -1137,7 +1274,8 @@ describe('Address Service', function() { }); it('will get outputs for an address and timestamp', function(done) { - var testStream = new EventEmitter(); + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; var args = { start: 15, end: 12, @@ -1146,10 +1284,10 @@ describe('Address Service', function() { var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - var gte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); - ops.gte.toString('hex').should.equal(gte.toString('hex')); - var lte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); - ops.lte.toString('hex').should.equal(lte.toString('hex')); + var gt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); + ops.gt.toString('hex').should.equal(gt.toString('hex')); + var lt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); + ops.lt.toString('hex').should.equal(lt.toString('hex')); createReadStreamCallCount++; return testStream; } @@ -1161,7 +1299,7 @@ describe('Address Service', function() { outputs[0].address.should.equal(address); outputs[0].txid.should.equal('125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87'); outputs[0].hashType.should.equal('pubkeyhash'); - outputs[0].hashType.should.equal(AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); + outputs[0].hashType.should.equal(constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); outputs[0].outputIndex.should.equal(1); outputs[0].satoshis.should.equal(4527773864); outputs[0].script.should.equal('76a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac'); @@ -1174,11 +1312,12 @@ describe('Address Service', function() { value: new Buffer('41f0de058a80000076a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac', 'hex') }; testStream.emit('data', data); - testStream.emit('close'); + testStream.push(null); }); it('should get outputs for an address', function(done) { - var readStream1 = new EventEmitter(); + var readStream1 = new stream.Readable(); + readStream1._read = function() { /* do nothing */ }; am.node.services.db.store = { createReadStream: sinon.stub().returns(readStream1) }; @@ -1234,11 +1373,12 @@ describe('Address Service', function() { readStream1.emit('data', data1); readStream1.emit('data', data2); - readStream1.emit('close'); + readStream1.push(null); }); it('should give an error if the readstream has an error', function(done) { - var readStream2 = new EventEmitter(); + var readStream2 = new stream.Readable(); + readStream2._read = function() { /* do nothing */ }; am.node.services.db.store = { createReadStream: sinon.stub().returns(readStream2) }; @@ -1251,7 +1391,7 @@ describe('Address Service', function() { readStream2.emit('error', new Error('readstreamerror')); setImmediate(function() { - readStream2.emit('close'); + readStream2.push(null); }); }); @@ -1261,8 +1401,9 @@ describe('Address Service', function() { // See https://github.com/bitpay/bitcore-node/issues/377 var address = '321jRYeWBrLBWr2j1KYnAFGico3GUdd5q7'; var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = AddressService.HASH_TYPES.REDEEMSCRIPT; - var testStream = new EventEmitter(); + var hashTypeBuffer = constants.HASH_TYPES.REDEEMSCRIPT; + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; var args = { start: 15, end: 12, @@ -1271,10 +1412,10 @@ describe('Address Service', function() { var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - var gte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); - ops.gte.toString('hex').should.equal(gte.toString('hex')); - var lte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); - ops.lte.toString('hex').should.equal(lte.toString('hex')); + var gt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); + ops.gt.toString('hex').should.equal(gt.toString('hex')); + var lt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); + ops.lt.toString('hex').should.equal(lt.toString('hex')); createReadStreamCallCount++; return testStream; } @@ -1286,7 +1427,7 @@ describe('Address Service', function() { outputs[0].address.should.equal(address); outputs[0].txid.should.equal('125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87'); outputs[0].hashType.should.equal('scripthash'); - outputs[0].hashType.should.equal(AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); + outputs[0].hashType.should.equal(constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); outputs[0].outputIndex.should.equal(1); outputs[0].satoshis.should.equal(4527773864); outputs[0].script.should.equal('a914038a213afdfc551fc658e9a2a58a86e98d69b68787'); @@ -1301,7 +1442,7 @@ describe('Address Service', function() { value: new Buffer('41f0de058a800000a914038a213afdfc551fc658e9a2a58a86e98d69b68787', 'hex') }; testStream.emit('data', data); - testStream.emit('close'); + testStream.push(null); }); it('should not print outputs for a p2pkh address, if the output was sent to a p2sh redeemScript', function(done) { @@ -1310,8 +1451,9 @@ describe('Address Service', function() { // See https://github.com/bitpay/bitcore-node/issues/377 var address = '321jRYeWBrLBWr2j1KYnAFGico3GUdd5q7'; var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = AddressService.HASH_TYPES.REDEEMSCRIPT; - var testStream = new EventEmitter(); + var hashTypeBuffer = constants.HASH_TYPES.REDEEMSCRIPT; + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; var args = { start: 15, end: 12, @@ -1322,10 +1464,10 @@ describe('Address Service', function() { // Verifying that the db query is looking for a redeemScript, *not* a p2pkh am.node.services.db.store = { createReadStream: function(ops) { - var gte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); - ops.gte.toString('hex').should.equal(gte.toString('hex')); - var lte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); - ops.lte.toString('hex').should.equal(lte.toString('hex')); + var gt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); + ops.gt.toString('hex').should.equal(gt.toString('hex')); + var lt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); + ops.lt.toString('hex').should.equal(lt.toString('hex')); createReadStreamCallCount++; return testStream; } @@ -1337,7 +1479,7 @@ describe('Address Service', function() { done(); }); createReadStreamCallCount.should.equal(1); - testStream.emit('close'); + testStream.push(null); }); }); @@ -1345,7 +1487,7 @@ describe('Address Service', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; + var hashTypeBuffer = constants.HASH_TYPES.PUBKEY; var db = { tip: { __height: 1 @@ -1391,14 +1533,16 @@ describe('Address Service', function() { throw err; } outputs.length.should.equal(1); - outputs[0].address.should.equal(address); - outputs[0].hashType.should.equal('pubkeyhash'); - outputs[0].txid.should.equal(txid); - outputs[0].outputIndex.should.equal(outputIndex); - outputs[0].height.should.equal(-1); - outputs[0].satoshis.should.equal(3); - outputs[0].script.should.equal('ac'); - outputs[0].confirmations.should.equal(0); + var output = outputs[0]; + output.address.should.equal(address); + output.hashType.should.equal('pubkeyhash'); + output.txid.should.equal(txid); + output.outputIndex.should.equal(outputIndex); + output.height.should.equal(-1); + output.satoshis.should.equal(3); + output.script.should.equal('ac'); + output.timestamp.should.equal(1452696715750); + output.confirmations.should.equal(0); done(); }); @@ -1408,7 +1552,7 @@ describe('Address Service', function() { var outputIndexBuffer = new Buffer(4); outputIndexBuffer.writeUInt32BE(outputIndex); var keyData = Buffer.concat([ - AddressService.MEMPREFIXES.OUTPUTS, + constants.MEMPREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, txidBuffer, @@ -1417,6 +1561,7 @@ describe('Address Service', function() { var valueData = Buffer.concat([ new Buffer('4008000000000000', 'hex'), + new Buffer('427523b78c1e6000', 'hex'), new Buffer('ac', 'hex') ]); @@ -1803,12 +1948,18 @@ describe('Address Service', function() { describe('#updateMempoolIndex/#removeMempoolIndex', function() { var am; var tx = Transaction().fromBuffer(txBuf); + var clock; - before(function() { + beforeEach(function() { am = new AddressService({ mempoolMemoryIndex: true, node: mocknode }); + clock = sinon.useFakeTimers(); + }); + + afterEach(function() { + clock.restore(); }); it('will update the input and output indexes', function() { @@ -1819,12 +1970,18 @@ describe('Address Service', function() { for (var i = 0; i < operations.length; i++) { operations[i].type.should.equal('put'); } - var expectedValue = '45202ffdeb8344af4dec07cddf0478485dc65cc7d08303e45959630c89b51ea200000002'; + var nowTime = new Date().getTime(); + var nowTimeBuffer = new Buffer(8); + nowTimeBuffer.writeDoubleBE(nowTime); + var expectedValue = '45202ffdeb8344af4dec07cddf0478485dc65cc7d08303e45959630c89b51ea200000002' + + nowTimeBuffer.toString('hex'); operations[7].value.toString('hex').should.equal(expectedValue); var matches = 0; + + for (var j = 0; j < operations.length; j++) { var match = Buffer.concat([ - AddressService.MEMPREFIXES.SPENTS, + constants.MEMPREFIXES.SPENTS, bitcore.Address('1JT7KDYwT9JY9o2vyqcKNSJgTWeKfV3ui8').hashBuffer ]).toString('hex'); @@ -1850,88 +2007,743 @@ describe('Address Service', function() { }); }); + describe('#getAddressSummary', function() { - var node = { - datadir: 'testdir', - network: Networks.testnet, - services: { - bitcoind: { - isSpent: sinon.stub().returns(false), - on: sinon.spy() - } - } - }; - var inputs = [ - { - 'txid': '9f183412de12a6c1943fc86c390174c1cde38d709217fdb59dcf540230fa58a6', - 'height': -1, - 'confirmations': 0, - 'addresses': { - 'mpkDdnLq26djg17s6cYknjnysAm3QwRzu2': { - 'outputIndexes': [], - 'inputIndexes': [ - 3 - ] + var clock; + beforeEach(function() { + clock = sinon.useFakeTimers(); + sinon.stub(log, 'warn'); + }); + afterEach(function() { + clock.restore(); + log.warn.restore(); + }); + it('will handle error from _getAddressConfirmedSummary', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() } }, - 'address': 'mpkDdnLq26djg17s6cYknjnysAm3QwRzu2' - } - ]; - - var outputs = [ - { - 'address': 'mpkDdnLq26djg17s6cYknjnysAm3QwRzu2', - 'txid': '689e9f543fa4aa5b2daa3b5bb65f9a00ad5aa1a2e9e1fc4e11061d85f2aa9bc5', - 'outputIndex': 0, - 'height': 556351, - 'satoshis': 3487110, - 'script': '76a914653b58493c2208481e0902a8ffb97b8112b13fe188ac', - 'confirmations': 13190 - } - ]; - - var as = new AddressService({ - mempoolMemoryIndex: true, - node: node - }); - as.getInputs = sinon.stub().callsArgWith(2, null, inputs); - as.getOutputs = sinon.stub().callsArgWith(2, null, outputs); - var key = Buffer.concat([ - new Buffer('689e9f543fa4aa5b2daa3b5bb65f9a00ad5aa1a2e9e1fc4e11061d85f2aa9bc5', 'hex'), - new Buffer(Array(4)) - ]).toString('binary'); - as.mempoolSpentIndex = {}; - as.mempoolSpentIndex[key] = true; - it('should handle unconfirmed and confirmed outputs and inputs', function(done) { - as.getAddressSummary('mpkDdnLq26djg17s6cYknjnysAm3QwRzu2', {}, function(err, summary) { - should.not.exist(err); - summary.totalReceived.should.equal(3487110); - summary.totalSpent.should.equal(0); - summary.balance.should.equal(3487110); - summary.unconfirmedBalance.should.equal(0); - summary.appearances.should.equal(1); - summary.unconfirmedAppearances.should.equal(1); - summary.txids.should.deep.equal( - [ - '9f183412de12a6c1943fc86c390174c1cde38d709217fdb59dcf540230fa58a6', - '689e9f543fa4aa5b2daa3b5bb65f9a00ad5aa1a2e9e1fc4e11061d85f2aa9bc5' - ] - ); + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + var options = {}; + addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, new Error('test')); + addressService.getAddressSummary(address, options, function(err) { + should.exist(err); + err.message.should.equal('test'); done(); }); }); - it('noTxList should not include txids array', function(done) { - as.getAddressSummary('mpkDdnLq26djg17s6cYknjnysAm3QwRzu2', {noTxList: true}, function(err, summary) { - should.not.exist(err); - summary.totalReceived.should.equal(3487110); - summary.totalSpent.should.equal(0); - summary.balance.should.equal(3487110); - summary.unconfirmedBalance.should.equal(0); - summary.appearances.should.equal(1); - summary.unconfirmedAppearances.should.equal(1); - should.not.exist(summary.txids); + it('will handle error from _getAddressMempoolSummary', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + var options = {}; + addressService._getAddressConfirmedSummary = sinon.stub().callsArg(2); + addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(2, new Error('test2')); + addressService.getAddressSummary(address, options, function(err) { + should.exist(err); + err.message.should.equal('test2'); + done(); + }); + }); + it('will pass cache and summary between functions correctly', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + var options = {}; + var cache = {}; + var summary = {}; + addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); + addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(3, null, cache); + addressService._transformAddressSummaryFromCache = sinon.stub().returns(summary); + addressService.getAddressSummary(address, options, function(err, sum) { + addressService._getAddressConfirmedSummary.callCount.should.equal(1); + addressService._getAddressMempoolSummary.callCount.should.equal(1); + addressService._getAddressMempoolSummary.args[0][2].should.equal(cache); + addressService._transformAddressSummaryFromCache.callCount.should.equal(1); + addressService._transformAddressSummaryFromCache.args[0][0].should.equal(cache); + sum.should.equal(summary); + done(); + }); + }); + it('will log if there is a slow query', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + var options = {}; + var cache = {}; + var summary = {}; + addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); + addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); + addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(3, null, cache); + addressService._transformAddressSummaryFromCache = sinon.stub().returns(summary); + addressService.getAddressSummary(address, options, function() { + log.warn.callCount.should.equal(2); + done(); + }); + clock.tick(6000); + }); + }); + + describe('#_getAddressConfirmedSummary', function() { + it('handle error from _getAddressConfirmedSummaryCache', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + tip: { + __height: 10 + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + var options = {}; + addressService._getAddressConfirmedSummaryCache = sinon.stub().callsArgWith(2, new Error('test')); + addressService._getAddressConfirmedSummary(address, options, function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); + + it('will NOT update cache if matches current tip', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + tip: { + __height: 10 + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + var options = {}; + var cache = { + height: 10 + }; + addressService._updateAddressConfirmedSummaryCache = sinon.stub(); + addressService._getAddressConfirmedSummaryCache = sinon.stub().callsArgWith(2, null, cache); + addressService._getAddressConfirmedSummary(address, options, function(err, cache) { + if (err) { + return done(err); + } + should.exist(cache); + addressService._updateAddressConfirmedSummaryCache.callCount.should.equal(0); + done(); + }); + }); + + it('will call _updateAddressConfirmedSummaryCache with correct arguments', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + tip: { + __height: 11 + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + var options = {}; + var cache = { + height: 10 + }; + addressService._updateAddressConfirmedSummaryCache = sinon.stub().callsArgWith(4, null, cache); + addressService._getAddressConfirmedSummaryCache = sinon.stub().callsArgWith(2, null, cache); + addressService._getAddressConfirmedSummary(address, options, function(err, cache) { + if (err) { + return done(err); + } + should.exist(cache); + addressService._updateAddressConfirmedSummaryCache.callCount.should.equal(1); + var args = addressService._updateAddressConfirmedSummaryCache.args[0]; + args[0].should.equal(address); + args[1].should.equal(options); + args[2].should.equal(cache); + args[3].should.equal(11); done(); }); }); }); + + describe('#_getAddressConfirmedSummaryCache', function() { + function shouldExistBasecache(cache) { + should.exist(cache); + should.not.exist(cache.height); + should.exist(cache.result); + cache.result.appearanceIds.should.deep.equal({}); + cache.result.totalReceived.should.equal(0); + cache.result.balance.should.equal(0); + cache.result.unconfirmedAppearanceIds.should.deep.equal({}); + cache.result.unconfirmedBalance.should.equal(0); + } + it('give base cache if "start" or "end" options are used (e.g. >= 0)', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub(), + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = { + start: 0, + end: 0 + }; + addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { + if (err) { + return done(err); + } + shouldExistBasecache(cache); + done(); + }); + }); + it('give base cache if "start" or "end" options are used (e.g. 10, 9)', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub(), + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = { + start: 10, + end: 9 + }; + addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { + if (err) { + return done(err); + } + shouldExistBasecache(cache); + done(); + }); + }); + + it('give base cache if cache does NOT exist', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub(), + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + addressService.summaryCache = {}; + addressService.summaryCache.get = sinon.stub().callsArgWith(2, new levelup.errors.NotFoundError()); + addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { + if (err) { + return done(err); + } + shouldExistBasecache(cache); + done(); + }); + }); + + it('give base cache if cached tip hash differs (e.g. reorg)', function(done) { + var hash = '000000002c05cc2e78923c34df87fd108b22221ac6076c18f3ade378a4d915e9'; + var testnode = { + services: { + bitcoind: { + on: sinon.stub(), + getBlockIndex: sinon.stub().returns({ + hash: '00000000700e92a916b46b8b91a14d1303d5d91ef0b09eecc3151fb958fd9a2e' + }) + }, + db: { + tip: { + hash: hash + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var txid = '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e'; + var options = {}; + var cache = { + height: 10, + hash: hash, + result: { + totalReceived: 100, + balance: 10, + txids: [txid], + appearanceIds: { + '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e': 9 + } + } + }; + var cacheBuffer = encoding.encodeSummaryCacheValue(cache, 10, hash); + addressService.summaryCache = {}; + addressService.summaryCache.get = sinon.stub().callsArgWith(2, null, cacheBuffer); + addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { + if (err) { + return done(err); + } + shouldExistBasecache(cache); + done(); + }); + }); + + it('handle error from levelup', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub(), + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + addressService.summaryCache = {}; + addressService.summaryCache.get = sinon.stub().callsArgWith(2, new Error('test')); + addressService._getAddressConfirmedSummaryCache(address, options, function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); + + it('call encode and decode with args and result', function(done) { + var hash = '000000002c05cc2e78923c34df87fd108b22221ac6076c18f3ade378a4d915e9'; + var testnode = { + services: { + bitcoind: { + on: sinon.stub(), + getBlockIndex: sinon.stub().returns({ + hash: hash + }) + }, + db: { + tip: { + hash: hash + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var txid = '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e'; + var options = {}; + var cache = { + height: 10, + hash: hash, + result: { + totalReceived: 100, + balance: 10, + txids: [txid], + appearanceIds: { + '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e': 9 + } + } + }; + var cacheBuffer = encoding.encodeSummaryCacheValue(cache, 10, hash); + addressService.summaryCache = {}; + addressService.summaryCache.get = sinon.stub().callsArgWith(2, null, cacheBuffer); + addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { + if (err) { + return done(err); + } + should.exist(cache); + cache.height.should.equal(10); + cache.hash.should.equal(hash); + should.exist(cache.result); + cache.result.totalReceived.should.equal(100); + cache.result.balance.should.equal(10); + cache.result.txids.should.deep.equal([txid]); + cache.result.appearanceIds.should.deep.equal({ + '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e': 9 + }); + done(); + }); + }); + }); + + describe('#_updateAddressConfirmedSummaryCache', function() { + it('will pass partial options to input/output summary query', function(done) { + var tipHeight = 12; + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + var cache = { + height: 10, + result: { + txids: [] + } + }; + as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, cache); + as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, cache); + as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); + as._saveAddressConfirmedSummaryCache = sinon.stub().callsArg(3, null, cache); + as._updateAddressConfirmedSummaryCache(address, options, cache, tipHeight, function(err, cache) { + if (err) { + return done(err); + } + as._getAddressConfirmedInputsSummary.callCount.should.equal(1); + as._getAddressConfirmedOutputsSummary.callCount.should.equal(1); + + as._getAddressConfirmedInputsSummary.args[0][2].start.should.equal(12); + as._getAddressConfirmedInputsSummary.args[0][2].end.should.equal(11); + + as._getAddressConfirmedOutputsSummary.args[0][2].start.should.equal(12); + as._getAddressConfirmedOutputsSummary.args[0][2].end.should.equal(11); + done(); + }); + }); + + it('will save cache if exceeds threshold and is NOT height query', function(done) { + var tipHeight = 12; + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + var cache = { + height: 10, + result: { + txids: [ + '9a816264c50910cbf57aa4637dde5f7fec03df642b822661e8bc9710475986b6', + '05c6b9ccf3fc4026391bf8f8a64b4784a95b930851359b8f85a4be7bb6bf6f1e' + ] + } + }; + as.summaryCacheThreshold = 1; + as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, cache); + as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, cache); + as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); + as._saveAddressConfirmedSummaryCache = sinon.stub().callsArg(3, null, cache); + as._updateAddressConfirmedSummaryCache(address, options, cache, tipHeight, function(err) { + if (err) { + return done(err); + } + as._saveAddressConfirmedSummaryCache.callCount.should.equal(1); + done(); + }); + }); + + it('will NOT save cache if exceeds threshold and IS height query', function(done) { + var tipHeight = 12; + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + var cache = { + result: { + txids: [] + } + }; + as.summaryCacheThreshold = 1; + as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, cache); + as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, cache); + as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); + as._saveAddressConfirmedSummaryCache = sinon.stub().callsArg(3, null, cache); + as._updateAddressConfirmedSummaryCache(address, options, cache, tipHeight, function(err) { + if (err) { + return done(err); + } + as._saveAddressConfirmedSummaryCache.callCount.should.equal(0); + done(); + }); + }); + + }); + + describe('#_getAddressConfirmedInputsSummary', function() { + it('will stream inputs and collect txids', function(done) { + var streamStub = new stream.Readable(); + streamStub._read = function() { /* do nothing */ }; + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var cache = { + height: 10, + result: { + appearanceIds: {} + } + }; + var options = {}; + var txid = 'f2cfc19d13f0c12199f70e420d84e2b3b1d4e499702aa9d737f8c24559c9ec47'; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + as.createInputsStream = sinon.stub().returns(streamStub); + as._getAddressConfirmedInputsSummary(address, cache, options, function(err, cache) { + if (err) { + return done(err); + } + cache.result.appearanceIds[txid].should.equal(10); + done(); + }); + + streamStub.emit('data', { + txid: txid, + height: 10 + }); + streamStub.push(null); + }); + it('handle stream error', function(done) { + var streamStub = new stream.Readable(); + streamStub._read = function() { /* do nothing */ }; + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var cache = {}; + var options = {}; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + as.createInputsStream = sinon.stub().returns(streamStub); + as._getAddressConfirmedInputsSummary(address, cache, options, function(err, cache) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + + streamStub.emit('error', new Error('test')); + streamStub.push(null); + }); + }); + + describe('#_getAddressConfirmedOutputsSummary', function() { + it('will stream inputs and collect txids', function(done) { + var streamStub = new stream.Readable(); + streamStub._read = function() { /* do nothing */ }; + var testnode = { + services: { + bitcoind: { + on: sinon.stub(), + isSpent: sinon.stub().returns(false) + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var cache = { + height: 10, + result: { + appearanceIds: {}, + unconfirmedAppearanceIds: {}, + balance: 0, + totalReceived: 0, + unconfirmedBalance: 0 + } + }; + var options = { + queryMempool: true + }; + var txid = 'f2cfc19d13f0c12199f70e420d84e2b3b1d4e499702aa9d737f8c24559c9ec47'; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + + as.createOutputsStream = sinon.stub().returns(streamStub); + + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey(new Buffer(txid, 'hex'), 2); + as.mempoolSpentIndex[spentIndexSyncKey] = true; + + as._getAddressConfirmedOutputsSummary(address, cache, options, function(err, cache) { + if (err) { + return done(err); + } + cache.result.appearanceIds[txid].should.equal(10); + cache.result.balance.should.equal(1000); + cache.result.totalReceived.should.equal(1000); + cache.result.unconfirmedBalance.should.equal(-1000); + done(); + }); + + streamStub.emit('data', { + txid: txid, + height: 10, + outputIndex: 2, + satoshis: 1000 + }); + streamStub.push(null); + }); + it('handle stream error', function(done) { + var streamStub = new stream.Readable(); + streamStub._read = function() { /* do nothing */ }; + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var cache = { + height: 10, + result: { + appearanceIds: {}, + unconfirmedAppearanceIds: {}, + balance: 0, + totalReceived: 0, + unconfirmedBalance: 0 + } + }; + var options = {}; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + as.createOutputsStream = sinon.stub().returns(streamStub); + as._getAddressConfirmedOutputsSummary(address, cache, options, function(err, cache) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + + streamStub.emit('error', new Error('test')); + streamStub.push(null); + }); + }); + + describe.skip('#_setAndSortTxidsFromAppearanceIds', function() { + }); + + describe.skip('#_saveAddressConfirmedSummaryCache', function() { + }); + + describe.skip('#_getAddressMempoolSummary', function() { + }); + + describe.skip('#_transformAddressSummaryFromCache', function() { + }); + }); From ead6c2f45fc768ca7792a64c5d7456edc4591745 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 14 Jan 2016 17:07:44 -0500 Subject: [PATCH 09/20] Address Service: Removed caching and added max query limits Querying addresses that have millions of transactions is supported however takes hundreds of seconds to fully calculate the balance. Creating a cache of previous results wasn't currently working because the `isSpent` query is always based on the current bitcoind tip. Thus the balance of the outputs would be included however wouldn't be removed when spent as the output wouldn't be checked again when querying for blocks past the last checkpoint. Including the satoshis in the inputs address index would make it possible to subtract the spent amount, however this degrades optimizations elsewhere. The syncing times or querying for addresses with 10,000 transactions per address. It may preferrable to have an additional address service that handles high-volume addresses be on an opt-in basis so that a custom running client could select high volume addresses to create optimizations for querying balances and history. The strategies for creating indexes differs on these use cases. --- lib/services/address/index.js | 219 ++++-------- test/services/address/index.unit.js | 511 ++-------------------------- 2 files changed, 92 insertions(+), 638 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 22c2420f..829d8414 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -44,12 +44,10 @@ var AddressService = function(options) { this.node.services.bitcoind.on('tx', this.transactionHandler.bind(this)); this.node.services.bitcoind.on('txleave', this.transactionLeaveHandler.bind(this)); - this.summaryCacheThreshold = options.summaryCacheThreshold || constants.SUMMARY_CACHE_THRESHOLD; this.maxInputsQueryLength = options.maxInputsQueryLength || constants.MAX_INPUTS_QUERY_LENGTH; this.maxOutputsQueryLength = options.maxOutputsQueryLength || constants.MAX_OUTPUTS_QUERY_LENGTH; this._setMempoolIndexPath(); - this._setSummaryCachePath(); if (options.mempoolMemoryIndex) { this.levelupStore = memdown; } else { @@ -98,19 +96,6 @@ AddressService.prototype.start = function(callback) { }, next ); - }, - function(next) { - self.summaryCache = levelup( - self.summaryCachePath, - { - db: self.levelupStore, - keyEncoding: 'binary', - valueEncoding: 'binary', - fillCache: false, - maxOpenFiles: 200 - }, - next - ); } ], callback); @@ -121,14 +106,6 @@ AddressService.prototype.stop = function(callback) { this.mempoolIndex.close(callback); }; -/** - * This function will set `this.summaryCachePath` based on `this.node.network`. - * @private - */ -AddressService.prototype._setSummaryCachePath = function() { - this.summaryCachePath = this._getDBPathFor('bitcore-addresssummary.db'); -}; - /** * This function will set `this.mempoolIndexPath` based on `this.node.network`. * @private @@ -1357,21 +1334,20 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb function(next) { self._getAddressConfirmedSummary(address, options, next); }, - function(cache, next) { - self._getAddressMempoolSummary(address, options, cache, next); + function(result, next) { + self._getAddressMempoolSummary(address, options, result, next); } - ], function(err, cache) { + ], function(err, result) { if (err) { return callback(err); } - var summary = self._transformAddressSummaryFromCache(cache, options); + var summary = self._transformAddressSummaryFromCache(result, options); var timeDelta = new Date() - startTime; if (timeDelta > 5000) { var seconds = Math.round(timeDelta / 1000); log.warn('Slow (' + seconds + 's) getAddressSummary request for address: ' + address.toString()); - log.warn('Address Summary:', summary); } callback(null, summary); @@ -1382,108 +1358,48 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb AddressService.prototype._getAddressConfirmedSummary = function(address, options, callback) { var self = this; - var tipHeight = this.node.services.db.tip.__height; - - self._getAddressConfirmedSummaryCache(address, options, function(err, cache) { - if (err) { - return callback(err); - } - // Immediately give cache is already current, otherwise update - if (cache && cache.height === tipHeight) { - return callback(null, cache); - } - self._updateAddressConfirmedSummaryCache(address, options, cache, tipHeight, callback); - }); -}; - -AddressService.prototype._getAddressConfirmedSummaryCache = function(address, options, callback) { - var self = this; - var baseCache = { - result: { - appearanceIds: {}, - totalReceived: 0, - balance: 0, - unconfirmedAppearanceIds: {}, - unconfirmedBalance: 0 - } + var baseResult = { + appearanceIds: {}, + totalReceived: 0, + balance: 0, + unconfirmedAppearanceIds: {}, + unconfirmedBalance: 0 }; - // Use the base cache if the "start" and "end" options have been used - // We only save and retrieve a cache for the summary of all history - if (options.start >= 0 || options.end >= 0) { - return callback(null, baseCache); - } - var key = encoding.encodeSummaryCacheKey(address); - this.summaryCache.get(key, { - valueEncoding: 'binary', - keyEncoding: 'binary' - }, function(err, buffer) { - if (err instanceof levelup.errors.NotFoundError) { - return callback(null, baseCache); - } else if (err) { - return callback(err); - } - var cache = encoding.decodeSummaryCacheValue(buffer); - - // Use base cache if the cached tip/height doesn't match (e.g. there has been a reorg) - var blockIndex = self.node.services.bitcoind.getBlockIndex(cache.height); - if (cache.hash !== blockIndex.hash) { - return callback(null, baseCache); - } - - callback(null, cache); - }); -}; - -AddressService.prototype._updateAddressConfirmedSummaryCache = function(address, options, cache, tipHeight, callback) { - var self = this; - - var optionsPartial = _.clone(options); - var isHeightQuery = (options.start >= 0 || options.end >= 0); - if (!isHeightQuery) { - // We will pick up from the last point cached and query for all blocks - // proceeding the cache - var cacheHeight = _.isUndefined(cache.height) ? 0 : cache.height + 1; - optionsPartial.start = tipHeight; - optionsPartial.end = cacheHeight; - } else { - $.checkState(_.isUndefined(cache.height)); - } async.waterfall([ function(next) { - self._getAddressConfirmedInputsSummary(address, cache, optionsPartial, next); + self._getAddressConfirmedInputsSummary(address, baseResult, options, next); }, - function(cache, next) { - self._getAddressConfirmedOutputsSummary(address, cache, optionsPartial, next); + function(result, next) { + self._getAddressConfirmedOutputsSummary(address, result, options, next); }, - function(cache, next) { - self._setAndSortTxidsFromAppearanceIds(cache, next); + function(result, next) { + self._setAndSortTxidsFromAppearanceIds(result, next); } - ], function(err, cache) { + ], callback); - // Skip saving the cache if the "start" or "end" options have been used, or - // if the transaction length does not exceed the caching threshold. - // We only want to cache full history results for addresses that have a large - // number of transactions. - var exceedsCacheThreshold = (cache.result.txids.length > self.summaryCacheThreshold); - if (exceedsCacheThreshold && !isHeightQuery) { - self._saveAddressConfirmedSummaryCache(address, cache, tipHeight, callback); - } else { - callback(null, cache); - } - - }); }; -AddressService.prototype._getAddressConfirmedInputsSummary = function(address, cache, options, callback) { +AddressService.prototype._getAddressConfirmedInputsSummary = function(address, result, options, callback) { $.checkArgument(address instanceof Address); var self = this; var error = null; + var count = 0; var inputsStream = self.createInputsStream(address, options); inputsStream.on('data', function(input) { var txid = input.txid; - cache.result.appearanceIds[txid] = input.height; + result.appearanceIds[txid] = input.height; + + count++; + + if (count > self.maxInputsQueryLength) { + log.warn('Tried to query too many inputs (' + self.maxInputsQueryLength + ') for summary of address ' + address.toString()); + error = new Error('Maximum number of inputs (' + self.maxInputsQueryLength + ') per query reached'); + inputsStream.pause(); + inputsStream.end(); + } + }); inputsStream.on('error', function(err) { @@ -1494,17 +1410,18 @@ AddressService.prototype._getAddressConfirmedInputsSummary = function(address, c if (error) { return callback(error); } - callback(null, cache); + callback(null, result); }); }; -AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, cache, options, callback) { +AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, result, options, callback) { $.checkArgument(address instanceof Address); - $.checkArgument(!_.isUndefined(cache.result) && - !_.isUndefined(cache.result.appearanceIds) && - !_.isUndefined(cache.result.unconfirmedAppearanceIds)); + $.checkArgument(!_.isUndefined(result) && + !_.isUndefined(result.appearanceIds) && + !_.isUndefined(result.unconfirmedAppearanceIds)); var self = this; + var count = 0; var outputStream = self.createOutputsStream(address, options); @@ -1515,13 +1432,12 @@ AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, // Bitcoind's isSpent only works for confirmed transactions var spentDB = self.node.services.bitcoind.isSpent(txid, outputIndex); - cache.result.totalReceived += output.satoshis; - cache.result.appearanceIds[txid] = output.height; + result.totalReceived += output.satoshis; + result.appearanceIds[txid] = output.height; if (!spentDB) { - cache.result.balance += output.satoshis; + result.balance += output.satoshis; } - // TODO: subtract if spent (because of cache)? if (options.queryMempool) { // Check to see if this output is spent in the mempool and if so @@ -1532,10 +1448,19 @@ AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, ); var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; if (spentMempool) { - cache.result.unconfirmedBalance -= output.satoshis; + result.unconfirmedBalance -= output.satoshis; } } + count++; + + if (count > self.maxOutputsQueryLength) { + log.warn('Tried to query too many outputs (' + self.maxOutputsQueryLength + ') for summary of address ' + address.toString()); + error = new Error('Maximum number of outputs (' + self.maxOutputsQueryLength + ') per query reached'); + outputStream.pause(); + outputStream.end(); + } + }); var error = null; @@ -1548,40 +1473,25 @@ AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, if (error) { return callback(error); } - callback(null, cache); + callback(null, result); }); }; -AddressService.prototype._setAndSortTxidsFromAppearanceIds = function(cache, callback) { - cache.result.txids = Object.keys(cache.result.appearanceIds); - cache.result.txids.sort(function(a, b) { - return cache.result.appearanceIds[a] - cache.result.appearanceIds[b]; +AddressService.prototype._setAndSortTxidsFromAppearanceIds = function(result, callback) { + result.txids = Object.keys(result.appearanceIds); + result.txids.sort(function(a, b) { + return result.appearanceIds[a] - result.appearanceIds[b]; }); - callback(null, cache); + callback(null, result); }; -AddressService.prototype._saveAddressConfirmedSummaryCache = function(address, cache, tipHeight, callback) { - - log.info('Saving address summary cache for: ' + address.toString() + 'at height: ' + tipHeight); - var key = encoding.encodeSummaryCacheKey(address); - var tipBlockIndex = this.node.services.bitcoind.getBlockIndex(tipHeight); - var value = encoding.encodeSummaryCacheValue(cache, tipHeight, tipBlockIndex.hash); - this.summaryCache.put(key, value, function(err) { - if (err) { - return callback(err); - } - callback(null, cache); - }); - -}; - -AddressService.prototype._getAddressMempoolSummary = function(address, options, cache, callback) { +AddressService.prototype._getAddressMempoolSummary = function(address, options, result, callback) { var self = this; // Skip if the options do not want to include the mempool if (!options.queryMempool) { - return callback(null, cache); + return callback(null, result); } var addressStr = address.toString(); @@ -1596,12 +1506,12 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, } for(var i = 0; i < mempoolInputs.length; i++) { var input = mempoolInputs[i]; - cache.result.unconfirmedAppearanceIds[input.txid] = input.timestamp; + result.unconfirmedAppearanceIds[input.txid] = input.timestamp; } - next(null, cache); + next(null, result); }); - }, function(cache, next) { + }, function(result, next) { self._getOutputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolOutputs) { if (err) { return next(err); @@ -1609,7 +1519,7 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, for(var i = 0; i < mempoolOutputs.length; i++) { var output = mempoolOutputs[i]; - cache.result.unconfirmedAppearanceIds[output.txid] = output.timestamp; + result.unconfirmedAppearanceIds[output.txid] = output.timestamp; var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( new Buffer(output.txid, 'hex'), // TODO: get buffer directly @@ -1618,19 +1528,18 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; // Only add this to the balance if it's not spent in the mempool already if (!spentMempool) { - cache.result.unconfirmedBalance += output.satoshis; + result.unconfirmedBalance += output.satoshis; } } - next(null, cache); + next(null, result); }); } ], callback); }; -AddressService.prototype._transformAddressSummaryFromCache = function(cache, options) { +AddressService.prototype._transformAddressSummaryFromCache = function(result, options) { - var result = cache.result; - var confirmedTxids = cache.result.txids; + var confirmedTxids = result.txids; var unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); var summary = { diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index 193a9f8a..a50f121b 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -100,7 +100,7 @@ describe('Address Service', function() { done(); }); }); - it('start levelup db for mempool and summary index', function(done) { + it('start levelup db for mempool', function(done) { var levelupStub = sinon.stub().callsArg(2); var TestAddressService = proxyquire('../../../lib/services/address', { 'fs': { @@ -117,7 +117,7 @@ describe('Address Service', function() { node: mocknode }); am.start(function() { - levelupStub.callCount.should.equal(2); + levelupStub.callCount.should.equal(1); var dbPath1 = levelupStub.args[0][0]; dbPath1.should.equal('testdir/testnet3/bitcore-addressmempool.db'); var options = levelupStub.args[0][1]; @@ -125,8 +125,6 @@ describe('Address Service', function() { options.keyEncoding.should.equal('binary'); options.valueEncoding.should.equal('binary'); options.fillCache.should.equal(false); - var dbPath2 = levelupStub.args[1][0]; - dbPath2.should.equal('testdir/testnet3/bitcore-addresssummary.db'); done(); }); }); @@ -2115,457 +2113,14 @@ describe('Address Service', function() { addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(3, null, cache); addressService._transformAddressSummaryFromCache = sinon.stub().returns(summary); addressService.getAddressSummary(address, options, function() { - log.warn.callCount.should.equal(2); + log.warn.callCount.should.equal(1); done(); }); clock.tick(6000); }); }); - describe('#_getAddressConfirmedSummary', function() { - it('handle error from _getAddressConfirmedSummaryCache', function(done) { - var testnode = { - services: { - bitcoind: { - on: sinon.stub() - }, - db: { - tip: { - __height: 10 - } - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; - var options = {}; - addressService._getAddressConfirmedSummaryCache = sinon.stub().callsArgWith(2, new Error('test')); - addressService._getAddressConfirmedSummary(address, options, function(err) { - should.exist(err); - err.message.should.equal('test'); - done(); - }); - }); - - it('will NOT update cache if matches current tip', function(done) { - var testnode = { - services: { - bitcoind: { - on: sinon.stub() - }, - db: { - tip: { - __height: 10 - } - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; - var options = {}; - var cache = { - height: 10 - }; - addressService._updateAddressConfirmedSummaryCache = sinon.stub(); - addressService._getAddressConfirmedSummaryCache = sinon.stub().callsArgWith(2, null, cache); - addressService._getAddressConfirmedSummary(address, options, function(err, cache) { - if (err) { - return done(err); - } - should.exist(cache); - addressService._updateAddressConfirmedSummaryCache.callCount.should.equal(0); - done(); - }); - }); - - it('will call _updateAddressConfirmedSummaryCache with correct arguments', function(done) { - var testnode = { - services: { - bitcoind: { - on: sinon.stub() - }, - db: { - tip: { - __height: 11 - } - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; - var options = {}; - var cache = { - height: 10 - }; - addressService._updateAddressConfirmedSummaryCache = sinon.stub().callsArgWith(4, null, cache); - addressService._getAddressConfirmedSummaryCache = sinon.stub().callsArgWith(2, null, cache); - addressService._getAddressConfirmedSummary(address, options, function(err, cache) { - if (err) { - return done(err); - } - should.exist(cache); - addressService._updateAddressConfirmedSummaryCache.callCount.should.equal(1); - var args = addressService._updateAddressConfirmedSummaryCache.args[0]; - args[0].should.equal(address); - args[1].should.equal(options); - args[2].should.equal(cache); - args[3].should.equal(11); - done(); - }); - }); - }); - - describe('#_getAddressConfirmedSummaryCache', function() { - function shouldExistBasecache(cache) { - should.exist(cache); - should.not.exist(cache.height); - should.exist(cache.result); - cache.result.appearanceIds.should.deep.equal({}); - cache.result.totalReceived.should.equal(0); - cache.result.balance.should.equal(0); - cache.result.unconfirmedAppearanceIds.should.deep.equal({}); - cache.result.unconfirmedBalance.should.equal(0); - } - it('give base cache if "start" or "end" options are used (e.g. >= 0)', function(done) { - var testnode = { - services: { - bitcoind: { - on: sinon.stub(), - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = { - start: 0, - end: 0 - }; - addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { - if (err) { - return done(err); - } - shouldExistBasecache(cache); - done(); - }); - }); - it('give base cache if "start" or "end" options are used (e.g. 10, 9)', function(done) { - var testnode = { - services: { - bitcoind: { - on: sinon.stub(), - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = { - start: 10, - end: 9 - }; - addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { - if (err) { - return done(err); - } - shouldExistBasecache(cache); - done(); - }); - }); - - it('give base cache if cache does NOT exist', function(done) { - var testnode = { - services: { - bitcoind: { - on: sinon.stub(), - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = {}; - addressService.summaryCache = {}; - addressService.summaryCache.get = sinon.stub().callsArgWith(2, new levelup.errors.NotFoundError()); - addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { - if (err) { - return done(err); - } - shouldExistBasecache(cache); - done(); - }); - }); - - it('give base cache if cached tip hash differs (e.g. reorg)', function(done) { - var hash = '000000002c05cc2e78923c34df87fd108b22221ac6076c18f3ade378a4d915e9'; - var testnode = { - services: { - bitcoind: { - on: sinon.stub(), - getBlockIndex: sinon.stub().returns({ - hash: '00000000700e92a916b46b8b91a14d1303d5d91ef0b09eecc3151fb958fd9a2e' - }) - }, - db: { - tip: { - hash: hash - } - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var txid = '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e'; - var options = {}; - var cache = { - height: 10, - hash: hash, - result: { - totalReceived: 100, - balance: 10, - txids: [txid], - appearanceIds: { - '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e': 9 - } - } - }; - var cacheBuffer = encoding.encodeSummaryCacheValue(cache, 10, hash); - addressService.summaryCache = {}; - addressService.summaryCache.get = sinon.stub().callsArgWith(2, null, cacheBuffer); - addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { - if (err) { - return done(err); - } - shouldExistBasecache(cache); - done(); - }); - }); - - it('handle error from levelup', function(done) { - var testnode = { - services: { - bitcoind: { - on: sinon.stub(), - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = {}; - addressService.summaryCache = {}; - addressService.summaryCache.get = sinon.stub().callsArgWith(2, new Error('test')); - addressService._getAddressConfirmedSummaryCache(address, options, function(err) { - should.exist(err); - err.message.should.equal('test'); - done(); - }); - }); - - it('call encode and decode with args and result', function(done) { - var hash = '000000002c05cc2e78923c34df87fd108b22221ac6076c18f3ade378a4d915e9'; - var testnode = { - services: { - bitcoind: { - on: sinon.stub(), - getBlockIndex: sinon.stub().returns({ - hash: hash - }) - }, - db: { - tip: { - hash: hash - } - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var txid = '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e'; - var options = {}; - var cache = { - height: 10, - hash: hash, - result: { - totalReceived: 100, - balance: 10, - txids: [txid], - appearanceIds: { - '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e': 9 - } - } - }; - var cacheBuffer = encoding.encodeSummaryCacheValue(cache, 10, hash); - addressService.summaryCache = {}; - addressService.summaryCache.get = sinon.stub().callsArgWith(2, null, cacheBuffer); - addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { - if (err) { - return done(err); - } - should.exist(cache); - cache.height.should.equal(10); - cache.hash.should.equal(hash); - should.exist(cache.result); - cache.result.totalReceived.should.equal(100); - cache.result.balance.should.equal(10); - cache.result.txids.should.deep.equal([txid]); - cache.result.appearanceIds.should.deep.equal({ - '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e': 9 - }); - done(); - }); - }); - }); - - describe('#_updateAddressConfirmedSummaryCache', function() { - it('will pass partial options to input/output summary query', function(done) { - var tipHeight = 12; - var testnode = { - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = {}; - var cache = { - height: 10, - result: { - txids: [] - } - }; - as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, cache); - as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, cache); - as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); - as._saveAddressConfirmedSummaryCache = sinon.stub().callsArg(3, null, cache); - as._updateAddressConfirmedSummaryCache(address, options, cache, tipHeight, function(err, cache) { - if (err) { - return done(err); - } - as._getAddressConfirmedInputsSummary.callCount.should.equal(1); - as._getAddressConfirmedOutputsSummary.callCount.should.equal(1); - - as._getAddressConfirmedInputsSummary.args[0][2].start.should.equal(12); - as._getAddressConfirmedInputsSummary.args[0][2].end.should.equal(11); - - as._getAddressConfirmedOutputsSummary.args[0][2].start.should.equal(12); - as._getAddressConfirmedOutputsSummary.args[0][2].end.should.equal(11); - done(); - }); - }); - - it('will save cache if exceeds threshold and is NOT height query', function(done) { - var tipHeight = 12; - var testnode = { - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = {}; - var cache = { - height: 10, - result: { - txids: [ - '9a816264c50910cbf57aa4637dde5f7fec03df642b822661e8bc9710475986b6', - '05c6b9ccf3fc4026391bf8f8a64b4784a95b930851359b8f85a4be7bb6bf6f1e' - ] - } - }; - as.summaryCacheThreshold = 1; - as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, cache); - as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, cache); - as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); - as._saveAddressConfirmedSummaryCache = sinon.stub().callsArg(3, null, cache); - as._updateAddressConfirmedSummaryCache(address, options, cache, tipHeight, function(err) { - if (err) { - return done(err); - } - as._saveAddressConfirmedSummaryCache.callCount.should.equal(1); - done(); - }); - }); - - it('will NOT save cache if exceeds threshold and IS height query', function(done) { - var tipHeight = 12; - var testnode = { - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = {}; - var cache = { - result: { - txids: [] - } - }; - as.summaryCacheThreshold = 1; - as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, cache); - as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, cache); - as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); - as._saveAddressConfirmedSummaryCache = sinon.stub().callsArg(3, null, cache); - as._updateAddressConfirmedSummaryCache(address, options, cache, tipHeight, function(err) { - if (err) { - return done(err); - } - as._saveAddressConfirmedSummaryCache.callCount.should.equal(0); - done(); - }); - }); - + describe.skip('#_getAddressConfirmedSummary', function() { }); describe('#_getAddressConfirmedInputsSummary', function() { @@ -2584,21 +2139,18 @@ describe('Address Service', function() { mempoolMemoryIndex: true, node: testnode }); - var cache = { - height: 10, - result: { - appearanceIds: {} - } + var result = { + appearanceIds: {} }; var options = {}; var txid = 'f2cfc19d13f0c12199f70e420d84e2b3b1d4e499702aa9d737f8c24559c9ec47'; var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); as.createInputsStream = sinon.stub().returns(streamStub); - as._getAddressConfirmedInputsSummary(address, cache, options, function(err, cache) { + as._getAddressConfirmedInputsSummary(address, result, options, function(err, result) { if (err) { return done(err); } - cache.result.appearanceIds[txid].should.equal(10); + result.appearanceIds[txid].should.equal(10); done(); }); @@ -2655,16 +2207,14 @@ describe('Address Service', function() { mempoolMemoryIndex: true, node: testnode }); - var cache = { - height: 10, - result: { - appearanceIds: {}, - unconfirmedAppearanceIds: {}, - balance: 0, - totalReceived: 0, - unconfirmedBalance: 0 - } + var result = { + appearanceIds: {}, + unconfirmedAppearanceIds: {}, + balance: 0, + totalReceived: 0, + unconfirmedBalance: 0 }; + var options = { queryMempool: true }; @@ -2676,14 +2226,14 @@ describe('Address Service', function() { var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey(new Buffer(txid, 'hex'), 2); as.mempoolSpentIndex[spentIndexSyncKey] = true; - as._getAddressConfirmedOutputsSummary(address, cache, options, function(err, cache) { + as._getAddressConfirmedOutputsSummary(address, result, options, function(err, cache) { if (err) { return done(err); } - cache.result.appearanceIds[txid].should.equal(10); - cache.result.balance.should.equal(1000); - cache.result.totalReceived.should.equal(1000); - cache.result.unconfirmedBalance.should.equal(-1000); + result.appearanceIds[txid].should.equal(10); + result.balance.should.equal(1000); + result.totalReceived.should.equal(1000); + result.unconfirmedBalance.should.equal(-1000); done(); }); @@ -2710,20 +2260,18 @@ describe('Address Service', function() { mempoolMemoryIndex: true, node: testnode }); - var cache = { - height: 10, - result: { - appearanceIds: {}, - unconfirmedAppearanceIds: {}, - balance: 0, - totalReceived: 0, - unconfirmedBalance: 0 - } + var result = { + appearanceIds: {}, + unconfirmedAppearanceIds: {}, + balance: 0, + totalReceived: 0, + unconfirmedBalance: 0 }; + var options = {}; var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); as.createOutputsStream = sinon.stub().returns(streamStub); - as._getAddressConfirmedOutputsSummary(address, cache, options, function(err, cache) { + as._getAddressConfirmedOutputsSummary(address, result, options, function(err, cache) { should.exist(err); err.message.should.equal('test'); done(); @@ -2737,9 +2285,6 @@ describe('Address Service', function() { describe.skip('#_setAndSortTxidsFromAppearanceIds', function() { }); - describe.skip('#_saveAddressConfirmedSummaryCache', function() { - }); - describe.skip('#_getAddressMempoolSummary', function() { }); From e79c00db105ab8dd77308e28393be3453d73543a Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 15 Jan 2016 12:42:10 -0500 Subject: [PATCH 10/20] Address Service: Updated tests and fixed various bugs --- lib/services/address/history.js | 85 ++++--- lib/services/address/index.js | 13 +- test/services/address/history.unit.js | 259 ++++---------------- test/services/address/index.unit.js | 327 +++++++++++++++++++++++++- 4 files changed, 424 insertions(+), 260 deletions(-) diff --git a/lib/services/address/history.js b/lib/services/address/history.js index eb689828..7c0cfc00 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -50,7 +50,7 @@ AddressHistory.prototype._mergeAndSortTxids = function(summaries) { delete summary.appearanceIds[key]; } for (var unconfirmedKey in summary.unconfirmedAppearanceIds) { - unconfirmedAppearanceIds[unconfirmedKey] = summary.unconfirmedAppearanceIds[key]; + unconfirmedAppearanceIds[unconfirmedKey] = summary.unconfirmedAppearanceIds[unconfirmedKey]; delete summary.unconfirmedAppearanceIds[key]; } } @@ -74,8 +74,6 @@ AddressHistory.prototype._mergeAndSortTxids = function(summaries) { */ AddressHistory.prototype.get = function(callback) { var self = this; - var totalCount; - if (this.addresses.length > this.maxAddressesQuery) { return callback(new Error('Maximum number of addresses (' + this.maxAddressQuery + ') exceeded')); } @@ -86,7 +84,7 @@ AddressHistory.prototype.get = function(callback) { if (err) { return callback(err); } - return finish(summary.txids); + return self.finish.call(self, summary.txids, callback); }); } else { var opts = _.clone(this.options); @@ -101,53 +99,54 @@ AddressHistory.prototype.get = function(callback) { return callback(err); } var txids = self._mergeAndSortTxids(summaries); - return finish(txids); + return self.finish.call(self, txids, callback); } ); } - function finish(allTxids) { - totalCount = allTxids.length; +}; - // Slice the page starting with the most recent - var txids; - if (self.options.from >= 0 && self.options.to >= 0) { - var fromOffset = totalCount - self.options.from; - var toOffset = totalCount - self.options.to; - txids = allTxids.slice(toOffset, fromOffset); - } else { - txids = allTxids; - } - - // Verify that this query isn't too long - if (txids.length > self.maxHistoryQueryLength) { - return callback(new Error( - 'Maximum length query (' + self.maxAddressQueryLength + ') exceeded for addresses:' + - self.address.join(',') - )); - } - - // Reverse to include most recent at the top - txids.reverse(); - - async.eachSeries( - txids, - function(txid, next) { - self.getDetailedInfo(txid, next); - }, - function(err) { - if (err) { - return callback(err); - } - callback(null, { - totalCount: totalCount, - items: self.detailedArray - }); - } - ); +AddressHistory.prototype.finish = function(allTxids, callback) { + var self = this; + var totalCount = allTxids.length; + // Slice the page starting with the most recent + var txids; + if (self.options.from >= 0 && self.options.to >= 0) { + var fromOffset = totalCount - self.options.from; + var toOffset = totalCount - self.options.to; + txids = allTxids.slice(toOffset, fromOffset); + } else { + txids = allTxids; } + // Verify that this query isn't too long + if (txids.length > self.maxHistoryQueryLength) { + return callback(new Error( + 'Maximum length query (' + self.maxAddressQueryLength + ') exceeded for addresses:' + + self.address.join(',') + )); + } + + // Reverse to include most recent at the top + txids.reverse(); + + async.eachSeries( + txids, + function(txid, next) { + self.getDetailedInfo(txid, next); + }, + function(err) { + if (err) { + return callback(err); + } + callback(null, { + totalCount: totalCount, + items: self.detailedArray + }); + } + ); + }; /** diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 829d8414..5dfebf57 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -760,6 +760,11 @@ AddressService.prototype.createInputsStream = function(addressStr, options) { inputStream.end(); }).pipe(inputStream); + + inputStream.on('end', function() { + stream.end(); + }); + return stream; }; @@ -976,6 +981,10 @@ AddressService.prototype.createOutputsStream = function(addressStr, options) { }) .pipe(outputStream); + outputStream.on('end', function() { + stream.end(); + }); + return stream; }; @@ -1342,7 +1351,7 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb return callback(err); } - var summary = self._transformAddressSummaryFromCache(result, options); + var summary = self._transformAddressSummaryFromResult(result, options); var timeDelta = new Date() - startTime; if (timeDelta > 5000) { @@ -1537,7 +1546,7 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, ], callback); }; -AddressService.prototype._transformAddressSummaryFromCache = function(result, options) { +AddressService.prototype._transformAddressSummaryFromResult = function(result, options) { var confirmedTxids = result.txids; var unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index 4745c624..37c3d697 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -35,220 +35,65 @@ describe('Address Service History', function() { }); }); - describe('#get', function() { - it('will complete the async each limit series', function(done) { - var addresses = [address]; - var summary = { - txids: [] - }; - var history = new AddressHistory({ - node: { - services: { - address: { - getAddressSummary: sinon.stub().callsArgWith(2, null, summary) - } + describe('#_mergeAndSortTxids', function() { + it('will merge and sort multiple summaries', function() { + var summaries = [ + { + totalReceived: 10000000, + totalSpent: 0, + balance: 10000000, + appearances: 2, + unconfirmedBalance: 20000000, + unconfirmedAppearances: 2, + appearanceIds: { + '56fafeb01961831b926558d040c246b97709fd700adcaa916541270583e8e579': 154, + 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce': 120 + }, + unconfirmedAppearanceIds: { + 'ec94d845c603f292a93b7c829811ac624b76e52b351617ca5a758e9d61a11681': 1452898347406, + 'ed11a08e3102f9610bda44c80c46781d97936a4290691d87244b1b345b39a693': 1452898331964 } }, - options: {}, - addresses: addresses - }); - var expected = [{}]; - history.detailedArray = expected; - history.getDetailedInfo = sinon.stub().callsArg(1); - history.get(function(err, results) { - if (err) { - throw err; + { + totalReceived: 59990000, + totalSpent: 0, + balance: 49990000, + appearances: 3, + unconfirmedBalance: 1000000, + unconfirmedAppearances: 3, + appearanceIds: { + 'bc992ad772eb02864db07ef248d31fb3c6826d25f1153ebf8c79df9b7f70fcf2': 156, + 'f3c1ba3ef86a0420d6102e40e2cfc8682632ab95d09d86a27f5d466b9fa9da47': 152, + 'f637384e9f81f18767ea50e00bce58fc9848b6588a1130529eebba22a410155f': 151 + }, + unconfirmedAppearanceIds: { + 'f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345': 1452897902377, + 'edc080f2084eed362aa488ccc873a24c378dc0979aa29b05767517b70569414a': 1452897971363, + 'f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9': 1452897923107 + } } - history.getDetailedInfo.callCount.should.equal(1); - history.combineTransactionInfo.callCount.should.equal(1); - results.should.deep.equal({ - totalCount: 1, - items: expected - }); - done(); - }); - }); - it('handle an error from getDetailedInfo', function(done) { + ]; + var node = {}; + var options = {}; var addresses = [address]; var history = new AddressHistory({ - node: {}, - options: {}, + node: node, + options: options, addresses: addresses }); - var expected = [{}]; - history.sortedArray = expected; - history.transactionInfo = [{}]; - history.getDetailedInfo = sinon.stub().callsArgWith(1, new Error('test')); - history.get(function(err) { - err.message.should.equal('test'); - done(); - }); - }); - }); - - describe('#_mergeAndSortTxids', function() { - it('will sort latest to oldest using height', function() { - var transactionInfo = [ - { - height: 276328 - }, - { - height: 273845, - }, - { - height: 555655 - }, - { - height: 325496 - }, - { - height: 329186 - }, - { - height: 534195 - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].height.should.equal(555655); - transactionInfo[1].height.should.equal(534195); - transactionInfo[2].height.should.equal(329186); - transactionInfo[3].height.should.equal(325496); - transactionInfo[4].height.should.equal(276328); - transactionInfo[5].height.should.equal(273845); - }); - it('mempool and tip with time in the future', function() { - var transactionInfo = [ - { - timestamp: 1442050425439, - height: 14, - }, - { - timestamp: 1442050424328, - height: -1 - }, - { - timestamp: 1442050424429, - height: -1 - }, - { - timestamp: 1442050425439, - height: 15 - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].height.should.equal(-1); - transactionInfo[0].timestamp.should.equal(1442050424429); - transactionInfo[1].height.should.equal(-1); - transactionInfo[1].timestamp.should.equal(1442050424328); - transactionInfo[2].height.should.equal(15); - transactionInfo[3].height.should.equal(14); - }); - it('tip with time in the future and mempool', function() { - var transactionInfo = [ - { - timestamp: 1442050425439, - height: 14, - }, - { - timestamp: 1442050424328, - height: -1 - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].height.should.equal(-1); - transactionInfo[1].height.should.equal(14); - }); - it('many transactions in the mempool', function() { - var transactionInfo = [ - { - timestamp: 1442259670462, - height: -1 - }, - { - timestamp: 1442259785114, - height: -1 - }, - { - timestamp: 1442259759896, - height: -1 - }, - { - timestamp: 1442259692601, - height: -1 - }, - { - timestamp: 1442259692601, - height: 100 - }, - { - timestamp: 1442259749463, - height: -1 - }, - { - timestamp: 1442259737719, - height: -1 - }, - { - timestamp: 1442259773138, - height: -1, - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].timestamp.should.equal(1442259785114); - transactionInfo[1].timestamp.should.equal(1442259773138); - transactionInfo[2].timestamp.should.equal(1442259759896); - transactionInfo[3].timestamp.should.equal(1442259749463); - transactionInfo[4].timestamp.should.equal(1442259737719); - transactionInfo[5].timestamp.should.equal(1442259692601); - transactionInfo[6].timestamp.should.equal(1442259670462); - transactionInfo[7].height.should.equal(100); - }); - it('mempool and mempool', function() { - var transactionInfo = [ - { - timestamp: 1442050424328, - height: -1 - }, - { - timestamp: 1442050425439, - height: -1, - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].timestamp.should.equal(1442050425439); - transactionInfo[1].timestamp.should.equal(1442050424328); - }); - it('mempool and mempool with the same timestamp', function() { - var transactionInfo = [ - { - timestamp: 1442050425439, - height: -1, - txid: '1', - }, - { - timestamp: 1442050425439, - height: -1, - txid: '2' - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].txid.should.equal('1'); - transactionInfo[1].txid.should.equal('2'); - }); - it('matching block heights', function() { - var transactionInfo = [ - { - height: 325496, - txid: '1', - }, - { - height: 325496, - txid: '2' - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].txid.should.equal('1'); - transactionInfo[1].txid.should.equal('2'); + var txids = history._mergeAndSortTxids(summaries); + txids.should.deep.equal([ + 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce', + 'f637384e9f81f18767ea50e00bce58fc9848b6588a1130529eebba22a410155f', + 'f3c1ba3ef86a0420d6102e40e2cfc8682632ab95d09d86a27f5d466b9fa9da47', + '56fafeb01961831b926558d040c246b97709fd700adcaa916541270583e8e579', + 'bc992ad772eb02864db07ef248d31fb3c6826d25f1153ebf8c79df9b7f70fcf2', + 'f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345', + 'f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9', + 'edc080f2084eed362aa488ccc873a24c378dc0979aa29b05767517b70569414a', + 'ed11a08e3102f9610bda44c80c46781d97936a4290691d87244b1b345b39a693', + 'ec94d845c603f292a93b7c829811ac624b76e52b351617ca5a758e9d61a11681' + ]); }); }); diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index a50f121b..c0635e4b 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -2080,13 +2080,13 @@ describe('Address Service', function() { var summary = {}; addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(3, null, cache); - addressService._transformAddressSummaryFromCache = sinon.stub().returns(summary); + addressService._transformAddressSummaryFromResult = sinon.stub().returns(summary); addressService.getAddressSummary(address, options, function(err, sum) { addressService._getAddressConfirmedSummary.callCount.should.equal(1); addressService._getAddressMempoolSummary.callCount.should.equal(1); addressService._getAddressMempoolSummary.args[0][2].should.equal(cache); - addressService._transformAddressSummaryFromCache.callCount.should.equal(1); - addressService._transformAddressSummaryFromCache.args[0][0].should.equal(cache); + addressService._transformAddressSummaryFromResult.callCount.should.equal(1); + addressService._transformAddressSummaryFromResult.args[0][0].should.equal(cache); sum.should.equal(summary); done(); }); @@ -2111,7 +2111,7 @@ describe('Address Service', function() { addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(3, null, cache); - addressService._transformAddressSummaryFromCache = sinon.stub().returns(summary); + addressService._transformAddressSummaryFromResult = sinon.stub().returns(summary); addressService.getAddressSummary(address, options, function() { log.warn.callCount.should.equal(1); done(); @@ -2120,7 +2120,119 @@ describe('Address Service', function() { }); }); - describe.skip('#_getAddressConfirmedSummary', function() { + describe('#_getAddressConfirmedSummary', function() { + it('will pass arguments correctly', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var result = {}; + as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, result); + as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, result); + as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, result); + as._getAddressConfirmedSummary(address, options, function(err) { + if (err) { + return done(err); + } + var expectedResult = { + appearanceIds: {}, + totalReceived: 0, + balance: 0, + unconfirmedAppearanceIds: {}, + unconfirmedBalance: 0 + }; + as._getAddressConfirmedInputsSummary.args[0][0].should.equal(address); + as._getAddressConfirmedInputsSummary.args[0][1].should.deep.equal(expectedResult); + as._getAddressConfirmedInputsSummary.args[0][2].should.deep.equal(options); + as._getAddressConfirmedOutputsSummary.args[0][0].should.equal(address); + as._getAddressConfirmedOutputsSummary.args[0][1].should.deep.equal(result); + as._getAddressConfirmedOutputsSummary.args[0][2].should.equal(options); + as._setAndSortTxidsFromAppearanceIds.args[0][0].should.equal(result); + done(); + }); + }); + it('will pass error correctly (inputs)', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var result = {}; + as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, new Error('test')); + as._getAddressConfirmedSummary(address, options, function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); + it('will pass error correctly (outputs)', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var result = {}; + as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, result); + as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, new Error('test')); + as._getAddressConfirmedSummary(address, options, function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); + it('will pass error correctly (sort)', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var result = {}; + as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, result); + as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, result); + as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, new Error('test')); + as._getAddressConfirmedSummary(address, options, function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); }); describe('#_getAddressConfirmedInputsSummary', function() { @@ -2282,13 +2394,212 @@ describe('Address Service', function() { }); }); - describe.skip('#_setAndSortTxidsFromAppearanceIds', function() { + describe('#_setAndSortTxidsFromAppearanceIds', function() { + it('will sort correctly', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var result = { + appearanceIds: { + '22488dbb99aed86e7081ac480e3459fa40ccab7ee18bef98b84b3cdce6bf05be': 200, + '1c413601acbd608240fc635b95886c3c1f76ec8589c3392a58b5715ceb618e93': 100, + '206d3834c010d46a2cf478cb1c5fe252be41f683c8a738e3ebe27f1aae67f505': 101 + } + }; + as._setAndSortTxidsFromAppearanceIds(result, function(err, result) { + if (err) { + return done(err); + } + should.exist(result.txids); + result.txids[0].should.equal('1c413601acbd608240fc635b95886c3c1f76ec8589c3392a58b5715ceb618e93'); + result.txids[1].should.equal('206d3834c010d46a2cf478cb1c5fe252be41f683c8a738e3ebe27f1aae67f505'); + result.txids[2].should.equal('22488dbb99aed86e7081ac480e3459fa40ccab7ee18bef98b84b3cdce6bf05be'); + done(); + }); + }); }); - describe.skip('#_getAddressMempoolSummary', function() { + describe('#_getAddressMempoolSummary', function() { + it('skip if options not enabled', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var resultBase = { + unconfirmedAppearanceIds: {}, + unconfirmedBalance: 0 + }; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + as._getAddressMempoolSummary(address, options, resultBase, function(err, result) { + if (err) { + return done(err); + } + Object.keys(result.unconfirmedAppearanceIds).length.should.equal(0); + result.unconfirmedBalance.should.equal(0); + done(); + }); + }); + it('include all txids and balance from inputs and outputs', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var resultBase = { + unconfirmedAppearanceIds: {}, + unconfirmedBalance: 0 + }; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = { + queryMempool: true + }; + var mempoolInputs = [ + { + address: '3NbU8XzUgKyuCgYgZEKsBtUvkTm2r7Xgwj', + hashType: 'scripthash', + txid: '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8', + inputIndex: 0, + timestamp: 1452874536321, + height: -1, + confirmations: 0 + } + ]; + var mempoolOutputs = [ + { + address: '3NbU8XzUgKyuCgYgZEKsBtUvkTm2r7Xgwj', + hashType: 'scripthash', + txid: '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d', + outputIndex: 0, + height: -1, + timestamp: 1452874521466, + satoshis: 131368318, + script: '76a9148c66db6e9f74b1db9c400eaa2aed3743417f38e688ac', + confirmations: 0 + }, + { + address: '3NbU8XzUgKyuCgYgZEKsBtUvkTm2r7Xgwj', + hashType: 'scripthash', + txid: '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247', + outputIndex: 0, + height: -1, + timestamp: 1452874521466, + satoshis: 131368318, + script: '76a9148c66db6e9f74b1db9c400eaa2aed3743417f38e688ac', + confirmations: 0 + } + ]; + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( + new Buffer(mempoolOutputs[1].txid, 'hex'), + 0 + ); + as.mempoolSpentIndex[spentIndexSyncKey] = true; + as._getInputsMempool = sinon.stub().callsArgWith(3, null, mempoolInputs); + as._getOutputsMempool = sinon.stub().callsArgWith(3, null, mempoolOutputs); + as._getAddressMempoolSummary(address, options, resultBase, function(err, result) { + if (err) { + return done(err); + } + var txid1 = '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8'; + var txid2 = '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d'; + var txid3 = '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247'; + result.unconfirmedAppearanceIds[txid1].should.equal(1452874536321); + result.unconfirmedAppearanceIds[txid2].should.equal(1452874521466); + result.unconfirmedAppearanceIds[txid3].should.equal(1452874521466); + result.unconfirmedBalance.should.equal(131368318); + done(); + }); + }); }); - describe.skip('#_transformAddressSummaryFromCache', function() { + describe('#_transformAddressSummaryFromResult', function() { + var result = { + totalReceived: 1000000, + balance: 500000, + txids: [ + '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8', + 'b1bfa8dbbde790cb46b9763ef3407c1a21c8264b67bfe224f462ec0e1f569e92' + ], + appearanceIds: { + 'b1bfa8dbbde790cb46b9763ef3407c1a21c8264b67bfe224f462ec0e1f569e92': 100000, + '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8': 200000 + }, + unconfirmedAppearanceIds: { + '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d': 1452874536321, + '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247': 1452874521466 + }, + unconfirmedBalance: 500000 + }; + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + it('will transform result into summary', function() { + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = {}; + var summary = as._transformAddressSummaryFromResult(result, options); + summary.totalReceived.should.equal(1000000); + summary.totalSpent.should.equal(500000); + summary.balance.should.equal(500000); + summary.appearances.should.equal(2); + summary.unconfirmedAppearances.should.equal(2); + summary.unconfirmedBalance.should.equal(500000); + summary.txids.length.should.equal(4); + }); + it('will omit txlist', function() { + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = { + noTxList: true + }; + var summary = as._transformAddressSummaryFromResult(result, options); + should.not.exist(summary.txids); + }); + it('will include full appearance ids', function() { + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = { + fullTxList: true + }; + var summary = as._transformAddressSummaryFromResult(result, options); + should.exist(summary.appearanceIds); + should.exist(summary.unconfirmedAppearanceIds); + }); }); }); From 3d9b6d5532b814bffa9d1a52bd992ca60a8630c0 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Jan 2016 12:59:49 -0500 Subject: [PATCH 11/20] Address Service: More tests for history --- lib/services/address/history.js | 12 +- lib/services/address/index.js | 2 + test/services/address/history.unit.js | 253 +++++++++++++++++++++++++- test/services/address/index.unit.js | 2 + 4 files changed, 256 insertions(+), 13 deletions(-) diff --git a/lib/services/address/history.js b/lib/services/address/history.js index 7c0cfc00..1d707279 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -75,7 +75,7 @@ AddressHistory.prototype._mergeAndSortTxids = function(summaries) { AddressHistory.prototype.get = function(callback) { var self = this; if (this.addresses.length > this.maxAddressesQuery) { - return callback(new Error('Maximum number of addresses (' + this.maxAddressQuery + ') exceeded')); + return callback(new TypeError('Maximum number of addresses (' + this.maxAddressesQuery + ') exceeded')); } if (this.addresses.length === 1) { @@ -84,7 +84,7 @@ AddressHistory.prototype.get = function(callback) { if (err) { return callback(err); } - return self.finish.call(self, summary.txids, callback); + return self._paginateWithDetails.call(self, summary.txids, callback); }); } else { var opts = _.clone(this.options); @@ -99,14 +99,14 @@ AddressHistory.prototype.get = function(callback) { return callback(err); } var txids = self._mergeAndSortTxids(summaries); - return self.finish.call(self, txids, callback); + return self._paginateWithDetails.call(self, txids, callback); } ); } }; -AddressHistory.prototype.finish = function(allTxids, callback) { +AddressHistory.prototype._paginateWithDetails = function(allTxids, callback) { var self = this; var totalCount = allTxids.length; @@ -123,8 +123,8 @@ AddressHistory.prototype.finish = function(allTxids, callback) { // Verify that this query isn't too long if (txids.length > self.maxHistoryQueryLength) { return callback(new Error( - 'Maximum length query (' + self.maxAddressQueryLength + ') exceeded for addresses:' + - self.address.join(',') + 'Maximum length query (' + self.maxHistoryQueryLength + ') exceeded for address(es): ' + + self.addresses.join(',') )); } diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 5dfebf57..9b93ad90 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -1507,6 +1507,8 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, var hashBuffer = address.hashBuffer; var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; + // TODO: Sort mempool by timestamp? + async.waterfall([ function(next) { self._getInputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolInputs) { diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index 37c3d697..868c0a70 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -35,6 +35,222 @@ describe('Address Service History', function() { }); }); + describe('#get', function() { + it('will give an error if length of addresses is too long', function(done) { + var node = {}; + var options = {}; + var addresses = []; + for (var i = 0; i < 101; i++) { + addresses.push(address); + } + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + history.get(function(err) { + should.exist(err); + err.message.match(/Maximum/); + done(); + }); + }); + it('give error from getAddressSummary with one address', function(done) { + var node = { + services: { + address: { + getAddressSummary: sinon.stub().callsArgWith(2, new Error('test')) + } + } + }; + var options = {}; + var addresses = [address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + history.get(function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); + it('give error from getAddressSummary with multiple addresses', function(done) { + var node = { + services: { + address: { + getAddressSummary: sinon.stub().callsArgWith(2, new Error('test2')) + } + } + }; + var options = {}; + var addresses = [address, address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + history.get(function(err) { + should.exist(err); + err.message.should.equal('test2'); + done(); + }); + }); + it('will query get address summary directly with one address', function(done) { + var txids = []; + var summary = { + txids: txids + }; + var node = { + services: { + address: { + getAddressSummary: sinon.stub().callsArgWith(2, null, summary) + } + } + }; + var options = {}; + var addresses = [address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + history._mergeAndSortTxids = sinon.stub(); + history._paginateWithDetails = sinon.stub().callsArg(1); + history.get(function() { + history.node.services.address.getAddressSummary.callCount.should.equal(1); + history.node.services.address.getAddressSummary.args[0][0].should.equal(address); + history.node.services.address.getAddressSummary.args[0][1].should.equal(options); + history._paginateWithDetails.callCount.should.equal(1); + history._paginateWithDetails.args[0][0].should.equal(txids); + history._mergeAndSortTxids.callCount.should.equal(0); + done(); + }); + }); + it('will merge multiple summaries with multiple addresses', function(done) { + var txids = []; + var summary = { + txids: txids + }; + var node = { + services: { + address: { + getAddressSummary: sinon.stub().callsArgWith(2, null, summary) + } + } + }; + var options = {}; + var addresses = [address, address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + history._mergeAndSortTxids = sinon.stub().returns(txids); + history._paginateWithDetails = sinon.stub().callsArg(1); + history.get(function() { + history.node.services.address.getAddressSummary.callCount.should.equal(2); + history.node.services.address.getAddressSummary.args[0][0].should.equal(address); + history.node.services.address.getAddressSummary.args[0][1].should.deep.equal({ + fullTxList: true + }); + history._paginateWithDetails.callCount.should.equal(1); + history._paginateWithDetails.args[0][0].should.equal(txids); + history._mergeAndSortTxids.callCount.should.equal(1); + done(); + }); + }); + }); + + describe('#_paginateWithDetails', function() { + it('slice txids based on "from" and "to" (3 to 30)', function() { + var node = {}; + var options = { + from: 3, + to: 30 + }; + var addresses = [address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + sinon.stub(history, 'getDetailedInfo', function(txid, next) { + this.detailedArray.push(txid); + next(); + }); + history._paginateWithDetails(txids, function(err, result) { + result.totalCount.should.equal(11); + result.items.should.deep.equal([7, 6, 5, 4, 3, 2, 1, 0]); + }); + }); + it('slice txids based on "from" and "to" (0 to 3)', function() { + var node = {}; + var options = { + from: 0, + to: 3 + }; + var addresses = [address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + sinon.stub(history, 'getDetailedInfo', function(txid, next) { + this.detailedArray.push(txid); + next(); + }); + history._paginateWithDetails(txids, function(err, result) { + result.totalCount.should.equal(11); + result.items.should.deep.equal([10, 9, 8]); + }); + }); + it('will given an error if the full details is too long', function() { + var node = {}; + var options = { + from: 0, + to: 3 + }; + var addresses = [address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + sinon.stub(history, 'getDetailedInfo', function(txid, next) { + this.detailedArray.push(txid); + next(); + }); + history.maxHistoryQueryLength = 1; + history._paginateWithDetails(txids, function(err) { + should.exist(err); + err.message.match(/Maximum/); + }); + }); + it('will give full result without pagination options', function() { + var node = {}; + var options = {}; + var addresses = [address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + sinon.stub(history, 'getDetailedInfo', function(txid, next) { + this.detailedArray.push(txid); + next(); + }); + history._paginateWithDetails(txids, function(err, result) { + result.totalCount.should.equal(11); + result.items.should.deep.equal([10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0]); + }); + }); + }); + describe('#_mergeAndSortTxids', function() { it('will merge and sort multiple summaries', function() { var summaries = [ @@ -98,27 +314,42 @@ describe('Address Service History', function() { }); describe('#getDetailedInfo', function() { - it('will add additional information to existing this.transactions', function() { + it('will add additional information to existing this.transactions', function(done) { var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; + var tx = { + populateInputs: sinon.stub().callsArg(2), + __height: 20, + __timestamp: 1453134151, + isCoinbase: sinon.stub().returns(false), + getFee: sinon.stub().returns(1000) + }; var history = new AddressHistory({ node: { services: { db: { - getTransactionWithBlockInfo: sinon.stub() + getTransactionWithBlockInfo: sinon.stub().callsArgWith(2, null, tx), + tip: { + __height: 300 + } } } }, options: {}, addresses: [] }); + history.getAddressDetailsForTransaction = sinon.stub().returns({ + addresses: {}, + satoshis: 1000, + }); history.getDetailedInfo(txid, function(err) { if (err) { throw err; } - history.node.services.db.getTransactionsWithBlockInfo.callCount.should.equal(0); + history.node.services.db.getTransactionWithBlockInfo.callCount.should.equal(1); + done(); }); }); - it('will handle error from getTransactionFromBlock', function() { + it('will handle error from getTransactionFromBlock', function(done) { var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; var history = new AddressHistory({ node: { @@ -133,9 +364,10 @@ describe('Address Service History', function() { }); history.getDetailedInfo(txid, function(err) { err.message.should.equal('test'); + done(); }); }); - it('will handle error from populateInputs', function() { + it('will handle error from populateInputs', function(done) { var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; var history = new AddressHistory({ node: { @@ -152,9 +384,10 @@ describe('Address Service History', function() { }); history.getDetailedInfo(txid, function(err) { err.message.should.equal('test'); + done(); }); }); - it('will set this.transactions with correct information', function() { + it('will set this.transactions with correct information', function(done) { // block #314159 // txid 30169e8bf78bc27c4014a7aba3862c60e2e3cce19e52f1909c8255e4b7b3174e // outputIndex 1 @@ -215,11 +448,16 @@ describe('Address Service History', function() { info.timestamp.should.equal(1407292005); info.fees.should.equal(20000); info.tx.should.equal(transaction); + done(); }); }); }); + + describe.skip('#getAddressDetailsForTransaction', function() { + }); + describe('#getConfirmationsDetail', function() { - it('the correct confirmations when included in the tip', function() { + it('the correct confirmations when included in the tip', function(done) { var history = new AddressHistory({ node: { services: { @@ -237,6 +475,7 @@ describe('Address Service History', function() { __height: 100 }; history.getConfirmationsDetail(transaction).should.equal(1); + done(); }); }); }); diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index c0635e4b..e855b38b 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -2534,6 +2534,8 @@ describe('Address Service', function() { done(); }); }); + it.skip('will sort txids by timestamp', function(done) { + }); }); describe('#_transformAddressSummaryFromResult', function() { From 687400eab2ae10184801cf428c4605867c9ea412 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Jan 2016 13:53:32 -0500 Subject: [PATCH 12/20] Address Service: Added test for history `getAddressDetailsForTransaction` --- test/services/address/history.unit.js | 61 ++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index 868c0a70..5f6266c0 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -2,6 +2,7 @@ var should = require('chai').should(); var sinon = require('sinon'); +var bitcore = require('bitcore-lib'); var Transaction = require('../../../lib/transaction'); var AddressHistory = require('../../../lib/services/address/history'); @@ -453,7 +454,65 @@ describe('Address Service History', function() { }); }); - describe.skip('#getAddressDetailsForTransaction', function() { + describe('#getAddressDetailsForTransaction', function() { + it('will calculate details for the transaction', function(done) { + /* jshint sub:true */ + var tx = bitcore.Transaction({ + 'hash': 'b12b3ae8489c5a566b629a3c62ce4c51c3870af550fb5dc77d715b669a91343c', + 'version': 1, + 'inputs': [ + { + 'prevTxId': 'a2b7ea824a92f4a4944686e67ec1001bc8785348b8c111c226f782084077b543', + 'outputIndex': 0, + 'sequenceNumber': 4294967295, + 'script': '47304402201b81c933297241960a57ae1b2952863b965ac8c9ec7466ff0b715712d27548d50220576e115b63864f003889443525f47c7cf0bc1e2b5108398da085b221f267ba2301210229766f1afa25ca499a51f8e01c292b0255a21a41bb6685564a1607a811ffe924', + 'scriptString': '71 0x304402201b81c933297241960a57ae1b2952863b965ac8c9ec7466ff0b715712d27548d50220576e115b63864f003889443525f47c7cf0bc1e2b5108398da085b221f267ba2301 33 0x0229766f1afa25ca499a51f8e01c292b0255a21a41bb6685564a1607a811ffe924', + 'output': { + 'satoshis': 1000000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + } + } + ], + 'outputs': [ + { + 'satoshis': 100000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + }, + { + 'satoshis': 200000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + }, + { + 'satoshis': 50000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + }, + { + 'satoshis': 300000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + }, + { + 'satoshis': 349990000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + } + ], + 'nLockTime': 0 + }); + var history = new AddressHistory({ + node: { + network: bitcore.Networks.testnet + }, + options: {}, + addresses: ['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'] + }); + var details = history.getAddressDetailsForTransaction(tx); + should.exist(details.addresses['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW']); + details.addresses['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'].inputIndexes.should.deep.equal([0]); + details.addresses['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'].outputIndexes.should.deep.equal([ + 0, 1, 2, 3, 4 + ]); + details.satoshis.should.equal(-10000); + done(); + }); }); describe('#getConfirmationsDetail', function() { From 62934b4b667f3118c91a5f596cea89ab14f8e8df Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Jan 2016 14:54:58 -0500 Subject: [PATCH 13/20] Address Service: Removed event listeners prior to stopping --- lib/services/address/index.js | 22 ++++++++++------------ test/services/address/index.unit.js | 14 +++++++++++++- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 9b93ad90..98617419 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -41,8 +41,10 @@ var AddressService = function(options) { this.subscriptions['address/transaction'] = {}; this.subscriptions['address/balance'] = {}; - this.node.services.bitcoind.on('tx', this.transactionHandler.bind(this)); - this.node.services.bitcoind.on('txleave', this.transactionLeaveHandler.bind(this)); + this._bitcoindTransactionListener = this.transactionHandler.bind(this); + this._bitcoindTransactionLeaveListener = this.transactionLeaveHandler.bind(this); + this.node.services.bitcoind.on('tx', this._bitcoindTransactionListener); + this.node.services.bitcoind.on('txleave', this._bitcoindTransactionLeaveListener); this.maxInputsQueryLength = options.maxInputsQueryLength || constants.MAX_INPUTS_QUERY_LENGTH; this.maxOutputsQueryLength = options.maxOutputsQueryLength || constants.MAX_OUTPUTS_QUERY_LENGTH; @@ -103,6 +105,8 @@ AddressService.prototype.start = function(callback) { AddressService.prototype.stop = function(callback) { // TODO Keep track of ongoing db requests before shutting down + this.node.services.bitcoind.removeListener('tx', this._bitcoindTransactionListener); + this.node.services.bitcoind.removeListener('txleave', this._bitcoindTransactionLeaveListener); this.mempoolIndex.close(callback); }; @@ -227,6 +231,10 @@ AddressService.prototype.transactionLeaveHandler = function(txInfo) { AddressService.prototype.transactionHandler = function(txInfo, callback) { var self = this; + if (this.node.stopping) { + return callback(); + } + // Basic transaction format is handled by the daemon // and we can safely assume the buffer is properly formatted. var tx = bitcore.Transaction().fromBuffer(txInfo.buffer); @@ -760,11 +768,6 @@ AddressService.prototype.createInputsStream = function(addressStr, options) { inputStream.end(); }).pipe(inputStream); - - inputStream.on('end', function() { - stream.end(); - }); - return stream; }; @@ -967,7 +970,6 @@ AddressService.prototype._getSpentMempool = function(txidBuffer, outputIndex, ca }; AddressService.prototype.createOutputsStream = function(addressStr, options) { - var outputStream = new OutputsTransformStream({ address: new Address(addressStr, this.node.network), tipHeight: this.node.services.db.tip.__height @@ -981,10 +983,6 @@ AddressService.prototype.createOutputsStream = function(addressStr, options) { }) .pipe(outputStream); - outputStream.on('end', function() { - stream.end(); - }); - return stream; }; diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index e855b38b..be0767eb 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -192,14 +192,26 @@ describe('Address Service', function() { describe('#stop', function() { it('will close mempool levelup', function(done) { + var testnode = { + network: Networks.testnet, + datadir: 'testdir', + db: mockdb, + services: { + bitcoind: { + on: sinon.stub(), + removeListener: sinon.stub() + } + } + }; var am = new AddressService({ mempoolMemoryIndex: true, - node: mocknode + node: testnode }); am.mempoolIndex = {}; am.mempoolIndex.close = sinon.stub().callsArg(0); am.stop(function() { am.mempoolIndex.close.callCount.should.equal(1); + am.node.services.bitcoind.removeListener.callCount.should.equal(2); done(); }); }); From a166b6af2320d452a89f0a1cca56500ba2ab1133 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Jan 2016 15:06:18 -0500 Subject: [PATCH 14/20] Address Service: Removed nolonger used constant for cache --- lib/services/address/constants.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js index 7cc8ef58..2cf334e9 100644 --- a/lib/services/address/constants.js +++ b/lib/services/address/constants.js @@ -41,10 +41,6 @@ exports.SPACER_HEIGHT_MAX = new Buffer('ffffffffff', 'hex'); exports.TIMESTAMP_MIN = new Buffer('0000000000000000', 'hex'); exports.TIMESTAMP_MAX = new Buffer('ffffffffffffffff', 'hex'); -// The total number of transactions that an address can receive before it will start -// to cache the summary to disk. -exports.SUMMARY_CACHE_THRESHOLD = 10000; - // The maximum number of inputs that can be queried at once exports.MAX_INPUTS_QUERY_LENGTH = 50000; // The maximum number of outputs that can be queried at once From d4f2df5c51b0023949f45613ae55ecac8032b344 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Jan 2016 15:55:09 -0500 Subject: [PATCH 15/20] Address Service: Sort mempool txids --- lib/services/address/index.js | 8 +++++--- test/services/address/index.unit.js | 18 ++++++++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 98617419..9ba17ea9 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -1490,6 +1490,10 @@ AddressService.prototype._setAndSortTxidsFromAppearanceIds = function(result, ca result.txids.sort(function(a, b) { return result.appearanceIds[a] - result.appearanceIds[b]; }); + result.unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); + result.unconfirmedTxids.sort(function(a, b) { + return result.unconfirmedAppearanceIds[a] - result.unconfirmedAppearanceIds[b]; + }); callback(null, result); }; @@ -1505,8 +1509,6 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, var hashBuffer = address.hashBuffer; var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; - // TODO: Sort mempool by timestamp? - async.waterfall([ function(next) { self._getInputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolInputs) { @@ -1549,7 +1551,7 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, AddressService.prototype._transformAddressSummaryFromResult = function(result, options) { var confirmedTxids = result.txids; - var unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); + var unconfirmedTxids = result.unconfirmedTxids; var summary = { totalReceived: result.totalReceived, diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index be0767eb..1941b6ea 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -2425,6 +2425,13 @@ describe('Address Service', function() { '22488dbb99aed86e7081ac480e3459fa40ccab7ee18bef98b84b3cdce6bf05be': 200, '1c413601acbd608240fc635b95886c3c1f76ec8589c3392a58b5715ceb618e93': 100, '206d3834c010d46a2cf478cb1c5fe252be41f683c8a738e3ebe27f1aae67f505': 101 + }, + unconfirmedAppearanceIds: { + 'ec94d845c603f292a93b7c829811ac624b76e52b351617ca5a758e9d61a11681': 1452898347406, + 'ed11a08e3102f9610bda44c80c46781d97936a4290691d87244b1b345b39a693': 1452898331964, + 'f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345': 1452897902377, + 'edc080f2084eed362aa488ccc873a24c378dc0979aa29b05767517b70569414a': 1452897971363, + 'f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9': 1452897923107 } }; as._setAndSortTxidsFromAppearanceIds(result, function(err, result) { @@ -2435,6 +2442,11 @@ describe('Address Service', function() { result.txids[0].should.equal('1c413601acbd608240fc635b95886c3c1f76ec8589c3392a58b5715ceb618e93'); result.txids[1].should.equal('206d3834c010d46a2cf478cb1c5fe252be41f683c8a738e3ebe27f1aae67f505'); result.txids[2].should.equal('22488dbb99aed86e7081ac480e3459fa40ccab7ee18bef98b84b3cdce6bf05be'); + result.unconfirmedTxids[0].should.equal('f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345'); + result.unconfirmedTxids[1].should.equal('f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9'); + result.unconfirmedTxids[2].should.equal('edc080f2084eed362aa488ccc873a24c378dc0979aa29b05767517b70569414a'); + result.unconfirmedTxids[3].should.equal('ed11a08e3102f9610bda44c80c46781d97936a4290691d87244b1b345b39a693'); + result.unconfirmedTxids[4].should.equal('ec94d845c603f292a93b7c829811ac624b76e52b351617ca5a758e9d61a11681'); done(); }); }); @@ -2546,8 +2558,6 @@ describe('Address Service', function() { done(); }); }); - it.skip('will sort txids by timestamp', function(done) { - }); }); describe('#_transformAddressSummaryFromResult', function() { @@ -2566,6 +2576,10 @@ describe('Address Service', function() { '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d': 1452874536321, '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247': 1452874521466 }, + unconfirmedTxids: [ + '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247', + '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d' + ], unconfirmedBalance: 500000 }; var testnode = { From e498e0fac2ebe30fa8d1444c6dbd14eef2f3222b Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Jan 2016 16:03:37 -0500 Subject: [PATCH 16/20] Address Service: Include default callback earlier --- lib/services/address/index.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 9ba17ea9..5c38d3ca 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -231,6 +231,14 @@ AddressService.prototype.transactionLeaveHandler = function(txInfo) { AddressService.prototype.transactionHandler = function(txInfo, callback) { var self = this; + if (!callback) { + callback = function(err) { + if (err) { + return log.error(err); + } + }; + } + if (this.node.stopping) { return callback(); } @@ -246,14 +254,6 @@ AddressService.prototype.transactionHandler = function(txInfo, callback) { this.transactionOutputHandler(messages, tx, i, !txInfo.mempool); } - if (!callback) { - callback = function(err) { - if (err) { - return log.error(err); - } - }; - } - function finish(err) { if (err) { return callback(err); From 45029030f1c64d8ba5411e8d14f73ef44bf45b27 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Jan 2016 16:16:53 -0500 Subject: [PATCH 17/20] Address Service: Sort after unconfirmed and confirmed --- lib/services/address/index.js | 6 +++--- test/services/address/index.unit.js | 31 ++++------------------------- 2 files changed, 7 insertions(+), 30 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 5c38d3ca..17199b25 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -1343,6 +1343,9 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb }, function(result, next) { self._getAddressMempoolSummary(address, options, result, next); + }, + function(result, next) { + self._setAndSortTxidsFromAppearanceIds(result, next); } ], function(err, result) { if (err) { @@ -1379,9 +1382,6 @@ AddressService.prototype._getAddressConfirmedSummary = function(address, options }, function(result, next) { self._getAddressConfirmedOutputsSummary(address, result, options, next); - }, - function(result, next) { - self._setAndSortTxidsFromAppearanceIds(result, next); } ], callback); diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index 1941b6ea..a35614df 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -2092,11 +2092,14 @@ describe('Address Service', function() { var summary = {}; addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(3, null, cache); + addressService._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); addressService._transformAddressSummaryFromResult = sinon.stub().returns(summary); addressService.getAddressSummary(address, options, function(err, sum) { addressService._getAddressConfirmedSummary.callCount.should.equal(1); addressService._getAddressMempoolSummary.callCount.should.equal(1); addressService._getAddressMempoolSummary.args[0][2].should.equal(cache); + addressService._setAndSortTxidsFromAppearanceIds.callCount.should.equal(1); + addressService._setAndSortTxidsFromAppearanceIds.args[0][0].should.equal(cache); addressService._transformAddressSummaryFromResult.callCount.should.equal(1); addressService._transformAddressSummaryFromResult.args[0][0].should.equal(cache); sum.should.equal(summary); @@ -2123,6 +2126,7 @@ describe('Address Service', function() { addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(3, null, cache); + addressService._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); addressService._transformAddressSummaryFromResult = sinon.stub().returns(summary); addressService.getAddressSummary(address, options, function() { log.warn.callCount.should.equal(1); @@ -2151,7 +2155,6 @@ describe('Address Service', function() { var result = {}; as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, result); as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, result); - as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, result); as._getAddressConfirmedSummary(address, options, function(err) { if (err) { return done(err); @@ -2169,7 +2172,6 @@ describe('Address Service', function() { as._getAddressConfirmedOutputsSummary.args[0][0].should.equal(address); as._getAddressConfirmedOutputsSummary.args[0][1].should.deep.equal(result); as._getAddressConfirmedOutputsSummary.args[0][2].should.equal(options); - as._setAndSortTxidsFromAppearanceIds.args[0][0].should.equal(result); done(); }); }); @@ -2220,31 +2222,6 @@ describe('Address Service', function() { done(); }); }); - it('will pass error correctly (sort)', function(done) { - var testnode = { - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = {}; - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var result = {}; - as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, result); - as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, result); - as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, new Error('test')); - as._getAddressConfirmedSummary(address, options, function(err) { - should.exist(err); - err.message.should.equal('test'); - done(); - }); - }); }); describe('#_getAddressConfirmedInputsSummary', function() { From 39f8355cd976082ac4185956fc7e1f60029f6c70 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 19 Jan 2016 14:07:28 -0500 Subject: [PATCH 18/20] Address Service: Bump maximum number of addresses default --- lib/services/address/constants.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js index 2cf334e9..84f9b32a 100644 --- a/lib/services/address/constants.js +++ b/lib/services/address/constants.js @@ -48,7 +48,7 @@ exports.MAX_OUTPUTS_QUERY_LENGTH = 50000; // The maximum number of transactions that can be queried at once exports.MAX_HISTORY_QUERY_LENGTH = 100; // The maximum number of addresses that can be queried at once -exports.MAX_ADDRESSES_QUERY = 100; +exports.MAX_ADDRESSES_QUERY = 10000; module.exports = exports; From a2acc0c80f289a6223d0b29724c6e3c036e1de0e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 26 Jan 2016 13:09:31 -0500 Subject: [PATCH 19/20] Address Service: Fixed test for max address limit --- test/services/address/history.unit.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index 5f6266c0..6ddb7369 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -49,6 +49,7 @@ describe('Address Service History', function() { options: options, addresses: addresses }); + history.maxAddressesQuery = 100; history.get(function(err) { should.exist(err); err.message.match(/Maximum/); From 3d7fb6f234e0d02a2ad6942abc56eb9fad62c3ae Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 26 Jan 2016 13:25:53 -0500 Subject: [PATCH 20/20] Address Service: End stream without pausing first There was an issue where streams would still be held open if "pause" was called before "end", this would lead to http requests from the insight-api not being returned with an error status as soon as possible but would instead stay open. --- lib/services/address/index.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 17199b25..61eed5ef 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -854,7 +854,6 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { if (inputs.length > self.maxInputsQueryLength) { log.warn('Tried to query too many inputs (' + self.maxInputsQueryLength + ') for address '+ addressStr); error = new Error('Maximum number of inputs (' + self.maxInputsQueryLength + ') per query reached'); - stream.pause(); stream.end(); } }); @@ -1076,7 +1075,6 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { if (outputs.length > self.maxOutputsQueryLength) { log.warn('Tried to query too many outputs (' + self.maxOutputsQueryLength + ') for address ' + addressStr); error = new Error('Maximum number of outputs (' + self.maxOutputsQueryLength + ') per query reached'); - stream.pause(); stream.end(); } }); @@ -1403,7 +1401,6 @@ AddressService.prototype._getAddressConfirmedInputsSummary = function(address, r if (count > self.maxInputsQueryLength) { log.warn('Tried to query too many inputs (' + self.maxInputsQueryLength + ') for summary of address ' + address.toString()); error = new Error('Maximum number of inputs (' + self.maxInputsQueryLength + ') per query reached'); - inputsStream.pause(); inputsStream.end(); } @@ -1464,7 +1461,6 @@ AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, if (count > self.maxOutputsQueryLength) { log.warn('Tried to query too many outputs (' + self.maxOutputsQueryLength + ') for summary of address ' + address.toString()); error = new Error('Maximum number of outputs (' + self.maxOutputsQueryLength + ') per query reached'); - outputStream.pause(); outputStream.end(); }