From 539b263c679bf2e1d0964268399280c77dccc23c Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 1 Oct 2015 16:06:44 -0400 Subject: [PATCH 1/3] Add spentTxId Index - To be able to query an inputTxId and inputIndex that spends an outputTxId and outputIndex - Extends the mempoolSpentIndex to include the inputTxId and inputIndex --- integration/regtest-node.js | 43 ++++++++- lib/services/address/index.js | 137 +++++++++++++++++++++++++--- test/services/address/index.unit.js | 52 +++++++++-- 3 files changed, 210 insertions(+), 22 deletions(-) diff --git a/integration/regtest-node.js b/integration/regtest-node.js index a0e4699c..2319d80a 100644 --- a/integration/regtest-node.js +++ b/integration/regtest-node.js @@ -34,6 +34,7 @@ var testKey; var client; var outputForIsSpentTest1; +var unspentOutputSpentTxId; describe('Node Functionality', function() { @@ -356,6 +357,8 @@ describe('Node Functionality', function() { tx.change(address); tx.sign(testKey); + unspentOutputSpentTxId = tx.id; + node.services.bitcoind.sendTransaction(tx.serialize()); function mineBlock(next) { @@ -733,9 +736,28 @@ describe('Node Functionality', function() { }); - describe('isSpent', function() { + describe('#getInputForOutput(db)', function() { + it('will get the input txid and input index', function(done) { + var txid = outputForIsSpentTest1.txid; + var outputIndex = outputForIsSpentTest1.outputIndex; + var options = { + queryMempool: true + }; + node.services.address.getInputForOutput(txid, outputIndex, options, function(err, result) { + result.inputTxId.should.equal(unspentOutputSpentTxId); + result.inputIndex.should.equal(0); + done(); + }); + }); + }); + + describe('#isSpent and #getInputForOutput(mempool)', function() { + var spentOutput; + var spentOutputInputTxId; it('will return true if an input is spent in a confirmed transaction', function(done) { - var result = node.services.bitcoind.isSpent(outputForIsSpentTest1.txid, outputForIsSpentTest1.outputIndex); + var txid = outputForIsSpentTest1.txid; + var outputIndex = outputForIsSpentTest1.outputIndex; + var result = node.services.bitcoind.isSpent(txid, outputIndex); result.should.equal(true); done(); }); @@ -755,6 +777,8 @@ describe('Node Functionality', function() { tx.sign(testKey); node.services.bitcoind.sendTransaction(tx.serialize()); + spentOutput = unspentOutput; + spentOutputInputTxId = tx.hash; setImmediate(function() { var result = node.services.bitcoind.isSpent(unspentOutput.txid, unspentOutput.outputIndex); @@ -763,6 +787,21 @@ describe('Node Functionality', function() { }); }); }); + + it('will get the input txid and input index (mempool)', function(done) { + var txid = spentOutput.txid; + var outputIndex = spentOutput.outputIndex; + var options = { + queryMempool: true + }; + node.services.address.getInputForOutput(txid, outputIndex, options, function(err, result) { + result.inputTxId.should.equal(spentOutputInputTxId); + result.inputIndex.should.equal(0); + done(); + }); + }); + }); + }); }); diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 94851a08..2d0b8b77 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -50,7 +50,8 @@ AddressService.dependencies = [ AddressService.PREFIXES = { OUTPUTS: new Buffer('02', 'hex'), - SPENTS: new Buffer('03', 'hex') + SPENTS: new Buffer('03', 'hex'), + SPENTSMAP: new Buffer('05', 'hex') }; AddressService.SPACER_MIN = new Buffer('00', 'hex'); @@ -65,6 +66,7 @@ AddressService.prototype.getAPIMethods = function() { ['getBalance', this, this.getBalance, 2], ['getOutputs', this, this.getOutputs, 2], ['getUnspentOutputs', this, this.getUnspentOutputs, 2], + ['getInputForOutput', this, this.getInputForOutput, 2], ['isSpent', this, this.isSpent, 2], ['getAddressHistory', this, this.getAddressHistory, 2], ['getAddressSummary', this, this.getAddressSummary, 1] @@ -174,13 +176,18 @@ AddressService.prototype.transactionHandler = function(txInfo) { * txid - A hex string of the transaction hash * inputIndex - A number of the corresponding input * - * mempoolSpentIndex, an object keyed by - + * mempoolSpentIndex, an object keyed by - with (buffer) values: + * inputTxId - A 32 byte buffer of the input txid + * inputIndex - 4 bytes stored as UInt32BE * * @param {Transaction} - An instance of a Bitcore Transaction */ AddressService.prototype.updateMempoolIndex = function(tx) { /* jshint maxstatements: 30 */ + var txid = tx.hash; + var txidBuffer = new Buffer(txid, 'hex'); + var outputLength = tx.outputs.length; for (var outputIndex = 0; outputIndex < outputLength; outputIndex++) { var output = tx.outputs[outputIndex]; @@ -203,7 +210,7 @@ AddressService.prototype.updateMempoolIndex = function(tx) { } this.mempoolOutputIndex[addressStr].push({ - txid: tx.hash, // TODO use buffer + txid: txid, outputIndex: outputIndex, satoshis: output.satoshis, script: output._scriptBuffer.toString('hex') //TODO use a buffer @@ -217,7 +224,14 @@ AddressService.prototype.updateMempoolIndex = function(tx) { // Update spent index var spentIndexKey = [input.prevTxId.toString('hex'), input.outputIndex].join('-'); - this.mempoolSpentIndex[spentIndexKey] = true; + + var inputIndexBuffer = new Buffer(4); + inputIndexBuffer.writeUInt32BE(inputIndex); + var inputIndexValue = Buffer.concat([ + txidBuffer, + inputIndexBuffer + ]); + this.mempoolSpentIndex[spentIndexKey] = inputIndexValue; var address = input.script.toAddress(this.node.network); if (!address) { @@ -313,6 +327,7 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { var tx = txs[i]; var txid = tx.id; + var txidBuffer = new Buffer(txid, 'hex'); var inputs = tx.inputs; var outputs = tx.outputs; @@ -340,7 +355,7 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { // can have a time that is previous to the previous block (however not // less than the mean of the 11 previous blocks) and not greater than 2 // hours in the future. - var key = this._encodeOutputKey(addressInfo.hashBuffer, height, txid, outputIndex); + var key = this._encodeOutputKey(addressInfo.hashBuffer, height, txidBuffer, outputIndex); var value = this._encodeOutputValue(output.satoshis, output._scriptBuffer); operations.push({ type: action, @@ -389,15 +404,28 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { continue; } + var prevTxIdBuffer = new Buffer(input.prevTxId, 'hex'); + // To be able to query inputs by address and spent height - var inputKey = this._encodeInputKey(inputHash, height, input.prevTxId, input.outputIndex); - var inputValue = this._encodeInputValue(txid, inputIndex); + var inputKey = this._encodeInputKey(inputHash, height, prevTxIdBuffer, input.outputIndex); + var inputValue = this._encodeInputValue(txidBuffer, inputIndex); operations.push({ type: action, key: inputKey, value: inputValue }); + + // To be able to search for an input spending an output + var inputKeyMap = this._encodeInputKeyMap(prevTxIdBuffer, input.outputIndex); + var inputValueMap = this._encodeInputValueMap(txidBuffer, inputIndex); + + operations.push({ + type: action, + key: inputKeyMap, + value: inputValueMap + }); + } } @@ -406,7 +434,7 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { }); }; -AddressService.prototype._encodeOutputKey = function(hashBuffer, height, txid, outputIndex) { +AddressService.prototype._encodeOutputKey = function(hashBuffer, height, txidBuffer, outputIndex) { var heightBuffer = new Buffer(4); heightBuffer.writeUInt32BE(height); var outputIndexBuffer = new Buffer(4); @@ -416,7 +444,7 @@ AddressService.prototype._encodeOutputKey = function(hashBuffer, height, txid, o hashBuffer, AddressService.SPACER_MIN, heightBuffer, - new Buffer(txid, 'hex'), //TODO get buffer directly from tx + txidBuffer, outputIndexBuffer ]); return key; @@ -486,11 +514,11 @@ AddressService.prototype._decodeInputKey = function(buffer) { }; }; -AddressService.prototype._encodeInputValue = function(txid, inputIndex) { +AddressService.prototype._encodeInputValue = function(txidBuffer, inputIndex) { var inputIndexBuffer = new Buffer(4); inputIndexBuffer.writeUInt32BE(inputIndex); return Buffer.concat([ - new Buffer(txid, 'hex'), + txidBuffer, inputIndexBuffer ]); }; @@ -504,6 +532,43 @@ AddressService.prototype._decodeInputValue = function(buffer) { }; }; +AddressService.prototype._encodeInputKeyMap = function(outputTxIdBuffer, outputIndex) { + var outputIndexBuffer = new Buffer(4); + outputIndexBuffer.writeUInt32BE(outputIndex); + return Buffer.concat([ + AddressService.PREFIXES.SPENTSMAP, + outputTxIdBuffer, + outputIndexBuffer + ]); +}; + +AddressService.prototype._decodeInputKeyMap = function(buffer) { + var txid = buffer.slice(1, 33); + var outputIndex = buffer.readUInt32BE(33); + return { + outputTxId: txid, + outputIndex: outputIndex + }; +}; + +AddressService.prototype._encodeInputValueMap = function(inputTxIdBuffer, inputIndex) { + var inputIndexBuffer = new Buffer(4); + inputIndexBuffer.writeUInt32BE(inputIndex); + return Buffer.concat([ + inputTxIdBuffer, + inputIndexBuffer + ]); +}; + +AddressService.prototype._decodeInputValueMap = function(buffer) { + var txid = buffer.slice(0, 32); + var inputIndex = buffer.readUInt32BE(32); + return { + inputTxId: txid, + inputIndex: inputIndex + }; +}; + /** * This function is responsible for emitting events to any subscribers to the * `address/transaction` event. @@ -659,6 +724,56 @@ AddressService.prototype.getBalance = function(address, queryMempool, callback) }); }; +/** + * Will give the input that spends an output if it exists with: + * inputTxId - The input txid hex string + * inputIndex - A number with the spending input index + * @param {String|Buffer} txid - The transaction hash with the output + * @param {Number} outputIndex - The output index in the transaction + * @param {Object} options + * @param {Object} options.queryMempool - Include mempool in results + * @param {Function} callback + */ +AddressService.prototype.getInputForOutput = function(txid, outputIndex, options, callback) { + $.checkArgument(_.isNumber(outputIndex)); + $.checkArgument(_.isObject(options)); + $.checkArgument(_.isFunction(callback)); + var self = this; + var txidBuffer; + if (Buffer.isBuffer(txid)) { + txidBuffer = txid; + } else { + txidBuffer = new Buffer(txid, 'hex'); + } + if (options.queryMempool) { + var spentIndexKey = [txid.toString('hex'), outputIndex].join('-'); + if (this.mempoolSpentIndex[spentIndexKey]) { + var mempoolValue = this.mempoolSpentIndex[spentIndexKey]; + var inputTxId = mempoolValue.slice(0, 32); + var inputIndex = mempoolValue.readUInt32BE(32); + return callback(null, { + inputTxId: inputTxId.toString('hex'), + inputIndex: inputIndex + }); + } + } + var key = this._encodeInputKeyMap(txidBuffer, outputIndex); + var dbOptions = { + valueEncoding: 'binary', + keyEncoding: 'binary' + }; + this.node.services.db.store.get(key, dbOptions, function(err, buffer) { + if (err) { + return callback(err); + } + var value = self._decodeInputValueMap(buffer); + callback(null, { + inputTxId: value.inputTxId.toString('hex'), + inputIndex: value.inputIndex + }); + }); +}; + /** * Will give inputs that spend previous outputs for an address as an object with: * address - The base58check encoded address diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index c0e17d13..14cc1eda 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -33,7 +33,7 @@ describe('Address Service', function() { it('should return the correct methods', function() { var am = new AddressService({node: mocknode}); var methods = am.getAPIMethods(); - methods.length.should.equal(6); + methods.length.should.equal(7); }); }); @@ -162,16 +162,19 @@ describe('Address Service', function() { am.blockHandler(block, true, function(err, operations) { should.not.exist(err); - operations.length.should.equal(81); + operations.length.should.equal(151); operations[0].type.should.equal('put'); operations[0].key.toString('hex').should.equal('0202a61d2066d19e9e2fd348a8320b7ebd4dd3ca2b00000543abfdbefe0d064729d85556bd3ab13c3a889b685d042499c02b4aa2064fb1e1692300000000'); operations[0].value.toString('hex').should.equal('41e2a49ec1c0000076a91402a61d2066d19e9e2fd348a8320b7ebd4dd3ca2b88ac'); operations[3].type.should.equal('put'); operations[3].key.toString('hex').should.equal('03fdbd324b28ea69e49c998816407dc055fb81d06e00000543ab3d7d5d98df753ef2a4f82438513c509e3b11f3e738e94a7234967b03a03123a900000020'); operations[3].value.toString('hex').should.equal('5780f3ee54889a0717152a01abee9a32cec1b0cdf8d5537a08c7bd9eeb6bfbca00000000'); - operations[64].type.should.equal('put'); - operations[64].key.toString('hex').should.equal('029780ccd5356e2acc0ee439ee04e0fe69426c752800000543abe66f3b989c790178de2fc1a5329f94c0d8905d0d3df4e7ecf0115e7f90a6283d00000001'); - operations[64].value.toString('hex').should.equal('4147a6b00000000076a9149780ccd5356e2acc0ee439ee04e0fe69426c752888ac'); + operations[4].type.should.equal('put'); + operations[4].key.toString('hex').should.equal('053d7d5d98df753ef2a4f82438513c509e3b11f3e738e94a7234967b03a03123a900000020'); + operations[4].value.toString('hex').should.equal('5780f3ee54889a0717152a01abee9a32cec1b0cdf8d5537a08c7bd9eeb6bfbca00000000'); + operations[121].type.should.equal('put'); + operations[121].key.toString('hex').should.equal('029780ccd5356e2acc0ee439ee04e0fe69426c752800000543abe66f3b989c790178de2fc1a5329f94c0d8905d0d3df4e7ecf0115e7f90a6283d00000001'); + operations[121].value.toString('hex').should.equal('4147a6b00000000076a9149780ccd5356e2acc0ee439ee04e0fe69426c752888ac'); done(); }); }); @@ -185,16 +188,16 @@ describe('Address Service', function() { }; am.blockHandler(block, false, function(err, operations) { should.not.exist(err); - operations.length.should.equal(81); + operations.length.should.equal(151); operations[0].type.should.equal('del'); operations[0].key.toString('hex').should.equal('0202a61d2066d19e9e2fd348a8320b7ebd4dd3ca2b00000543abfdbefe0d064729d85556bd3ab13c3a889b685d042499c02b4aa2064fb1e1692300000000'); operations[0].value.toString('hex').should.equal('41e2a49ec1c0000076a91402a61d2066d19e9e2fd348a8320b7ebd4dd3ca2b88ac'); operations[3].type.should.equal('del'); operations[3].key.toString('hex').should.equal('03fdbd324b28ea69e49c998816407dc055fb81d06e00000543ab3d7d5d98df753ef2a4f82438513c509e3b11f3e738e94a7234967b03a03123a900000020'); operations[3].value.toString('hex').should.equal('5780f3ee54889a0717152a01abee9a32cec1b0cdf8d5537a08c7bd9eeb6bfbca00000000'); - operations[64].type.should.equal('del'); - operations[64].key.toString('hex').should.equal('029780ccd5356e2acc0ee439ee04e0fe69426c752800000543abe66f3b989c790178de2fc1a5329f94c0d8905d0d3df4e7ecf0115e7f90a6283d00000001'); - operations[64].value.toString('hex').should.equal('4147a6b00000000076a9149780ccd5356e2acc0ee439ee04e0fe69426c752888ac'); + operations[121].type.should.equal('del'); + operations[121].key.toString('hex').should.equal('029780ccd5356e2acc0ee439ee04e0fe69426c752800000543abe66f3b989c790178de2fc1a5329f94c0d8905d0d3df4e7ecf0115e7f90a6283d00000001'); + operations[121].value.toString('hex').should.equal('4147a6b00000000076a9149780ccd5356e2acc0ee439ee04e0fe69426c752888ac'); done(); }); }); @@ -208,6 +211,7 @@ describe('Address Service', function() { }, transactions: [ { + id: '3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', inputs: [], outputs: [ { @@ -263,6 +267,36 @@ describe('Address Service', function() { }); }); + describe('#_encodeInputKeyMap/#_decodeInputKeyMap roundtrip', function() { + var encoded; + var outputTxIdBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); + it('encode key', function() { + var am = new AddressService({node: mocknode}); + encoded = am._encodeInputKeyMap(outputTxIdBuffer, 13); + }); + it('decode key', function() { + var am = new AddressService({node: mocknode}); + var key = am._decodeInputKeyMap(encoded); + key.outputTxId.toString('hex').should.equal(outputTxIdBuffer.toString('hex')); + key.outputIndex.should.equal(13); + }); + }); + + describe('#_encodeInputValueMap/#_decodeInputValueMap roundtrip', function() { + var encoded; + var inputTxIdBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); + it('encode key', function() { + var am = new AddressService({node: mocknode}); + encoded = am._encodeInputValueMap(inputTxIdBuffer, 7); + }); + it('decode key', function() { + var am = new AddressService({node: mocknode}); + var key = am._decodeInputValueMap(encoded); + key.inputTxId.toString('hex').should.equal(inputTxIdBuffer.toString('hex')); + key.inputIndex.should.equal(7); + }); + }); + describe('#transactionEventHandler', function() { it('will emit a transaction if there is a subscriber', function(done) { var am = new AddressService({node: mocknode}); From 27e90ef41aa71778162111469e0d6ad80306b1b7 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 1 Oct 2015 23:50:06 -0400 Subject: [PATCH 2/3] Give false if spent information not available. --- lib/services/address/index.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 2d0b8b77..d08ddc3e 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -8,6 +8,7 @@ var log = index.log; var errors = index.errors; var Transaction = require('../../transaction'); var bitcore = require('bitcore'); +var levelup = require('levelup'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; var Hash = bitcore.crypto.Hash; @@ -763,7 +764,9 @@ AddressService.prototype.getInputForOutput = function(txid, outputIndex, options keyEncoding: 'binary' }; this.node.services.db.store.get(key, dbOptions, function(err, buffer) { - if (err) { + if (err instanceof levelup.errors.NotFoundError) { + return callback(null, false); + } else if (err) { return callback(err); } var value = self._decodeInputValueMap(buffer); From da9d856da344de1a6e8518b275624d9d57b3f64a Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 2 Oct 2015 10:56:28 -0400 Subject: [PATCH 3/3] Add comments to describe each prefix. --- lib/services/address/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index d08ddc3e..75878d70 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -50,9 +50,9 @@ AddressService.dependencies = [ ]; AddressService.PREFIXES = { - OUTPUTS: new Buffer('02', 'hex'), - SPENTS: new Buffer('03', 'hex'), - SPENTSMAP: new Buffer('05', 'hex') + OUTPUTS: new Buffer('02', 'hex'), // Query outputs by address and/or height + SPENTS: new Buffer('03', 'hex'), // Query inputs by address and/or height + SPENTSMAP: new Buffer('05', 'hex') // Get the input that spends an output }; AddressService.SPACER_MIN = new Buffer('00', 'hex');