From bf64e834c6773f4a5a1e088bdbce2da0ce44d781 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 5 Apr 2016 15:53:38 -0400 Subject: [PATCH] main: add spentindex option --- qa/pull-tester/rpc-tests.sh | 1 + qa/rpc-tests/spentindex.py | 73 +++++++++++++++++++++++++++++++ src/init.cpp | 1 + src/main.cpp | 86 ++++++++++++++++++++++++++++--------- src/main.h | 65 ++++++++++++++++++++++++++++ src/rpc/misc.cpp | 41 ++++++++++++++++++ src/rpcserver.h | 1 + src/txdb.cpp | 17 ++++++++ src/txdb.h | 4 ++ 9 files changed, 268 insertions(+), 21 deletions(-) create mode 100755 qa/rpc-tests/spentindex.py diff --git a/qa/pull-tester/rpc-tests.sh b/qa/pull-tester/rpc-tests.sh index badb640e2..68138ff06 100755 --- a/qa/pull-tester/rpc-tests.sh +++ b/qa/pull-tester/rpc-tests.sh @@ -49,6 +49,7 @@ testScripts=( 'reindex.py' 'addressindex.py' 'timestampindex.py' + 'spentindex.py' 'decodescript.py' 'blockchain.py' 'disablewallet.py' diff --git a/qa/rpc-tests/spentindex.py b/qa/rpc-tests/spentindex.py new file mode 100755 index 000000000..0fe98dcbd --- /dev/null +++ b/qa/rpc-tests/spentindex.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python2 +# Copyright (c) 2014-2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test addressindex generation and fetching +# + +import time +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +from test_framework.script import * +from test_framework.mininode import * +import binascii + +class SpentIndexTest(BitcoinTestFramework): + + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 4) + + def setup_network(self): + self.nodes = [] + # Nodes 0/1 are "wallet" nodes + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-spentindex"])) + # Nodes 2/3 are used for testing + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug", "-spentindex"])) + self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-spentindex"])) + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) + connect_nodes(self.nodes[0], 3) + + self.is_network_split = False + self.sync_all() + + def run_test(self): + print "Mining blocks..." + self.nodes[0].generate(105) + self.sync_all() + + chain_height = self.nodes[1].getblockcount() + assert_equal(chain_height, 105) + + # Check that + print "Testing spent index..." + + privkey = "cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG" + address = "mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW" + addressHash = "0b2f0a0c31bfe0406b0ccc1381fdbe311946dadc".decode("hex") + scriptPubKey = CScript([OP_DUP, OP_HASH160, addressHash, OP_EQUALVERIFY, OP_CHECKSIG]) + unspent = self.nodes[0].listunspent() + tx = CTransaction() + amount = unspent[0]["amount"] * 100000000 + tx.vin = [CTxIn(COutPoint(int(unspent[0]["txid"], 16), unspent[0]["vout"]))] + tx.vout = [CTxOut(amount, scriptPubKey)] + tx.rehash() + + signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + txid = self.nodes[0].sendrawtransaction(signed_tx["hex"], True) + self.nodes[0].generate(1) + self.sync_all() + + info = self.nodes[1].getspentinfo({"txid": unspent[0]["txid"], "index": unspent[0]["vout"]}) + assert_equal(info["txid"], txid) + assert_equal(info["index"], 0) + + print "Passed\n" + + +if __name__ == '__main__': + SpentIndexTest().main() diff --git a/src/init.cpp b/src/init.cpp index 12140ee1d..2c57f73a6 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -373,6 +373,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-addressindex", strprintf(_("Maintain a full address index, used to query for the balance, txids and unspent outputs for addresses (default: %u)"), DEFAULT_ADDRESSINDEX)); strUsage += HelpMessageOpt("-timestampindex", strprintf(_("Maintain a timestamp index for block hashes, used to query blocks hashes by a range of timestamps (default: %u)"), DEFAULT_TIMESTAMPINDEX)); + strUsage += HelpMessageOpt("-spentindex", strprintf(_("Maintain a full spent index, used to query the spending txid and input index for an outpoint (default: %u)"), DEFAULT_SPENTINDEX)); strUsage += HelpMessageGroup(_("Connection options:")); strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); diff --git a/src/main.cpp b/src/main.cpp index a9d797da1..ec9f1aa15 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -69,6 +69,7 @@ bool fReindex = false; bool fTxIndex = false; bool fAddressIndex = false; bool fTimestampIndex = false; +bool fSpentIndex = false; bool fHavePruned = false; bool fPruneMode = false; bool fIsBareMultisigStd = true; @@ -1598,6 +1599,17 @@ bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::v return true; } +bool GetSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) +{ + if (!fSpentIndex) + return error("spent index not enabled"); + + if (!pblocktree->ReadSpentIndex(key, value)) + return error("unable to get spent info"); + + return true; +} + bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex, int start, int end) { @@ -2259,6 +2271,7 @@ bool DisconnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex std::vector > addressIndex; std::vector > addressUnspentIndex; + std::vector > spentIndex; // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { @@ -2350,8 +2363,14 @@ bool DisconnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex if (fAddressIndex) { std::vector > addressIndex; + const CTxIn input = tx.vin[j]; + + if (fSpentIndex) { + // undo and delete the spent index + spentIndex.push_back(make_pair(CSpentIndexKey(input.prevout.hash, input.prevout.n), CSpentIndexValue())); + } + if (fAddressIndex) { - const CTxIn input = tx.vin[j]; const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); @@ -2376,6 +2395,7 @@ bool DisconnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex continue; } } + } } } @@ -2570,6 +2590,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin blockundo.vtxundo.reserve(block.vtx.size() - 1); std::vector > addressIndex; std::vector > addressUnspentIndex; + std::vector > spentIndex; // Construct the incremental merkle tree at the current // block position, @@ -2619,31 +2640,43 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin return state.DoS(100, error("ConnectBlock(): JoinSplit requirements not met"), REJECT_INVALID, "bad-txns-joinsplit-requirements-not-met"); - if (fAddressIndex) + if (fAddressIndex || fSpentIndex) { for (size_t j = 0; j < tx.vin.size(); j++) { + const CTxIn input = tx.vin[j]; - const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); - if (prevout.scriptPubKey.IsPayToScriptHash()) { - vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); - // record spending activity - addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); - - // remove address from unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); - } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { - vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); - - // record spending activity - addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); - - // remove address from unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); - - } else { - continue; + if (fSpentIndex) { + // add the spent index to determine the txid and input that spent an output + spentIndex.push_back(make_pair(CSpentIndexKey(input.prevout.hash, input.prevout.n), CSpentIndexValue(txhash, j, pindex->nHeight))); } + + if (fAddressIndex) { + + const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); + + if (prevout.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); + + // record spending activity + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + + // remove address from unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + + // record spending activity + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + + // remove address from unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); + + } else { + continue; + } + } + } } @@ -2798,6 +2831,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin } } + if (fSpentIndex) + if (!pblocktree->UpdateSpentIndex(spentIndex)) + return AbortNode(state, "Failed to write transaction index"); + if (fTimestampIndex) if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) return AbortNode(state, "Failed to write timestamp index"); @@ -4296,6 +4333,10 @@ bool static LoadBlockIndexDB() pblocktree->ReadFlag("timestampindex", fTimestampIndex); LogPrintf("%s: timestamp index %s\n", __func__, fTimestampIndex ? "enabled" : "disabled"); + // Check whether we have a spent index + pblocktree->ReadFlag("spentindex", fSpentIndex); + LogPrintf("%s: spent index %s\n", __func__, fSpentIndex ? "enabled" : "disabled"); + // Fill in-memory data BOOST_FOREACH(const PAIRTYPE(uint256, CBlockIndex*)& item, mapBlockIndex) { @@ -4638,6 +4679,9 @@ bool InitBlockIndex() { fTimestampIndex = GetBoolArg("-timestampindex", DEFAULT_TIMESTAMPINDEX); pblocktree->WriteFlag("timestampindex", fTimestampIndex); + fSpentIndex = GetBoolArg("-spentindex", DEFAULT_SPENTINDEX); + pblocktree->WriteFlag("spentindex", fSpentIndex); + LogPrintf("Initializing databases...\n"); // Only add the genesis block if not reindexing (in which case we reuse the one already on disk) diff --git a/src/main.h b/src/main.h index c19f32718..71acc01a3 100644 --- a/src/main.h +++ b/src/main.h @@ -101,6 +101,7 @@ static const unsigned int MAX_REJECT_MESSAGE_LENGTH = 111; static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60; static const bool DEFAULT_ADDRESSINDEX = false; static const bool DEFAULT_TIMESTAMPINDEX = false; +static const bool DEFAULT_SPENTINDEX = false; // Sanity check the magic numbers when we change them BOOST_STATIC_ASSERT(DEFAULT_BLOCK_MAX_SIZE <= MAX_BLOCK_SIZE); @@ -270,6 +271,69 @@ struct CNodeStateStats { std::vector vHeightInFlight; }; +struct CSpentIndexKey { + uint256 txid; + unsigned int outputIndex; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(txid); + READWRITE(outputIndex); + } + + CSpentIndexKey(uint256 t, unsigned int i) { + txid = t; + outputIndex = i; + } + + CSpentIndexKey() { + SetNull(); + } + + void SetNull() { + txid.SetNull(); + outputIndex = 0; + } + +}; + +struct CSpentIndexValue { + uint256 txid; + unsigned int inputIndex; + int blockHeight; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(txid); + READWRITE(inputIndex); + READWRITE(blockHeight); + } + + CSpentIndexValue(uint256 t, unsigned int i, int h) { + txid = t; + inputIndex = i; + blockHeight = h; + } + + CSpentIndexValue() { + SetNull(); + } + + void SetNull() { + txid.SetNull(); + inputIndex = 0; + blockHeight = 0; + } + + bool IsNull() const { + return txid.IsNull(); + } +}; + struct CTimestampIndexIteratorKey { unsigned int timestamp; @@ -687,6 +751,7 @@ public: }; bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes); +bool GetSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value); bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex, int start = 0, int end = 0); diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index d5cf2309e..4e0442aeb 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -500,6 +500,8 @@ static const CRPCCommand commands[] = { "addressindex", "getaddressutxos", &getaddressutxos, false }, /* insight explorer */ { "addressindex", "getaddressmempool", &getaddressmempool, false }, /* insight explorer */ + { "blockchain", "getspentinfo", &getspentinfo, true }, /* insight explorer */ + /* Not shown in help */ { "hidden", "setmocktime", &setmocktime, true }, }; @@ -813,3 +815,42 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) return result; } + +UniValue getspentinfo(const UniValue& params, bool fHelp) +{ + + if (fHelp || params.size() != 1 || !params[0].isObject()) + throw runtime_error( + "getspentinfo\n" + "\nReturns the txid and index where an output is spent.\n" + "\nResult\n" + "{\n" + " \"txid\" (string) The transaction id\n" + " \"index\" (number) The spending input index\n" + " ,...\n" + "}\n" + ); + + UniValue txidValue = find_value(params[0].get_obj(), "txid"); + UniValue indexValue = find_value(params[0].get_obj(), "index"); + + if (!txidValue.isStr() || !indexValue.isNum()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid txid or index"); + } + + uint256 txid = ParseHashV(txidValue, "txid"); + int outputIndex = indexValue.get_int(); + + CSpentIndexKey key(txid, outputIndex); + CSpentIndexValue value; + + if (!GetSpentIndex(key, value)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unable to get spent info"); + } + + UniValue obj(UniValue::VOBJ); + obj.push_back(Pair("txid", value.txid.GetHex())); + obj.push_back(Pair("index", (int)value.inputIndex)); + + return obj; +} diff --git a/src/rpcserver.h b/src/rpcserver.h index 5507366a6..9cadc5be3 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -282,6 +282,7 @@ extern UniValue verifychain(const UniValue& params, bool fHelp); extern UniValue getchaintips(const UniValue& params, bool fHelp); extern UniValue invalidateblock(const UniValue& params, bool fHelp); extern UniValue reconsiderblock(const UniValue& params, bool fHelp); +extern UniValue getspentinfo(const UniValue& params, bool fHelp); extern UniValue getblocksubsidy(const UniValue& params, bool fHelp); diff --git a/src/txdb.cpp b/src/txdb.cpp index 0616d61db..7d01c9484 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -29,6 +29,7 @@ static const char DB_TXINDEX = 't'; static const char DB_ADDRESSINDEX = 'd'; static const char DB_ADDRESSUNSPENTINDEX = 'u'; static const char DB_TIMESTAMPINDEX = 'S'; +static const char DB_SPENTINDEX = 'p'; static const char DB_BLOCK_INDEX = 'b'; static const char DB_BEST_BLOCK = 'B'; @@ -296,6 +297,22 @@ bool CBlockTreeDB::WriteTxIndex(const std::vector return WriteBatch(batch); } +bool CBlockTreeDB::ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) { + return Read(make_pair(DB_SPENTINDEX, key), value); +} + +bool CBlockTreeDB::UpdateSpentIndex(const std::vector >&vect) { + CDBBatch batch(*this); + for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) { + if (it->second.IsNull()) { + batch.Erase(make_pair(DB_SPENTINDEX, it->first)); + } else { + batch.Write(make_pair(DB_SPENTINDEX, it->first), it->second); + } + } + return WriteBatch(batch); +} + bool CBlockTreeDB::UpdateAddressUnspentIndex(const std::vector >&vect) { CDBBatch batch(*this); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) { diff --git a/src/txdb.h b/src/txdb.h index 5c55ae3b2..3e9d6a078 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -23,6 +23,8 @@ struct CAddressIndexKey; struct CAddressIndexIteratorKey; struct CTimestampIndexKey; struct CTimestampIndexIteratorKey; +struct CSpentIndexKey; +struct CSpentIndexValue; class uint256; //! -dbcache default (MiB) @@ -76,6 +78,8 @@ public: bool ReadReindexing(bool &fReindex); bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); bool WriteTxIndex(const std::vector > &list); + bool ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value); + bool UpdateSpentIndex(const std::vector >&vect); bool UpdateAddressUnspentIndex(const std::vector >&vect); bool ReadAddressUnspentIndex(uint160 addressHash, int type, std::vector > &vect);