From 2f8d20ba76d5cedfb045f392dc1c812e18524e04 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 4 Apr 2016 16:37:43 -0400 Subject: [PATCH] main: mempool address index Zcash: Integrated into older mempool code --- qa/rpc-tests/addressindex.py | 40 ++++++++++++++++++- src/addressindex.h | 77 ++++++++++++++++++++++++++++++++++++ src/main.cpp | 3 ++ src/rpc/misc.cpp | 47 ++++++++++++++++++++++ src/rpcserver.h | 1 + src/txmempool.cpp | 74 ++++++++++++++++++++++++++++++++++ src/txmempool.h | 16 ++++++++ 7 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 src/addressindex.h diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index ad79c7416..d28cd393f 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -7,6 +7,7 @@ # Test addressindex generation and fetching # +import time from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * from test_framework.script import * @@ -25,7 +26,7 @@ class AddressIndexTest(BitcoinTestFramework): self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-addressindex"])) # Nodes 2/3 are used for testing - self.nodes.append(start_node(2, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug", "-addressindex"])) self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-addressindex"])) connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[0], 2) @@ -207,6 +208,43 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(utxos3[1]["height"], 264) assert_equal(utxos3[2]["height"], 265) + # Check mempool indexing + print "Testing mempool indexing..." + + privKey3 = "cVfUn53hAbRrDEuMexyfgDpZPhF7KqXpS8UZevsyTDaugB7HZ3CD" + address3 = "mw4ynwhS7MmrQ27hr82kgqu7zryNDK26JB" + addressHash3 = "aa9872b5bbcdb511d89e0e11aa27da73fd2c3f50".decode("hex") + scriptPubKey3 = CScript([OP_DUP, OP_HASH160, addressHash3, OP_EQUALVERIFY, OP_CHECKSIG]) + unspent = self.nodes[2].listunspent() + + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(unspent[0]["txid"], 16), unspent[0]["vout"]))] + amount = unspent[0]["amount"] * 100000000 + tx.vout = [CTxOut(amount, scriptPubKey3)] + tx.rehash() + signed_tx = self.nodes[2].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + memtxid1 = self.nodes[2].sendrawtransaction(signed_tx["hex"], True) + time.sleep(2) + + tx2 = CTransaction() + tx2.vin = [CTxIn(COutPoint(int(unspent[1]["txid"], 16), unspent[1]["vout"]))] + amount = unspent[1]["amount"] * 100000000 + tx2.vout = [CTxOut(amount, scriptPubKey3)] + tx2.rehash() + signed_tx2 = self.nodes[2].signrawtransaction(binascii.hexlify(tx2.serialize()).decode("utf-8")) + memtxid2 = self.nodes[2].sendrawtransaction(signed_tx2["hex"], True) + time.sleep(2) + + mempool = self.nodes[2].getaddressmempool({"addresses": [address3]}) + assert_equal(len(mempool), 2) + assert_equal(mempool[0]["txid"], memtxid1) + assert_equal(mempool[1]["txid"], memtxid2) + + self.nodes[2].generate(1); + self.sync_all(); + mempool2 = self.nodes[2].getaddressmempool({"addresses": [address3]}) + assert_equal(len(mempool2), 0) + print "Passed\n" diff --git a/src/addressindex.h b/src/addressindex.h new file mode 100644 index 000000000..b6e3587c1 --- /dev/null +++ b/src/addressindex.h @@ -0,0 +1,77 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ADDRESSINDEX_H +#define BITCOIN_ADDRESSINDEX_H + +#include "uint256.h" +#include "amount.h" + +struct CMempoolAddressDelta +{ + int64_t time; + CAmount amount; + uint256 prevhash; + unsigned int prevout; + + CMempoolAddressDelta(int64_t t, CAmount a, uint256 hash, unsigned int out) { + time = t; + amount = a; + prevhash = hash; + prevout = out; + } + + CMempoolAddressDelta(int64_t t, CAmount a) { + time = t; + amount = a; + prevhash.SetNull(); + prevout = 0; + } +}; + +struct CMempoolAddressDeltaKey +{ + int type; + uint160 addressBytes; + uint256 txhash; + unsigned int index; + bool spending; + + CMempoolAddressDeltaKey(int addressType, uint160 addressHash, uint256 hash, unsigned int i, bool s) { + type = addressType; + addressBytes = addressHash; + txhash = hash; + index = i; + spending = s; + } + + CMempoolAddressDeltaKey(int addressType, uint160 addressHash) { + type = addressType; + addressBytes = addressHash; + txhash.SetNull(); + index = 0; + } +}; + +struct CMempoolAddressDeltaKeyCompare +{ + bool operator()(const CMempoolAddressDeltaKey& a, const CMempoolAddressDeltaKey& b) { + if (a.type == b.type) { + if (a.addressBytes == b.addressBytes) { + if (a.txhash == b.txhash) { + return a.index < b.index; + } else { + return a.txhash < b.txhash; + } + } else { + return a.addressBytes < b.addressBytes; + } + } else { + return a.type < b.type; + } + } +}; + +#endif // BITCOIN_ADDRESSINDEX_H diff --git a/src/main.cpp b/src/main.cpp index 418c2cfd5..a9d797da1 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1577,6 +1577,9 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa // Store transaction in memory pool.addUnchecked(hash, entry, !IsInitialBlockDownload()); + if (fAddressIndex) { + pool.addAddressIndex(entry, view); + } } SyncWithWallets(tx, NULL); diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 037842e2f..0ca4ed2cd 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -11,6 +11,7 @@ #include "netbase.h" #include "rpc/server.h" #include "timedata.h" +#include "txmempool.h" #include "util.h" #ifdef ENABLE_WALLET #include "wallet/wallet.h" @@ -497,6 +498,7 @@ static const CRPCCommand commands[] = { "addressindex", "getaddressbalance", &getaddressbalance, false }, /* insight explorer */ { "addressindex", "getaddressdeltas", &getaddressdeltas, false }, /* insight explorer */ { "addressindex", "getaddressutxos", &getaddressutxos, false }, /* insight explorer */ + { "addressindex", "getaddressmempool", &getaddressmempool, false }, /* insight explorer */ /* Not shown in help */ { "hidden", "setmocktime", &setmocktime, true }, @@ -549,6 +551,51 @@ bool heightSort(std::pair a, return a.second.blockHeight < b.second.blockHeight; } +bool timestampSort(std::pair a, + std::pair b) { + return a.second.time < b.second.time; +} + +UniValue getaddressmempool(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 1) + throw runtime_error( + "getaddressmempool\n" + "\nReturns all mempool deltas for an address (requires addressindex to be enabled).\n" + ); + + std::vector > addresses; + + if (!getAddressesFromParams(params, addresses)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + + std::vector > indexes; + + if (!mempool.getAddressIndex(addresses, indexes)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + + std::sort(indexes.begin(), indexes.end(), timestampSort); + + UniValue result(UniValue::VARR); + + for (std::vector >::iterator it = indexes.begin(); + it != indexes.end(); it++) { + + UniValue delta(UniValue::VOBJ); + delta.push_back(Pair("addressType", (int)it->first.type)); + delta.push_back(Pair("addressHash", it->first.addressBytes.GetHex())); + delta.push_back(Pair("txid", it->first.txhash.GetHex())); + delta.push_back(Pair("index", (int)it->first.index)); + delta.push_back(Pair("satoshis", it->second.amount)); + delta.push_back(Pair("timestamp", it->second.time)); + result.push_back(delta); + } + + return result; +} + UniValue getaddressutxos(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 1) diff --git a/src/rpcserver.h b/src/rpcserver.h index ea568dd9a..5507366a6 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -173,6 +173,7 @@ extern std::string HelpExampleRpc(const std::string& methodname, const std::stri extern void EnsureWalletIsUnlocked(); extern UniValue getconnectioncount(const UniValue& params, bool fHelp); // in rpcnet.cpp +extern UniValue getaddressmempool(const UniValue& params, bool fHelp); extern UniValue getaddressutxos(const UniValue& params, bool fHelp); extern UniValue getaddressdeltas(const UniValue& params, bool fHelp); extern UniValue getaddresstxids(const UniValue& params, bool fHelp); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index cf04ad577..6ff66f0dd 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -121,6 +121,79 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, return true; } +void CTxMemPool::addAddressIndex(const CTxMemPoolEntry &entry, const CCoinsViewCache &view) +{ + LOCK(cs); + const CTransaction& tx = entry.GetTx(); + std::vector inserted; + + uint256 txhash = tx.GetHash(); + for (unsigned int j = 0; j < tx.vin.size(); j++) { + const CTxIn input = tx.vin[j]; + const CTxOut &prevout = view.GetOutputFor(input); + if (prevout.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); + CMempoolAddressDeltaKey key(2, uint160(hashBytes), txhash, j, true); + CMempoolAddressDelta delta(entry.GetTime(), prevout.nValue * -1, input.prevout.hash, input.prevout.n); + mapAddress.insert(make_pair(key, delta)); + inserted.push_back(key); + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + CMempoolAddressDeltaKey key(1, uint160(hashBytes), txhash, j, true); + CMempoolAddressDelta delta(entry.GetTime(), prevout.nValue * -1, input.prevout.hash, input.prevout.n); + mapAddress.insert(make_pair(key, delta)); + inserted.push_back(key); + } + } + + for (unsigned int k = 0; k < tx.vout.size(); k++) { + const CTxOut &out = tx.vout[k]; + if (out.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + CMempoolAddressDeltaKey key(2, uint160(hashBytes), txhash, k, false); + mapAddress.insert(make_pair(key, CMempoolAddressDelta(entry.GetTime(), out.nValue))); + inserted.push_back(key); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + std::pair ret; + CMempoolAddressDeltaKey key(1, uint160(hashBytes), txhash, k, false); + mapAddress.insert(make_pair(key, CMempoolAddressDelta(entry.GetTime(), out.nValue))); + inserted.push_back(key); + } + } + + mapAddressInserted.insert(make_pair(txhash, inserted)); +} + +bool CTxMemPool::getAddressIndex(std::vector > &addresses, + std::vector > &results) +{ + LOCK(cs); + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { + addressDeltaMap::iterator ait = mapAddress.lower_bound(CMempoolAddressDeltaKey((*it).second, (*it).first)); + while (ait != mapAddress.end() && (*ait).first.addressBytes == (*it).first) { + results.push_back(*ait); + ait++; + } + } + return true; +} + +bool CTxMemPool::removeAddressIndex(const uint256 txhash) +{ + LOCK(cs); + addressDeltaMapInserted::iterator it = mapAddressInserted.find(txhash); + + if (it != mapAddressInserted.end()) { + std::vector keys = (*it).second; + for (std::vector::iterator mit = keys.begin(); mit != keys.end(); mit++) { + mapAddress.erase(*mit); + } + mapAddressInserted.erase(it); + } + + return true; +} void CTxMemPool::remove(const CTransaction &origTx, std::list& removed, bool fRecursive) { @@ -172,6 +245,7 @@ void CTxMemPool::remove(const CTransaction &origTx, std::list& rem mapTx.erase(hash); nTransactionsUpdated++; minerPolicyEstimator->removeTx(hash); + removeAddressIndex(hash); } } } diff --git a/src/txmempool.h b/src/txmempool.h index ec8a8518a..54b258dc2 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -8,6 +8,7 @@ #include +#include "addressindex.h" #include "amount.h" #include "coins.h" #include "primitives/transaction.h" @@ -152,6 +153,15 @@ public: mutable CCriticalSection cs; indexed_transaction_set mapTx; + +private: + typedef std::map addressDeltaMap; + addressDeltaMap mapAddress; + + typedef std::map > addressDeltaMapInserted; + addressDeltaMapInserted mapAddressInserted; + +public: std::map mapNextTx; std::map > mapDeltas; @@ -168,6 +178,12 @@ public: void setSanityCheck(double dFrequency = 1.0) { nCheckFrequency = static_cast(dFrequency * 4294967295.0); } bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate = true); + + void addAddressIndex(const CTxMemPoolEntry &entry, const CCoinsViewCache &view); + bool getAddressIndex(std::vector > &addresses, + std::vector > &results); + bool removeAddressIndex(const uint256 txhash); + void remove(const CTransaction &tx, std::list& removed, bool fRecursive = false); void removeWithAnchor(const uint256 &invalidRoot, ShieldedType type); void removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight, int flags);