Auto merge of #3849 - LarryRuane:3708-spentindex, r=daira

add SpentIndex changes needed for bitcore block explorer

Addresses #3708, follow-on to #3837, note there are no tests yet (will come later with RPC interfaces).

src/Makefile.am

@@ -189,6 +189,7 @@ BITCOIN_CORE_H = \
   script/sign.h \
   script/standard.h \
   serialize.h \
+  spentindex.h \
   streams.h \
   support/allocators/secure.h \
   support/allocators/zeroafterfree.h \

src/main.cpp

@@ -69,6 +69,7 @@ bool fReindex = false;
 bool fTxIndex = false;
 bool fInsightExplorer = false;  // insightexplorer
 bool fAddressIndex = false;     // insightexplorer
+bool fSpentIndex = false;       // insightexplorer
 bool fHavePruned = false;
 bool fPruneMode = false;
 bool fIsBareMultisigStd = true;
@@ -2229,10 +2230,10 @@ enum DisconnectResult
 
 /** Undo the effects of this block (with given index) on the UTXO set represented by coins.
  *  When UNCLEAN or FAILED is returned, view is left in an indeterminate state.
- *  The addressIndex will be updated if requested.
+ *  The addressIndex and spentIndex will be updated if requested.
  */
 static DisconnectResult DisconnectBlock(const CBlock& block, CValidationState& state,
-    const CBlockIndex* pindex, CCoinsViewCache& view, const bool fUpdateAddressIndex)
+    const CBlockIndex* pindex, CCoinsViewCache& view, bool const updateIndices)
 {
     assert(pindex->GetBlockHash() == view.GetBestBlock());
 
@@ -2255,6 +2256,7 @@ static DisconnectResult DisconnectBlock(const CBlock& block, CValidationState& s
     }
     std::vector<CAddressIndexDbEntry> addressIndex;
     std::vector<CAddressUnspentDbEntry> addressUnspentIndex;
+    std::vector<CSpentIndexDbEntry> spentIndex;
 
     // undo transactions in reverse order
     for (int i = block.vtx.size() - 1; i >= 0; i--) {
@@ -2263,7 +2265,7 @@ static DisconnectResult DisconnectBlock(const CBlock& block, CValidationState& s
 
         // insightexplorer
         // https://github.com/bitpay/bitcoin/commit/017f548ea6d89423ef568117447e61dd5707ec42#diff-7ec3c68a81efff79b6ca22ac1f1eabbaR2236
-        if (fAddressIndex && fUpdateAddressIndex) {
+        if (fAddressIndex && updateIndices) {
             for (unsigned int k = tx.vout.size(); k-- > 0;) {
                 const CTxOut &out = tx.vout[k];
                 int const scriptType = out.scriptPubKey.Type();
@@ -2320,8 +2322,8 @@ static DisconnectResult DisconnectBlock(const CBlock& block, CValidationState& s
 
                 // insightexplorer
                 // https://github.com/bitpay/bitcoin/commit/017f548ea6d89423ef568117447e61dd5707ec42#diff-7ec3c68a81efff79b6ca22ac1f1eabbaR2304
-                if (fAddressIndex && fUpdateAddressIndex) {
+                const CTxIn input = tx.vin[j];
-                    const CTxIn input = tx.vin[j];
+                if (fAddressIndex && updateIndices) {
                     const CTxOut &prevout = view.GetOutputFor(input);
                     int const scriptType = prevout.scriptPubKey.Type();
                     if (scriptType > 0) {
@@ -2338,6 +2340,13 @@ static DisconnectResult DisconnectBlock(const CBlock& block, CValidationState& s
                             CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey, undo.nHeight)));
                     }
                 }
+                // insightexplorer
+                if (fSpentIndex && updateIndices) {
+                    // undo and delete the spent index
+                    spentIndex.push_back(make_pair(
+                        CSpentIndexKey(input.prevout.hash, input.prevout.n),
+                        CSpentIndexValue()));
+                }
             }
         }
     }
@@ -2360,7 +2369,7 @@ static DisconnectResult DisconnectBlock(const CBlock& block, CValidationState& s
     view.SetBestBlock(pindex->pprev->GetBlockHash());
 
     // insightexplorer
-    if (fAddressIndex && fUpdateAddressIndex) {
+    if (fAddressIndex && updateIndices) {
         if (!pblocktree->EraseAddressIndex(addressIndex)) {
             AbortNode(state, "Failed to delete address index");
             return DISCONNECT_FAILED;
@@ -2370,6 +2379,13 @@ static DisconnectResult DisconnectBlock(const CBlock& block, CValidationState& s
             return DISCONNECT_FAILED;
         }
     }
+    // insightexplorer
+    if (fSpentIndex && updateIndices) {
+        if (!pblocktree->UpdateSpentIndex(spentIndex)) {
+            AbortNode(state, "Failed to write transaction index");
+            return DISCONNECT_FAILED;
+        }
+    }
     return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN;
 }
 
@@ -2568,6 +2584,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
     blockundo.vtxundo.reserve(block.vtx.size() - 1);
     std::vector<CAddressIndexDbEntry> addressIndex;
     std::vector<CAddressUnspentDbEntry> addressUnspentIndex;
+    std::vector<CSpentIndexDbEntry> spentIndex;
 
     // Construct the incremental merkle tree at the current
     // block position,
@@ -2619,15 +2636,14 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
 
             // insightexplorer
             // https://github.com/bitpay/bitcoin/commit/017f548ea6d89423ef568117447e61dd5707ec42#diff-7ec3c68a81efff79b6ca22ac1f1eabbaR2597
-            if (fAddressIndex) {
+            if (fAddressIndex || fSpentIndex) {
                 for (size_t j = 0; j < tx.vin.size(); j++) {
 
                     const CTxIn input = tx.vin[j];
                     const CTxOut &prevout = view.GetOutputFor(tx.vin[j]);
                     int const scriptType = prevout.scriptPubKey.Type();
-                    if (scriptType > 0) {
+                    const uint160 addrHash = prevout.scriptPubKey.AddressHash();
-                        uint160 const addrHash = prevout.scriptPubKey.AddressHash();
+                    if (fAddressIndex && scriptType > 0) {
-
                         // record spending activity
                         addressIndex.push_back(make_pair(
                             CAddressIndexKey(scriptType, addrHash, pindex->nHeight, i, hash, j, true),
@@ -2638,6 +2654,15 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
                             CAddressUnspentKey(scriptType, addrHash, input.prevout.hash, input.prevout.n),
                             CAddressUnspentValue()));
                     }
+                    if (fSpentIndex) {
+                        // Add the spent index to determine the txid and input that spent an output
+                        // and to find the amount and address from an input.
+                        // If we do not recognize the script type, we still add an entry to the
+                        // spentindex db, with a script type of 0 and addrhash of all zeroes.
+                        spentIndex.push_back(make_pair(
+                            CSpentIndexKey(input.prevout.hash, input.prevout.n),
+                            CSpentIndexValue(hash, j, pindex->nHeight, prevout.nValue, scriptType, addrHash)));
+                    }
                 }
             }
 
@@ -2785,6 +2810,12 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
             return AbortNode(state, "Failed to write address unspent index");
         }
     }
+    // insightexplorer
+    if (fSpentIndex) {
+        if (!pblocktree->UpdateSpentIndex(spentIndex)) {
+            return AbortNode(state, "Failed to write spent index");
+        }
+    }
 
     // add this block to the view's block chain
     view.SetBestBlock(pindex->GetBlockHash());
@@ -4340,6 +4371,7 @@ bool static LoadBlockIndexDB()
     pblocktree->ReadFlag("insightexplorer", fInsightExplorer);
     LogPrintf("%s: insight explorer %s\n", __func__, fAddressIndex ? "enabled" : "disabled");
     fAddressIndex = fInsightExplorer;
+    fSpentIndex = fInsightExplorer;
 
     // Fill in-memory data
     BOOST_FOREACH(const PAIRTYPE(uint256, CBlockIndex*)& item, mapBlockIndex)
@@ -4682,6 +4714,7 @@ bool InitBlockIndex() {
     fInsightExplorer = GetBoolArg("-insightexplorer", false);
     pblocktree->WriteFlag("insightexplorer", fInsightExplorer);
     fAddressIndex = fInsightExplorer;
+    fSpentIndex = fInsightExplorer;
 
     LogPrintf("Initializing databases...\n");
 

src/main.h

@@ -27,6 +27,7 @@
 #include "txmempool.h"
 #include "uint256.h"
 #include "addressindex.h"
+#include "spentindex.h"
 
 #include <algorithm>
 #include <exception>
@@ -143,6 +144,9 @@ extern bool fInsightExplorer;
 // Maintain a full address index, used to query for the balance, txids and unspent outputs for addresses
 extern bool fAddressIndex;
 
+// Maintain a full spent index, used to query the spending txid and input index for an outpoint
+extern bool fSpentIndex;
+
 // END insightexplorer
 
 extern bool fIsBareMultisigStd;

src/script/script.cpp

@@ -262,8 +262,9 @@ uint160 CScript::AddressHash() const
     else if (this->IsPayToScriptHash())
         start = 2;
     else {
-        // unknown script type; return an empty vector
+        // unknown script type; return zeros
         vector<unsigned char> hashBytes;
+        hashBytes.resize(20);
         return uint160(hashBytes);
     }
     vector<unsigned char> hashBytes(this->begin()+start, this->begin()+start+20);

src/spentindex.h

@@ -0,0 +1,97 @@
+// Copyright (c) 2009-2010 Satoshi Nakamoto
+// Copyright (c) 2009-2015 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_SPENTINDEX_H
+#define BITCOIN_SPENTINDEX_H
+
+#include "uint256.h"
+#include "amount.h"
+
+struct CSpentIndexKey {
+    uint256 txid;
+    unsigned int outputIndex;
+
+    ADD_SERIALIZE_METHODS;
+
+    template <typename Stream, typename Operation>
+    inline void SerializationOp(Stream& s, Operation ser_action) {
+        READWRITE(txid);
+        READWRITE(outputIndex);
+    }
+
+    CSpentIndexKey(uint256 t, unsigned int i) {
+        txid = t;
+        outputIndex = i;
+    }
+
+    CSpentIndexKey() {
+        SetNull();
+    }
+
+    void SetNull() {
+        txid.SetNull();
+        outputIndex = 0;
+    }
+};
+
+struct CSpentIndexValue {
+    uint256 txid;
+    unsigned int inputIndex;
+    int blockHeight;
+    CAmount satoshis;
+    int addressType;
+    uint160 addressHash;
+
+    ADD_SERIALIZE_METHODS;
+
+    template <typename Stream, typename Operation>
+    inline void SerializationOp(Stream& s, Operation ser_action) {
+        READWRITE(txid);
+        READWRITE(inputIndex);
+        READWRITE(blockHeight);
+        READWRITE(satoshis);
+        READWRITE(addressType);
+        READWRITE(addressHash);
+    }
+
+    CSpentIndexValue(uint256 t, unsigned int i, int h, CAmount s, int type, uint160 a) {
+        txid = t;
+        inputIndex = i;
+        blockHeight = h;
+        satoshis = s;
+        addressType = type;
+        addressHash = a;
+    }
+
+    CSpentIndexValue() {
+        SetNull();
+    }
+
+    void SetNull() {
+        txid.SetNull();
+        inputIndex = 0;
+        blockHeight = 0;
+        satoshis = 0;
+        addressType = 0;
+        addressHash.SetNull();
+    }
+
+    bool IsNull() const {
+        return txid.IsNull();
+    }
+};
+
+struct CSpentIndexKeyCompare
+{
+    bool operator()(const CSpentIndexKey& a, const CSpentIndexKey& b) const {
+        if (a.txid == b.txid) {
+            return a.outputIndex < b.outputIndex;
+        } else {
+            return a.txid < b.txid;
+        }
+    }
+};
+
+#endif // BITCOIN_SPENTINDEX_H

src/txdb.cpp

@@ -38,7 +38,7 @@ static const char DB_LAST_BLOCK = 'l';
 // insightexplorer
 static const char DB_ADDRESSINDEX = 'd';
 static const char DB_ADDRESSUNSPENTINDEX = 'u';
-
+static const char DB_SPENTINDEX = 'p';
 
 CCoinsViewDB::CCoinsViewDB(std::string dbName, size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / dbName, nCacheSize, fMemory, fWipe) {
 }
@@ -374,6 +374,22 @@ bool CBlockTreeDB::ReadAddressIndex(
     }
     return true;
 }
+
+bool CBlockTreeDB::ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) {
+    return Read(make_pair(DB_SPENTINDEX, key), value);
+}
+
+bool CBlockTreeDB::UpdateSpentIndex(const std::vector<CSpentIndexDbEntry> &vect) {
+    CDBBatch batch(*this);
+    for (std::vector<CSpentIndexDbEntry>::const_iterator it=vect.begin(); it!=vect.end(); it++) {
+        if (it->second.IsNull()) {
+            batch.Erase(make_pair(DB_SPENTINDEX, it->first));
+        } else {
+            batch.Write(make_pair(DB_SPENTINDEX, it->first), it->second);
+        }
+    }
+    return WriteBatch(batch);
+}
 // END insightexplorer
 
 bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) {

src/txdb.h

@@ -24,9 +24,12 @@ struct CAddressUnspentValue;
 struct CAddressIndexKey;
 struct CAddressIndexIteratorKey;
 struct CAddressIndexIteratorHeightKey;
+struct CSpentIndexKey;
+struct CSpentIndexValue;
 
 typedef std::pair<CAddressUnspentKey, CAddressUnspentValue> CAddressUnspentDbEntry;
 typedef std::pair<CAddressIndexKey, CAmount> CAddressIndexDbEntry;
+typedef std::pair<CSpentIndexKey, CSpentIndexValue> CSpentIndexDbEntry;
 // END insightexplorer
 
 class uint256;
@@ -89,6 +92,8 @@ public:
     bool WriteAddressIndex(const std::vector<CAddressIndexDbEntry> &vect);
     bool EraseAddressIndex(const std::vector<CAddressIndexDbEntry> &vect);
     bool ReadAddressIndex(uint160 addressHash, int type, std::vector<CAddressIndexDbEntry> &addressIndex, int start = 0, int end = 0);
+    bool ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value);
+    bool UpdateSpentIndex(const std::vector<CSpentIndexDbEntry> &vect);
     // END insightexplorer
 
     bool WriteFlag(const std::string &name, bool fValue);