From 4fcfdc9b21aebb8ddc08faa5583c3a3e0e7da242 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 22 Mar 2016 18:11:04 -0400 Subject: [PATCH] main: add block timestamp index --- qa/pull-tester/rpc-tests.sh | 1 + qa/rpc-tests/timestampindex.py | 51 +++++++++++++++++++++++++++ src/main.cpp | 28 +++++++++++++-- src/main.h | 63 ++++++++++++++++++++++++++++++++++ src/rpc/blockchain.cpp | 32 +++++++++++++++++ src/rpcserver.h | 1 + src/txdb.cpp | 27 +++++++++++++++ src/txdb.h | 4 +++ 8 files changed, 205 insertions(+), 2 deletions(-) create mode 100755 qa/rpc-tests/timestampindex.py diff --git a/qa/pull-tester/rpc-tests.sh b/qa/pull-tester/rpc-tests.sh index 80d5ee6ed..badb640e2 100755 --- a/qa/pull-tester/rpc-tests.sh +++ b/qa/pull-tester/rpc-tests.sh @@ -48,6 +48,7 @@ testScripts=( 'nodehandling.py' 'reindex.py' 'addressindex.py' + 'timestampindex.py' 'decodescript.py' 'blockchain.py' 'disablewallet.py' diff --git a/qa/rpc-tests/timestampindex.py b/qa/rpc-tests/timestampindex.py new file mode 100755 index 000000000..46ff710bd --- /dev/null +++ b/qa/rpc-tests/timestampindex.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python2 +# Copyright (c) 2014-2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test timestampindex generation and fetching +# + +import time + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + + +class TimestampIndexTest(BitcoinTestFramework): + + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 4) + + def setup_network(self): + self.nodes = [] + # Nodes 0/1 are "wallet" nodes + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-timestampindex"])) + # Nodes 2/3 are used for testing + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-timestampindex"])) + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) + connect_nodes(self.nodes[0], 3) + + self.is_network_split = False + self.sync_all() + + def run_test(self): + print "Mining 5 blocks..." + blockhashes = self.nodes[0].generate(5) + low = self.nodes[0].getblock(blockhashes[0])["time"] + high = self.nodes[0].getblock(blockhashes[4])["time"] + self.sync_all() + print "Checking timestamp index..." + hashes = self.nodes[1].getblockhashes(high, low) + assert_equal(len(hashes), 5) + assert_equal(sorted(blockhashes), sorted(hashes)) + print "Passed\n" + + +if __name__ == '__main__': + TimestampIndexTest().main() diff --git a/src/main.cpp b/src/main.cpp index 0464ecb38..7a2619f6f 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -68,6 +68,7 @@ bool fImporting = false; bool fReindex = false; bool fTxIndex = false; bool fAddressIndex = false; +bool fTimestampIndex = false; bool fHavePruned = false; bool fPruneMode = false; bool fIsBareMultisigStd = true; @@ -1583,6 +1584,17 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa return true; } +bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes) +{ + if (!fTimestampIndex) + return error("Timestamp index not enabled"); + + if (!pblocktree->ReadTimestampIndex(high, low, hashes)) + return error("Unable to get hashes for timestamps"); + + return true; +} + bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex) { if (!fAddressIndex) @@ -2655,8 +2667,12 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin return AbortNode(state, "Failed to write transaction index"); if (fAddressIndex) - if (!pblocktree->WriteAddressIndex(addressIndex)) - return AbortNode(state, "Failed to write address index"); + if (!pblocktree->WriteAddressIndex(addressIndex)) + return AbortNode(state, "Failed to write address index"); + + if (fTimestampIndex) + if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) + return AbortNode(state, "Failed to write timestamp index"); // add this block to the view's block chain view.SetBestBlock(pindex->GetBlockHash()); @@ -4148,6 +4164,10 @@ bool static LoadBlockIndexDB() pblocktree->ReadFlag("addressindex", fAddressIndex); LogPrintf("%s: address index %s\n", __func__, fAddressIndex ? "enabled" : "disabled"); + // Check whether we have a timestamp index + pblocktree->ReadFlag("timestampindex", fTimestampIndex); + LogPrintf("%s: timestamp index %s\n", __func__, fTimestampIndex ? "enabled" : "disabled"); + // Fill in-memory data BOOST_FOREACH(const PAIRTYPE(uint256, CBlockIndex*)& item, mapBlockIndex) { @@ -4486,6 +4506,10 @@ bool InitBlockIndex() { fAddressIndex = GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX); pblocktree->WriteFlag("addressindex", fAddressIndex); + // Use the provided setting for -timestampindex in the new database + fTimestampIndex = GetBoolArg("-timestampindex", DEFAULT_TIMESTAMPINDEX); + pblocktree->WriteFlag("timestampindex", fTimestampIndex); + LogPrintf("Initializing databases...\n"); // Only add the genesis block if not reindexing (in which case we reuse the one already on disk) diff --git a/src/main.h b/src/main.h index 5b73d0266..11f89b922 100644 --- a/src/main.h +++ b/src/main.h @@ -100,6 +100,7 @@ static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; static const unsigned int MAX_REJECT_MESSAGE_LENGTH = 111; static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60; static const bool DEFAULT_ADDRESSINDEX = false; +static const bool DEFAULT_TIMESTAMPINDEX = false; // Sanity check the magic numbers when we change them BOOST_STATIC_ASSERT(DEFAULT_BLOCK_MAX_SIZE <= MAX_BLOCK_SIZE); @@ -269,6 +270,67 @@ struct CNodeStateStats { std::vector vHeightInFlight; }; +struct CTimestampIndexIteratorKey { + unsigned int timestamp; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 4; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata32be(s, timestamp); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + timestamp = ser_readdata32be(s); + } + + CTimestampIndexIteratorKey(unsigned int time) { + timestamp = time; + } + + CTimestampIndexIteratorKey() { + SetNull(); + } + + void SetNull() { + timestamp = 0; + } +}; + +struct CTimestampIndexKey { + unsigned int timestamp; + uint256 blockHash; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 36; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata32be(s, timestamp); + blockHash.Serialize(s, nType, nVersion); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + timestamp = ser_readdata32be(s); + blockHash.Unserialize(s, nType, nVersion); + } + + CTimestampIndexKey(unsigned int time, uint256 hash) { + timestamp = time; + blockHash = hash; + } + + CTimestampIndexKey() { + SetNull(); + } + + void SetNull() { + timestamp = 0; + blockHash.SetNull(); + } +}; + struct CAddressIndexKey { unsigned int type; uint160 hashBytes; @@ -521,6 +583,7 @@ public: ScriptError GetScriptError() const { return error; } }; +bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes); bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); /** Functions for disk access for blocks */ diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 9a9b0fd57..979d44fd0 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -313,6 +313,38 @@ UniValue getrawmempool(const UniValue& params, bool fHelp) return mempoolToJSON(fVerbose); } +UniValue getblockhashes(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 2) + throw runtime_error( + "getblockhashes timestamp\n" + "\nReturns array of hashes of blocks within the timestamp range provided.\n" + "\nArguments:\n" + "1. high (numeric, required) The newer block timestamp\n" + "2. low (numeric, required) The older block timestamp\n" + "\nResult:\n" + "[" + " \"hash\" (string) The block hash\n" + "]" + "\nExamples:\n" + ); + + unsigned int high = params[0].get_int(); + unsigned int low = params[1].get_int(); + std::vector blockHashes; + + if (!GetTimestampIndex(high, low, blockHashes)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for block hashes"); + } + + UniValue result(UniValue::VARR); + for (std::vector::const_iterator it=blockHashes.begin(); it!=blockHashes.end(); it++) { + result.push_back(it->GetHex()); + } + + return result; +} + UniValue getblockhash(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 1) diff --git a/src/rpcserver.h b/src/rpcserver.h index 56c344a39..2e757c4ed 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -269,6 +269,7 @@ extern UniValue getdifficulty(const UniValue& params, bool fHelp); extern UniValue settxfee(const UniValue& params, bool fHelp); extern UniValue getmempoolinfo(const UniValue& params, bool fHelp); extern UniValue getrawmempool(const UniValue& params, bool fHelp); +extern UniValue getblockhashes(const UniValue& params, bool fHelp); extern UniValue getblockhash(const UniValue& params, bool fHelp); extern UniValue getblockheader(const UniValue& params, bool fHelp); extern UniValue getblock(const UniValue& params, bool fHelp); diff --git a/src/txdb.cpp b/src/txdb.cpp index 0a0b355d0..8eb146b29 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -27,6 +27,7 @@ static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; static const char DB_ADDRESSINDEX = 'd'; +static const char DB_TIMESTAMPINDEX = 'S'; static const char DB_BLOCK_INDEX = 'b'; static const char DB_BEST_BLOCK = 'B'; @@ -326,6 +327,32 @@ bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, std::vector &hashes) { + + boost::scoped_ptr pcursor(NewIterator()); + + pcursor->Seek(make_pair(DB_TIMESTAMPINDEX, CTimestampIndexIteratorKey(low))); + + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + std::pair key; + if (pcursor->GetKey(key) && key.first == DB_TIMESTAMPINDEX && key.second.timestamp <= high) { + hashes.push_back(key.second.blockHash); + pcursor->Next(); + } else { + break; + } + } + + return true; +} + bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) { return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0'); } diff --git a/src/txdb.h b/src/txdb.h index 2726611d0..8d16f3820 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -19,6 +19,8 @@ class CBlockIndex; struct CDiskTxPos; struct CAddressIndexKey; struct CAddressIndexIteratorKey; +struct CTimestampIndexKey; +struct CTimestampIndexIteratorKey; class uint256; //! -dbcache default (MiB) @@ -74,6 +76,8 @@ public: bool WriteTxIndex(const std::vector > &list); bool WriteAddressIndex(const std::vector > &vect); bool ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); + bool WriteTimestampIndex(const CTimestampIndexKey ×tampIndex); + bool ReadTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &vect); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); bool LoadBlockIndexGuts();