diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 58098bdec..db9776727 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -38,7 +38,7 @@ for i in range(1,len(sys.argv)): buildDir = BUILDDIR os.environ["BITCOIND"] = buildDir + '/src/bitcoind' + EXEEXT os.environ["BITCOINCLI"] = buildDir + '/src/bitcoin-cli' + EXEEXT - + #Disable Windows tests by default if EXEEXT == ".exe" and "-win" not in opts: print "Win tests currently disabled. Use -win option to enable" @@ -67,6 +67,7 @@ testScripts = [ 'reindex.py', 'decodescript.py', 'p2p-fullblocktest.py', + 'blockchain.py', ] testScriptsExt = [ 'bipdersig-p2p.py', @@ -98,10 +99,10 @@ if(ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1): rpcTestDir = buildDir + '/qa/rpc-tests/' #Run Tests for i in range(len(testScripts)): - if (len(opts) == 0 or (len(opts) == 1 and "-win" in opts ) or '-extended' in opts + if (len(opts) == 0 or (len(opts) == 1 and "-win" in opts ) or '-extended' in opts or testScripts[i] in opts or re.sub(".py$", "", testScripts[i]) in opts ): - print "Running testscript " + testScripts[i] + "..." - subprocess.call(rpcTestDir + testScripts[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True) + print "Running testscript " + testScripts[i] + "..." + subprocess.call(rpcTestDir + testScripts[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True) #exit if help is called so we print just one set of instructions p = re.compile(" -h| --help") if p.match(passOn): @@ -109,9 +110,9 @@ if(ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1): #Run Extended Tests for i in range(len(testScriptsExt)): - if ('-extended' in opts or testScriptsExt[i] in opts + if ('-extended' in opts or testScriptsExt[i] in opts or re.sub(".py$", "", testScriptsExt[i]) in opts): - print "Running 2nd level testscript " + testScriptsExt[i] + "..." - subprocess.call(rpcTestDir + testScriptsExt[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True) + print "Running 2nd level testscript " + testScriptsExt[i] + "..." + subprocess.call(rpcTestDir + testScriptsExt[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True) else: print "No rpc tests to run. Wallet, utils, and bitcoind must all be enabled" diff --git a/qa/rpc-tests/blockchain.py b/qa/rpc-tests/blockchain.py new file mode 100755 index 000000000..a5c98b777 --- /dev/null +++ b/qa/rpc-tests/blockchain.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python2 +# Copyright (c) 2014 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test RPC calls related to blockchain state. +# + +import decimal + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + initialize_chain, + assert_equal, + start_nodes, + connect_nodes_bi, +) + +class BlockchainTest(BitcoinTestFramework): + """ + Test blockchain-related RPC calls: + + - gettxoutsetinfo + + """ + + def setup_chain(self): + print("Initializing test directory " + self.options.tmpdir) + initialize_chain(self.options.tmpdir) + + def setup_network(self, split=False): + self.nodes = start_nodes(2, self.options.tmpdir) + connect_nodes_bi(self.nodes, 0, 1) + self.is_network_split = False + self.sync_all() + + def run_test(self): + node = self.nodes[0] + res = node.gettxoutsetinfo() + + assert_equal(res[u'total_amount'], decimal.Decimal('8725.00000000')) + assert_equal(res[u'transactions'], 200) + assert_equal(res[u'height'], 200) + assert_equal(res[u'txouts'], 200) + assert_equal(res[u'bytes_serialized'], 13000), + assert_equal(len(res[u'bestblock']), 64) + assert_equal(len(res[u'hash_serialized']), 64) + + +if __name__ == '__main__': + BlockchainTest().main() diff --git a/src/leveldbwrapper.cpp b/src/leveldbwrapper.cpp index ce96b5c8a..32c9345be 100644 --- a/src/leveldbwrapper.cpp +++ b/src/leveldbwrapper.cpp @@ -131,7 +131,7 @@ std::vector CLevelDBWrapper::CreateObfuscateKey() const bool CLevelDBWrapper::IsEmpty() { - boost::scoped_ptr it(NewIterator()); + boost::scoped_ptr it(NewIterator()); it->SeekToFirst(); return !(it->Valid()); } @@ -145,3 +145,10 @@ std::string CLevelDBWrapper::GetObfuscateKeyHex() const { return HexStr(obfuscate_key); } + +CLevelDBIterator::~CLevelDBIterator() { delete piter; } +bool CLevelDBIterator::Valid() { return piter->Valid(); } +void CLevelDBIterator::SeekToFirst() { piter->SeekToFirst(); } +void CLevelDBIterator::SeekToLast() { piter->SeekToLast(); } +void CLevelDBIterator::Next() { piter->Next(); } +void CLevelDBIterator::Prev() { piter->Prev(); } diff --git a/src/leveldbwrapper.h b/src/leveldbwrapper.h index f5c463830..60101e18c 100644 --- a/src/leveldbwrapper.h +++ b/src/leveldbwrapper.h @@ -32,13 +32,13 @@ class CLevelDBBatch private: leveldb::WriteBatch batch; - const std::vector obfuscate_key; + const std::vector *obfuscate_key; public: /** * @param[in] obfuscate_key If passed, XOR data with this key. */ - CLevelDBBatch(const std::vector& obfuscate_key) : obfuscate_key(obfuscate_key) { }; + CLevelDBBatch(const std::vector *obfuscate_key) : obfuscate_key(obfuscate_key) { }; template void Write(const K& key, const V& value) @@ -51,7 +51,7 @@ public: CDataStream ssValue(SER_DISK, CLIENT_VERSION); ssValue.reserve(ssValue.GetSerializeSize(value)); ssValue << value; - ssValue.Xor(obfuscate_key); + ssValue.Xor(*obfuscate_key); leveldb::Slice slValue(&ssValue[0], ssValue.size()); batch.Put(slKey, slValue); @@ -68,7 +68,72 @@ public: batch.Delete(slKey); } }; + +class CLevelDBIterator +{ +private: + leveldb::Iterator *piter; + const std::vector *obfuscate_key; +public: + + /** + * @param[in] piterIn The original leveldb iterator. + * @param[in] obfuscate_key If passed, XOR data with this key. + */ + CLevelDBIterator(leveldb::Iterator *piterIn, const std::vector* obfuscate_key) : + piter(piterIn), obfuscate_key(obfuscate_key) { }; + ~CLevelDBIterator(); + + bool Valid(); + + void SeekToFirst(); + void SeekToLast(); + + template void Seek(const K& key) { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey.reserve(ssKey.GetSerializeSize(key)); + ssKey << key; + leveldb::Slice slKey(&ssKey[0], ssKey.size()); + piter->Seek(slKey); + } + + void Next(); + void Prev(); + + template bool GetKey(K& key) { + leveldb::Slice slKey = piter->key(); + try { + CDataStream ssKey(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION); + ssKey >> key; + } catch(std::exception &e) { + return false; + } + return true; + } + + unsigned int GetKeySize() { + return piter->key().size(); + } + + template bool GetValue(V& value) { + leveldb::Slice slValue = piter->value(); + try { + CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION); + ssValue.Xor(*obfuscate_key); + ssValue >> value; + } catch(std::exception &e) { + return false; + } + return true; + } + + unsigned int GetValueSize() { + return piter->value().size(); + } + +}; + class CLevelDBWrapper { private: @@ -145,7 +210,7 @@ public: template bool Write(const K& key, const V& value, bool fSync = false) throw(leveldb_error) { - CLevelDBBatch batch(obfuscate_key); + CLevelDBBatch batch(&obfuscate_key); batch.Write(key, value); return WriteBatch(batch, fSync); } @@ -172,7 +237,7 @@ public: template bool Erase(const K& key, bool fSync = false) throw(leveldb_error) { - CLevelDBBatch batch(obfuscate_key); + CLevelDBBatch batch(&obfuscate_key); batch.Erase(key); return WriteBatch(batch, fSync); } @@ -187,14 +252,13 @@ public: bool Sync() throw(leveldb_error) { - CLevelDBBatch batch(obfuscate_key); + CLevelDBBatch batch(&obfuscate_key); return WriteBatch(batch, true); } - // not exactly clean encapsulation, but it's easiest for now - leveldb::Iterator* NewIterator() + CLevelDBIterator *NewIterator() { - return pdb->NewIterator(iteroptions); + return new CLevelDBIterator(pdb->NewIterator(iteroptions), &obfuscate_key); } /** diff --git a/src/test/leveldbwrapper_tests.cpp b/src/test/leveldbwrapper_tests.cpp index db04f3ea4..606313b00 100644 --- a/src/test/leveldbwrapper_tests.cpp +++ b/src/test/leveldbwrapper_tests.cpp @@ -46,7 +46,86 @@ BOOST_AUTO_TEST_CASE(leveldbwrapper) BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); } } - + +// Test batch operations +BOOST_AUTO_TEST_CASE(leveldbwrapper_batch) +{ + // Perform tests both obfuscated and non-obfuscated. + for (int i = 0; i < 2; i++) { + bool obfuscate = (bool)i; + path ph = temp_directory_path() / unique_path(); + CLevelDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); + + char key = 'i'; + uint256 in = GetRandHash(); + char key2 = 'j'; + uint256 in2 = GetRandHash(); + char key3 = 'k'; + uint256 in3 = GetRandHash(); + + uint256 res; + CLevelDBBatch batch(&dbw.GetObfuscateKey()); + + batch.Write(key, in); + batch.Write(key2, in2); + batch.Write(key3, in3); + + // Remove key3 before it's even been written + batch.Erase(key3); + + dbw.WriteBatch(batch); + + BOOST_CHECK(dbw.Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); + BOOST_CHECK(dbw.Read(key2, res)); + BOOST_CHECK_EQUAL(res.ToString(), in2.ToString()); + + // key3 never should've been written + BOOST_CHECK(dbw.Read(key3, res) == false); + } +} + +BOOST_AUTO_TEST_CASE(leveldbwrapper_iterator) +{ + // Perform tests both obfuscated and non-obfuscated. + for (int i = 0; i < 2; i++) { + bool obfuscate = (bool)i; + path ph = temp_directory_path() / unique_path(); + CLevelDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); + + // The two keys are intentionally chosen for ordering + char key = 'j'; + uint256 in = GetRandHash(); + BOOST_CHECK(dbw.Write(key, in)); + char key2 = 'k'; + uint256 in2 = GetRandHash(); + BOOST_CHECK(dbw.Write(key2, in2)); + + boost::scoped_ptr it(const_cast(&dbw)->NewIterator()); + + // Be sure to seek past the obfuscation key (if it exists) + it->Seek(key); + + char key_res; + uint256 val_res; + + it->GetKey(key_res); + it->GetValue(val_res); + BOOST_CHECK_EQUAL(key_res, key); + BOOST_CHECK_EQUAL(val_res.ToString(), in.ToString()); + + it->Next(); + + it->GetKey(key_res); + it->GetValue(val_res); + BOOST_CHECK_EQUAL(key_res, key2); + BOOST_CHECK_EQUAL(val_res.ToString(), in2.ToString()); + + it->Next(); + BOOST_CHECK_EQUAL(it->Valid(), false); + } +} + // Test that we do not obfuscation if there is existing data. BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) { diff --git a/src/txdb.cpp b/src/txdb.cpp index 9738dea03..5723c9244 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -49,7 +49,7 @@ uint256 CCoinsViewDB::GetBestBlock() const { } bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { - CLevelDBBatch batch(db.GetObfuscateKey()); + CLevelDBBatch batch(&db.GetObfuscateKey()); size_t count = 0; size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { @@ -98,8 +98,8 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ - boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); - pcursor->SeekToFirst(); + boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); + pcursor->Seek('c'); CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); stats.hashBlock = GetBestBlock(); @@ -107,22 +107,10 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { CAmount nTotalAmount = 0; while (pcursor->Valid()) { boost::this_thread::interruption_point(); - try { - leveldb::Slice slKey = pcursor->key(); - CDataStream ssKey(slKey.data(), slKey.data()+slKey.size(), SER_DISK, CLIENT_VERSION); - char chType; - ssKey >> chType; - if (chType == DB_COINS) { - leveldb::Slice slValue = pcursor->value(); - CDataStream ssValue(slValue.data(), slValue.data()+slValue.size(), SER_DISK, CLIENT_VERSION); - CCoins coins; - ssValue >> coins; - uint256 txhash; - ssKey >> txhash; - ss << txhash; - ss << VARINT(coins.nVersion); - ss << (coins.fCoinBase ? 'c' : 'n'); - ss << VARINT(coins.nHeight); + std::pair key; + CCoins coins; + if (pcursor->GetKey(key) && key.first == 'c') { + if (pcursor->GetValue(coins)) { stats.nTransactions++; for (unsigned int i=0; iGetKeySize(); ss << VARINT(0); + } else { + return error("CCoinsViewDB::GetStats() : unable to read value"); } - pcursor->Next(); - } catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } else { + break; } + pcursor->Next(); } { LOCK(cs_main); @@ -151,7 +141,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { } bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { - CLevelDBBatch batch(GetObfuscateKey()); + CLevelDBBatch batch(&GetObfuscateKey()); for (std::vector >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second); } @@ -167,7 +157,7 @@ bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) { } bool CBlockTreeDB::WriteTxIndex(const std::vector >&vect) { - CLevelDBBatch batch(GetObfuscateKey()); + CLevelDBBatch batch(&GetObfuscateKey()); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) batch.Write(make_pair(DB_TXINDEX, it->first), it->second); return WriteBatch(batch); @@ -187,26 +177,17 @@ bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) { bool CBlockTreeDB::LoadBlockIndexGuts() { - boost::scoped_ptr pcursor(NewIterator()); + boost::scoped_ptr pcursor(NewIterator()); - CDataStream ssKeySet(SER_DISK, CLIENT_VERSION); - ssKeySet << make_pair(DB_BLOCK_INDEX, uint256()); - pcursor->Seek(ssKeySet.str()); + pcursor->Seek(make_pair('b', uint256())); // Load mapBlockIndex while (pcursor->Valid()) { boost::this_thread::interruption_point(); - try { - leveldb::Slice slKey = pcursor->key(); - CDataStream ssKey(slKey.data(), slKey.data()+slKey.size(), SER_DISK, CLIENT_VERSION); - char chType; - ssKey >> chType; - if (chType == DB_BLOCK_INDEX) { - leveldb::Slice slValue = pcursor->value(); - CDataStream ssValue(slValue.data(), slValue.data()+slValue.size(), SER_DISK, CLIENT_VERSION); - CDiskBlockIndex diskindex; - ssValue >> diskindex; - + std::pair key; + if (pcursor->GetKey(key) && key.first == 'b') { + CDiskBlockIndex diskindex; + if (pcursor->GetValue(diskindex)) { // Construct block index object CBlockIndex* pindexNew = InsertBlockIndex(diskindex.GetBlockHash()); pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev); @@ -227,10 +208,10 @@ bool CBlockTreeDB::LoadBlockIndexGuts() pcursor->Next(); } else { - break; // if shutdown requested or finished loading block index + return error("LoadBlockIndex() : failed to read value"); } - } catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } else { + break; } }