diff --git a/doc/release-notes.md b/doc/release-notes.md index a29094b51..2cb9f9413 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -4,3 +4,11 @@ release-notes at release time) Notable changes =============== +Merkle branches removed from wallet +----------------------------------- + +Previously, every wallet transaction stored a Merkle branch to prove its +presence in blocks. This wasn't being used for more than an expensive +sanity check. Since 5.1.0, these are no longer stored. When loading a +5.1.0 wallet into an older version, it will automatically rescan to avoid +failed checks. diff --git a/src/Makefile.am b/src/Makefile.am index 03d7c13aa..ebc84bf89 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -212,6 +212,7 @@ BITCOIN_CORE_H = \ compressor.h \ consensus/consensus.h \ consensus/funding.h \ + consensus/merkle.h \ consensus/params.h \ consensus/upgrades.h \ consensus/validation.h \ @@ -449,6 +450,7 @@ libbitcoin_common_a_SOURCES = \ coins.cpp \ compressor.cpp \ consensus/funding.cpp \ + consensus/merkle.cpp \ consensus/params.cpp \ consensus/upgrades.cpp \ core_read.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 0a1a8dce2..73ed751db 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -84,6 +84,7 @@ BITCOIN_TESTS =\ test/dbwrapper_tests.cpp \ test/main_tests.cpp \ test/mempool_tests.cpp \ + test/merkle_tests.cpp \ test/miner_tests.cpp \ test/multisig_tests.cpp \ test/net_tests.cpp \ diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 01bdc329a..4525cf577 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -4,6 +4,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or https://www.opensource.org/licenses/mit-license.php . +#include "chainparams.h" +#include "consensus/merkle.h" #include "key_io.h" #include "main.h" #include "crypto/equihash.h" @@ -43,7 +45,7 @@ static CBlock CreateGenesisBlock(const char* pszTimestamp, const CScript& genesi genesis.nVersion = nVersion; genesis.vtx.push_back(txNew); genesis.hashPrevBlock.SetNull(); - genesis.hashMerkleRoot = genesis.BuildMerkleTree(); + genesis.hashMerkleRoot = BlockMerkleRoot(genesis); return genesis; } diff --git a/src/consensus/merkle.cpp b/src/consensus/merkle.cpp new file mode 100644 index 000000000..7abfe1452 --- /dev/null +++ b/src/consensus/merkle.cpp @@ -0,0 +1,172 @@ +#include "merkle.h" +#include "hash.h" +#include "util/strencodings.h" + +/* WARNING! If you're reading this because you're learning about crypto + and/or designing a new system that will use merkle trees, keep in mind + that the following merkle tree algorithm has a serious flaw related to + duplicate txids, resulting in a vulnerability (CVE-2012-2459). + + The reason is that if the number of hashes in the list at a given time + is odd, the last one is duplicated before computing the next level (which + is unusual in Merkle trees). This results in certain sequences of + transactions leading to the same merkle root. For example, these two + trees: + + A A + / \ / \ + B C B C + / \ | / \ / \ + D E F D E F F + / \ / \ / \ / \ / \ / \ / \ + 1 2 3 4 5 6 1 2 3 4 5 6 5 6 + + for transaction lists [1,2,3,4,5,6] and [1,2,3,4,5,6,5,6] (where 5 and + 6 are repeated) result in the same root hash A (because the hash of both + of (F) and (F,F) is C). + + The vulnerability results from being able to send a block with such a + transaction list, with the same merkle root, and the same block hash as + the original without duplication, resulting in failed validation. If the + receiving node proceeds to mark that block as permanently invalid + however, it will fail to accept further unmodified (and thus potentially + valid) versions of the same block. We defend against this by detecting + the case where we would hash two identical hashes at the end of the list + together, and treating that identically to the block having an invalid + merkle root. Assuming no double-SHA256 collisions, this will detect all + known ways of changing the transactions without affecting the merkle + root. +*/ + +/* This implements a constant-space merkle root/path calculator, limited to 2^32 leaves. */ +static void MerkleComputation(const std::vector& leaves, uint256* proot, bool* pmutated, uint32_t branchpos, std::vector* pbranch) { + if (pbranch) pbranch->clear(); + if (leaves.size() == 0) { + if (pmutated) *pmutated = false; + if (proot) *proot = uint256(); + return; + } + bool mutated = false; + // count is the number of leaves processed so far. + uint32_t count = 0; + // inner is an array of eagerly computed subtree hashes, indexed by tree + // level (0 being the leaves). + // For example, when count is 25 (11001 in binary), inner[4] is the hash of + // the first 16 leaves, inner[3] of the next 8 leaves, and inner[0] equal to + // the last leaf. The other inner entries are undefined. + uint256 inner[32]; + // Which position in inner is a hash that depends on the matching leaf. + int matchlevel = -1; + // First process all leaves into 'inner' values. + while (count < leaves.size()) { + uint256 h = leaves[count]; + bool matchh = count == branchpos; + count++; + int level; + // For each of the lower bits in count that are 0, do 1 step. Each + // corresponds to an inner value that existed before processing the + // current leaf, and each needs a hash to combine it. + for (level = 0; !(count & (((uint32_t)1) << level)); level++) { + if (pbranch) { + if (matchh) { + pbranch->push_back(inner[level]); + } else if (matchlevel == level) { + pbranch->push_back(h); + matchh = true; + } + } + mutated |= (inner[level] == h); + CHash256().Write(inner[level].begin(), 32).Write(h.begin(), 32).Finalize(h.begin()); + } + // Store the resulting hash at inner position level. + inner[level] = h; + if (matchh) { + matchlevel = level; + } + } + // Do a final 'sweep' over the rightmost branch of the tree to process + // odd levels, and reduce everything to a single top value. + // Level is the level (counted from the bottom) up to which we've sweeped. + int level = 0; + // As long as bit number level in count is zero, skip it. It means there + // is nothing left at this level. + while (!(count & (((uint32_t)1) << level))) { + level++; + } + uint256 h = inner[level]; + bool matchh = matchlevel == level; + while (count != (((uint32_t)1) << level)) { + // If we reach this point, h is an inner value that is not the top. + // We combine it with itself (Bitcoin's special rule for odd levels in + // the tree) to produce a higher level one. + if (pbranch && matchh) { + pbranch->push_back(h); + } + CHash256().Write(h.begin(), 32).Write(h.begin(), 32).Finalize(h.begin()); + // Increment count to the value it would have if two entries at this + // level had existed. + count += (((uint32_t)1) << level); + level++; + // And propagate the result upwards accordingly. + while (!(count & (((uint32_t)1) << level))) { + if (pbranch) { + if (matchh) { + pbranch->push_back(inner[level]); + } else if (matchlevel == level) { + pbranch->push_back(h); + matchh = true; + } + } + CHash256().Write(inner[level].begin(), 32).Write(h.begin(), 32).Finalize(h.begin()); + level++; + } + } + // Return result. + if (pmutated) *pmutated = mutated; + if (proot) *proot = h; +} + +uint256 ComputeMerkleRoot(const std::vector& leaves, bool* mutated) { + uint256 hash; + MerkleComputation(leaves, &hash, mutated, -1, NULL); + return hash; +} + +std::vector ComputeMerkleBranch(const std::vector& leaves, uint32_t position) { + std::vector ret; + MerkleComputation(leaves, NULL, NULL, position, &ret); + return ret; +} + +uint256 ComputeMerkleRootFromBranch(const uint256& leaf, const std::vector& vMerkleBranch, uint32_t nIndex) { + uint256 hash = leaf; + for (std::vector::const_iterator it = vMerkleBranch.begin(); it != vMerkleBranch.end(); ++it) { + if (nIndex & 1) { + hash = Hash(BEGIN(*it), END(*it), BEGIN(hash), END(hash)); + } else { + hash = Hash(BEGIN(hash), END(hash), BEGIN(*it), END(*it)); + } + nIndex >>= 1; + } + return hash; +} + +uint256 BlockMerkleRoot(const CBlock& block, bool* mutated) +{ + std::vector leaves; + leaves.resize(block.vtx.size()); + for (size_t s = 0; s < block.vtx.size(); s++) { + leaves[s] = block.vtx[s].GetHash(); + } + return ComputeMerkleRoot(leaves, mutated); +} + +std::vector BlockMerkleBranch(const CBlock& block, uint32_t position) +{ + std::vector leaves; + leaves.resize(block.vtx.size()); + for (size_t s = 0; s < block.vtx.size(); s++) { + leaves[s] = block.vtx[s].GetHash(); + } + return ComputeMerkleBranch(leaves, position); +} diff --git a/src/consensus/merkle.h b/src/consensus/merkle.h new file mode 100644 index 000000000..f50d60ce1 --- /dev/null +++ b/src/consensus/merkle.h @@ -0,0 +1,32 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CONSENSUS_MERKLE_H +#define BITCOIN_CONSENSUS_MERKLE_H + +#include +#include + +#include "primitives/transaction.h" +#include "primitives/block.h" +#include "uint256.h" + +uint256 ComputeMerkleRoot(const std::vector& leaves, bool* mutated = NULL); +std::vector ComputeMerkleBranch(const std::vector& leaves, uint32_t position); +uint256 ComputeMerkleRootFromBranch(const uint256& leaf, const std::vector& branch, uint32_t position); + +/* + * Compute the Merkle root of the transactions in a block. + * *mutated is set to true if a duplicated subtree was found. + */ +uint256 BlockMerkleRoot(const CBlock& block, bool* mutated = NULL); + +/* + * Compute the Merkle branch for the tree of transactions in a block, for a + * given position. + * This can be verified using ComputeMerkleRootFromBranch. + */ +std::vector BlockMerkleBranch(const CBlock& block, uint32_t position); + +#endif // BITCOIN_CONSENSUS_MERKLE_H diff --git a/src/core_memusage.h b/src/core_memusage.h index 706997229..7b6d9c42e 100644 --- a/src/core_memusage.h +++ b/src/core_memusage.h @@ -57,7 +57,7 @@ static inline size_t RecursiveDynamicUsage(const CMutableTransaction& tx) { } static inline size_t RecursiveDynamicUsage(const CBlock& block) { - size_t mem = memusage::DynamicUsage(block.vtx) + memusage::DynamicUsage(block.vMerkleTree); + size_t mem = memusage::DynamicUsage(block.vtx); for (std::vector::const_iterator it = block.vtx.begin(); it != block.vtx.end(); it++) { mem += RecursiveDynamicUsage(*it); } diff --git a/src/gtest/test_validation.cpp b/src/gtest/test_validation.cpp index ec56c5072..b4acf495b 100644 --- a/src/gtest/test_validation.cpp +++ b/src/gtest/test_validation.cpp @@ -1,6 +1,7 @@ #include #include +#include "consensus/merkle.h" #include "consensus/upgrades.h" #include "consensus/validation.h" #include "main.h" @@ -154,7 +155,7 @@ TEST(Validation, ContextualCheckInputsDetectsOldBranchId) { // Create a fake block. It doesn't need to contain any transactions; we just // need it to be in the global state when our fake view is used. CBlock block; - block.hashMerkleRoot = block.BuildMerkleTree(); + block.hashMerkleRoot = BlockMerkleRoot(block); auto blockHash = block.GetHash(); CBlockIndex fakeIndex {block}; mapBlockIndex.insert(std::make_pair(blockHash, &fakeIndex)); @@ -226,14 +227,14 @@ TEST(Validation, ReceivedBlockTransactions) { // Create a fake genesis block CBlock block1; block1.vtx.push_back(GetValidSproutReceive(sk, 5, true)); - block1.hashMerkleRoot = block1.BuildMerkleTree(); + block1.hashMerkleRoot = BlockMerkleRoot(block1); CBlockIndex fakeIndex1 {block1}; // Create a fake child block CBlock block2; block2.hashPrevBlock = block1.GetHash(); block2.vtx.push_back(GetValidSproutReceive(sk, 10, true)); - block2.hashMerkleRoot = block2.BuildMerkleTree(); + block2.hashMerkleRoot = BlockMerkleRoot(block2); CBlockIndex fakeIndex2 {block2}; fakeIndex2.pprev = &fakeIndex1; diff --git a/src/main.cpp b/src/main.cpp index 28e4fb9a3..4bbab4c64 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -14,6 +14,7 @@ #include "checkqueue.h" #include "consensus/consensus.h" #include "consensus/funding.h" +#include "consensus/merkle.h" #include "consensus/upgrades.h" #include "consensus/validation.h" #include "deprecation.h" @@ -4729,6 +4730,9 @@ bool CheckBlock(const CBlock& block, { // These are checks that are independent of context. + if (block.fChecked) + return true; + // Check that the header is valid (particularly PoW). This is mostly // redundant with the call in AcceptBlockHeader. if (!CheckBlockHeader(block, state, chainparams, fCheckPOW)) @@ -4737,7 +4741,7 @@ bool CheckBlock(const CBlock& block, // Check the merkle root. if (fCheckMerkleRoot) { bool mutated; - uint256 hashMerkleRoot2 = block.BuildMerkleTree(&mutated); + uint256 hashMerkleRoot2 = BlockMerkleRoot(block, &mutated); if (block.hashMerkleRoot != hashMerkleRoot2) return state.DoS(100, error("CheckBlock(): hashMerkleRoot mismatch"), REJECT_INVALID, "bad-txnmrklroot", true); @@ -4787,6 +4791,9 @@ bool CheckBlock(const CBlock& block, return state.DoS(100, error("CheckBlock(): out-of-bounds SigOpCount"), REJECT_INVALID, "bad-blk-sigops", true); + if (fCheckPOW && fCheckMerkleRoot) + block.fChecked = true; + return true; } diff --git a/src/miner.cpp b/src/miner.cpp index cd8102ac2..71128b868 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -13,6 +13,7 @@ #include "chainparams.h" #include "consensus/consensus.h" #include "consensus/funding.h" +#include "consensus/merkle.h" #include "consensus/upgrades.h" #include "consensus/validation.h" #ifdef ENABLE_MINING @@ -867,7 +868,7 @@ void IncrementExtraNonce( assert(txCoinbase.vin[0].scriptSig.size() <= 100); pblock->vtx[0] = txCoinbase; - pblock->hashMerkleRoot = pblock->BuildMerkleTree(); + pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); if (consensusParams.NetworkUpgradeActive(nHeight, Consensus::UPGRADE_NU5)) { pblocktemplate->hashAuthDataRoot = pblock->BuildAuthDataMerkleTree(); pblock->hashBlockCommitments = DeriveBlockCommitmentsHash( diff --git a/src/primitives/block.cpp b/src/primitives/block.cpp index 7b433f3e3..95bdb5a48 100644 --- a/src/primitives/block.cpp +++ b/src/primitives/block.cpp @@ -37,100 +37,6 @@ uint256 CBlockHeader::GetHash() const return SerializeHash(*this); } -uint256 CBlock::BuildMerkleTree(bool* fMutated) const -{ - /* WARNING! If you're reading this because you're learning about crypto - and/or designing a new system that will use merkle trees, keep in mind - that the following merkle tree algorithm has a serious flaw related to - duplicate txids, resulting in a vulnerability (CVE-2012-2459). - - The reason is that if the number of hashes in the list at a given time - is odd, the last one is duplicated before computing the next level (which - is unusual in Merkle trees). This results in certain sequences of - transactions leading to the same merkle root. For example, these two - trees: - - A A - / \ / \ - B C B C - / \ | / \ / \ - D E F D E F F - / \ / \ / \ / \ / \ / \ / \ - 1 2 3 4 5 6 1 2 3 4 5 6 5 6 - - for transaction lists [1,2,3,4,5,6] and [1,2,3,4,5,6,5,6] (where 5 and - 6 are repeated) result in the same root hash A (because the hash of both - of (F) and (F,F) is C). - - The vulnerability results from being able to send a block with such a - transaction list, with the same merkle root, and the same block hash as - the original without duplication, resulting in failed validation. If the - receiving node proceeds to mark that block as permanently invalid - however, it will fail to accept further unmodified (and thus potentially - valid) versions of the same block. We defend against this by detecting - the case where we would hash two identical hashes at the end of the list - together, and treating that identically to the block having an invalid - merkle root. Assuming no double-SHA256 collisions, this will detect all - known ways of changing the transactions without affecting the merkle - root. - */ - vMerkleTree.clear(); - vMerkleTree.reserve(vtx.size() * 2 + 16); // Safe upper bound for the number of total nodes. - for (std::vector::const_iterator it(vtx.begin()); it != vtx.end(); ++it) - vMerkleTree.push_back(it->GetHash()); - int j = 0; - bool mutated = false; - for (int nSize = vtx.size(); nSize > 1; nSize = (nSize + 1) / 2) - { - for (int i = 0; i < nSize; i += 2) - { - int i2 = std::min(i+1, nSize-1); - if (i2 == i + 1 && i2 + 1 == nSize && vMerkleTree[j+i] == vMerkleTree[j+i2]) { - // Two identical hashes at the end of the list at a particular level. - mutated = true; - } - vMerkleTree.push_back(Hash(BEGIN(vMerkleTree[j+i]), END(vMerkleTree[j+i]), - BEGIN(vMerkleTree[j+i2]), END(vMerkleTree[j+i2]))); - } - j += nSize; - } - if (fMutated) { - *fMutated = mutated; - } - return (vMerkleTree.empty() ? uint256() : vMerkleTree.back()); -} - -std::vector CBlock::GetMerkleBranch(int nIndex) const -{ - if (vMerkleTree.empty()) - BuildMerkleTree(); - std::vector vMerkleBranch; - int j = 0; - for (int nSize = vtx.size(); nSize > 1; nSize = (nSize + 1) / 2) - { - int i = std::min(nIndex^1, nSize-1); - vMerkleBranch.push_back(vMerkleTree[j+i]); - nIndex >>= 1; - j += nSize; - } - return vMerkleBranch; -} - -uint256 CBlock::CheckMerkleBranch(uint256 hash, const std::vector& vMerkleBranch, int nIndex) -{ - if (nIndex == -1) - return uint256(); - for (std::vector::const_iterator it(vMerkleBranch.begin()); it != vMerkleBranch.end(); ++it) - { - if (nIndex & 1) - hash = Hash(BEGIN(*it), END(*it), BEGIN(hash), END(hash)); - else - hash = Hash(BEGIN(hash), END(hash), BEGIN(*it), END(*it)); - nIndex >>= 1; - } - return hash; -} - // Return 0 if x == 0, otherwise the smallest power of 2 greater than or equal to x. // Algorithm based on . static uint64_t next_pow2(uint64_t x) @@ -193,9 +99,5 @@ std::string CBlock::ToString() const { s << " " << vtx[i].ToString() << "\n"; } - s << " vMerkleTree: "; - for (unsigned int i = 0; i < vMerkleTree.size(); i++) - s << " " << vMerkleTree[i].ToString(); - s << "\n"; return s.str(); } diff --git a/src/primitives/block.h b/src/primitives/block.h index eedac8b87..5ea2ad974 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -90,7 +90,7 @@ public: std::vector vtx; // memory only - mutable std::vector vMerkleTree; + mutable bool fChecked; CBlock() { @@ -115,7 +115,7 @@ public: { CBlockHeader::SetNull(); vtx.clear(); - vMerkleTree.clear(); + fChecked = false; } CBlockHeader GetBlockHeader() const @@ -132,15 +132,6 @@ public: return block; } - // Build the in-memory merkle tree for this block and return the merkle root. - // If non-NULL, *mutated is set to whether mutation was detected in the merkle - // tree (a duplication of transactions in the block leading to an identical - // merkle root). - uint256 BuildMerkleTree(bool* mutated = NULL) const; - - std::vector GetMerkleBranch(int nIndex) const; - static uint256 CheckMerkleBranch(uint256 hash, const std::vector& vMerkleBranch, int nIndex); - // Build the authorizing data Merkle tree for this block and return its // root. uint256 BuildAuthDataMerkleTree() const; diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 3ca7dac28..3badfe574 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -8,6 +8,7 @@ #include "chainparams.h" #include "consensus/consensus.h" #include "consensus/funding.h" +#include "consensus/merkle.h" #include "consensus/validation.h" #include "core_io.h" #ifdef ENABLE_MINING @@ -789,7 +790,7 @@ UniValue getblocktemplate(const UniValue& params, bool fHelp) // block template returned by this RPC is used unmodified. Otherwise, // these values must be recomputed. UniValue defaults(UniValue::VOBJ); - defaults.pushKV("merkleroot", pblock->BuildMerkleTree().GetHex()); + defaults.pushKV("merkleroot", BlockMerkleRoot(*pblock).GetHex()); defaults.pushKV("chainhistoryroot", pblocktemplate->hashChainHistoryRoot.GetHex()); if (consensus.NetworkUpgradeActive(pindexPrev->nHeight+1, Consensus::UPGRADE_NU5)) { defaults.pushKV("authdataroot", pblocktemplate->hashAuthDataRoot.GetHex()); diff --git a/src/test/main_tests.cpp b/src/test/main_tests.cpp index 8f75df036..65e68e8cd 100644 --- a/src/test/main_tests.cpp +++ b/src/test/main_tests.cpp @@ -123,5 +123,4 @@ BOOST_AUTO_TEST_CASE(test_combiner_all) Test.disconnect(&ReturnTrue); BOOST_CHECK(Test()); } - BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/merkle_tests.cpp b/src/test/merkle_tests.cpp new file mode 100644 index 000000000..7c074e378 --- /dev/null +++ b/src/test/merkle_tests.cpp @@ -0,0 +1,136 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "consensus/merkle.h" +#include "test/test_bitcoin.h" +#include "test_random.h" + +#include + +BOOST_FIXTURE_TEST_SUITE(merkle_tests, TestingSetup) + +// Older version of the merkle root computation code, for comparison. +static uint256 BlockBuildMerkleTree(const CBlock& block, bool* fMutated, std::vector& vMerkleTree) +{ + vMerkleTree.clear(); + vMerkleTree.reserve(block.vtx.size() * 2 + 16); // Safe upper bound for the number of total nodes. + for (std::vector::const_iterator it(block.vtx.begin()); it != block.vtx.end(); ++it) + vMerkleTree.push_back(it->GetHash()); + int j = 0; + bool mutated = false; + for (int nSize = block.vtx.size(); nSize > 1; nSize = (nSize + 1) / 2) + { + for (int i = 0; i < nSize; i += 2) + { + int i2 = std::min(i+1, nSize-1); + if (i2 == i + 1 && i2 + 1 == nSize && vMerkleTree[j+i] == vMerkleTree[j+i2]) { + // Two identical hashes at the end of the list at a particular level. + mutated = true; + } + vMerkleTree.push_back(Hash(vMerkleTree[j+i].begin(), vMerkleTree[j+i].end(), + vMerkleTree[j+i2].begin(), vMerkleTree[j+i2].end())); + } + j += nSize; + } + if (fMutated) { + *fMutated = mutated; + } + return (vMerkleTree.empty() ? uint256() : vMerkleTree.back()); +} + +// Older version of the merkle branch computation code, for comparison. +static std::vector BlockGetMerkleBranch(const CBlock& block, const std::vector& vMerkleTree, int nIndex) +{ + std::vector vMerkleBranch; + int j = 0; + for (int nSize = block.vtx.size(); nSize > 1; nSize = (nSize + 1) / 2) + { + int i = std::min(nIndex^1, nSize-1); + vMerkleBranch.push_back(vMerkleTree[j+i]); + nIndex >>= 1; + j += nSize; + } + return vMerkleBranch; +} + +static inline int ctz(uint32_t i) { + if (i == 0) return 0; + int j = 0; + while (!(i & 1)) { + j++; + i >>= 1; + } + return j; +} + +BOOST_AUTO_TEST_CASE(merkle_test) +{ + for (int i = 0; i < 32; i++) { + // Try 32 block sizes: all sizes from 0 to 16 inclusive, and then 15 random sizes. + int ntx = (i <= 16) ? i : 17 + (insecure_rand() % 4000); + // Try up to 3 mutations. + for (int mutate = 0; mutate <= 3; mutate++) { + int duplicate1 = mutate >= 1 ? 1 << ctz(ntx) : 0; // The last how many transactions to duplicate first. + if (duplicate1 >= ntx) break; // Duplication of the entire tree results in a different root (it adds a level). + int ntx1 = ntx + duplicate1; // The resulting number of transactions after the first duplication. + int duplicate2 = mutate >= 2 ? 1 << ctz(ntx1) : 0; // Likewise for the second mutation. + if (duplicate2 >= ntx1) break; + int ntx2 = ntx1 + duplicate2; + int duplicate3 = mutate >= 3 ? 1 << ctz(ntx2) : 0; // And for the the third mutation. + if (duplicate3 >= ntx2) break; + int ntx3 = ntx2 + duplicate3; + // Build a block with ntx different transactions. + CBlock block; + block.vtx.resize(ntx); + for (int j = 0; j < ntx; j++) { + CMutableTransaction mtx; + mtx.nLockTime = j; + block.vtx[j] = mtx; + } + // Compute the root of the block before mutating it. + bool unmutatedMutated = false; + uint256 unmutatedRoot = BlockMerkleRoot(block, &unmutatedMutated); + BOOST_CHECK(unmutatedMutated == false); + // Optionally mutate by duplicating the last transactions, resulting in the same merkle root. + block.vtx.resize(ntx3); + for (int j = 0; j < duplicate1; j++) { + block.vtx[ntx + j] = block.vtx[ntx + j - duplicate1]; + } + for (int j = 0; j < duplicate2; j++) { + block.vtx[ntx1 + j] = block.vtx[ntx1 + j - duplicate2]; + } + for (int j = 0; j < duplicate3; j++) { + block.vtx[ntx2 + j] = block.vtx[ntx2 + j - duplicate3]; + } + // Compute the merkle root and merkle tree using the old mechanism. + bool oldMutated = false; + std::vector merkleTree; + uint256 oldRoot = BlockBuildMerkleTree(block, &oldMutated, merkleTree); + // Compute the merkle root using the new mechanism. + bool newMutated = false; + uint256 newRoot = BlockMerkleRoot(block, &newMutated); + BOOST_CHECK(oldRoot == newRoot); + BOOST_CHECK(newRoot == unmutatedRoot); + BOOST_CHECK((newRoot == uint256()) == (ntx == 0)); + BOOST_CHECK(oldMutated == newMutated); + BOOST_CHECK(newMutated == !!mutate); + // If no mutation was done (once for every ntx value), try up to 16 branches. + if (mutate == 0) { + for (int loop = 0; loop < std::min(ntx, 16); loop++) { + // If ntx <= 16, try all branches. Otherise, try 16 random ones. + int mtx = loop; + if (ntx > 16) { + mtx = insecure_rand() % ntx; + } + std::vector newBranch = BlockMerkleBranch(block, mtx); + std::vector oldBranch = BlockGetMerkleBranch(block, merkleTree, mtx); + BOOST_CHECK(oldBranch == newBranch); + BOOST_CHECK(ComputeMerkleRootFromBranch(block.vtx[mtx].GetHash(), newBranch, mtx) == oldRoot); + } + } + } + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index f3e9acc47..0684f62e6 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -4,6 +4,7 @@ // file COPYING or https://www.opensource.org/licenses/mit-license.php . #include "arith_uint256.h" +#include "consensus/merkle.h" #include "consensus/validation.h" #include "main.h" #include "miner.h" @@ -189,7 +190,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) pblock->vtx[0] = CTransaction(txCoinbase); if (txFirst.size() < 2) txFirst.push_back(new CTransaction(pblock->vtx[0])); - pblock->hashMerkleRoot = pblock->BuildMerkleTree(); + pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); pblock->nNonce = uint256S(blockinfo[i].nonce_hex); pblock->nSolution = ParseHex(blockinfo[i].solution_hex); diff --git a/src/test/pmt_tests.cpp b/src/test/pmt_tests.cpp index 29473a6cb..f5229c73b 100644 --- a/src/test/pmt_tests.cpp +++ b/src/test/pmt_tests.cpp @@ -3,6 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or https://www.opensource.org/licenses/mit-license.php . +#include "consensus/merkle.h" #include "merkleblock.h" #include "serialize.h" #include "streams.h" @@ -49,7 +50,7 @@ BOOST_AUTO_TEST_CASE(pmt_test1) } // calculate actual merkle root and height - uint256 merkleRoot1 = block.BuildMerkleTree(); + uint256 merkleRoot1 = BlockMerkleRoot(block); std::vector vTxid(nTx, uint256()); for (unsigned int j=0; jhashMerkleRoot) - return 0; - fMerkleVerified = true; - } - pindexRet = pindex; return chainActive.Height() - pindex->nHeight + 1; } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index e435d0d46..eba2460dc 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -372,13 +372,8 @@ private: public: uint256 hashBlock; - std::vector vMerkleBranch; int nIndex; - // memory only - mutable bool fMerkleVerified; - - CMerkleTx() { Init(); @@ -393,13 +388,13 @@ public: { hashBlock = uint256(); nIndex = -1; - fMerkleVerified = false; } ADD_SERIALIZE_METHODS; template inline void SerializationOp(Stream& s, Operation ser_action) { + std::vector vMerkleBranch; // For compatibility with older versions. READWRITE(*(CTransaction*)this); READWRITE(hashBlock); READWRITE(vMerkleBranch); diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 9bf56152e..b02adceaf 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -276,12 +276,14 @@ bool CWalletDB::EraseWatchOnly(const CScript &dest) bool CWalletDB::WriteBestBlock(const CBlockLocator& locator) { nWalletDBUpdateCounter++; - return Write(std::string("bestblock"), locator); + Write(std::string("bestblock"), CBlockLocator()); // Write empty block locator so versions that require a merkle branch automatically rescan + return Write(std::string("bestblock_nomerkle"), locator); } bool CWalletDB::ReadBestBlock(CBlockLocator& locator) { - return Read(std::string("bestblock"), locator); + if (Read(std::string("bestblock"), locator) && !locator.vHave.empty()) return true; + return Read(std::string("bestblock_nomerkle"), locator); } bool CWalletDB::WriteOrderPosNext(int64_t nOrderPosNext)