2014-12-16 17:47:57 -08:00
|
|
|
// Copyright (c) 2012-2014 The Bitcoin Core developers
|
2023-01-23 10:31:54 -08:00
|
|
|
// Copyright (c) 2019-2023 The Zcash developers
|
2014-10-30 17:43:19 -07:00
|
|
|
// Distributed under the MIT software license, see the accompanying
|
2019-07-18 07:16:09 -07:00
|
|
|
// file COPYING or https://www.opensource.org/licenses/mit-license.php .
|
2012-08-12 20:26:27 -07:00
|
|
|
|
|
|
|
#include "bloom.h"
|
2013-04-12 22:13:08 -07:00
|
|
|
|
2014-11-18 13:03:02 -08:00
|
|
|
#include "primitives/transaction.h"
|
2014-10-28 14:47:18 -07:00
|
|
|
#include "hash.h"
|
2014-08-22 18:35:51 -07:00
|
|
|
#include "script/script.h"
|
|
|
|
#include "script/standard.h"
|
2015-07-19 12:43:34 -07:00
|
|
|
#include "random.h"
|
2014-10-22 12:08:30 -07:00
|
|
|
#include "streams.h"
|
2012-08-12 20:26:27 -07:00
|
|
|
|
2013-04-12 22:13:08 -07:00
|
|
|
#include <math.h>
|
|
|
|
#include <stdlib.h>
|
|
|
|
|
2019-05-21 15:47:44 -07:00
|
|
|
#include <algorithm>
|
2014-08-22 20:09:47 -07:00
|
|
|
|
2012-08-12 20:26:27 -07:00
|
|
|
#define LN2SQUARED 0.4804530139182014246671025263266649717305529515945455
|
|
|
|
#define LN2 0.6931471805599453094172321214581765680755001343602552
|
|
|
|
|
2017-02-12 21:39:48 -08:00
|
|
|
CBloomFilter::CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweakIn, unsigned char nFlagsIn) :
|
2015-04-24 10:14:45 -07:00
|
|
|
/**
|
|
|
|
* The ideal size for a bloom filter with a given number of elements and false positive rate is:
|
|
|
|
* - nElements * log(fp rate) / ln(2)^2
|
|
|
|
* We ignore filter parameters which will create a bloom filter larger than the protocol limits
|
|
|
|
*/
|
2017-01-27 00:43:41 -08:00
|
|
|
vData(std::min((unsigned int)(-1 / LN2SQUARED * nElements * log(nFPRate)), MAX_BLOOM_FILTER_SIZE * 8) / 8),
|
2015-04-24 10:14:45 -07:00
|
|
|
/**
|
|
|
|
* The ideal number of hash functions is filter size * ln(2) / number of elements
|
|
|
|
* Again, we ignore filter parameters which will create a bloom filter with more hash functions than the protocol limits
|
|
|
|
* See https://en.wikipedia.org/wiki/Bloom_filter for an explanation of these formulas
|
|
|
|
*/
|
2017-01-27 00:43:41 -08:00
|
|
|
nHashFuncs(std::min((unsigned int)(vData.size() * 8 / nElements * LN2), MAX_HASH_FUNCS)),
|
2015-04-24 10:14:45 -07:00
|
|
|
nTweak(nTweakIn),
|
|
|
|
nFlags(nFlagsIn)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2012-08-12 20:26:27 -07:00
|
|
|
inline unsigned int CBloomFilter::Hash(unsigned int nHashNum, const std::vector<unsigned char>& vDataToHash) const
|
|
|
|
{
|
|
|
|
// 0xFBA4C795 chosen as it guarantees a reasonable bit difference between nHashNum values.
|
2012-11-02 15:33:50 -07:00
|
|
|
return MurmurHash3(nHashNum * 0xFBA4C795 + nTweak, vDataToHash) % (vData.size() * 8);
|
2012-08-12 20:26:27 -07:00
|
|
|
}
|
|
|
|
|
2017-01-27 00:43:41 -08:00
|
|
|
void CBloomFilter::insert(const std::vector<unsigned char>& vKey)
|
2012-08-12 20:26:27 -07:00
|
|
|
{
|
2020-04-28 10:19:34 -07:00
|
|
|
if (vData.empty()) // Avoid divide-by-zero (CVE-2013-5700)
|
2013-02-24 17:36:59 -08:00
|
|
|
return;
|
2012-08-12 20:26:27 -07:00
|
|
|
for (unsigned int i = 0; i < nHashFuncs; i++)
|
|
|
|
{
|
|
|
|
unsigned int nIndex = Hash(i, vKey);
|
|
|
|
// Sets bit nIndex of vData
|
2014-03-19 21:21:23 -07:00
|
|
|
vData[nIndex >> 3] |= (1 << (7 & nIndex));
|
2012-08-12 20:26:27 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void CBloomFilter::insert(const COutPoint& outpoint)
|
|
|
|
{
|
|
|
|
CDataStream stream(SER_NETWORK, PROTOCOL_VERSION);
|
|
|
|
stream << outpoint;
|
2017-01-27 00:43:41 -08:00
|
|
|
std::vector<unsigned char> data(stream.begin(), stream.end());
|
2012-08-12 20:26:27 -07:00
|
|
|
insert(data);
|
|
|
|
}
|
|
|
|
|
|
|
|
void CBloomFilter::insert(const uint256& hash)
|
|
|
|
{
|
2017-01-27 00:43:41 -08:00
|
|
|
std::vector<unsigned char> data(hash.begin(), hash.end());
|
2012-08-12 20:26:27 -07:00
|
|
|
insert(data);
|
|
|
|
}
|
|
|
|
|
2017-01-27 00:43:41 -08:00
|
|
|
bool CBloomFilter::contains(const std::vector<unsigned char>& vKey) const
|
2012-08-12 20:26:27 -07:00
|
|
|
{
|
2020-04-28 10:19:34 -07:00
|
|
|
if (vData.empty()) // Avoid divide-by-zero (CVE-2013-5700)
|
2013-02-24 17:36:59 -08:00
|
|
|
return true;
|
2012-08-12 20:26:27 -07:00
|
|
|
for (unsigned int i = 0; i < nHashFuncs; i++)
|
|
|
|
{
|
|
|
|
unsigned int nIndex = Hash(i, vKey);
|
|
|
|
// Checks bit nIndex of vData
|
2014-03-19 21:21:23 -07:00
|
|
|
if (!(vData[nIndex >> 3] & (1 << (7 & nIndex))))
|
2012-08-12 20:26:27 -07:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool CBloomFilter::contains(const COutPoint& outpoint) const
|
|
|
|
{
|
|
|
|
CDataStream stream(SER_NETWORK, PROTOCOL_VERSION);
|
|
|
|
stream << outpoint;
|
2017-01-27 00:43:41 -08:00
|
|
|
std::vector<unsigned char> data(stream.begin(), stream.end());
|
2012-08-12 20:26:27 -07:00
|
|
|
return contains(data);
|
|
|
|
}
|
|
|
|
|
|
|
|
bool CBloomFilter::contains(const uint256& hash) const
|
|
|
|
{
|
2017-01-27 00:43:41 -08:00
|
|
|
std::vector<unsigned char> data(hash.begin(), hash.end());
|
2012-08-12 20:26:27 -07:00
|
|
|
return contains(data);
|
|
|
|
}
|
|
|
|
|
|
|
|
bool CBloomFilter::IsWithinSizeConstraints() const
|
|
|
|
{
|
|
|
|
return vData.size() <= MAX_BLOOM_FILTER_SIZE && nHashFuncs <= MAX_HASH_FUNCS;
|
|
|
|
}
|
|
|
|
|
2014-06-09 01:02:00 -07:00
|
|
|
bool CBloomFilter::IsRelevantAndUpdate(const CTransaction& tx)
|
2012-08-12 20:26:27 -07:00
|
|
|
{
|
2012-08-18 20:38:28 -07:00
|
|
|
bool fFound = false;
|
2012-08-12 20:26:27 -07:00
|
|
|
// Match if the filter contains the hash of tx
|
|
|
|
// for finding tx when they appear in a block
|
2020-04-28 10:19:34 -07:00
|
|
|
if (vData.empty()) // zero-size = "match-all" filter
|
2013-08-18 20:21:06 -07:00
|
|
|
return true;
|
2016-08-30 12:49:38 -07:00
|
|
|
const uint256& hash = tx.GetHash();
|
2012-08-12 20:26:30 -07:00
|
|
|
if (contains(hash))
|
2012-08-18 20:38:28 -07:00
|
|
|
fFound = true;
|
2012-08-12 20:26:27 -07:00
|
|
|
|
2012-08-18 20:38:28 -07:00
|
|
|
for (unsigned int i = 0; i < tx.vout.size(); i++)
|
2012-08-12 20:26:27 -07:00
|
|
|
{
|
2012-08-18 20:38:28 -07:00
|
|
|
const CTxOut& txout = tx.vout[i];
|
2012-08-12 20:26:27 -07:00
|
|
|
// Match if the filter contains any arbitrary script data element in any scriptPubKey in tx
|
2012-08-18 20:38:28 -07:00
|
|
|
// If this matches, also add the specific output that was matched.
|
|
|
|
// This means clients don't have to update the filter themselves when a new relevant tx
|
|
|
|
// is discovered in order to find spending transactions, which avoids round-tripping and race conditions.
|
2012-08-12 20:26:27 -07:00
|
|
|
CScript::const_iterator pc = txout.scriptPubKey.begin();
|
2017-01-27 00:43:41 -08:00
|
|
|
std::vector<unsigned char> data;
|
2012-08-12 20:26:27 -07:00
|
|
|
while (pc < txout.scriptPubKey.end())
|
|
|
|
{
|
|
|
|
opcodetype opcode;
|
|
|
|
if (!txout.scriptPubKey.GetOp(pc, opcode, data))
|
|
|
|
break;
|
|
|
|
if (data.size() != 0 && contains(data))
|
2012-08-18 20:38:28 -07:00
|
|
|
{
|
|
|
|
fFound = true;
|
2013-01-10 17:23:28 -08:00
|
|
|
if ((nFlags & BLOOM_UPDATE_MASK) == BLOOM_UPDATE_ALL)
|
|
|
|
insert(COutPoint(hash, i));
|
|
|
|
else if ((nFlags & BLOOM_UPDATE_MASK) == BLOOM_UPDATE_P2PUBKEY_ONLY)
|
|
|
|
{
|
|
|
|
txnouttype type;
|
2017-01-27 00:43:41 -08:00
|
|
|
std::vector<std::vector<unsigned char> > vSolutions;
|
2013-01-10 17:23:28 -08:00
|
|
|
if (Solver(txout.scriptPubKey, type, vSolutions) &&
|
|
|
|
(type == TX_PUBKEY || type == TX_MULTISIG))
|
|
|
|
insert(COutPoint(hash, i));
|
|
|
|
}
|
2012-08-18 20:38:28 -07:00
|
|
|
break;
|
|
|
|
}
|
2012-08-12 20:26:27 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-08-18 20:38:28 -07:00
|
|
|
if (fFound)
|
|
|
|
return true;
|
|
|
|
|
2017-06-01 18:18:57 -07:00
|
|
|
for (const CTxIn& txin : tx.vin)
|
2012-08-12 20:26:27 -07:00
|
|
|
{
|
|
|
|
// Match if the filter contains an outpoint tx spends
|
|
|
|
if (contains(txin.prevout))
|
|
|
|
return true;
|
|
|
|
|
|
|
|
// Match if the filter contains any arbitrary script data element in any scriptSig in tx
|
|
|
|
CScript::const_iterator pc = txin.scriptSig.begin();
|
2017-01-27 00:43:41 -08:00
|
|
|
std::vector<unsigned char> data;
|
2012-08-12 20:26:27 -07:00
|
|
|
while (pc < txin.scriptSig.end())
|
|
|
|
{
|
|
|
|
opcodetype opcode;
|
|
|
|
if (!txin.scriptSig.GetOp(pc, opcode, data))
|
|
|
|
break;
|
|
|
|
if (data.size() != 0 && contains(data))
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
2013-08-18 20:21:06 -07:00
|
|
|
|
2017-02-12 21:39:48 -08:00
|
|
|
CRollingBloomFilter::CRollingBloomFilter(const unsigned int nElements, const double fpRate)
|
2015-04-24 10:14:45 -07:00
|
|
|
{
|
2015-11-27 04:20:29 -08:00
|
|
|
double logFpRate = log(fpRate);
|
|
|
|
/* The optimal number of hash functions is log(fpRate) / log(0.5), but
|
|
|
|
* restrict it to the range 1-50. */
|
|
|
|
nHashFuncs = std::max(1, std::min((int)round(logFpRate / log(0.5)), 50));
|
|
|
|
/* In this rolling bloom filter, we'll store between 2 and 3 generations of nElements / 2 entries. */
|
|
|
|
nEntriesPerGeneration = (nElements + 1) / 2;
|
|
|
|
uint32_t nMaxElements = nEntriesPerGeneration * 3;
|
|
|
|
/* The maximum fpRate = pow(1.0 - exp(-nHashFuncs * nMaxElements / nFilterBits), nHashFuncs)
|
|
|
|
* => pow(fpRate, 1.0 / nHashFuncs) = 1.0 - exp(-nHashFuncs * nMaxElements / nFilterBits)
|
|
|
|
* => 1.0 - pow(fpRate, 1.0 / nHashFuncs) = exp(-nHashFuncs * nMaxElements / nFilterBits)
|
|
|
|
* => log(1.0 - pow(fpRate, 1.0 / nHashFuncs)) = -nHashFuncs * nMaxElements / nFilterBits
|
|
|
|
* => nFilterBits = -nHashFuncs * nMaxElements / log(1.0 - pow(fpRate, 1.0 / nHashFuncs))
|
|
|
|
* => nFilterBits = -nHashFuncs * nMaxElements / log(1.0 - exp(logFpRate / nHashFuncs))
|
|
|
|
*/
|
|
|
|
uint32_t nFilterBits = (uint32_t)ceil(-1.0 * nHashFuncs * nMaxElements / log(1.0 - exp(logFpRate / nHashFuncs)));
|
|
|
|
data.clear();
|
2016-04-24 09:37:29 -07:00
|
|
|
/* For each data element we need to store 2 bits. If both bits are 0, the
|
|
|
|
* bit is treated as unset. If the bits are (01), (10), or (11), the bit is
|
|
|
|
* treated as set in generation 1, 2, or 3 respectively.
|
|
|
|
* These bits are stored in separate integers: position P corresponds to bit
|
|
|
|
* (P & 63) of the integers data[(P >> 6) * 2] and data[(P >> 6) * 2 + 1]. */
|
|
|
|
data.resize(((nFilterBits + 63) / 64) << 1);
|
2015-07-27 09:58:00 -07:00
|
|
|
reset();
|
2015-04-24 10:14:45 -07:00
|
|
|
}
|
|
|
|
|
2015-11-27 04:20:29 -08:00
|
|
|
/* Similar to CBloomFilter::Hash */
|
2016-04-24 09:37:29 -07:00
|
|
|
static inline uint32_t RollingBloomHash(unsigned int nHashNum, uint32_t nTweak, const std::vector<unsigned char>& vDataToHash) {
|
|
|
|
return MurmurHash3(nHashNum * 0xFBA4C795 + nTweak, vDataToHash);
|
2015-11-27 04:20:29 -08:00
|
|
|
}
|
|
|
|
|
replace modulus with FastMod
Replaces the slow modulo operation with a much faster 32bit multiplication & shift. This works
because the hash should be uniformly distributed between 0 and 2^32-1. This speeds up the benchmark
by a factor of about 1.3:
RollingBloom, 5, 1500000, 3.73733, 4.97569e-07, 4.99002e-07, 4.98372e-07 # before
RollingBloom, 5, 1500000, 2.86842, 3.81630e-07, 3.83730e-07, 3.82473e-07 # FastMod
Be aware that this changes the position of the bits that are toggled, so this should probably
not be used for CBloomFilter which is serialized.
(cherry picked from commit 9aac9f90d5e56752cc6cbfac48063ad29a01143c)
2018-05-06 04:55:33 -07:00
|
|
|
|
|
|
|
// A replacement for x % n. This assumes that x and n are 32bit integers, and x is a uniformly random distributed 32bit value
|
|
|
|
// which should be the case for a good hash.
|
|
|
|
// See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/
|
|
|
|
static inline uint32_t FastMod(uint32_t x, size_t n) {
|
|
|
|
return ((uint64_t)x * (uint64_t)n) >> 32;
|
|
|
|
}
|
|
|
|
|
2015-04-24 10:14:45 -07:00
|
|
|
void CRollingBloomFilter::insert(const std::vector<unsigned char>& vKey)
|
|
|
|
{
|
2015-11-27 04:20:29 -08:00
|
|
|
if (nEntriesThisGeneration == nEntriesPerGeneration) {
|
|
|
|
nEntriesThisGeneration = 0;
|
|
|
|
nGeneration++;
|
|
|
|
if (nGeneration == 4) {
|
|
|
|
nGeneration = 1;
|
|
|
|
}
|
2017-02-13 22:39:51 -08:00
|
|
|
uint64_t nGenerationMask1 = 0 - (uint64_t)(nGeneration & 1);
|
|
|
|
uint64_t nGenerationMask2 = 0 - (uint64_t)(nGeneration >> 1);
|
2015-11-27 04:20:29 -08:00
|
|
|
/* Wipe old entries that used this generation number. */
|
2016-04-24 09:37:29 -07:00
|
|
|
for (uint32_t p = 0; p < data.size(); p += 2) {
|
|
|
|
uint64_t p1 = data[p], p2 = data[p + 1];
|
|
|
|
uint64_t mask = (p1 ^ nGenerationMask1) | (p2 ^ nGenerationMask2);
|
|
|
|
data[p] = p1 & mask;
|
|
|
|
data[p + 1] = p2 & mask;
|
2015-11-27 04:20:29 -08:00
|
|
|
}
|
2015-04-24 10:14:45 -07:00
|
|
|
}
|
2015-11-27 04:20:29 -08:00
|
|
|
nEntriesThisGeneration++;
|
|
|
|
|
|
|
|
for (int n = 0; n < nHashFuncs; n++) {
|
2016-04-24 09:37:29 -07:00
|
|
|
uint32_t h = RollingBloomHash(n, nTweak, vKey);
|
|
|
|
int bit = h & 0x3F;
|
replace modulus with FastMod
Replaces the slow modulo operation with a much faster 32bit multiplication & shift. This works
because the hash should be uniformly distributed between 0 and 2^32-1. This speeds up the benchmark
by a factor of about 1.3:
RollingBloom, 5, 1500000, 3.73733, 4.97569e-07, 4.99002e-07, 4.98372e-07 # before
RollingBloom, 5, 1500000, 2.86842, 3.81630e-07, 3.83730e-07, 3.82473e-07 # FastMod
Be aware that this changes the position of the bits that are toggled, so this should probably
not be used for CBloomFilter which is serialized.
(cherry picked from commit 9aac9f90d5e56752cc6cbfac48063ad29a01143c)
2018-05-06 04:55:33 -07:00
|
|
|
/* FastMod works with the upper bits of h, so it is safe to ignore that the lower bits of h are already used for bit. */
|
|
|
|
uint32_t pos = FastMod(h, data.size());
|
2016-04-24 09:37:29 -07:00
|
|
|
/* The lowest bit of pos is ignored, and set to zero for the first bit, and to one for the second. */
|
|
|
|
data[pos & ~1] = (data[pos & ~1] & ~(((uint64_t)1) << bit)) | ((uint64_t)(nGeneration & 1)) << bit;
|
|
|
|
data[pos | 1] = (data[pos | 1] & ~(((uint64_t)1) << bit)) | ((uint64_t)(nGeneration >> 1)) << bit;
|
2015-04-24 10:14:45 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-07-17 03:42:43 -07:00
|
|
|
void CRollingBloomFilter::insert(const uint256& hash)
|
|
|
|
{
|
2017-01-27 00:43:41 -08:00
|
|
|
std::vector<unsigned char> vData(hash.begin(), hash.end());
|
2016-09-02 09:19:01 -07:00
|
|
|
insert(vData);
|
2015-07-17 03:42:43 -07:00
|
|
|
}
|
|
|
|
|
2015-04-24 10:14:45 -07:00
|
|
|
bool CRollingBloomFilter::contains(const std::vector<unsigned char>& vKey) const
|
|
|
|
{
|
2015-11-27 04:20:29 -08:00
|
|
|
for (int n = 0; n < nHashFuncs; n++) {
|
2016-04-24 09:37:29 -07:00
|
|
|
uint32_t h = RollingBloomHash(n, nTweak, vKey);
|
|
|
|
int bit = h & 0x3F;
|
replace modulus with FastMod
Replaces the slow modulo operation with a much faster 32bit multiplication & shift. This works
because the hash should be uniformly distributed between 0 and 2^32-1. This speeds up the benchmark
by a factor of about 1.3:
RollingBloom, 5, 1500000, 3.73733, 4.97569e-07, 4.99002e-07, 4.98372e-07 # before
RollingBloom, 5, 1500000, 2.86842, 3.81630e-07, 3.83730e-07, 3.82473e-07 # FastMod
Be aware that this changes the position of the bits that are toggled, so this should probably
not be used for CBloomFilter which is serialized.
(cherry picked from commit 9aac9f90d5e56752cc6cbfac48063ad29a01143c)
2018-05-06 04:55:33 -07:00
|
|
|
uint32_t pos = FastMod(h, data.size());
|
2016-04-24 09:37:29 -07:00
|
|
|
/* If the relevant bit is not set in either data[pos & ~1] or data[pos | 1], the filter does not contain vKey */
|
|
|
|
if (!(((data[pos & ~1] | data[pos | 1]) >> bit) & 1)) {
|
2015-11-27 04:20:29 -08:00
|
|
|
return false;
|
|
|
|
}
|
2015-04-24 10:14:45 -07:00
|
|
|
}
|
2015-11-27 04:20:29 -08:00
|
|
|
return true;
|
2015-04-24 10:14:45 -07:00
|
|
|
}
|
|
|
|
|
2015-07-17 03:42:43 -07:00
|
|
|
bool CRollingBloomFilter::contains(const uint256& hash) const
|
|
|
|
{
|
2017-01-27 00:43:41 -08:00
|
|
|
std::vector<unsigned char> vData(hash.begin(), hash.end());
|
2016-09-02 09:19:01 -07:00
|
|
|
return contains(vData);
|
2015-07-17 03:42:43 -07:00
|
|
|
}
|
|
|
|
|
2015-07-27 09:58:00 -07:00
|
|
|
void CRollingBloomFilter::reset()
|
2015-04-24 10:14:45 -07:00
|
|
|
{
|
2015-11-27 04:20:29 -08:00
|
|
|
nTweak = GetRand(std::numeric_limits<unsigned int>::max());
|
|
|
|
nEntriesThisGeneration = 0;
|
|
|
|
nGeneration = 1;
|
2019-05-21 15:47:44 -07:00
|
|
|
std::fill(data.begin(), data.end(), 0);
|
2015-04-24 10:14:45 -07:00
|
|
|
}
|