diff --git a/doc/release-notes.md b/doc/release-notes.md index f16eec32a..9272d427c 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -1,2 +1,21 @@ (note: this is a temporary file, to be added-to by anybody, and moved to release-notes at release time) + +New RPC methods +=============== + +Fee/Priority estimation +----------------------- + +estimatefee nblocks : Returns approximate fee-per-1,000-bytes needed for +a transaction to be confirmed within nblocks. Returns -1 if not enough +transactions have been observed to compute a good estimate. + +estimatepriority nblocks : Returns approximate priority needed for +a zero-fee transaction to confirm within nblocks. Returns -1 if not +enough free transactions have been observed to compute a good +estimate. + +Statistics used to estimate fees and priorities are saved in the +data directory in the 'fee_estimates.dat' file just before +program shutdown, and are read in at startup. diff --git a/qa/rpc-tests/smartfees.py b/qa/rpc-tests/smartfees.py new file mode 100755 index 000000000..e8abbfba1 --- /dev/null +++ b/qa/rpc-tests/smartfees.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python + +# +# Test fee estimation code +# + +# Add python-bitcoinrpc to module search path: +import os +import sys +sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "python-bitcoinrpc")) + +import json +import random +import shutil +import subprocess +import tempfile +import traceback + +from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException +from util import * + + +def run_test(nodes, test_dir): + nodes.append(start_node(0, test_dir, + ["-debug=mempool", "-debug=estimatefee"])) + # Node1 mines small-but-not-tiny blocks, and allows free transactions. + # NOTE: the CreateNewBlock code starts counting block size at 1,000 bytes, + # so blockmaxsize of 2,000 is really just 1,000 bytes (room enough for + # 6 or 7 transactions) + nodes.append(start_node(1, test_dir, + ["-blockprioritysize=1500", "-blockmaxsize=2000", + "-debug=mempool", "-debug=estimatefee"])) + connect_nodes(nodes[1], 0) + + # Node2 is a stingy miner, that + # produces very small blocks (room for only 3 or so transactions) + node2args = [ "-blockprioritysize=0", "-blockmaxsize=1500", + "-debug=mempool", "-debug=estimatefee"] + nodes.append(start_node(2, test_dir, node2args)) + connect_nodes(nodes[2], 0) + + sync_blocks(nodes) + + # Prime the memory pool with pairs of transactions + # (high-priority, random fee and zero-priority, random fee) + min_fee = Decimal("0.001") + fees_per_kb = []; + for i in range(12): + (txid, txhex, fee) = random_zeropri_transaction(nodes, Decimal("1.1"), + min_fee, min_fee, 20) + tx_kbytes = (len(txhex)/2)/1000.0 + fees_per_kb.append(float(fee)/tx_kbytes) + + # Mine blocks with node2 until the memory pool clears: + count_start = nodes[2].getblockcount() + while len(nodes[2].getrawmempool()) > 0: + nodes[2].setgenerate(True, 1) + sync_blocks(nodes) + + all_estimates = [ nodes[0].estimatefee(i) for i in range(1,20) ] + print("Fee estimates, super-stingy miner: "+str([str(e) for e in all_estimates])) + + # Estimates should be within the bounds of what transactions fees actually were: + delta = 1.0e-6 # account for rounding error + for e in filter(lambda x: x >= 0, all_estimates): + if float(e)+delta < min(fees_per_kb) or float(e)-delta > max(fees_per_kb): + raise AssertionError("Estimated fee (%f) out of range (%f,%f)"%(float(e), min_fee_kb, max_fee_kb)) + + # Generate transactions while mining 30 more blocks, this time with node1: + for i in range(30): + for j in range(random.randrange(6-4,6+4)): + (txid, txhex, fee) = random_transaction(nodes, Decimal("1.1"), + Decimal("0.0"), min_fee, 20) + tx_kbytes = (len(txhex)/2)/1000.0 + fees_per_kb.append(float(fee)/tx_kbytes) + nodes[1].setgenerate(True, 1) + sync_blocks(nodes) + + all_estimates = [ nodes[0].estimatefee(i) for i in range(1,20) ] + print("Fee estimates, more generous miner: "+str([ str(e) for e in all_estimates])) + for e in filter(lambda x: x >= 0, all_estimates): + if float(e)+delta < min(fees_per_kb) or float(e)-delta > max(fees_per_kb): + raise AssertionError("Estimated fee (%f) out of range (%f,%f)"%(float(e), min_fee_kb, max_fee_kb)) + + # Finish by mining a normal-sized block: + while len(nodes[0].getrawmempool()) > 0: + nodes[0].setgenerate(True, 1) + sync_blocks(nodes) + + final_estimates = [ nodes[0].estimatefee(i) for i in range(1,20) ] + print("Final fee estimates: "+str([ str(e) for e in final_estimates])) + +def main(): + import optparse + + parser = optparse.OptionParser(usage="%prog [options]") + parser.add_option("--nocleanup", dest="nocleanup", default=False, action="store_true", + help="Leave bitcoinds and test.* datadir on exit or error") + parser.add_option("--srcdir", dest="srcdir", default="../../src", + help="Source directory containing bitcoind/bitcoin-cli (default: %default%)") + parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="test"), + help="Root directory for datadirs") + (options, args) = parser.parse_args() + + os.environ['PATH'] = options.srcdir+":"+os.environ['PATH'] + + check_json_precision() + + success = False + nodes = [] + try: + print("Initializing test directory "+options.tmpdir) + print(" node0 running at: 127.0.0.1:%d"%(p2p_port(0))) + if not os.path.isdir(options.tmpdir): + os.makedirs(options.tmpdir) + initialize_chain(options.tmpdir) + + run_test(nodes, options.tmpdir) + + success = True + + except AssertionError as e: + print("Assertion failed: "+e.message) + except Exception as e: + print("Unexpected exception caught during testing: "+str(e)) + traceback.print_tb(sys.exc_info()[2]) + + if not options.nocleanup: + print("Cleaning up") + stop_nodes(nodes) + wait_bitcoinds() + shutil.rmtree(options.tmpdir) + + if success: + print("Tests successful") + sys.exit(0) + else: + print("Failed") + sys.exit(1) + +if __name__ == '__main__': + main() diff --git a/qa/rpc-tests/util.py b/qa/rpc-tests/util.py index eab526daf..eded098c7 100644 --- a/qa/rpc-tests/util.py +++ b/qa/rpc-tests/util.py @@ -12,6 +12,7 @@ sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "python from decimal import Decimal import json +import random import shutil import subprocess import time @@ -70,6 +71,7 @@ def initialize_datadir(dir, n): f.write("rpcpassword=rt\n"); f.write("port="+str(p2p_port(n))+"\n"); f.write("rpcport="+str(rpc_port(n))+"\n"); + return datadir def initialize_chain(test_dir): """ @@ -82,7 +84,7 @@ def initialize_chain(test_dir): devnull = open("/dev/null", "w+") # Create cache directories, run bitcoinds: for i in range(4): - initialize_datadir("cache", i) + datadir=initialize_datadir("cache", i) args = [ "bitcoind", "-keypool=1", "-datadir="+datadir ] if i > 0: args.append("-connect=127.0.0.1:"+str(p2p_port(0))) @@ -140,25 +142,28 @@ def _rpchost_to_args(rpchost): rv += ['-rpcport=' + rpcport] return rv -def start_nodes(num_nodes, dir, extra_args=None, rpchost=None): - # Start bitcoinds, and wait for RPC interface to be up and running: +def start_node(i, dir, extra_args=None, rpchost=None): + """ + Start a bitcoind and return RPC connection to it + """ + datadir = os.path.join(dir, "node"+str(i)) + args = [ "bitcoind", "-datadir="+datadir, "-keypool=1" ] + if extra_args is not None: args.extend(extra_args) + bitcoind_processes.append(subprocess.Popen(args)) devnull = open("/dev/null", "w+") - for i in range(num_nodes): - datadir = os.path.join(dir, "node"+str(i)) - args = [ "bitcoind", "-datadir="+datadir, "-keypool=1" ] - if extra_args is not None: - args += extra_args[i] - bitcoind_processes.append(subprocess.Popen(args)) - subprocess.check_call([ "bitcoin-cli", "-datadir="+datadir] + - _rpchost_to_args(rpchost) + - ["-rpcwait", "getblockcount"], stdout=devnull) + subprocess.check_call([ "bitcoin-cli", "-datadir="+datadir] + + _rpchost_to_args(rpchost) + + ["-rpcwait", "getblockcount"], stdout=devnull) devnull.close() - # Create&return JSON-RPC connections - rpc_connections = [] - for i in range(num_nodes): - url = "http://rt:rt@%s:%d" % (rpchost or '127.0.0.1', rpc_port(i)) - rpc_connections.append(AuthServiceProxy(url)) - return rpc_connections + url = "http://rt:rt@%s:%d" % (rpchost or '127.0.0.1', rpc_port(i)) + return AuthServiceProxy(url) + +def start_nodes(num_nodes, dir, extra_args=None, rpchost=None): + """ + Start multiple bitcoinds, return RPC connections to them + """ + if extra_args is None: extra_args = [ None for i in range(num_nodes) ] + return [ start_node(i, dir, extra_args[i], rpchost) for i in range(num_nodes) ] def debug_log(dir, n_node): return os.path.join(dir, "node"+str(n_node), "regtest", "debug.log") @@ -178,6 +183,108 @@ def connect_nodes(from_connection, node_num): ip_port = "127.0.0.1:"+str(p2p_port(node_num)) from_connection.addnode(ip_port, "onetry") +def find_output(node, txid, amount): + """ + Return index to output of txid with value amount + Raises exception if there is none. + """ + txdata = node.getrawtransaction(txid, 1) + for i in range(len(txdata["vout"])): + if txdata["vout"][i]["value"] == amount: + return i + raise RuntimeError("find_output txid %s : %s not found"%(txid,str(amount))) + +def gather_inputs(from_node, amount_needed): + """ + Return a random set of unspent txouts that are enough to pay amount_needed + """ + utxo = from_node.listunspent(1) + random.shuffle(utxo) + inputs = [] + total_in = Decimal("0.00000000") + while total_in < amount_needed and len(utxo) > 0: + t = utxo.pop() + total_in += t["amount"] + inputs.append({ "txid" : t["txid"], "vout" : t["vout"], "address" : t["address"] } ) + if total_in < amount_needed: + raise RuntimeError("Insufficient funds: need %d, have %d"%(amount+fee*2, total_in)) + return (total_in, inputs) + +def make_change(from_node, amount_in, amount_out, fee): + """ + Create change output(s), return them + """ + outputs = {} + amount = amount_out+fee + change = amount_in - amount + if change > amount*2: + # Create an extra change output to break up big inputs + outputs[from_node.getnewaddress()] = float(change/2) + change = change/2 + if change > 0: + outputs[from_node.getnewaddress()] = float(change) + return outputs + +def send_zeropri_transaction(from_node, to_node, amount, fee): + """ + Create&broadcast a zero-priority transaction. + Returns (txid, hex-encoded-txdata) + Ensures transaction is zero-priority by first creating a send-to-self, + then using it's output + """ + + # Create a send-to-self with confirmed inputs: + self_address = from_node.getnewaddress() + (total_in, inputs) = gather_inputs(from_node, amount+fee*2) + outputs = make_change(from_node, total_in, amount+fee, fee) + outputs[self_address] = float(amount+fee) + + self_rawtx = from_node.createrawtransaction(inputs, outputs) + self_signresult = from_node.signrawtransaction(self_rawtx) + self_txid = from_node.sendrawtransaction(self_signresult["hex"], True) + + vout = find_output(from_node, self_txid, amount+fee) + # Now immediately spend the output to create a 1-input, 1-output + # zero-priority transaction: + inputs = [ { "txid" : self_txid, "vout" : vout } ] + outputs = { to_node.getnewaddress() : float(amount) } + + rawtx = from_node.createrawtransaction(inputs, outputs) + signresult = from_node.signrawtransaction(rawtx) + txid = from_node.sendrawtransaction(signresult["hex"], True) + + return (txid, signresult["hex"]) + +def random_zeropri_transaction(nodes, amount, min_fee, fee_increment, fee_variants): + """ + Create a random zero-priority transaction. + Returns (txid, hex-encoded-transaction-data, fee) + """ + from_node = random.choice(nodes) + to_node = random.choice(nodes) + fee = min_fee + fee_increment*random.randint(0,fee_variants) + (txid, txhex) = send_zeropri_transaction(from_node, to_node, amount, fee) + return (txid, txhex, fee) + +def random_transaction(nodes, amount, min_fee, fee_increment, fee_variants): + """ + Create a random transaction. + Returns (txid, hex-encoded-transaction-data, fee) + """ + from_node = random.choice(nodes) + to_node = random.choice(nodes) + fee = min_fee + fee_increment*random.randint(0,fee_variants) + + (total_in, inputs) = gather_inputs(from_node, amount+fee) + outputs = make_change(from_node, total_in, amount, fee) + outputs[to_node.getnewaddress()] = float(amount) + + rawtx = from_node.createrawtransaction(inputs, outputs) + signresult = from_node.signrawtransaction(rawtx) + txid = from_node.sendrawtransaction(signresult["hex"], True) + + return (txid, signresult["hex"], fee) + def assert_equal(thing1, thing2): if thing1 != thing2: raise AssertionError("%s != %s"%(str(thing1),str(thing2))) diff --git a/src/core.h b/src/core.h index 9fccffc4b..0e5912934 100644 --- a/src/core.h +++ b/src/core.h @@ -120,6 +120,7 @@ class CFeeRate private: int64_t nSatoshisPerK; // unit is satoshis-per-1,000-bytes public: + CFeeRate() : nSatoshisPerK(0) { } explicit CFeeRate(int64_t _nSatoshisPerK): nSatoshisPerK(_nSatoshisPerK) { } CFeeRate(int64_t nFeePaid, size_t nSize); CFeeRate(const CFeeRate& other) { nSatoshisPerK = other.nSatoshisPerK; } @@ -132,6 +133,8 @@ public: friend bool operator==(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK == b.nSatoshisPerK; } std::string ToString() const; + + IMPLEMENT_SERIALIZE( READWRITE(nSatoshisPerK); ) }; diff --git a/src/init.cpp b/src/init.cpp index 1aad679b3..aca5e0c9a 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -59,6 +59,7 @@ enum BindFlags { BF_REPORT_ERROR = (1U << 1) }; +static const char* FEE_ESTIMATES_FILENAME="fee_estimates.dat"; ////////////////////////////////////////////////////////////////////////////// // @@ -121,6 +122,14 @@ void Shutdown() #endif StopNode(); UnregisterNodeSignals(GetNodeSignals()); + + boost::filesystem::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; + CAutoFile est_fileout = CAutoFile(fopen(est_path.string().c_str(), "wb"), SER_DISK, CLIENT_VERSION); + if (est_fileout) + mempool.WriteFeeEstimates(est_fileout); + else + LogPrintf("failed to write fee estimates"); + { LOCK(cs_main); #ifdef ENABLE_WALLET @@ -933,6 +942,11 @@ bool AppInit2(boost::thread_group& threadGroup) return false; } + boost::filesystem::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; + CAutoFile est_filein = CAutoFile(fopen(est_path.string().c_str(), "rb"), SER_DISK, CLIENT_VERSION); + if (est_filein) + mempool.ReadFeeEstimates(est_filein); + // ********************************************************* Step 8: load wallet #ifdef ENABLE_WALLET if (fDisableWallet) { diff --git a/src/main.cpp b/src/main.cpp index 9d6c01d64..429473d8f 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -851,6 +851,7 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa CCoinsView dummy; CCoinsViewCache view(dummy); + int64_t nValueIn = 0; { LOCK(pool.cs); CCoinsViewMemPool viewMemPool(*pcoinsTip, pool); @@ -879,6 +880,8 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa // Bring the best block into scope view.GetBestBlock(); + nValueIn = view.GetValueIn(tx); + // we have all inputs cached now, so switch back to dummy, so we don't need to keep lock on mempool view.SetBackend(dummy); } @@ -891,7 +894,6 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa // you should add code here to check that the transaction does a // reasonable number of ECDSA signature verifications. - int64_t nValueIn = view.GetValueIn(tx); int64_t nValueOut = tx.GetValueOut(); int64_t nFees = nValueIn-nValueOut; double dPriority = view.GetPriority(tx, chainActive.Height()); @@ -2017,11 +2019,7 @@ bool static ConnectTip(CValidationState &state, CBlockIndex *pindexNew) { return false; // Remove conflicting transactions from the mempool. list txConflicted; - BOOST_FOREACH(const CTransaction &tx, block.vtx) { - list unused; - mempool.remove(tx, unused); - mempool.removeConflicts(tx, txConflicted); - } + mempool.removeForBlock(block.vtx, pindexNew->nHeight, txConflicted); mempool.check(pcoinsTip); // Update chainActive & related variables. UpdateTip(pindexNew); diff --git a/src/main.h b/src/main.h index 3fccd32a2..23c866037 100644 --- a/src/main.h +++ b/src/main.h @@ -292,13 +292,6 @@ unsigned int GetLegacySigOpCount(const CTransaction& tx); unsigned int GetP2SHSigOpCount(const CTransaction& tx, CCoinsViewCache& mapInputs); -inline bool AllowFree(double dPriority) -{ - // Large (in bytes) low-priority (new, small-coin) transactions - // need a fee. - return dPriority > COIN * 144 / 250; -} - // Check whether all inputs of this transaction are valid (no double spends, scripts & sigs, amounts) // This does not modify the UTXO set. If pvChecks is not NULL, script checks are pushed onto it // instead of being performed inline. diff --git a/src/rpcclient.cpp b/src/rpcclient.cpp index 4f3c39ce9..b89a95ad1 100644 --- a/src/rpcclient.cpp +++ b/src/rpcclient.cpp @@ -176,6 +176,8 @@ Array RPCConvertValues(const std::string &strMethod, const std::vector 1) ConvertTo(params[1]); if (strMethod == "keypoolrefill" && n > 0) ConvertTo(params[0]); if (strMethod == "getrawmempool" && n > 0) ConvertTo(params[0]); + if (strMethod == "estimatefee" && n > 0) ConvertTo(params[0]); + if (strMethod == "estimatepriority" && n > 0) ConvertTo(params[0]); return params; } diff --git a/src/rpcmining.cpp b/src/rpcmining.cpp index 23876c603..dd148c6af 100644 --- a/src/rpcmining.cpp +++ b/src/rpcmining.cpp @@ -15,6 +15,7 @@ #endif #include +#include #include "json/json_spirit_utils.h" #include "json/json_spirit_value.h" @@ -626,3 +627,63 @@ Value submitblock(const Array& params, bool fHelp) return Value::null; } + +Value estimatefee(const Array& params, bool fHelp) +{ + if (fHelp || params.size() != 1) + throw runtime_error( + "estimatefee nblocks\n" + "\nEstimates the approximate fee per kilobyte\n" + "needed for a transaction to get confirmed\n" + "within nblocks blocks.\n" + "\nArguments:\n" + "1. nblocks (numeric)\n" + "\nResult:\n" + "n : (numeric) estimated fee-per-kilobyte\n" + "\n" + "-1.0 is returned if not enough transactions and\n" + "blocks have been observed to make an estimate.\n" + "\nExample:\n" + + HelpExampleCli("estimatefee", "6") + ); + + RPCTypeCheck(params, boost::assign::list_of(int_type)); + + int nBlocks = params[0].get_int(); + if (nBlocks < 1) + nBlocks = 1; + + CFeeRate feeRate = mempool.estimateFee(nBlocks); + if (feeRate == CFeeRate(0)) + return -1.0; + + return ValueFromAmount(feeRate.GetFeePerK()); +} + +Value estimatepriority(const Array& params, bool fHelp) +{ + if (fHelp || params.size() != 1) + throw runtime_error( + "estimatepriority nblocks\n" + "\nEstimates the approximate priority\n" + "a zero-fee transaction needs to get confirmed\n" + "within nblocks blocks.\n" + "\nArguments:\n" + "1. nblocks (numeric)\n" + "\nResult:\n" + "n : (numeric) estimated priority\n" + "\n" + "-1.0 is returned if not enough transactions and\n" + "blocks have been observed to make an estimate.\n" + "\nExample:\n" + + HelpExampleCli("estimatepriority", "6") + ); + + RPCTypeCheck(params, boost::assign::list_of(int_type)); + + int nBlocks = params[0].get_int(); + if (nBlocks < 1) + nBlocks = 1; + + return mempool.estimatePriority(nBlocks); +} diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 2534a9dcf..72a7fe83e 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -268,6 +268,8 @@ static const CRPCCommand vRPCCommands[] = { "createmultisig", &createmultisig, true, true , false }, { "validateaddress", &validateaddress, true, false, false }, /* uses wallet if enabled */ { "verifymessage", &verifymessage, false, false, false }, + { "estimatefee", &estimatefee, true, true, false }, + { "estimatepriority", &estimatepriority, true, true, false }, #ifdef ENABLE_WALLET /* Wallet */ diff --git a/src/rpcserver.h b/src/rpcserver.h index e8cd2cd0f..73e8b9426 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -133,6 +133,8 @@ extern json_spirit::Value getmininginfo(const json_spirit::Array& params, bool f extern json_spirit::Value getwork(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getblocktemplate(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value submitblock(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value estimatefee(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value estimatepriority(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getnewaddress(const json_spirit::Array& params, bool fHelp); // in rpcwallet.cpp extern json_spirit::Value getaccountaddress(const json_spirit::Array& params, bool fHelp); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 64c9eac73..4bf01d484 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -6,6 +6,8 @@ #include "core.h" #include "txmempool.h" +#include + using namespace std; CTxMemPoolEntry::CTxMemPoolEntry() @@ -35,12 +37,311 @@ CTxMemPoolEntry::GetPriority(unsigned int currentHeight) const return dResult; } +// +// Keep track of fee/priority for transactions confirmed within N blocks +// +class CBlockAverage +{ +private: + boost::circular_buffer feeSamples; + boost::circular_buffer prioritySamples; + + template std::vector buf2vec(boost::circular_buffer buf) const + { + std::vector vec(buf.begin(), buf.end()); + return vec; + } + +public: + CBlockAverage() : feeSamples(100), prioritySamples(100) { } + + void RecordFee(const CFeeRate& feeRate) { + feeSamples.push_back(feeRate); + } + + void RecordPriority(double priority) { + prioritySamples.push_back(priority); + } + + size_t FeeSamples() const { return feeSamples.size(); } + size_t GetFeeSamples(std::vector& insertInto) const + { + BOOST_FOREACH(const CFeeRate& f, feeSamples) + insertInto.push_back(f); + return feeSamples.size(); + } + size_t PrioritySamples() const { return prioritySamples.size(); } + size_t GetPrioritySamples(std::vector& insertInto) const + { + BOOST_FOREACH(double d, prioritySamples) + insertInto.push_back(d); + return prioritySamples.size(); + } + + // Used as belt-and-suspenders check when reading to detect + // file corruption + bool AreSane(const std::vector& vecFee) + { + BOOST_FOREACH(CFeeRate fee, vecFee) + { + if (fee < CFeeRate(0)) + return false; + if (fee.GetFee(1000) > CTransaction::minRelayTxFee.GetFee(1000) * 10000) + return false; + } + return true; + } + bool AreSane(const std::vector vecPriority) + { + BOOST_FOREACH(double priority, vecPriority) + { + if (priority < 0) + return false; + } + return true; + } + + void Write(CAutoFile& fileout) const + { + std::vector vecFee = buf2vec(feeSamples); + fileout << vecFee; + std::vector vecPriority = buf2vec(prioritySamples); + fileout << vecPriority; + } + + void Read(CAutoFile& filein) { + std::vector vecFee; + filein >> vecFee; + if (AreSane(vecFee)) + feeSamples.insert(feeSamples.end(), vecFee.begin(), vecFee.end()); + else + throw runtime_error("Corrupt fee value in estimates file."); + std::vector vecPriority; + filein >> vecPriority; + if (AreSane(vecPriority)) + prioritySamples.insert(prioritySamples.end(), vecPriority.begin(), vecPriority.end()); + else + throw runtime_error("Corrupt priority value in estimates file."); + if (feeSamples.size() + prioritySamples.size() > 0) + LogPrint("estimatefee", "Read %d fee samples and %d priority samples\n", + feeSamples.size(), prioritySamples.size()); + } +}; + +class CMinerPolicyEstimator +{ +private: + // Records observed averages transactions that confirmed within one block, two blocks, + // three blocks etc. + std::vector history; + std::vector sortedFeeSamples; + std::vector sortedPrioritySamples; + + int nBestSeenHeight; + + // nBlocksAgo is 0 based, i.e. transactions that confirmed in the highest seen block are + // nBlocksAgo == 0, transactions in the block before that are nBlocksAgo == 1 etc. + void seenTxConfirm(CFeeRate feeRate, double dPriority, int nBlocksAgo) + { + // Last entry records "everything else". + int nBlocksTruncated = min(nBlocksAgo, (int) history.size() - 1); + assert(nBlocksTruncated >= 0); + + // We need to guess why the transaction was included in a block-- either + // because it is high-priority or because it has sufficient fees. + bool sufficientFee = (feeRate > CTransaction::minRelayTxFee); + bool sufficientPriority = AllowFree(dPriority); + const char* assignedTo = "unassigned"; + if (sufficientFee && !sufficientPriority) + { + history[nBlocksTruncated].RecordFee(feeRate); + assignedTo = "fee"; + } + else if (sufficientPriority && !sufficientFee) + { + history[nBlocksTruncated].RecordPriority(dPriority); + assignedTo = "priority"; + } + else + { + // Neither or both fee and priority sufficient to get confirmed: + // don't know why they got confirmed. + } + LogPrint("estimatefee", "Seen TX confirm: %s : %s fee/%g priority, took %d blocks\n", + assignedTo, feeRate.ToString(), dPriority, nBlocksAgo); + } + +public: + CMinerPolicyEstimator(int nEntries) : nBestSeenHeight(0) + { + history.resize(nEntries); + } + + void seenBlock(const std::vector& entries, int nBlockHeight) + { + if (nBlockHeight <= nBestSeenHeight) + { + // Ignore side chains and re-orgs; assuming they are random + // they don't affect the estimate. + // And if an attacker can re-org the chain at will, then + // you've got much bigger problems than "attacker can influence + // transaction fees." + return; + } + nBestSeenHeight = nBlockHeight; + + // Fill up the history buckets based on how long transactions took + // to confirm. + std::vector > entriesByConfirmations; + entriesByConfirmations.resize(history.size()); + BOOST_FOREACH(const CTxMemPoolEntry& entry, entries) + { + // How many blocks did it take for miners to include this transaction? + int delta = nBlockHeight - entry.GetHeight(); + if (delta <= 0) + { + // Re-org made us lose height, this should only happen if we happen + // to re-org on a difficulty transition point: very rare! + continue; + } + if ((delta-1) >= (int)history.size()) + delta = history.size(); // Last bucket is catch-all + entriesByConfirmations[delta-1].push_back(&entry); + } + for (size_t i = 0; i < entriesByConfirmations.size(); i++) + { + std::vector &e = entriesByConfirmations.at(i); + // Insert at most 10 random entries per bucket, otherwise a single block + // can dominate an estimate: + if (e.size() > 10) { + std::random_shuffle(e.begin(), e.end()); + e.resize(10); + } + BOOST_FOREACH(const CTxMemPoolEntry* entry, e) + { + // Fees are stored and reported as BTC-per-kb: + CFeeRate feeRate(entry->GetFee(), entry->GetTxSize()); + double dPriority = entry->GetPriority(entry->GetHeight()); // Want priority when it went IN + seenTxConfirm(feeRate, dPriority, i); + } + } + for (size_t i = 0; i < history.size(); i++) { + if (history[i].FeeSamples() + history[i].PrioritySamples() > 0) + LogPrint("estimatefee", "estimates: for confirming within %d blocks based on %d/%d samples, fee=%s, prio=%g\n", + i, + history[i].FeeSamples(), history[i].PrioritySamples(), + estimateFee(i+1).ToString(), estimatePriority(i+1)); + } + sortedFeeSamples.clear(); + sortedPrioritySamples.clear(); + } + + // Can return CFeeRate(0) if we don't have any data for that many blocks back. nBlocksToConfirm is 1 based. + CFeeRate estimateFee(int nBlocksToConfirm) + { + nBlocksToConfirm--; + + if (nBlocksToConfirm < 0 || nBlocksToConfirm >= (int)history.size()) + return CFeeRate(0); + + if (sortedFeeSamples.size() == 0) + { + for (size_t i = 0; i < history.size(); i++) + history.at(i).GetFeeSamples(sortedFeeSamples); + std::sort(sortedFeeSamples.begin(), sortedFeeSamples.end(), + std::greater()); + } + if (sortedFeeSamples.size() == 0) + return CFeeRate(0); + + int nBucketSize = history.at(nBlocksToConfirm).FeeSamples(); + + // Estimates should not increase as number of confirmations goes up, + // but the estimates are noisy because confirmations happen discretely + // in blocks. To smooth out the estimates, use all samples in the history + // and use the nth highest where n is (number of samples in previous bucket + + // half the samples in nBlocksToConfirm bucket): + size_t nPrevSize = 0; + for (int i = 0; i < nBlocksToConfirm; i++) + nPrevSize += history.at(i).FeeSamples(); + size_t index = min(nPrevSize + nBucketSize/2, sortedFeeSamples.size()-1); + return sortedFeeSamples[index]; + } + double estimatePriority(int nBlocksToConfirm) + { + nBlocksToConfirm--; + + if (nBlocksToConfirm < 0 || nBlocksToConfirm >= (int)history.size()) + return -1; + + if (sortedPrioritySamples.size() == 0) + { + for (size_t i = 0; i < history.size(); i++) + history.at(i).GetPrioritySamples(sortedPrioritySamples); + std::sort(sortedPrioritySamples.begin(), sortedPrioritySamples.end(), + std::greater()); + } + if (sortedPrioritySamples.size() == 0) + return -1.0; + + int nBucketSize = history.at(nBlocksToConfirm).PrioritySamples(); + + // Estimates should not increase as number of confirmations needed goes up, + // but the estimates are noisy because confirmations happen discretely + // in blocks. To smooth out the estimates, use all samples in the history + // and use the nth highest where n is (number of samples in previous buckets + + // half the samples in nBlocksToConfirm bucket). + size_t nPrevSize = 0; + for (int i = 0; i < nBlocksToConfirm; i++) + nPrevSize += history.at(i).PrioritySamples(); + size_t index = min(nPrevSize + nBucketSize/2, sortedFeeSamples.size()-1); + return sortedPrioritySamples[index]; + } + + void Write(CAutoFile& fileout) const + { + fileout << nBestSeenHeight; + fileout << history.size(); + BOOST_FOREACH(const CBlockAverage& entry, history) + { + entry.Write(fileout); + } + } + + void Read(CAutoFile& filein) + { + filein >> nBestSeenHeight; + size_t numEntries; + filein >> numEntries; + history.clear(); + for (size_t i = 0; i < numEntries; i++) + { + CBlockAverage entry; + entry.Read(filein); + history.push_back(entry); + } + } +}; + + CTxMemPool::CTxMemPool() { // Sanity checks off by default for performance, because otherwise // accepting transactions becomes O(N^2) where N is the number // of transactions in the pool fSanityCheck = false; + + // 25 blocks is a compromise between using a lot of disk/memory and + // trying to give accurate estimates to people who might be willing + // to wait a day or two to save a fraction of a penny in fees. + // Confirmation times for very-low-fee transactions that take more + // than an hour or three to confirm are highly variable. + minerPolicyEstimator = new CMinerPolicyEstimator(25); +} + +CTxMemPool::~CTxMemPool() +{ + delete minerPolicyEstimator; } void CTxMemPool::pruneSpent(const uint256 &hashTx, CCoins &coins) @@ -128,6 +429,28 @@ void CTxMemPool::removeConflicts(const CTransaction &tx, std::list } } +// Called when a block is connected. Removes from mempool and updates the miner fee estimator. +void CTxMemPool::removeForBlock(const std::vector& vtx, unsigned int nBlockHeight, + std::list& conflicts) +{ + LOCK(cs); + std::vector entries; + BOOST_FOREACH(const CTransaction& tx, vtx) + { + uint256 hash = tx.GetHash(); + if (mapTx.count(hash)) + entries.push_back(mapTx[hash]); + } + minerPolicyEstimator->seenBlock(entries, nBlockHeight); + BOOST_FOREACH(const CTransaction& tx, vtx) + { + std::list dummy; + remove(tx, dummy, false); + removeConflicts(tx, conflicts); + } +} + + void CTxMemPool::clear() { LOCK(cs); @@ -195,6 +518,53 @@ bool CTxMemPool::lookup(uint256 hash, CTransaction& result) const return true; } +CFeeRate CTxMemPool::estimateFee(int nBlocks) const +{ + LOCK(cs); + return minerPolicyEstimator->estimateFee(nBlocks); +} +double CTxMemPool::estimatePriority(int nBlocks) const +{ + LOCK(cs); + return minerPolicyEstimator->estimatePriority(nBlocks); +} + +bool +CTxMemPool::WriteFeeEstimates(CAutoFile& fileout) const +{ + try { + LOCK(cs); + fileout << 99900; // version required to read: 0.9.99 or later + fileout << CLIENT_VERSION; // version that wrote the file + minerPolicyEstimator->Write(fileout); + } + catch (std::exception &e) { + LogPrintf("CTxMemPool::WriteFeeEstimates() : unable to write policy estimator data (non-fatal)"); + return false; + } + return true; +} + +bool +CTxMemPool::ReadFeeEstimates(CAutoFile& filein) +{ + try { + int nVersionRequired, nVersionThatWrote; + filein >> nVersionRequired >> nVersionThatWrote; + if (nVersionRequired > CLIENT_VERSION) + return error("CTxMemPool::ReadFeeEstimates() : up-version (%d) fee estimate file", nVersionRequired); + + LOCK(cs); + minerPolicyEstimator->Read(filein); + } + catch (std::exception &e) { + LogPrintf("CTxMemPool::ReadFeeEstimates() : unable to read policy estimator data (non-fatal)"); + return false; + } + return true; +} + + CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView &baseIn, CTxMemPool &mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { } bool CCoinsViewMemPool::GetCoins(const uint256 &txid, CCoins &coins) { diff --git a/src/txmempool.h b/src/txmempool.h index 4509e9577..b2915aa84 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -11,6 +11,13 @@ #include "core.h" #include "sync.h" +inline bool AllowFree(double dPriority) +{ + // Large (in bytes) low-priority (new, small-coin) transactions + // need a fee. + return dPriority > COIN * 144 / 250; +} + /** Fake height value used in CCoins to signify they are only in the memory pool (since 0.8) */ static const unsigned int MEMPOOL_HEIGHT = 0x7FFFFFFF; @@ -41,6 +48,8 @@ public: unsigned int GetHeight() const { return nHeight; } }; +class CMinerPolicyEstimator; + /* * CTxMemPool stores valid-according-to-the-current-best-chain * transactions that may be included in the next block. @@ -56,6 +65,7 @@ class CTxMemPool private: bool fSanityCheck; // Normally false, true if -checkmempool or -regtest unsigned int nTransactionsUpdated; + CMinerPolicyEstimator* minerPolicyEstimator; public: mutable CCriticalSection cs; @@ -63,6 +73,7 @@ public: std::map mapNextTx; CTxMemPool(); + ~CTxMemPool(); /* * If sanity-checking is turned on, check makes sure the pool is @@ -76,6 +87,8 @@ public: bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry); void remove(const CTransaction &tx, std::list& removed, bool fRecursive = false); void removeConflicts(const CTransaction &tx, std::list& removed); + void removeForBlock(const std::vector& vtx, unsigned int nBlockHeight, + std::list& conflicts); void clear(); void queryHashes(std::vector& vtxid); void pruneSpent(const uint256& hash, CCoins &coins); @@ -95,6 +108,16 @@ public: } bool lookup(uint256 hash, CTransaction& result) const; + + // Estimate fee rate needed to get into the next + // nBlocks + CFeeRate estimateFee(int nBlocks) const; + // Estimate priority needed to get into the next + // nBlocks + double estimatePriority(int nBlocks) const; + // Write/Read estimates to disk + bool WriteFeeEstimates(CAutoFile& fileout) const; + bool ReadFeeEstimates(CAutoFile& filein); }; /** CCoinsView that brings transactions from a memorypool into view.