change(rpc): Select getblocktemplate RPC transactions according to ZIP-317 (#5724)

* Split the conventional fee check into its own method

This will be used for block production and relaying

* Move getblocktemplate transaction selection into a new zip317 module

* Add a block_production_fee_weight field to VerifiedUnminedTx

* Add a custom Zebra minimum transaction weight for block production

* Implement ZIP-317 transaction selection for block production

* Split weighted index setup into its own function

* Split picking a transaction into its own function

Cargo.lock

@@ -5458,6 +5458,7 @@ dependencies = [
  "num_cpus",
  "proptest",
  "proptest-derive",
+ "rand 0.8.5",
  "serde",
  "serde_json",
  "thiserror",

zebra-chain/src/transaction/unmined.rs

@@ -290,7 +290,7 @@ impl From<&Arc<Transaction>> for UnminedTx {
 /// A verified unmined transaction, and the corresponding transaction fee.
 ///
 /// This transaction has been fully verified, in the context of the mempool.
-#[derive(Clone, Debug, Eq, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 #[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))]
 pub struct VerifiedUnminedTx {
     /// The unmined transaction.
@@ -302,6 +302,14 @@ pub struct VerifiedUnminedTx {
     /// The number of legacy signature operations in this transaction's
     /// transparent inputs and outputs.
     pub legacy_sigop_count: u64,
+
+    /// The block production fee weight for `transaction`, as defined by [ZIP-317].
+    ///
+    /// This is not consensus-critical, so we use `f32` for efficient calculations
+    /// when the mempool holds a large number of transactions.
+    ///
+    /// [ZIP-317]: https://zips.z.cash/zip-0317#block-production
+    pub block_production_fee_weight: f32,
 }
 
 impl fmt::Display for VerifiedUnminedTx {
@@ -315,19 +323,31 @@ impl fmt::Display for VerifiedUnminedTx {
 }
 
 impl VerifiedUnminedTx {
-    /// Create a new verified unmined transaction from a transaction, its fee and the legacy sigop count.
+    /// Create a new verified unmined transaction from an unmined transaction,
+    /// its miner fee, and its legacy sigop count.
     pub fn new(
         transaction: UnminedTx,
         miner_fee: Amount<NonNegative>,
         legacy_sigop_count: u64,
     ) -> Self {
+        let block_production_fee_weight =
+            zip317::block_production_fee_weight(&transaction, miner_fee);
+
         Self {
             transaction,
             miner_fee,
             legacy_sigop_count,
+            block_production_fee_weight,
         }
     }
 
+    /// Returns `true` if the transaction pays at least the [ZIP-317] conventional fee.
+    ///
+    /// [ZIP-317]: https://zips.z.cash/zip-0317#mempool-size-limiting
+    pub fn pays_conventional_fee(&self) -> bool {
+        self.miner_fee >= self.transaction.conventional_fee
+    }
+
     /// The cost in bytes of the transaction, as defined in [ZIP-401].
     ///
     /// A reflection of the work done by the network in processing them (proof
@@ -365,7 +385,7 @@ impl VerifiedUnminedTx {
     pub fn eviction_weight(&self) -> u64 {
         let mut cost = self.cost();
 
-        if self.miner_fee < self.transaction.conventional_fee {
+        if !self.pays_conventional_fee() {
             cost += MEMPOOL_TRANSACTION_LOW_FEE_PENALTY
         }
 

zebra-chain/src/transaction/unmined/zip317.rs

@@ -1,18 +1,15 @@
-//! The [ZIP-317 conventional fee calculation](https://zips.z.cash/zip-0317#fee-calculation)
-//! for [UnminedTx]s.
+//! An implementation of the [ZIP-317] fee calculations for [UnminedTx]s:
+//! - [conventional fee](https://zips.z.cash/zip-0317#fee-calculation)
+//! - [block production transaction weight](https://zips.z.cash/zip-0317#block-production)
 
 use std::cmp::max;
 
 use crate::{
     amount::{Amount, NonNegative},
     serialization::ZcashSerialize,
-    transaction::Transaction,
+    transaction::{Transaction, UnminedTx},
 };
 
-// For doc links
-#[allow(unused_imports)]
-use crate::transaction::UnminedTx;
-
 /// The marginal fee for the ZIP-317 fee calculation, in zatoshis per logical action.
 //
 // TODO: allow Amount<NonNegative> in constants
@@ -27,6 +24,26 @@ const P2PKH_STANDARD_INPUT_SIZE: usize = 150;
 /// The standard size of p2pkh outputs for the ZIP-317 fee calculation, in bytes.
 const P2PKH_STANDARD_OUTPUT_SIZE: usize = 34;
 
+/// The recommended weight cap for ZIP-317 block production.
+const MAX_BLOCK_PRODUCTION_WEIGHT: f32 = 4.0;
+
+/// Zebra's custom minimum weight for ZIP-317 block production,
+/// based on half the [ZIP-203] recommended transaction expiry height of 40 blocks.
+///
+/// This ensures all transactions have a non-zero probability of being mined,
+/// which simplifies our implementation.
+///
+/// If blocks are full, this makes it likely that very low fee transactions
+/// will be mined:
+/// - after approximately 20 blocks delay,
+/// - but before they expire.
+///
+/// Note: Small transactions that pay the legacy ZIP-313 conventional fee have twice this weight.
+/// If blocks are full, they will be mined after approximately 10 blocks delay.
+///
+/// [ZIP-203]: https://zips.z.cash/zip-0203#changes-for-blossom>
+const MIN_BLOCK_PRODUCTION_WEIGHT: f32 = 1.0 / 20.0;
+
 /// Returns the conventional fee for `transaction`, as defined by [ZIP-317].
 ///
 /// [ZIP-317]: https://zips.z.cash/zip-0317#fee-calculation
@@ -72,6 +89,18 @@ pub fn conventional_fee(transaction: &Transaction) -> Amount<NonNegative> {
     conventional_fee.expect("conventional fee is positive and limited by serialized size limit")
 }
 
+/// Returns the block production fee weight for `transaction`, as defined by [ZIP-317].
+///
+/// [ZIP-317]: https://zips.z.cash/zip-0317#block-production
+pub fn block_production_fee_weight(transaction: &UnminedTx, miner_fee: Amount<NonNegative>) -> f32 {
+    let miner_fee = i64::from(miner_fee) as f32;
+    let conventional_fee = i64::from(transaction.conventional_fee) as f32;
+
+    let uncapped_weight = miner_fee / conventional_fee;
+
+    uncapped_weight.clamp(MIN_BLOCK_PRODUCTION_WEIGHT, MAX_BLOCK_PRODUCTION_WEIGHT)
+}
+
 /// Divide `quotient` by `divisor`, rounding the result up to the nearest integer.
 ///
 /// # Correctness

zebra-consensus/src/transaction.rs

@@ -117,7 +117,7 @@ pub enum Request {
 
 /// The response type for the transaction verifier service.
 /// Responses identify the transaction that was verified.
-#[derive(Clone, Debug, Eq, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum Response {
     /// A response to a block transaction verification request.
     Block {

zebra-rpc/Cargo.toml

@@ -14,6 +14,7 @@ default = []
 
 # Experimental mining RPC support
 getblocktemplate-rpcs = [
+    "rand",
     "zebra-consensus/getblocktemplate-rpcs",
     "zebra-state/getblocktemplate-rpcs",
     "zebra-node-services/getblocktemplate-rpcs",
@@ -54,6 +55,10 @@ tracing-futures = "0.2.5"
 hex = { version = "0.4.3", features = ["serde"] }
 serde = { version = "1.0.148", features = ["serde_derive"] }
 
+# Experimental feature getblocktemplate-rpcs
+rand = { version = "0.8.5", package = "rand", optional = true }
+
+# Test-only feature proptest-impl
 proptest = { version = "0.10.1", optional = true }
 proptest-derive = { version = "0.3.0", optional = true }
 

zebra-rpc/src/methods/get_block_template_rpcs.rs

@@ -40,7 +40,9 @@ use crate::methods::{
 
 pub mod config;
 pub mod constants;
+
 pub(crate) mod types;
+pub(crate) mod zip317;
 
 /// The max estimated distance to the chain tip for the getblocktemplate method
 // Set to 30 in case the local time is a little ahead.
@@ -315,7 +317,7 @@ where
                 });
             }
 
-            let mempool_txs = select_mempool_transactions(mempool).await?;
+            let mempool_txs = zip317::select_mempool_transactions(mempool).await?;
 
             let miner_fee = miner_fee(&mempool_txs);
 
@@ -482,37 +484,6 @@ where
 
 // get_block_template support methods
 
-/// Returns selected transactions in the `mempool`, or an error if the mempool has failed.
-///
-/// TODO: select transactions according to ZIP-317 (#5473)
-pub async fn select_mempool_transactions<Mempool>(
-    mempool: Mempool,
-) -> Result<Vec<VerifiedUnminedTx>>
-where
-    Mempool: Service<
-            mempool::Request,
-            Response = mempool::Response,
-            Error = zebra_node_services::BoxError,
-        > + 'static,
-    Mempool::Future: Send,
-{
-    let response = mempool
-        .oneshot(mempool::Request::FullTransactions)
-        .await
-        .map_err(|error| Error {
-            code: ErrorCode::ServerError(0),
-            message: error.to_string(),
-            data: None,
-        })?;
-
-    if let mempool::Response::FullTransactions(transactions) = response {
-        // TODO: select transactions according to ZIP-317 (#5473)
-        Ok(transactions)
-    } else {
-        unreachable!("unmatched response to a mempool::FullTransactions request");
-    }
-}
-
 /// Returns the total miner fee for `mempool_txs`.
 pub fn miner_fee(mempool_txs: &[VerifiedUnminedTx]) -> Amount<NonNegative> {
     let miner_fee: amount::Result<Amount<NonNegative>> =

zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs

@@ -0,0 +1,186 @@
+//! The [ZIP-317 block production algorithm](https://zips.z.cash/zip-0317#block-production).
+//!
+//! This is recommended algorithm, so these calculations are not consensus-critical,
+//! or standardised across node implementations:
+//! > it is sufficient to use floating point arithmetic to calculate the argument to `floor`
+//! > when computing `size_target`, since there is no consensus requirement for this to be
+//! > exactly the same between implementations.
+
+use jsonrpc_core::{Error, ErrorCode, Result};
+use rand::{
+    distributions::{Distribution, WeightedIndex},
+    prelude::thread_rng,
+};
+use tower::{Service, ServiceExt};
+
+use zebra_chain::{block::MAX_BLOCK_BYTES, transaction::VerifiedUnminedTx};
+use zebra_consensus::MAX_BLOCK_SIGOPS;
+use zebra_node_services::mempool;
+
+/// Selects mempool transactions for block production according to [ZIP-317].
+///
+/// Returns selected transactions from the `mempool`, or an error if the mempool has failed.
+///
+/// [ZIP-317]: https://zips.z.cash/zip-0317#block-production
+pub async fn select_mempool_transactions<Mempool>(
+    mempool: Mempool,
+) -> Result<Vec<VerifiedUnminedTx>>
+where
+    Mempool: Service<
+            mempool::Request,
+            Response = mempool::Response,
+            Error = zebra_node_services::BoxError,
+        > + 'static,
+    Mempool::Future: Send,
+{
+    let mempool_transactions = fetch_mempool_transactions(mempool).await?;
+
+    // Setup the transaction lists.
+    let (conventional_fee_txs, low_fee_txs): (Vec<_>, Vec<_>) = mempool_transactions
+        .into_iter()
+        .partition(VerifiedUnminedTx::pays_conventional_fee);
+
+    // Set up limit tracking
+    let mut selected_txs = Vec::new();
+    let mut remaining_block_sigops = MAX_BLOCK_SIGOPS;
+    let mut remaining_block_bytes: usize = MAX_BLOCK_BYTES.try_into().expect("fits in memory");
+
+    if let Some((conventional_fee_tx_weights, _total_weight)) =
+        setup_fee_weighted_index(&conventional_fee_txs)
+    {
+        let mut conventional_fee_tx_weights = Some(conventional_fee_tx_weights);
+
+        // > Repeat while there is any mempool transaction that:
+        // > - pays at least the conventional fee,
+        // > - is within the block sigop limit, and
+        // > - fits in the block...
+        while let Some(tx_weights) = conventional_fee_tx_weights {
+            // > Pick one of those transactions at random with probability in direct proportion
+            // > to its weight, and add it to the block.
+            let (tx_weights, candidate_tx) =
+                choose_transaction_weighted_random(&conventional_fee_txs, tx_weights);
+            conventional_fee_tx_weights = tx_weights;
+
+            if candidate_tx.legacy_sigop_count <= remaining_block_sigops
+                && candidate_tx.transaction.size <= remaining_block_bytes
+            {
+                selected_txs.push(candidate_tx.clone());
+
+                remaining_block_sigops -= candidate_tx.legacy_sigop_count;
+                remaining_block_bytes -= candidate_tx.transaction.size;
+            }
+        }
+    }
+
+    // > Let `N` be the number of remaining transactions with `tx.weight < 1`.
+    // > Calculate their sum of weights.
+    if let Some((low_fee_tx_weights, remaining_weight)) = setup_fee_weighted_index(&low_fee_txs) {
+        let low_fee_tx_count = low_fee_txs.len() as f32;
+
+        // > Calculate `size_target = ...`
+        //
+        // We track the remaining bytes within our scaled quota,
+        // so there is no need to actually calculate `size_target` or `size_of_block_so_far`.
+        let average_remaining_weight = remaining_weight / low_fee_tx_count;
+
+        let remaining_block_bytes =
+            remaining_block_bytes as f32 * average_remaining_weight.min(1.0);
+        let mut remaining_block_bytes = remaining_block_bytes as usize;
+
+        let mut low_fee_tx_weights = Some(low_fee_tx_weights);
+
+        while let Some(tx_weights) = low_fee_tx_weights {
+            // > Pick a transaction with probability in direct proportion to its weight...
+            let (tx_weights, candidate_tx) =
+                choose_transaction_weighted_random(&low_fee_txs, tx_weights);
+            low_fee_tx_weights = tx_weights;
+
+            // > and add it to the block. If that transaction would exceed the `size_target`
+            // > or the block sigop limit, stop without adding it.
+            if candidate_tx.legacy_sigop_count > remaining_block_sigops
+                || candidate_tx.transaction.size > remaining_block_bytes
+            {
+                // We've exceeded the scaled quota size limit, or the absolute sigop limit
+                break;
+            }
+
+            selected_txs.push(candidate_tx.clone());
+
+            remaining_block_sigops -= candidate_tx.legacy_sigop_count;
+            remaining_block_bytes -= candidate_tx.transaction.size;
+        }
+    }
+
+    Ok(selected_txs)
+}
+
+/// Fetch the transactions that are currently in `mempool`.
+async fn fetch_mempool_transactions<Mempool>(mempool: Mempool) -> Result<Vec<VerifiedUnminedTx>>
+where
+    Mempool: Service<
+            mempool::Request,
+            Response = mempool::Response,
+            Error = zebra_node_services::BoxError,
+        > + 'static,
+    Mempool::Future: Send,
+{
+    let response = mempool
+        .oneshot(mempool::Request::FullTransactions)
+        .await
+        .map_err(|error| Error {
+            code: ErrorCode::ServerError(0),
+            message: error.to_string(),
+            data: None,
+        })?;
+
+    if let mempool::Response::FullTransactions(transactions) = response {
+        Ok(transactions)
+    } else {
+        unreachable!("unmatched response to a mempool::FullTransactions request")
+    }
+}
+
+/// Returns a fee-weighted index and the total weight of `transactions`.
+///
+/// Returns `None` if there are no transactions, or if the weights are invalid.
+fn setup_fee_weighted_index(
+    transactions: &[VerifiedUnminedTx],
+) -> Option<(WeightedIndex<f32>, f32)> {
+    if transactions.is_empty() {
+        return None;
+    }
+
+    let tx_weights: Vec<f32> = transactions
+        .iter()
+        .map(|tx| tx.block_production_fee_weight)
+        .collect();
+    let total_tx_weight: f32 = tx_weights.iter().sum();
+
+    // Setup the transaction weights.
+    let tx_weights = WeightedIndex::new(tx_weights).ok()?;
+
+    Some((tx_weights, total_tx_weight))
+}
+
+/// Choose a transaction from `transactions`, using the previously set up `weighted_index`.
+///
+/// If some transactions have not yet been chosen, returns the weighted index and the transaction.
+/// Otherwise, just returns the transaction.
+fn choose_transaction_weighted_random(
+    transactions: &[VerifiedUnminedTx],
+    mut weighted_index: WeightedIndex<f32>,
+) -> (Option<WeightedIndex<f32>>, VerifiedUnminedTx) {
+    let candidate_position = weighted_index.sample(&mut thread_rng());
+    let candidate_tx = transactions[candidate_position].clone();
+
+    // Only pick each transaction once, by setting picked transaction weights to zero
+    if weighted_index
+        .update_weights(&[(candidate_position, &0.0)])
+        .is_err()
+    {
+        // All weights are zero, so each transaction has either been selected or rejected
+        (None, candidate_tx)
+    } else {
+        (Some(weighted_index), candidate_tx)
+    }
+}

zebrad/src/components/mempool/storage/verified_set.rs

@@ -161,7 +161,7 @@ impl VerifiedSet {
                 .collect();
 
             let dist = WeightedIndex::new(weights)
-                .expect("there is at least one weight and all weights are valid");
+                .expect("there is at least one weight, all weights are non-negative, and the total is positive");
 
             Some(self.remove(dist.sample(&mut thread_rng())))
         }