add(metrics): Track mempool actions and size bucketed by weight (copy of #6972, credit @str4d) (#7019)

* metrics: Track mempool actions and size bucketed by weight

* Fix tests

* draft fix tests

* fix `fix_arbitrary_generated_action_overflows`

* add some docs

* manually derive arbitrary

* remove unused import

---------

Co-authored-by: Jack Grigg <jack@electriccoin.co>
Co-authored-by: Marek <mail@marek.onl>
Co-authored-by: Alfredo Garcia <oxarbitrage@gmail.com>

zebra-chain/src/transaction/arbitrary.rs

@@ -30,7 +30,9 @@ use crate::{
 
 use itertools::Itertools;
 
-use super::{FieldNotPresent, JoinSplitData, LockTime, Memo, Transaction, UnminedTx};
+use super::{
+    FieldNotPresent, JoinSplitData, LockTime, Memo, Transaction, UnminedTx, VerifiedUnminedTx,
+};
 
 /// The maximum number of arbitrary transactions, inputs, or outputs.
 ///
@@ -783,6 +785,52 @@ impl Arbitrary for UnminedTx {
     type Strategy = BoxedStrategy<Self>;
 }
 
+impl Arbitrary for VerifiedUnminedTx {
+    type Parameters = ();
+
+    fn arbitrary_with(_args: Self::Parameters) -> Self::Strategy {
+        (
+            any::<UnminedTx>(),
+            any::<Amount<NonNegative>>(),
+            any::<u64>(),
+            any::<(u16, u16)>().prop_map(|(unpaid_actions, conventional_actions)| {
+                (
+                    unpaid_actions % conventional_actions.saturating_add(1),
+                    conventional_actions,
+                )
+            }),
+            any::<f32>(),
+        )
+            .prop_map(
+                |(
+                    transaction,
+                    miner_fee,
+                    legacy_sigop_count,
+                    (conventional_actions, mut unpaid_actions),
+                    fee_weight_ratio,
+                )| {
+                    if unpaid_actions > conventional_actions {
+                        unpaid_actions = conventional_actions;
+                    }
+
+                    let conventional_actions = conventional_actions as u32;
+                    let unpaid_actions = unpaid_actions as u32;
+
+                    Self {
+                        transaction,
+                        miner_fee,
+                        legacy_sigop_count,
+                        conventional_actions,
+                        unpaid_actions,
+                        fee_weight_ratio,
+                    }
+                },
+            )
+            .boxed()
+    }
+    type Strategy = BoxedStrategy<Self>;
+}
+
 // Utility functions
 
 /// Convert `trans` into a fake v5 transaction,

zebra-chain/src/transaction/unmined.rs

@@ -325,7 +325,6 @@ impl From<&Arc<Transaction>> for UnminedTx {
 //
 // This struct can't be `Eq`, because it contains a `f32`.
 #[derive(Clone, PartialEq)]
-#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))]
 pub struct VerifiedUnminedTx {
     /// The unmined transaction.
     pub transaction: UnminedTx,
@@ -337,6 +336,13 @@ pub struct VerifiedUnminedTx {
     /// transparent inputs and outputs.
     pub legacy_sigop_count: u64,
 
+    /// The number of conventional actions for `transaction`, as defined by [ZIP-317].
+    ///
+    /// The number of actions is limited by [`MAX_BLOCK_BYTES`], so it fits in a u32.
+    ///
+    /// [ZIP-317]: https://zips.z.cash/zip-0317#block-production
+    pub conventional_actions: u32,
+
     /// The number of unpaid actions for `transaction`,
     /// as defined by [ZIP-317] for block production.
     ///
@@ -381,6 +387,7 @@ impl VerifiedUnminedTx {
         legacy_sigop_count: u64,
     ) -> Result<Self, zip317::Error> {
         let fee_weight_ratio = zip317::conventional_fee_weight_ratio(&transaction, miner_fee);
+        let conventional_actions = zip317::conventional_actions(&transaction.transaction);
         let unpaid_actions = zip317::unpaid_actions(&transaction, miner_fee);
 
         zip317::mempool_checks(unpaid_actions, miner_fee, transaction.size)?;
@@ -390,6 +397,7 @@ impl VerifiedUnminedTx {
             miner_fee,
             legacy_sigop_count,
             fee_weight_ratio,
+            conventional_actions,
             unpaid_actions,
         })
     }

zebra-chain/src/transaction/unmined/zip317.rs

@@ -133,7 +133,7 @@ pub fn conventional_fee_weight_ratio(
 /// as defined by [ZIP-317].
 ///
 /// [ZIP-317]: https://zips.z.cash/zip-0317#fee-calculation
-fn conventional_actions(transaction: &Transaction) -> u32 {
+pub fn conventional_actions(transaction: &Transaction) -> u32 {
     let tx_in_total_size: usize = transaction
         .inputs()
         .iter()

zebra-rpc/src/methods/tests/vectors.rs

@@ -1185,7 +1185,7 @@ async fn rpc_getblocktemplate_mining_address(use_p2pkh: bool) {
         block::{Hash, MAX_BLOCK_BYTES, ZCASH_BLOCK_VERSION},
         chain_sync_status::MockSyncStatus,
         serialization::DateTime32,
-        transaction::VerifiedUnminedTx,
+        transaction::{zip317, VerifiedUnminedTx},
         work::difficulty::{CompactDifficulty, ExpandedDifficulty, U256},
     };
     use zebra_consensus::MAX_BLOCK_SIGOPS;
@@ -1441,10 +1441,13 @@ async fn rpc_getblocktemplate_mining_address(use_p2pkh: bool) {
         conventional_fee: 0.try_into().unwrap(),
     };
 
+    let conventional_actions = zip317::conventional_actions(&unmined_tx.transaction);
+
     let verified_unmined_tx = VerifiedUnminedTx {
         transaction: unmined_tx,
         miner_fee: 0.try_into().unwrap(),
         legacy_sigop_count: 0,
+        conventional_actions,
         unpaid_actions: 0,
         fee_weight_ratio: 1.0,
     };

zebrad/src/components/mempool/storage/verified_set.rs

@@ -286,10 +286,110 @@ impl VerifiedSet {
     }
 
     fn update_metrics(&mut self) {
+        // Track the sum of unpaid actions within each transaction (as they are subject to the
+        // unpaid action limit). Transactions that have weight >= 1 have no unpaid actions by
+        // definition.
+        let mut unpaid_actions_with_weight_lt20pct = 0;
+        let mut unpaid_actions_with_weight_lt40pct = 0;
+        let mut unpaid_actions_with_weight_lt60pct = 0;
+        let mut unpaid_actions_with_weight_lt80pct = 0;
+        let mut unpaid_actions_with_weight_lt1 = 0;
+
+        // Track the total number of paid actions across all transactions in the mempool. This
+        // added to the bucketed unpaid actions above is equal to the total number of conventional
+        // actions in the mempool.
+        let mut paid_actions = 0;
+
+        // Track the sum of transaction sizes (the metric by which they are mainly limited) across
+        // several buckets.
+        let mut size_with_weight_lt1 = 0;
+        let mut size_with_weight_eq1 = 0;
+        let mut size_with_weight_gt1 = 0;
+        let mut size_with_weight_gt2 = 0;
+        let mut size_with_weight_gt3 = 0;
+
+        for entry in self.full_transactions() {
+            paid_actions += entry.conventional_actions - entry.unpaid_actions;
+
+            if entry.fee_weight_ratio > 3.0 {
+                size_with_weight_gt3 += entry.transaction.size;
+            } else if entry.fee_weight_ratio > 2.0 {
+                size_with_weight_gt2 += entry.transaction.size;
+            } else if entry.fee_weight_ratio > 1.0 {
+                size_with_weight_gt1 += entry.transaction.size;
+            } else if entry.fee_weight_ratio == 1.0 {
+                size_with_weight_eq1 += entry.transaction.size;
+            } else {
+                size_with_weight_lt1 += entry.transaction.size;
+                if entry.fee_weight_ratio < 0.2 {
+                    unpaid_actions_with_weight_lt20pct += entry.unpaid_actions;
+                } else if entry.fee_weight_ratio < 0.4 {
+                    unpaid_actions_with_weight_lt40pct += entry.unpaid_actions;
+                } else if entry.fee_weight_ratio < 0.6 {
+                    unpaid_actions_with_weight_lt60pct += entry.unpaid_actions;
+                } else if entry.fee_weight_ratio < 0.8 {
+                    unpaid_actions_with_weight_lt80pct += entry.unpaid_actions;
+                } else {
+                    unpaid_actions_with_weight_lt1 += entry.unpaid_actions;
+                }
+            }
+        }
+
+        metrics::gauge!(
+            "zcash.mempool.actions.unpaid",
+            unpaid_actions_with_weight_lt20pct as f64,
+            "bk" => "< 0.2",
+        );
+        metrics::gauge!(
+            "zcash.mempool.actions.unpaid",
+            unpaid_actions_with_weight_lt40pct as f64,
+            "bk" => "< 0.4",
+        );
+        metrics::gauge!(
+            "zcash.mempool.actions.unpaid",
+            unpaid_actions_with_weight_lt60pct as f64,
+            "bk" => "< 0.6",
+        );
+        metrics::gauge!(
+            "zcash.mempool.actions.unpaid",
+            unpaid_actions_with_weight_lt80pct as f64,
+            "bk" => "< 0.8",
+        );
+        metrics::gauge!(
+            "zcash.mempool.actions.unpaid",
+            unpaid_actions_with_weight_lt1 as f64,
+            "bk" => "< 1",
+        );
+        metrics::gauge!("zcash.mempool.actions.paid", paid_actions as f64);
         metrics::gauge!(
             "zcash.mempool.size.transactions",
             self.transaction_count() as f64,
         );
+        metrics::gauge!(
+            "zcash.mempool.size.weighted",
+            size_with_weight_lt1 as f64,
+            "bk" => "< 1",
+        );
+        metrics::gauge!(
+            "zcash.mempool.size.weighted",
+            size_with_weight_eq1 as f64,
+            "bk" => "1",
+        );
+        metrics::gauge!(
+            "zcash.mempool.size.weighted",
+            size_with_weight_gt1 as f64,
+            "bk" => "> 1",
+        );
+        metrics::gauge!(
+            "zcash.mempool.size.weighted",
+            size_with_weight_gt2 as f64,
+            "bk" => "> 2",
+        );
+        metrics::gauge!(
+            "zcash.mempool.size.weighted",
+            size_with_weight_gt3 as f64,
+            "bk" => "> 3",
+        );
         metrics::gauge!(
             "zcash.mempool.size.bytes",
             self.transactions_serialized_size as f64,