Make chain tip updates and access more efficient (#2695)

* Store precalculated transactions in an `Arc`

Transaction `Hash`es are 32 bytes,
and the minimun transparent transaction size is 54 bytes.
So a full 2MB block can create 1.1MB of transaction hashes.

We use an `Arc` to avoid repeatedly cloning that much data.

* Remove the unused `Block` from `ChainTipBlock`

This drops the block as soon as it isn't needed any more.

Previously, it would stick around until every `ChainTipReceiver`
dropped their `ChainTipBlock`, even if they didn't use the `Block`
at all.

zebra-chain/src/chain_tip.rs

@@ -1,5 +1,7 @@
 //! Chain tip interfaces.
 
+use std::sync::Arc;
+
 use crate::{block, transaction};
 
 /// An interface for querying the chain tip.
@@ -18,7 +20,7 @@ pub trait ChainTip {
     ///
     /// All transactions with these mined IDs should be rejected from the mempool,
     /// even if their authorizing data is different.
-    fn best_tip_mined_transaction_ids(&self) -> Vec<transaction::Hash>;
+    fn best_tip_mined_transaction_ids(&self) -> Arc<[transaction::Hash]>;
 }
 
 /// A chain tip that is always empty.
@@ -34,7 +36,7 @@ impl ChainTip for NoChainTip {
         None
     }
 
-    fn best_tip_mined_transaction_ids(&self) -> Vec<transaction::Hash> {
+    fn best_tip_mined_transaction_ids(&self) -> Arc<[transaction::Hash]> {
-        Vec::new()
+        Arc::new([])
     }
 }

zebra-consensus/src/block.rs

@@ -155,11 +155,8 @@ where
             // the header binds to the transactions in the blocks.
 
             // Precomputing this avoids duplicating transaction hash computations.
-            let transaction_hashes = block
+            let transaction_hashes: Arc<[_]> =
-                .transactions
+                block.transactions.iter().map(|t| t.hash()).collect();
-                .iter()
-                .map(|t| t.hash())
-                .collect::<Vec<_>>();
 
             check::merkle_root_validity(network, &block, &transaction_hashes)?;
 

zebra-state/src/arbitrary.rs

@@ -20,8 +20,8 @@ impl Prepare for Arc<Block> {
         let block = self;
         let hash = block.hash();
         let height = block.coinbase_height().unwrap();
-        let transaction_hashes: Vec<_> = block.transactions.iter().map(|tx| tx.hash()).collect();
+        let transaction_hashes: Arc<[_]> = block.transactions.iter().map(|tx| tx.hash()).collect();
-        let new_outputs = transparent::new_ordered_outputs(&block, transaction_hashes.as_slice());
+        let new_outputs = transparent::new_ordered_outputs(&block, &transaction_hashes);
 
         PreparedBlock {
             block,

zebra-state/src/request.rs

@@ -80,7 +80,7 @@ pub struct PreparedBlock {
     /// earlier transaction.
     pub new_outputs: HashMap<transparent::OutPoint, transparent::OrderedUtxo>,
     /// A precomputed list of the hashes of the transactions in this block.
-    pub transaction_hashes: Vec<transaction::Hash>,
+    pub transaction_hashes: Arc<[transaction::Hash]>,
 }
 
 /// A contextually validated block, ready to be committed directly to the finalized state with
@@ -93,7 +93,7 @@ pub struct ContextuallyValidBlock {
     pub(crate) hash: block::Hash,
     pub(crate) height: block::Height,
     pub(crate) new_outputs: HashMap<transparent::OutPoint, transparent::Utxo>,
-    pub(crate) transaction_hashes: Vec<transaction::Hash>,
+    pub(crate) transaction_hashes: Arc<[transaction::Hash]>,
     /// The sum of the chain value pool changes of all transactions in this block.
     pub(crate) chain_value_pool_change: ValueBalance<NegativeAllowed>,
 }
@@ -110,7 +110,7 @@ pub struct FinalizedBlock {
     pub(crate) hash: block::Hash,
     pub(crate) height: block::Height,
     pub(crate) new_outputs: HashMap<transparent::OutPoint, transparent::Utxo>,
-    pub(crate) transaction_hashes: Vec<transaction::Hash>,
+    pub(crate) transaction_hashes: Arc<[transaction::Hash]>,
 }
 
 impl From<&PreparedBlock> for PreparedBlock {
@@ -165,12 +165,8 @@ impl From<Arc<Block>> for FinalizedBlock {
             .coinbase_height()
             .expect("finalized blocks must have a valid coinbase height");
         let hash = block.hash();
-        let transaction_hashes = block
+        let transaction_hashes: Arc<[_]> = block.transactions.iter().map(|tx| tx.hash()).collect();
-            .transactions
+        let new_outputs = transparent::new_outputs(&block, &transaction_hashes);
-            .iter()
-            .map(|tx| tx.hash())
-            .collect::<Vec<_>>();
-        let new_outputs = transparent::new_outputs(&block, transaction_hashes.as_slice());
 
         Self {
             block,

zebra-state/src/service/chain_tip.rs

@@ -2,11 +2,7 @@ use std::sync::Arc;
 
 use tokio::sync::watch;
 
-use zebra_chain::{
+use zebra_chain::{block, chain_tip::ChainTip, transaction};
-    block::{self, Block},
-    chain_tip::ChainTip,
-    transaction,
-};
 
 use crate::{request::ContextuallyValidBlock, FinalizedBlock};
 
@@ -21,19 +17,18 @@ type ChainTipData = Option<ChainTipBlock>;
 /// Used to efficiently update the [`ChainTipSender`].
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ChainTipBlock {
-    pub(crate) block: Arc<Block>,
     pub(crate) hash: block::Hash,
     pub(crate) height: block::Height,
 
     /// The mined transaction IDs of the transactions in `block`,
     /// in the same order as `block.transactions`.
-    pub(crate) transaction_hashes: Vec<transaction::Hash>,
+    pub(crate) transaction_hashes: Arc<[transaction::Hash]>,
 }
 
 impl From<ContextuallyValidBlock> for ChainTipBlock {
     fn from(contextually_valid: ContextuallyValidBlock) -> Self {
         let ContextuallyValidBlock {
-            block,
+            block: _,
             hash,
             height,
             new_outputs: _,
@@ -41,7 +36,6 @@ impl From<ContextuallyValidBlock> for ChainTipBlock {
             chain_value_pool_change: _,
         } = contextually_valid;
         Self {
-            block,
             hash,
             height,
             transaction_hashes,
@@ -52,14 +46,13 @@ impl From<ContextuallyValidBlock> for ChainTipBlock {
 impl From<FinalizedBlock> for ChainTipBlock {
     fn from(finalized: FinalizedBlock) -> Self {
         let FinalizedBlock {
-            block,
+            block: _,
             hash,
             height,
             new_outputs: _,
             transaction_hashes,
         } = finalized;
         Self {
-            block,
             hash,
             height,
             transaction_hashes,
@@ -180,11 +173,11 @@ impl ChainTip for ChainTipReceiver {
     ///
     /// All transactions with these mined IDs should be rejected from the mempool,
     /// even if their authorizing data is different.
-    fn best_tip_mined_transaction_ids(&self) -> Vec<transaction::Hash> {
+    fn best_tip_mined_transaction_ids(&self) -> Arc<[transaction::Hash]> {
         self.receiver
             .borrow()
             .as_ref()
             .map(|block| block.transaction_hashes.clone())
-            .unwrap_or_default()
+            .unwrap_or_else(|| Arc::new([]))
     }
 }

zebra-state/src/service/chain_tip/tests/prop.rs

@@ -34,17 +34,17 @@ proptest! {
         for update in tip_updates {
             match update {
                 BlockUpdate::Finalized(block) => {
-                    let block = block.map(FinalizedBlock::from).map(ChainTipBlock::from);
+                    let chain_tip = block.clone().map(FinalizedBlock::from).map(ChainTipBlock::from);
-                    chain_tip_sender.set_finalized_tip(block.clone());
+                    chain_tip_sender.set_finalized_tip(chain_tip.clone());
-                    if block.is_some() {
+                    if let Some(block) = block {
-                        latest_finalized_tip = block;
+                        latest_finalized_tip = Some((chain_tip, block));
                     }
                 }
                 BlockUpdate::NonFinalized(block) => {
-                    let block = block.map(FinalizedBlock::from).map(ChainTipBlock::from);
+                    let chain_tip = block.clone().map(FinalizedBlock::from).map(ChainTipBlock::from);
-                    chain_tip_sender.set_best_non_finalized_tip(block.clone());
+                    chain_tip_sender.set_best_non_finalized_tip(chain_tip.clone());
-                    if block.is_some() {
+                    if let Some(block) = block {
-                        latest_non_finalized_tip = block;
+                        latest_non_finalized_tip = Some((chain_tip, block));
                         seen_non_finalized_tip = true;
                     }
                 }
@@ -57,18 +57,37 @@ proptest! {
             latest_finalized_tip
         };
 
-        let expected_height = expected_tip.as_ref().and_then(|block| block.block.coinbase_height());
+        let chain_tip_height = expected_tip
+            .as_ref()
+            .and_then(|(chain_tip, _block)| chain_tip.as_ref())
+            .map(|chain_tip| chain_tip.height);
+        let expected_height = expected_tip.as_ref().and_then(|(_chain_tip, block)| block.coinbase_height());
+        prop_assert_eq!(chain_tip_receiver.best_tip_height(), chain_tip_height);
         prop_assert_eq!(chain_tip_receiver.best_tip_height(), expected_height);
 
-        let expected_hash = expected_tip.as_ref().map(|block| block.block.hash());
+        let chain_tip_hash = expected_tip
+            .as_ref()
+            .and_then(|(chain_tip, _block)| chain_tip.as_ref())
+            .map(|chain_tip| chain_tip.hash);
+        let expected_hash = expected_tip.as_ref().map(|(_chain_tip, block)| block.hash());
+        prop_assert_eq!(chain_tip_receiver.best_tip_hash(), chain_tip_hash);
         prop_assert_eq!(chain_tip_receiver.best_tip_hash(), expected_hash);
 
-        let expected_transaction_ids: Vec<_> = expected_tip
+        let chain_tip_transaction_ids = expected_tip
+            .as_ref()
+            .and_then(|(chain_tip, _block)| chain_tip.as_ref())
+            .map(|chain_tip| chain_tip.transaction_hashes.clone())
+            .unwrap_or_else(|| Arc::new([]));
+        let expected_transaction_ids = expected_tip
             .as_ref()
             .iter()
-            .flat_map(|block| block.block.transactions.clone())
+            .flat_map(|(_chain_tip, block)| block.transactions.clone())
             .map(|transaction| transaction.hash())
             .collect();
+        prop_assert_eq!(
+            chain_tip_receiver.best_tip_mined_transaction_ids(),
+            chain_tip_transaction_ids
+        );
         prop_assert_eq!(
             chain_tip_receiver.best_tip_mined_transaction_ids(),
             expected_transaction_ids

zebra-state/src/service/chain_tip/tests/vectors.rs

@@ -1,3 +1,5 @@
+use std::iter;
+
 use zebra_chain::chain_tip::{ChainTip, NoChainTip};
 
 use super::super::ChainTipSender;
@@ -10,7 +12,7 @@ fn best_tip_is_initially_empty() {
     assert_eq!(chain_tip_receiver.best_tip_hash(), None);
     assert_eq!(
         chain_tip_receiver.best_tip_mined_transaction_ids(),
-        Vec::new()
+        iter::empty().collect()
     );
 }
 
@@ -22,6 +24,6 @@ fn empty_chain_tip_is_empty() {
     assert_eq!(chain_tip_receiver.best_tip_hash(), None);
     assert_eq!(
         chain_tip_receiver.best_tip_mined_transaction_ids(),
-        Vec::new()
+        iter::empty().collect()
     );
 }

zebra-state/src/service/finalized_state.rs

@@ -355,7 +355,7 @@ impl FinalizedState {
             for (transaction_index, (transaction, transaction_hash)) in block
                 .transactions
                 .iter()
-                .zip(transaction_hashes.into_iter())
+                .zip(transaction_hashes.iter())
                 .enumerate()
             {
                 let transaction_location = TransactionLocation {