add(rpc): note tree sizes to `getblock` api (#7278)

* add the basics

* add some docs, move code

* upgrade compact formats to https://github.com/zcash/lightwalletd/blob/v0.4.15/walletrpc/compact_formats.proto

* add a test for in sync chain

* test changing to ecc lightwalletd

* revert change of lightwalletd repo (already merged to main)

* add debug log to see whats going on with the test

* change log to tracing::info

* remove log line

---------

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

zebra-rpc/src/methods.rs

@@ -744,11 +744,68 @@ where
                 //        this needs a new state request for the height -> hash index
                 let height = hash_or_height.height();
 
+                // Sapling trees
+                //
+                // # Concurrency
+                //
+                // We look up by block hash so the hash, transaction IDs, and confirmations
+                // are consistent.
+                let request = zebra_state::ReadRequest::SaplingTree(hash.into());
+                let response = state
+                    .ready()
+                    .and_then(|service| service.call(request))
+                    .await
+                    .map_err(|error| Error {
+                        code: ErrorCode::ServerError(0),
+                        message: error.to_string(),
+                        data: None,
+                    })?;
+
+                let sapling_note_commitment_tree_count = match response {
+                    zebra_state::ReadResponse::SaplingTree(Some(nct)) => nct.count(),
+                    zebra_state::ReadResponse::SaplingTree(None) => 0,
+                    _ => unreachable!("unmatched response to a SaplingTree request"),
+                };
+
+                // Orchard trees
+                //
+                // # Concurrency
+                //
+                // We look up by block hash so the hash, transaction IDs, and confirmations
+                // are consistent.
+                let request = zebra_state::ReadRequest::OrchardTree(hash.into());
+                let response = state
+                    .ready()
+                    .and_then(|service| service.call(request))
+                    .await
+                    .map_err(|error| Error {
+                        code: ErrorCode::ServerError(0),
+                        message: error.to_string(),
+                        data: None,
+                    })?;
+
+                let orchard_note_commitment_tree_count = match response {
+                    zebra_state::ReadResponse::OrchardTree(Some(nct)) => nct.count(),
+                    zebra_state::ReadResponse::OrchardTree(None) => 0,
+                    _ => unreachable!("unmatched response to a OrchardTree request"),
+                };
+
+                let sapling = SaplingTrees {
+                    size: sapling_note_commitment_tree_count,
+                };
+
+                let orchard = OrchardTrees {
+                    size: orchard_note_commitment_tree_count,
+                };
+
+                let trees = GetBlockTrees { sapling, orchard };
+
                 Ok(GetBlock::Object {
                     hash: GetBlockHash(hash),
                     confirmations,
                     height,
                     tx,
+                    trees,
                 })
             } else {
                 Err(Error {
@@ -1362,6 +1419,9 @@ pub enum GetBlock {
         //
         // TODO: use a typed Vec<transaction::Hash> here
         tx: Vec<String>,
+
+        /// Information about the note commitment trees.
+        trees: GetBlockTrees,
     },
 }
 
@@ -1524,6 +1584,39 @@ impl GetRawTransaction {
     }
 }
 
+/// Information about the sapling and orchard note commitment trees if any.
+#[derive(Copy, Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
+pub struct GetBlockTrees {
+    #[serde(skip_serializing_if = "SaplingTrees::is_empty")]
+    sapling: SaplingTrees,
+    #[serde(skip_serializing_if = "OrchardTrees::is_empty")]
+    orchard: OrchardTrees,
+}
+
+/// Sapling note commitment tree information.
+#[derive(Copy, Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
+pub struct SaplingTrees {
+    size: u64,
+}
+
+impl SaplingTrees {
+    fn is_empty(&self) -> bool {
+        self.size == 0
+    }
+}
+
+/// Orchard note commitment tree information.
+#[derive(Copy, Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
+pub struct OrchardTrees {
+    size: u64,
+}
+
+impl OrchardTrees {
+    fn is_empty(&self) -> bool {
+        self.size == 0
+    }
+}
+
 /// Check if provided height range is valid for address indexes.
 fn check_height_range(start: Height, end: Height, chain_height: Height) -> Result<()> {
     if start == Height(0) || end == Height(0) {

zebra-rpc/src/methods/tests/snapshots/get_block_verbose_hash_verbosity_1@mainnet_10.snap

@@ -7,5 +7,6 @@ expression: block
   "confirmations": 10,
   "tx": [
     "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609"
-  ]
+  ],
+  "trees": {}
 }

zebra-rpc/src/methods/tests/snapshots/get_block_verbose_hash_verbosity_1@testnet_10.snap

@@ -7,5 +7,6 @@ expression: block
   "confirmations": 10,
   "tx": [
     "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75"
-  ]
+  ],
+  "trees": {}
 }

zebra-rpc/src/methods/tests/snapshots/get_block_verbose_hash_verbosity_default@mainnet_10.snap

@@ -7,5 +7,6 @@ expression: block
   "confirmations": 10,
   "tx": [
     "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609"
-  ]
+  ],
+  "trees": {}
 }

zebra-rpc/src/methods/tests/snapshots/get_block_verbose_hash_verbosity_default@testnet_10.snap

@@ -7,5 +7,6 @@ expression: block
   "confirmations": 10,
   "tx": [
     "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75"
-  ]
+  ],
+  "trees": {}
 }

zebra-rpc/src/methods/tests/snapshots/get_block_verbose_height_verbosity_1@mainnet_10.snap

@@ -8,5 +8,6 @@ expression: block
   "height": 1,
   "tx": [
     "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609"
-  ]
+  ],
+  "trees": {}
 }

zebra-rpc/src/methods/tests/snapshots/get_block_verbose_height_verbosity_1@testnet_10.snap

@@ -8,5 +8,6 @@ expression: block
   "height": 1,
   "tx": [
     "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75"
-  ]
+  ],
+  "trees": {}
 }

zebra-rpc/src/methods/tests/snapshots/get_block_verbose_height_verbosity_default@mainnet_10.snap

@@ -8,5 +8,6 @@ expression: block
   "height": 1,
   "tx": [
     "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609"
-  ]
+  ],
+  "trees": {}
 }

zebra-rpc/src/methods/tests/snapshots/get_block_verbose_height_verbosity_default@testnet_10.snap

@@ -8,5 +8,6 @@ expression: block
   "height": 1,
   "tx": [
     "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75"
-  ]
+  ],
+  "trees": {}
 }

zebra-rpc/src/methods/tests/vectors.rs

@@ -121,6 +121,11 @@ async fn rpc_getblock() {
         assert_eq!(get_block, expected_result);
     }
 
+    // Create empty note commitment tree information.
+    let sapling = SaplingTrees { size: 0 };
+    let orchard = OrchardTrees { size: 0 };
+    let trees = GetBlockTrees { sapling, orchard };
+
     // Make height calls with verbosity=1 and check response
     for (i, block) in blocks.iter().enumerate() {
         let get_block = rpc
@@ -139,6 +144,7 @@ async fn rpc_getblock() {
                     .iter()
                     .map(|tx| tx.hash().encode_hex())
                     .collect(),
+                trees,
             }
         );
     }
@@ -161,6 +167,7 @@ async fn rpc_getblock() {
                     .iter()
                     .map(|tx| tx.hash().encode_hex())
                     .collect(),
+                trees,
             }
         );
     }
@@ -183,6 +190,7 @@ async fn rpc_getblock() {
                     .iter()
                     .map(|tx| tx.hash().encode_hex())
                     .collect(),
+                trees,
             }
         );
     }
@@ -205,6 +213,7 @@ async fn rpc_getblock() {
                     .iter()
                     .map(|tx| tx.hash().encode_hex())
                     .collect(),
+                trees,
             }
         );
     }

zebrad/tests/common/lightwalletd/proto/compact_formats.proto

@@ -1,4 +1,4 @@
-// Copyright (c) 2019-2020 The Zcash developers
+// Copyright (c) 2019-2021 The Zcash developers
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or https://www.opensource.org/licenses/mit-license.php .
 
@@ -6,39 +6,50 @@ syntax = "proto3";
 package cash.z.wallet.sdk.rpc;
 option go_package = "lightwalletd/walletrpc";
 option swift_prefix = "";
+
 // Remember that proto3 fields are all optional. A field that is not present will be set to its zero value.
 // bytes fields of hashes are in canonical little-endian format.
 
+// ChainMetadata represents information about the state of the chain as of a given block.
+message ChainMetadata {
+    uint32 saplingCommitmentTreeSize = 1; // the size of the Sapling note commitment tree as of the end of this block
+    uint32 orchardCommitmentTreeSize = 2; // the size of the Orchard note commitment tree as of the end of this block
+}
+
 // CompactBlock is a packaging of ONLY the data from a block that's needed to:
 //   1. Detect a payment to your shielded Sapling address
 //   2. Detect a spend of your shielded Sapling notes
 //   3. Update your witnesses to generate new Sapling spend proofs.
 message CompactBlock {
-    uint32 protoVersion = 1;    // the version of this wire format, for storage
-    uint64 height = 2;          // the height of this block
-    bytes hash = 3;             // the ID (hash) of this block, same as in block explorers
-    bytes prevHash = 4;         // the ID (hash) of this block's predecessor
-    uint32 time = 5;            // Unix epoch time when the block was mined
-    bytes header = 6;           // (hash, prevHash, and time) OR (full header)
-    repeated CompactTx vtx = 7; // zero or more compact transactions from this block
+    uint32 protoVersion = 1;         // the version of this wire format, for storage
+    uint64 height = 2;               // the height of this block
+    bytes hash = 3;                  // the ID (hash) of this block, same as in block explorers
+    bytes prevHash = 4;              // the ID (hash) of this block's predecessor
+    uint32 time = 5;                 // Unix epoch time when the block was mined
+    bytes header = 6;                // (hash, prevHash, and time) OR (full header)
+    repeated CompactTx vtx = 7;      // zero or more compact transactions from this block
+    ChainMetadata chainMetadata = 8; // information about the state of the chain as of this block
 }
 
 // CompactTx contains the minimum information for a wallet to know if this transaction
 // is relevant to it (either pays to it or spends from it) via shielded elements
 // only. This message will not encode a transparent-to-transparent transaction.
 message CompactTx {
+    // Index and hash will allow the receiver to call out to chain
+    // explorers or other data structures to retrieve more information
+    // about this transaction.
     uint64 index = 1;   // the index within the full block
     bytes hash = 2;     // the ID (hash) of this transaction, same as in block explorers
 
     // The transaction fee: present if server can provide. In the case of a
     // stateless server and a transaction with transparent inputs, this will be
     // unset because the calculation requires reference to prior transactions.
-    // in a pure-Sapling context, the fee will be calculable as:
-    //    valueBalance + (sum(vPubNew) - sum(vPubOld) - sum(tOut))
+    // If there are no transparent inputs, the fee will be calculable as:
+    //    valueBalanceSapling + valueBalanceOrchard + sum(vPubNew) - sum(vPubOld) - sum(tOut)
     uint32 fee = 3;
 
-    repeated CompactSaplingSpend spends = 4;   // inputs
-    repeated CompactSaplingOutput outputs = 5; // outputs
+    repeated CompactSaplingSpend spends = 4;
+    repeated CompactSaplingOutput outputs = 5;
     repeated CompactOrchardAction actions = 6;
 }
 
@@ -48,11 +59,14 @@ message CompactSaplingSpend {
     bytes nf = 1;   // nullifier (see the Zcash protocol specification)
 }
 
-// output is a Sapling Output Description as described in section 7.4 of the
-// Zcash protocol spec. Total size is 948.
+// output encodes the `cmu` field, `ephemeralKey` field, and a 52-byte prefix of the
+// `encCiphertext` field of a Sapling Output Description. These fields are described in
+// section 7.4 of the Zcash protocol spec:
+// https://zips.z.cash/protocol/protocol.pdf#outputencodingandconsensus
+// Total size is 116 bytes.
 message CompactSaplingOutput {
     bytes cmu = 1;          // note commitment u-coordinate
-    bytes epk = 2;          // ephemeral public key
+    bytes ephemeralKey = 2; // ephemeral public key
     bytes ciphertext = 3;   // first 52 bytes of ciphertext
 }
 
@@ -62,5 +76,5 @@ message CompactOrchardAction {
     bytes nullifier = 1;        // [32] The nullifier of the input note
     bytes cmx = 2;              // [32] The x-coordinate of the note commitment for the output note
     bytes ephemeralKey = 3;     // [32] An encoding of an ephemeral Pallas public key
-    bytes ciphertext = 4;       // [52] The note plaintext component of the encCiphertext field
+    bytes ciphertext = 4;       // [52] The first 52 bytes of the encCiphertext field
 }

zebrad/tests/common/lightwalletd/wallet_grpc_test.rs

@@ -39,7 +39,7 @@ use color_eyre::eyre::Result;
 use zebra_chain::{
     block::Block,
     parameters::Network,
-    parameters::NetworkUpgrade::{self, Canopy},
+    parameters::NetworkUpgrade::{Nu5, Sapling},
     serialization::ZcashDeserializeInto,
 };
 
@@ -145,27 +145,43 @@ pub async fn run() -> Result<()> {
         .await?
         .into_inner();
 
-    // As we are using a pretty much synchronized blockchain, we can assume the tip is above the Canopy network upgrade
-    assert!(block_tip.height > Canopy.activation_height(network).unwrap().0 as u64);
+    // Get `Sapling` activation height.
+    let sapling_activation_height = Sapling.activation_height(network).unwrap().0 as u64;
 
-    // `lightwalletd` only supports post-Sapling blocks, so we begin at the
-    // Sapling activation height.
-    let sapling_activation_height = NetworkUpgrade::Sapling
-        .activation_height(network)
-        .unwrap()
-        .0 as u64;
+    // As we are using a pretty much synchronized blockchain, we can assume the tip is above the Nu5 network upgrade
+    assert!(block_tip.height > Nu5.activation_height(network).unwrap().0 as u64);
 
-    // Call `GetBlock` with block 1 height
-    let block_one = rpc_client
+    // The first block in the mainnet that has sapling and orchard information.
+    let block_with_trees = 1687107;
+
+    // Call `GetBlock` with `block_with_trees`.
+    let get_block_response = rpc_client
         .get_block(BlockId {
-            height: sapling_activation_height,
+            height: block_with_trees,
             hash: vec![],
         })
         .await?
         .into_inner();
 
-    // Make sure we got block 1 back
-    assert_eq!(block_one.height, sapling_activation_height);
+    // Make sure we got block `block_with_trees` back
+    assert_eq!(get_block_response.height, block_with_trees);
+
+    // Testing the `trees` field of `GetBlock`.
+    assert_eq!(
+        get_block_response
+            .chain_metadata
+            .clone()
+            .unwrap()
+            .sapling_commitment_tree_size,
+        1170439
+    );
+    assert_eq!(
+        get_block_response
+            .chain_metadata
+            .unwrap()
+            .orchard_commitment_tree_size,
+        2
+    );
 
     // Call `GetBlockRange` with the range starting at block 1 up to block 10
     let mut block_range = rpc_client