add(scan): Adds gRPC reflection and documents how to use the zebra-scan gRPC server (#8288)

* adds tonic-reflection

* adds listen_addr to log

* Adds user guide for scan server to zebra book

* fixes typo

* Applies suggestions from code review

* update link

Co-authored-by: Marek <mail@marek.onl>

---------

Co-authored-by: Alfredo Garcia <oxarbitrage@gmail.com>
Co-authored-by: Marek <mail@marek.onl>

Cargo.lock

@@ -4648,6 +4648,19 @@ dependencies = [
  "syn 2.0.50",
 ]
 
+[[package]]
+name = "tonic-reflection"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "548c227bd5c0fae5925812c4ec6c66ffcfced23ea370cb823f4d18f0fc1cb6a7"
+dependencies = [
+ "prost",
+ "prost-types",
+ "tokio",
+ "tokio-stream",
+ "tonic 0.11.0",
+]
+
 [[package]]
 name = "tower"
 version = "0.4.13"
@@ -5798,6 +5811,7 @@ dependencies = [
  "tokio-stream",
  "tonic 0.11.0",
  "tonic-build 0.11.0",
+ "tonic-reflection",
  "tower",
  "zcash_primitives",
  "zebra-chain",

book/src/SUMMARY.md

@@ -17,6 +17,7 @@
     - [Testnet Mining with s-nomp](user/mining-testnet-s-nomp.md)
     - [Mining with Zebra in Docker](user/mining-docker.md)
   - [Shielded Scanning](user/shielded-scan.md)
+    - [Shielded Scanning gRPC Server](user/shielded-scan-grpc-server.md)
   - [Kibana blockchain explorer](user/elasticsearch.md)
   - [Forking the Zcash Testnet with Zebra](user/fork-zebra-testnet.md)
   - [Troubleshooting](user/troubleshooting.md)

book/src/user/shielded-scan-grpc-server.md

@@ -0,0 +1,148 @@
+# Zebra Shielded Scanning gRPC Server
+
+## Get Started
+
+### Setup
+
+After setting up [Zebra Shielded Scanning](https://zebra.zfnd.org/user/shielded-scan.html), add a `listen_addr` field to the shielded-scan configuration:
+
+```toml
+[shielded_scan]
+listen_addr = "127.0.0.1:8231"
+```
+
+Then, run `zebrad` to start the scan gRPC server.
+
+Making requests to the server will also require a gRPC client, the examples here use `grpcurl`, though any gRPC client should work.
+
+[See installation instructions for `grpcurl` here](https://github.com/fullstorydev/grpcurl?tab=readme-ov-file#installation).
+
+The types can be accessed through the `zebra-grpc` crate's root `scanner` module for clients in a Rust environment, and the [`scanner.proto` file here](https://github.com/ZcashFoundation/zebra/blob/main/zebra-grpc/proto/scanner.proto) can be used to build types in other environments.
+
+### Usage
+
+To check that the gRPC server is running, try calling `scanner.Scanner/GetInfo`, for example with `grpcurl`:
+
+```bash
+grpcurl -plaintext '127.0.0.1:8231' scanner.Scanner/GetInfo
+```
+
+The response should look like:
+
+```
+{
+  "minSaplingBirthdayHeight": 419200
+}
+```
+
+An example request to the `Scan` method with `grpcurl` would look like: 
+
+```bash
+grpcurl -plaintext -d '{ "keys": { "key": ["sapling_extended_full_viewing_key"] } }' '127.0.0.1:8231' scanner.Scanner/Scan
+```
+
+This will start scanning for transactions in Zebra's state and in new blocks as they're validated.
+
+Or, to use the scanner gRPC server without streaming, try calling `RegisterKeys` with your Sapling extended full viewing key, waiting for the scanner to cache some results, then calling `GetResults`:
+
+```bash
+grpcurl -plaintext -d '{ "keys": { "key": ["sapling_extended_full_viewing_key"] } }' '127.0.0.1:8231' scanner.Scanner/RegisterKeys
+grpcurl -plaintext -d '{ "keys": ["sapling_extended_full_viewing_key"] }' '127.0.0.1:8231' scanner.Scanner/GetResults
+```
+
+## gRPC Reflection
+
+To see all of the provided methods with `grpcurl`, try:
+
+```bash
+grpcurl -plaintext '127.0.0.1:8231' list scanner.Scanner
+```
+
+This will list the paths to each method in the `Scanner` service:
+```
+scanner.Scanner.ClearResults
+scanner.Scanner.DeleteKeys
+scanner.Scanner.GetInfo
+scanner.Scanner.GetResults
+scanner.Scanner.RegisterKeys
+```
+
+To see the the request and response types for a method, for example the `GetResults` method, try:
+
+
+```bash
+grpcurl -plaintext '127.0.0.1:8231' describe scanner.Scanner.GetResults \
+&& grpcurl -plaintext '127.0.0.1:8231' describe scanner.GetResultsRequest \
+&& grpcurl -plaintext '127.0.0.1:8231' describe scanner.GetResultsResponse \
+&& grpcurl -plaintext '127.0.0.1:8231' describe scanner.Results \
+&& grpcurl -plaintext '127.0.0.1:8231' describe scanner.Transactions \
+&& grpcurl -plaintext '127.0.0.1:8231' describe scanner.Transaction
+```
+
+The response should be the request and response types for the `GetResults` method:
+
+```
+scanner.Scanner.GetResults is a method:
+// Get all data we have stored for the given keys.
+rpc GetResults ( .scanner.GetResultsRequest ) returns ( .scanner.GetResultsResponse );
+scanner.GetResultsRequest is a message:
+// A request for getting results for a set of keys.
+message GetResultsRequest {
+  // Keys for which to get results.
+  repeated string keys = 1;
+}
+scanner.GetResultsResponse is a message:
+// A set of responses for each provided key of a GetResults call.
+message GetResultsResponse {
+  // Results for each key.
+  map<string, .scanner.Results> results = 1;
+}
+scanner.Results is a message:
+// A result for a single key.
+message Results {
+  // A height, transaction id map
+  map<uint32, .scanner.Transactions> by_height = 1;
+}
+scanner.Transactions is a message:
+// A vector of transaction hashes
+message Transactions {
+  // Transactions
+  repeated Transaction transactions = 1;
+}
+scanner.Transaction is a message:
+// Transaction data
+message Transaction {
+  // The transaction hash/id
+  string hash = 1;
+}
+```
+
+## Methods
+
+<!-- TODO: Add a reference to zebra-grpc method docs -->
+
+---
+#### GetInfo
+
+Returns basic information about the `zebra-scan` instance.
+
+#### RegisterKeys
+
+Starts scanning for a set of keys, with optional start heights, and caching the results.
+Cached results can later be retrieved by calling the `GetResults` or `Scan` methods.
+
+#### DeleteKeys
+
+Stops scanning transactions for a set of keys. Deletes the keys and their cached results for the keys from zebra-scan.
+
+#### GetResults
+
+Returns cached results for a set of keys.
+
+#### ClearResults
+
+Deletes any cached results for a set of keys.
+
+#### Scan
+
+Starts scanning for a set of keys and returns a stream of results.

zebra-grpc/Cargo.toml

@@ -18,6 +18,7 @@ categories = ["cryptography::cryptocurrencies"]
 
 futures-util = "0.3.28"
 tonic = "0.11.0"
+tonic-reflection = "0.11.0"
 prost = "0.12.3"
 serde = { version = "1.0.196", features = ["serde_derive"] }
 tokio = { version = "1.36.0", features = ["macros", "rt-multi-thread"] }

zebra-grpc/build.rs

@@ -1,10 +1,16 @@
 //! Compile proto files
 
+use std::{env, path::PathBuf};
+
 fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
+
     tonic_build::configure()
         .btree_map(["."])
         .protoc_arg("--experimental_allow_proto3_optional")
         .type_attribute(".", "#[derive(serde::Deserialize, serde::Serialize)]")
+        .file_descriptor_set_path(out_dir.join("scanner_descriptor.bin"))
         .compile(&["proto/scanner.proto"], &[""])?;
+
     Ok(())
 }

zebra-grpc/src/lib.rs

@@ -12,4 +12,7 @@ mod tests;
 /// The generated scanner proto
 pub mod scanner {
     tonic::include_proto!("scanner");
+
+    pub(crate) const FILE_DESCRIPTOR_SET: &[u8] =
+        tonic::include_file_descriptor_set!("scanner_descriptor");
 }

zebra-grpc/src/server.rs

@@ -417,8 +417,13 @@ where
     <ScanService as tower::Service<ScanServiceRequest>>::Future: Send,
 {
     let service = ScannerRPC { scan_service };
+    let reflection_service = tonic_reflection::server::Builder::configure()
+        .register_encoded_file_descriptor_set(crate::scanner::FILE_DESCRIPTOR_SET)
+        .build()
+        .unwrap();
 
     Server::builder()
+        .add_service(reflection_service)
         .add_service(ScannerServer::new(service))
         .serve(listen_addr)
         .await?;

zebra-scan/src/init.rs

@@ -28,7 +28,7 @@ pub async fn init_with_server(
         .service(ScanService::new(&config, network, state, chain_tip_change).await);
 
     // TODO: move this to zebra-grpc init() function and include addr
-    info!("starting scan gRPC server");
+    info!(?listen_addr, "starting scan gRPC server");
 
     // Start the gRPC server.
     zebra_grpc::server::init(listen_addr, scan_service).await?;