tests(scanner): Move zebra scanner tests to binary (#8659)

* readd the scanner tests in the new binary context

* remove commented out tests from zebrad

* add Cargo.lock

* add local copy of `ZECPAGES_SAPLING_VIEWING_KEY` to binary scanning tests

* moves scanner-results-reader to zebra-scan, adds zebrad bin to zebra-scan, adds a short sleep before trying to connect to gRPC server in test

* always include zebrad dependency in zebra-scan

* renames zebrad bin in zebra-scan to zebrad-for-scanner

* skip test when a cached state is missing

* remove scanning config from zebrad

* move `scan_task_commands` out of zebrad acceptance tests

* remove scanner config from zebrad

* do not panic if env var is not there

* re enable start_scan_where left, remove shielded-scan feature foir scanning tests in CI

* change app names

* uncomment test

* fix the scanner tests in CI by changing the test command

* add log needed by CI?

* move test

* rename trait, docs fix

---------

Co-authored-by: Arya <aryasolhi@gmail.com>

.github/workflows/sub-ci-integration-tests-gcp.yml

@@ -516,32 +516,31 @@ jobs:
       zebra_state_dir: "zebrad-cache"
     secrets: inherit
 
-  # TODO: Move this test once we have the new scanner binary.
+  # Test that the scanner can continue scanning where it was left when zebrad restarts.
-  # # Test that the scanner can continue scanning where it was left when zebrad restarts.
+  #
-  # #
+  # Runs:
-  # # Runs:
+  # - after every PR is merged to `main`
-  # # - after every PR is merged to `main`
+  # - on every PR update
-  # # - on every PR update
+  #
-  # #
+  # If the state version has changed, waits for the new cached states to be created.
-  # # If the state version has changed, waits for the new cached states to be created.
+  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
-  # # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
+  scan-start-where-left-test:
-  # scan-start-where-left-test:
+    name: Scan starts where left
-  #   name: Scan starts where left
+    needs: [test-full-sync, get-available-disks]
-  #   needs: [test-full-sync, get-available-disks]
+    uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
-  #   uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
+    if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
-  #   if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
+    with:
-  #   with:
+      app_name: zebra-scan
-  #     app_name: zebrad
+      test_id: scan-start-where-left
-  #     test_id: scan-start-where-left
+      test_description: Test that the scanner can continue scanning where it was left when zebrad restarts.
-  #     test_description: Test that the scanner can continue scanning where it was left when zebrad restarts.
+      test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCAN_START_WHERE_LEFT=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache"
-  #     test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCAN_START_WHERE_LEFT=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache"
+      needs_zebra_state: true
-  #     needs_zebra_state: true
+      needs_lwd_state: false
-  #     needs_lwd_state: false
+      saves_to_disk: true
-  #     saves_to_disk: true
+      disk_suffix: tip
-  #     disk_suffix: tip
+      root_state_path: "/var/cache"
-  #     root_state_path: "/var/cache"
+      zebra_state_dir: "zebrad-cache"
-  #     zebra_state_dir: "zebrad-cache"
+    secrets: inherit
-  #   secrets: inherit
 
   # Test that the scan task registers keys, deletes keys, and subscribes to results for keys while running.
   #
@@ -557,7 +556,7 @@ jobs:
     uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
     if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
     with:
-      app_name: zebrad
+      app_name: zebra-scan
       test_id: scan-task-commands
       test_description: Test that the scan task registers keys, deletes keys, and subscribes to results for keys while running.
       test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCAN_TASK_COMMANDS=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache"
@@ -589,7 +588,7 @@ jobs:
         lightwalletd-grpc-test,
         get-block-template-test,
         submit-block-test,
-        # scan-start-where-left-test,
+        scan-start-where-left-test,
         scan-task-commands-test,
       ]
     # Only open tickets for failed scheduled jobs, manual workflow runs, or `main` branch merges.

Cargo.lock

@@ -6016,9 +6016,11 @@ dependencies = [
  "ff",
  "futures",
  "group",
+ "hex",
  "indexmap 2.2.6",
  "insta",
  "itertools 0.13.0",
+ "jsonrpc",
  "jubjub",
  "lazy_static",
  "proptest",
@@ -6029,7 +6031,10 @@ dependencies = [
  "serde",
  "serde_json",
  "structopt",
+ "tempfile",
  "tokio",
+ "toml 0.8.14",
+ "tonic 0.11.0",
  "tower",
  "tracing",
  "tracing-subscriber",
@@ -6044,6 +6049,7 @@ dependencies = [
  "zebra-rpc",
  "zebra-state",
  "zebra-test",
+ "zebrad",
 ]
 
 [[package]]
@@ -6159,7 +6165,6 @@ dependencies = [
  "zebra-chain",
  "zebra-node-services",
  "zebra-rpc",
- "zebra-scan",
 ]
 
 [[package]]
@@ -6226,7 +6231,6 @@ dependencies = [
  "zebra-network",
  "zebra-node-services",
  "zebra-rpc",
- "zebra-scan",
  "zebra-state",
  "zebra-test",
  "zebra-utils",

docker/entrypoint.sh

@@ -347,14 +347,14 @@ case "$1" in
         run_cargo_test "${ENTRYPOINT_FEATURES}" "submit_block"
 
       elif [[ "${TEST_SCAN_START_WHERE_LEFT}" -eq "1" ]]; then
-        # Test that the scanner can continue scanning where it was left when zebrad restarts.
+        # Test that the scanner can continue scanning where it was left when zebra-scanner restarts.
         check_directory_files "${ZEBRA_CACHED_STATE_DIR}"
-        run_cargo_test "shielded-scan" "scan_start_where_left"
+        exec cargo test --locked --release --features "zebra-test" --package zebra-scan -- --nocapture --include-ignored scan_start_where_left
-      
+
       elif [[ "${TEST_SCAN_TASK_COMMANDS}" -eq "1" ]]; then
-        # Test that the scanner can continue scanning where it was left when zebrad restarts.
+        # Test that the scan task commands are working.
         check_directory_files "${ZEBRA_CACHED_STATE_DIR}"
-        run_cargo_test "shielded-scan" "scan_task_commands"
+        exec cargo test --locked --release --features "zebra-test" --package zebra-scan -- --nocapture --include-ignored scan_task_commands
 
       else
           exec "$@"

zebra-scan/Cargo.toml

@@ -23,6 +23,15 @@ required-features = ["proptest-impl"]
 name = "zebra-scanner"
 path = "src/bin/scanner/main.rs"
 
+[[bin]]
+name = "scanning-results-reader"
+path = "src/bin/scanning-results-reader/main.rs"
+required-features = ["results-reader"]
+
+[[bin]] # Bin to run zebrad, used in scanner tests
+name = "zebrad-for-scanner"
+path = "src/bin/zebrad-for-scanner/main.rs"
+
 [features]
 
 # Production features that activate extra dependencies, or extra features in dependencies
@@ -44,7 +53,10 @@ proptest-impl = [
 ]
 
 # Needed for the zebra-scanner binary.
-shielded-scan = []
+results-reader = [
+  "jsonrpc",
+  "hex"
+]
 
 [dependencies]
 
@@ -91,6 +103,11 @@ structopt = "0.3.26"
 lazy_static = "1.4.0"
 serde_json = "1.0.117"
 
+jsonrpc = { version = "0.18.0", optional = true }
+hex = { version = "0.4.3", optional = true }
+
+zebrad = { path = "../zebrad", version = "1.8.0" }
+
 [dev-dependencies]
 insta = { version = "1.39.0", features = ["ron", "redactions"] }
 tokio = { version = "1.37.0", features = ["test-util"] }
@@ -102,7 +119,11 @@ ff = "0.13.0"
 group = "0.13.0"
 jubjub = "0.10.0"
 rand = "0.8.5"
+tempfile = "3.10.1"
 zcash_note_encryption = "0.4.0"
+toml = "0.8.13"
+tonic = "0.11.0"
 
 zebra-state = { path = "../zebra-state", version = "1.0.0-beta.38", features = ["proptest-impl"] }
 zebra-test = { path = "../zebra-test", version = "1.0.0-beta.38" }
+

zebra-scan/src/bin/zebrad-for-scanner/main.rs

@@ -0,0 +1,8 @@
+//! Main entry point for Zebrad, to be used in zebra-scan tests
+
+use zebrad::application::{boot, APPLICATION};
+
+/// Process entry point for `zebrad`
+fn main() {
+    boot(&APPLICATION);
+}

zebra-scan/tests/scan_task_commands.rs

@@ -5,12 +5,14 @@
 //!
 //! export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state"
 //! cargo test scan_task_commands --features="shielded-scan" -- --ignored --nocapture
+#![allow(dead_code)]
 
 use std::{fs, time::Duration};
 
 use color_eyre::{eyre::eyre, Result};
-
 use tokio::sync::mpsc::error::TryRecvError;
+use tower::{util::BoxService, Service};
+
 use zebra_chain::{
     block::Height,
     chain_tip::ChainTip,
@@ -20,13 +22,12 @@ use zebra_chain::{
 use zebra_scan::{
     service::ScanTask,
     storage::{db::SCANNER_DATABASE_KIND, Storage},
-    tests::ZECPAGES_SAPLING_VIEWING_KEY,
 };
 
-use crate::common::{
+use zebra_state::{ChainTipChange, LatestChainTip};
-    cached_state::start_state_service_with_cache_dir, launch::can_spawn_zebrad_for_test_type,
+
-    test_type::TestType,
+pub type BoxStateService =
-};
+    BoxService<zebra_state::Request, zebra_state::Response, zebra_state::BoxError>;
 
 /// The minimum required tip height for the cached state in this test.
 const REQUIRED_MIN_TIP_HEIGHT: Height = Height(1_000_000);
@@ -38,6 +39,9 @@ const WAIT_FOR_RESULTS_DURATION: Duration = Duration::from_secs(60);
 /// A block height where a scan result can be found with the [`ZECPAGES_SAPLING_VIEWING_KEY`]
 const EXPECTED_RESULT_HEIGHT: Height = Height(780_532);
 
+/// The extended Sapling viewing key of [ZECpages](https://zecpages.com/boardinfo)
+const ZECPAGES_SAPLING_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz";
+
 /// Initialize Zebra's state service with a cached state, then:
 /// - Start the scan task,
 /// - Add a new key,
@@ -48,24 +52,23 @@ const EXPECTED_RESULT_HEIGHT: Height = Height(780_532);
 pub(crate) async fn run() -> Result<()> {
     let _init_guard = zebra_test::init();
 
-    let test_type = TestType::UpdateZebraCachedStateNoRpc;
-    let test_name = "scan_task_commands";
     let network = Network::Mainnet;
 
-    // Skip the test unless the user specifically asked for it and there is a zebrad_state_path
+    // Logs the network as zebrad would as part of the metadata when starting up.
-    if !can_spawn_zebrad_for_test_type(test_name, test_type, true) {
+    // This is currently needed for the 'Check startup logs' step in CI to pass.
-        return Ok(());
+    tracing::info!("Zcash network: {network}");
-    }
 
-    tracing::info!(
+    let zebrad_state_path = match std::env::var_os("ZEBRA_CACHED_STATE_DIR") {
-        ?network,
+        None => {
-        ?test_type,
+            tracing::error!("ZEBRA_CACHED_STATE_DIR is not set");
-        "running scan_subscribe_results test using zebra state service",
+            return Ok(());
-    );
+        }
+        Some(path) => std::path::PathBuf::from(path),
+    };
 
-    let zebrad_state_path = test_type
+    // Remove the scan directory before starting.
-        .zebrad_state_path(test_name)
+    let scan_db_path = zebrad_state_path.join(SCANNER_DATABASE_KIND);
-        .expect("already checked that there is a cached state path");
+    fs::remove_dir_all(std::path::Path::new(&scan_db_path)).ok();
 
     let mut scan_config = zebra_scan::Config::default();
     scan_config
@@ -73,14 +76,6 @@ pub(crate) async fn run() -> Result<()> {
         .cache_dir
         .clone_from(&zebrad_state_path);
 
-    // Logs the network as zebrad would as part of the metadata when starting up.
-    // This is currently needed for the 'Check startup logs' step in CI to pass.
-    tracing::info!("Zcash network: {network}");
-
-    // Remove the scan directory before starting.
-    let scan_db_path = zebrad_state_path.join(SCANNER_DATABASE_KIND);
-    fs::remove_dir_all(std::path::Path::new(&scan_db_path)).ok();
-
     let (_state_service, _read_state_service, latest_chain_tip, chain_tip_change) =
         start_state_service_with_cache_dir(&network, zebrad_state_path.clone()).await?;
 
@@ -88,7 +83,8 @@ pub(crate) async fn run() -> Result<()> {
         cache_dir: zebrad_state_path.clone(),
         ..zebra_state::Config::default()
     };
-    let (read_state, _db, _) = zebra_state::init_read_only(state_config, &network);
+
+    let (read_state, _, _) = zebra_state::init_read_only(state_config.clone(), &network);
 
     let chain_tip_height = latest_chain_tip
         .best_tip_height()
@@ -158,7 +154,6 @@ pub(crate) async fn run() -> Result<()> {
                     TryRecvError::Disconnected,
                     "any result senders should have been dropped"
                 );
-
                 break;
             }
         }
@@ -166,3 +161,25 @@ pub(crate) async fn run() -> Result<()> {
 
     Ok(())
 }
+
+pub async fn start_state_service_with_cache_dir(
+    network: &Network,
+    cache_dir: impl Into<std::path::PathBuf>,
+) -> Result<(
+    BoxStateService,
+    impl Service<
+        zebra_state::ReadRequest,
+        Response = zebra_state::ReadResponse,
+        Error = zebra_state::BoxError,
+    >,
+    LatestChainTip,
+    ChainTipChange,
+)> {
+    let config = zebra_state::Config {
+        cache_dir: cache_dir.into(),
+        ..zebra_state::Config::default()
+    };
+
+    // These tests don't need UTXOs to be verified efficiently, because they use cached states.
+    Ok(zebra_state::init(config, network, Height::MAX, 0))
+}

zebra-scan/tests/scanner.rs

@@ -0,0 +1,283 @@
+//! `zebra-scanner` binary tests.
+use tempfile::TempDir;
+
+use zebra_grpc::scanner::{scanner_client::ScannerClient, Empty};
+use zebra_test::{
+    args,
+    command::{Arguments, TestDirExt},
+    prelude::*,
+};
+
+use std::{io::Write, path::Path};
+
+mod scan_task_commands;
+
+/// The extended Sapling viewing key of [ZECpages](https://zecpages.com/boardinfo)
+const ZECPAGES_SAPLING_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz";
+
+/// Test the scanner binary with the `--help` flag.
+#[test]
+fn scanner_help() -> eyre::Result<()> {
+    let _init_guard = zebra_test::init();
+
+    let testdir = testdir()?;
+
+    let child = testdir.spawn_scanner_child(args!["--help"])?;
+    let output = child.wait_with_output()?;
+    let output = output.assert_success()?;
+
+    output.stdout_line_contains("USAGE:")?;
+
+    Ok(())
+}
+
+/// Test that the scanner binary starts the scan task.
+///
+/// This test creates a new zebrad cache dir by using a `zebrad` binary specified in the `CARGO_BIN_EXE_zebrad` env variable.
+///
+/// To run it locally, one way is:
+///
+/// ```
+/// RUST_LOG=info cargo test scan_binary_starts -- --include-ignored --nocapture
+/// ```
+#[tokio::test]
+#[cfg(not(target_os = "windows"))]
+async fn scan_binary_starts() -> Result<()> {
+    use std::time::Duration;
+
+    let _init_guard = zebra_test::init();
+
+    // Create a directory to dump test data into it.
+    let test_dir = testdir()?;
+
+    // Create a config for zebrad with a cache directory inside the test directory.
+    let mut config = zebrad::config::ZebradConfig::default();
+    config.state.cache_dir = test_dir.path().join("zebra").to_path_buf();
+    let config_file = test_dir.path().join("zebrad.toml");
+    std::fs::File::create(config_file.clone())?.write_all(toml::to_string(&config)?.as_bytes())?;
+
+    // Start the node just to make sure a cache is created.
+    let mut zebrad =
+        test_dir.spawn_zebrad_child(args!["-c", config_file.clone().to_str().unwrap(), "start"])?;
+    zebrad.expect_stdout_line_matches("Opened Zebra state cache at .*")?;
+
+    // Kill the node now that we have a valid zebra node cache.
+    zebrad.kill(false)?;
+    let output = zebrad.wait_with_output()?;
+
+    // Make sure the command was killed
+    output.assert_was_killed()?;
+
+    // Create a new directory for the scanner
+    let test_dir = testdir()?;
+
+    // Create arguments for the scanner
+    let key = serde_json::json!({"key": ZECPAGES_SAPLING_VIEWING_KEY}).to_string();
+    let listen_addr = "127.0.0.1:8231";
+    let rpc_listen_addr = "127.0.0.1:8232";
+    let zebrad_cache_dir = config.state.cache_dir.clone();
+    let scanning_cache_dir = test_dir.path().join("scanner").to_path_buf();
+    let args = args![
+        "--zebrad-cache-dir",
+        zebrad_cache_dir.to_str().unwrap(),
+        "--scanning-cache-dir",
+        scanning_cache_dir.to_str().unwrap(),
+        "--sapling-keys-to-scan",
+        key,
+        "--listen-addr",
+        listen_addr,
+        "--zebra-rpc-listen-addr",
+        rpc_listen_addr
+    ];
+
+    // Start the scaner using another test directory.
+    let mut zebra_scanner = test_dir.spawn_scanner_child(args)?;
+
+    // Check scanner was started.
+    zebra_scanner.expect_stdout_line_matches("loaded Zebra scanner cache")?;
+
+    // Wait for the scanner's gRPC server to start up
+    tokio::time::sleep(Duration::from_secs(1)).await;
+
+    // Check if the grpc server is up
+    let mut client = ScannerClient::connect(format!("http://{listen_addr}")).await?;
+    let request = tonic::Request::new(Empty {});
+    client.get_info(request).await?;
+
+    // Look for 2 scanner notices indicating we are below sapling activation.
+    zebra_scanner.expect_stdout_line_matches("scanner is waiting for Sapling activation. Current tip: [0-9]{1,4}, Sapling activation: 419200")?;
+    zebra_scanner.expect_stdout_line_matches("scanner is waiting for Sapling activation. Current tip: [0-9]{1,4}, Sapling activation: 419200")?;
+
+    // Kill the scanner.
+    zebra_scanner.kill(true)?;
+
+    // Check that scan task started and the scanning is done.
+    let output = zebra_scanner.wait_with_output()?;
+
+    // Make sure the command was killed
+    output.assert_was_killed()?;
+    output.assert_failure()?;
+
+    Ok(())
+}
+
+/// Test that the scanner can continue scanning where it was left when zebrad restarts.
+///
+/// Needs a cache state close to the tip. A possible way to run it locally is:
+///
+/// export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state"
+/// RUST_LOG=info cargo test scan_start_where_left -- --ignored --nocapture
+///
+/// The test will run zebrad with a key to scan, scan the first few blocks after sapling and then stops.
+/// Then it will restart zebrad and check that it resumes scanning where it was left.
+#[ignore]
+#[tokio::test]
+#[cfg(not(target_os = "windows"))]
+async fn scan_start_where_left() -> Result<()> {
+    use ZECPAGES_SAPLING_VIEWING_KEY;
+
+    let _init_guard = zebra_test::init();
+
+    let Ok(zebrad_cachedir) = std::env::var("ZEBRA_CACHED_STATE_DIR") else {
+        tracing::info!("skipping scan_start_where_left test due to missing cached state, \
+                        please set a ZEBRA_CACHED_STATE_DIR env var with a populated and valid path to run this test");
+        return Ok(());
+    };
+
+    // Logs the network as zebrad would as part of the metadata when starting up.
+    // This is currently needed for the 'Check startup logs' step in CI to pass.
+    let network = zebra_chain::parameters::Network::Mainnet;
+    tracing::info!("Zcash network: {network}");
+
+    let scanning_cache_dir = testdir()?.path().join("scanner").to_path_buf();
+
+    // Create arguments for the scanner
+    let key = serde_json::json!({"key": ZECPAGES_SAPLING_VIEWING_KEY}).to_string();
+    let listen_addr = "127.0.0.1:18231";
+    let rpc_listen_addr = "127.0.0.1:18232";
+    let args = args![
+        "--zebrad-cache-dir",
+        zebrad_cachedir,
+        "--scanning-cache-dir",
+        scanning_cache_dir.to_str().unwrap(),
+        "--sapling-keys-to-scan",
+        key,
+        "--listen-addr",
+        listen_addr,
+        "--zebra-rpc-listen-addr",
+        rpc_listen_addr
+    ];
+
+    // Start the scaner using another test directory.
+    let mut zebra_scanner: TestChild<TempDir> = testdir()?.spawn_scanner_child(args.clone())?;
+
+    // Check scanner was started.
+    zebra_scanner.expect_stdout_line_matches("loaded Zebra scanner cache")?;
+
+    // The first time
+    zebra_scanner.expect_stdout_line_matches(
+        r"Scanning the blockchain for key 0, started at block 419200, now at block 420000",
+    )?;
+
+    // Make sure scanner scans a few blocks.
+    zebra_scanner.expect_stdout_line_matches(
+        r"Scanning the blockchain for key 0, started at block 419200, now at block 430000",
+    )?;
+    zebra_scanner.expect_stdout_line_matches(
+        r"Scanning the blockchain for key 0, started at block 419200, now at block 440000",
+    )?;
+
+    // Kill the scanner.
+    zebra_scanner.kill(true)?;
+
+    // Check that scan task started and the first scanning is done.
+    let output = zebra_scanner.wait_with_output()?;
+
+    // Make sure the command was killed
+    output.assert_was_killed()?;
+    output.assert_failure()?;
+
+    // Start the node again with the same arguments.
+    let mut zebra_scanner: TestChild<TempDir> = testdir()?.spawn_scanner_child(args)?;
+
+    // Resuming message.
+    zebra_scanner.expect_stdout_line_matches(
+        "Last scanned height for key number 0 is 439000, resuming at 439001",
+    )?;
+    zebra_scanner.expect_stdout_line_matches("loaded Zebra scanner cache")?;
+
+    // Start scanning where it was left.
+    zebra_scanner.expect_stdout_line_matches(
+        r"Scanning the blockchain for key 0, started at block 439001, now at block 440000",
+    )?;
+    zebra_scanner.expect_stdout_line_matches(
+        r"Scanning the blockchain for key 0, started at block 439001, now at block 450000",
+    )?;
+
+    // Kill the scanner.
+    zebra_scanner.kill(true)?;
+
+    // Check that scan task started and the second scanning is done.
+    let output = zebra_scanner.wait_with_output()?;
+
+    // Make sure the command was killed
+    output.assert_was_killed()?;
+    output.assert_failure()?;
+
+    Ok(())
+}
+
+/// Tests successful:
+/// - Registration of a new key,
+/// - Subscription to scan results of new key, and
+/// - Deletion of keys
+/// in the scan task
+/// See [`common::shielded_scan::scan_task_commands`] for more information.
+///
+/// Example of how to run the scan_task_commands test locally:
+///
+/// ```console
+/// RUST_LOG=info ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state cargo test scan_task_commands -- --include-ignored --nocapture
+/// ```
+#[tokio::test]
+#[ignore]
+async fn scan_task_commands() -> Result<()> {
+    scan_task_commands::run().await
+}
+
+/// Create a temporary directory for testing `zebra-scanner`.
+pub fn testdir() -> eyre::Result<TempDir> {
+    tempfile::Builder::new()
+        .prefix("zebra_scanner_tests")
+        .tempdir()
+        .map_err(Into::into)
+}
+
+/// Extension trait for methods on `tempfile::TempDir` for using it as a test
+/// directory for `zebra-scanner`.
+pub trait ScannerTestDirExt
+where
+    Self: AsRef<Path> + Sized,
+{
+    /// Spawn `zebra-scanner` with `args` as a child process in this test directory.
+    fn spawn_scanner_child(self, args: Arguments) -> Result<TestChild<Self>>;
+
+    /// Spawn `zebrad` with `args` as a child process in this test directory.
+    fn spawn_zebrad_child(self, args: Arguments) -> Result<TestChild<Self>>;
+}
+
+impl<T> ScannerTestDirExt for T
+where
+    Self: TestDirExt + AsRef<Path> + Sized,
+{
+    fn spawn_scanner_child(self, extra_args: Arguments) -> Result<TestChild<Self>> {
+        let mut args = Arguments::new();
+        args.merge_with(extra_args);
+        self.spawn_child_with_command(env!("CARGO_BIN_EXE_zebra-scanner"), args)
+    }
+    fn spawn_zebrad_child(self, extra_args: Arguments) -> Result<TestChild<Self>> {
+        let mut args = Arguments::new();
+        args.merge_with(extra_args);
+        self.spawn_child_with_command(env!("CARGO_BIN_EXE_zebrad-for-scanner"), args)
+    }
+}

zebra-utils/Cargo.toml

@@ -36,11 +36,6 @@ name = "block-template-to-proposal"
 path = "src/bin/block-template-to-proposal/main.rs"
 required-features = ["getblocktemplate-rpcs"]
 
-[[bin]]
-name = "scanning-results-reader"
-path = "src/bin/scanning-results-reader/main.rs"
-required-features = ["shielded-scan"]
-
 [[bin]]
 name = "openapi-generator"
 path = "src/bin/openapi-generator/main.rs"
@@ -76,7 +71,6 @@ shielded-scan = [
     "jsonrpc",
     "zcash_primitives",
     "zcash_client_backend",
-    "zebra-scan"
 ]
 
 openapi-generator = [
@@ -102,7 +96,6 @@ thiserror = "1.0.61"
 
 zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.38" }
 zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.38" }
-zebra-scan = { path = "../zebra-scan", version = "0.1.0-alpha.7", optional = true }
 
 # These crates are needed for the block-template-to-proposal binary
 zebra-rpc = { path = "../zebra-rpc", version = "1.0.0-beta.38", optional = true }

zebrad/Cargo.toml

@@ -77,9 +77,6 @@ internal-miner = [
     "zebra-rpc/getblocktemplate-rpcs",
 ]
 
-# Experimental shielded blockchain scanning
-shielded-scan = ["zebra-scan"]
-
 # Experimental elasticsearch indexing
 elasticsearch = [
     "zebra-state/elasticsearch",
@@ -127,7 +124,6 @@ proptest-impl = [
     "zebra-state/proptest-impl",
     "zebra-network/proptest-impl",
     "zebra-chain/proptest-impl",
-    "zebra-scan?/proptest-impl",
 ]
 
 # Build the zebra-checkpoints utility for checkpoint generation tests
@@ -165,9 +161,6 @@ zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.3
 zebra-rpc = { path = "../zebra-rpc", version = "1.0.0-beta.38" }
 zebra-state = { path = "../zebra-state", version = "1.0.0-beta.38" }
 
-# Experimental shielded-scan feature
-zebra-scan = { path = "../zebra-scan", version = "0.1.0-alpha.7", optional = true }
-
 # Required for crates.io publishing, but it's only used in tests
 zebra-utils = { path = "../zebra-utils", version = "1.0.0-beta.38", optional = true }
 
@@ -283,7 +276,6 @@ color-eyre = { version = "0.6.3" }
 zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.38", features = ["proptest-impl"] }
 zebra-consensus = { path = "../zebra-consensus", version = "1.0.0-beta.38", features = ["proptest-impl"] }
 zebra-network = { path = "../zebra-network", version = "1.0.0-beta.38", features = ["proptest-impl"] }
-zebra-scan = { path = "../zebra-scan", version = "0.1.0-alpha.7", features = ["proptest-impl"] }
 zebra-state = { path = "../zebra-state", version = "1.0.0-beta.38", features = ["proptest-impl"] }
 
 zebra-test = { path = "../zebra-test", version = "1.0.0-beta.38" }

zebrad/src/config.rs

@@ -54,8 +54,4 @@ pub struct ZebradConfig {
     #[serde(skip_serializing_if = "zebra_rpc::config::mining::Config::skip_getblocktemplate")]
     /// Mining configuration
     pub mining: zebra_rpc::config::mining::Config,
-
-    #[cfg(feature = "shielded-scan")]
-    /// Scanner configuration
-    pub shielded_scan: zebra_scan::config::Config,
 }

zebrad/tests/acceptance.rs

@@ -120,14 +120,6 @@
 //!
 //! Please refer to the documentation of each test for more information.
 //!
-//! ## Shielded scanning tests
-//!
-//! Example of how to run the scan_task_commands test:
-//!
-//! ```console
-//! ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state cargo test scan_task_commands --features shielded-scan --release -- --ignored --nocapture
-//! ```
-//!
 //! ## Checkpoint Generation Tests
 //!
 //! Generate checkpoints on mainnet and testnet using a cached state:
@@ -2891,213 +2883,6 @@ async fn fully_synced_rpc_z_getsubtreesbyindex_snapshot_test() -> Result<()> {
     Ok(())
 }
 
-// TODO: Move this test once we have the new scanner binary.
-// /// Test that the scanner task gets started when the node starts.
-// #[test]
-// #[cfg(feature = "shielded-scan")]
-// fn scan_task_starts() -> Result<()> {
-//     use indexmap::IndexMap;
-//     use zebra_scan::tests::ZECPAGES_SAPLING_VIEWING_KEY;
-
-//     let _init_guard = zebra_test::init();
-
-//     let test_type = TestType::LaunchWithEmptyState {
-//         launches_lightwalletd: false,
-//     };
-//     let mut config = default_test_config(&Mainnet)?;
-//     let mut keys = IndexMap::new();
-//     keys.insert(ZECPAGES_SAPLING_VIEWING_KEY.to_string(), 1);
-//     config.shielded_scan.sapling_keys_to_scan = keys;
-
-//     // Start zebra with the config.
-//     let mut zebrad = testdir()?
-//         .with_exact_config(&config)?
-//         .spawn_child(args!["start"])?
-//         .with_timeout(test_type.zebrad_timeout());
-
-//     // Check scanner was started.
-//     zebrad.expect_stdout_line_matches("loaded Zebra scanner cache")?;
-
-//     // Look for 2 scanner notices indicating we are below sapling activation.
-//     zebrad.expect_stdout_line_matches("scanner is waiting for Sapling activation. Current tip: [0-9]{1,4}, Sapling activation: 419200")?;
-//     zebrad.expect_stdout_line_matches("scanner is waiting for Sapling activation. Current tip: [0-9]{1,4}, Sapling activation: 419200")?;
-
-//     // Kill the node.
-//     zebrad.kill(false)?;
-
-//     // Check that scan task started and the first scanning is done.
-//     let output = zebrad.wait_with_output()?;
-
-//     // Make sure the command was killed
-//     output.assert_was_killed()?;
-//     output.assert_failure()?;
-
-//     Ok(())
-// }
-
-// TODO: Move this test once we have the new scanner binary.
-// /// Test that the scanner gRPC server starts when the node starts.
-// #[tokio::test]
-// #[cfg(all(feature = "shielded-scan", not(target_os = "windows")))]
-// async fn scan_rpc_server_starts() -> Result<()> {
-//     use zebra_grpc::scanner::{scanner_client::ScannerClient, Empty};
-
-//     let _init_guard = zebra_test::init();
-
-//     let test_type = TestType::LaunchWithEmptyState {
-//         launches_lightwalletd: false,
-//     };
-
-//     let port = random_known_port();
-//     let listen_addr = format!("127.0.0.1:{port}");
-//     let mut config = default_test_config(&Mainnet)?;
-//     config.shielded_scan.listen_addr = Some(listen_addr.parse()?);
-
-//     // Start zebra with the config.
-//     let mut zebrad = testdir()?
-//         .with_exact_config(&config)?
-//         .spawn_child(args!["start"])?
-//         .with_timeout(test_type.zebrad_timeout());
-
-//     // Wait until gRPC server is starting.
-//     tokio::time::sleep(LAUNCH_DELAY).await;
-//     zebrad.expect_stdout_line_matches("starting scan gRPC server")?;
-//     tokio::time::sleep(Duration::from_secs(1)).await;
-
-//     let mut client = ScannerClient::connect(format!("http://{listen_addr}")).await?;
-
-//     let request = tonic::Request::new(Empty {});
-
-//     client.get_info(request).await?;
-
-//     // Kill the node.
-//     zebrad.kill(false)?;
-
-//     // Check that scan task started and the first scanning is done.
-//     let output = zebrad.wait_with_output()?;
-
-//     // Make sure the command was killed
-//     output.assert_was_killed()?;
-//     output.assert_failure()?;
-
-//     Ok(())
-// }
-
-// TODO: Move this test once we have the new scanner binary.
-// /// Test that the scanner can continue scanning where it was left when zebrad restarts.
-// ///
-// /// Needs a cache state close to the tip. A possible way to run it locally is:
-// ///
-// /// export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state"
-// /// cargo test scan_start_where_left --features="shielded-scan" -- --ignored --nocapture
-// ///
-// /// The test will run zebrad with a key to scan, scan the first few blocks after sapling and then stops.
-// /// Then it will restart zebrad and check that it resumes scanning where it was left.
-// ///
-// /// Note: This test will remove all the contents you may have in the ZEBRA_CACHED_STATE_DIR/private-scan directory
-// /// so it can start with an empty scanning state.
-// #[ignore]
-// #[test]
-// #[cfg(feature = "shielded-scan")]
-// fn scan_start_where_left() -> Result<()> {
-//     use indexmap::IndexMap;
-//     use zebra_scan::{storage::db::SCANNER_DATABASE_KIND, tests::ZECPAGES_SAPLING_VIEWING_KEY};
-
-//     let _init_guard = zebra_test::init();
-
-//     // use `UpdateZebraCachedStateNoRpc` as the test type to make sure a zebrad cache state is available.
-//     let test_type = TestType::UpdateZebraCachedStateNoRpc;
-//     if let Some(cache_dir) = test_type.zebrad_state_path("scan test") {
-//         // Add a key to the config
-//         let mut config = default_test_config(&Mainnet)?;
-//         let mut keys = IndexMap::new();
-//         keys.insert(ZECPAGES_SAPLING_VIEWING_KEY.to_string(), 1);
-//         config.shielded_scan.sapling_keys_to_scan = keys;
-
-//         // Add the cache dir to shielded scan, make it the same as the zebrad cache state.
-//         config
-//             .shielded_scan
-//             .db_config_mut()
-//             .cache_dir
-//             .clone_from(&cache_dir);
-//         config.shielded_scan.db_config_mut().ephemeral = false;
-
-//         // Add the cache dir to state.
-//         config.state.cache_dir.clone_from(&cache_dir);
-//         config.state.ephemeral = false;
-
-//         // Remove the scan directory before starting.
-//         let scan_db_path = cache_dir.join(SCANNER_DATABASE_KIND);
-//         fs::remove_dir_all(std::path::Path::new(&scan_db_path)).ok();
-
-//         // Start zebra with the config.
-//         let mut zebrad = testdir()?
-//             .with_exact_config(&config)?
-//             .spawn_child(args!["start"])?
-//             .with_timeout(test_type.zebrad_timeout());
-
-//         // Check scanner was started.
-//         zebrad.expect_stdout_line_matches("loaded Zebra scanner cache")?;
-
-//         // The first time
-//         zebrad.expect_stdout_line_matches(
-//             r"Scanning the blockchain for key 0, started at block 419200, now at block 420000",
-//         )?;
-
-//         // Make sure scanner scans a few blocks.
-//         zebrad.expect_stdout_line_matches(
-//             r"Scanning the blockchain for key 0, started at block 419200, now at block 430000",
-//         )?;
-//         zebrad.expect_stdout_line_matches(
-//             r"Scanning the blockchain for key 0, started at block 419200, now at block 440000",
-//         )?;
-
-//         // Kill the node.
-//         zebrad.kill(false)?;
-//         let output = zebrad.wait_with_output()?;
-
-//         // Make sure the command was killed
-//         output.assert_was_killed()?;
-//         output.assert_failure()?;
-
-//         // Start the node again.
-//         let mut zebrad = testdir()?
-//             .with_exact_config(&config)?
-//             .spawn_child(args!["start"])?
-//             .with_timeout(test_type.zebrad_timeout());
-
-//         // Resuming message.
-//         zebrad.expect_stdout_line_matches(
-//             "Last scanned height for key number 0 is 439000, resuming at 439001",
-//         )?;
-//         zebrad.expect_stdout_line_matches("loaded Zebra scanner cache")?;
-
-//         // Start scanning where it was left.
-//         zebrad.expect_stdout_line_matches(
-//             r"Scanning the blockchain for key 0, started at block 439001, now at block 440000",
-//         )?;
-//         zebrad.expect_stdout_line_matches(
-//             r"Scanning the blockchain for key 0, started at block 439001, now at block 450000",
-//         )?;
-//     }
-
-//     Ok(())
-// }
-
-// TODO: Add this test to CI (#8236)
-/// Tests successful:
-/// - Registration of a new key,
-/// - Subscription to scan results of new key, and
-/// - Deletion of keys
-/// in the scan task
-/// See [`common::shielded_scan::scan_task_commands`] for more information.
-#[tokio::test]
-#[ignore]
-#[cfg(feature = "shielded-scan")]
-async fn scan_task_commands() -> Result<()> {
-    common::shielded_scan::scan_task_commands::run().await
-}
-
 /// Checks that the Regtest genesis block can be validated.
 #[tokio::test]
 async fn validate_regtest_genesis_block() {

zebrad/tests/common/mod.rs

@@ -26,6 +26,3 @@ pub mod get_block_template_rpcs;
 
 #[cfg(feature = "getblocktemplate-rpcs")]
 pub mod regtest;
-
-#[cfg(feature = "shielded-scan")]
-pub mod shielded_scan;

zebrad/tests/common/shielded_scan.rs

@@ -1,3 +0,0 @@
-//! Acceptance tests for `shielded-scan`` feature in zebrad.
-
-pub(crate) mod scan_task_commands;