zecfoundation
/
zebra
mirror of https://github.com/ZcashFoundation/zebra.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

tests(scanner): Move zebra scanner tests to binary (#8659) 

* readd the scanner tests in the new binary context

* remove commented out tests from zebrad

* add Cargo.lock

* add local copy of `ZECPAGES_SAPLING_VIEWING_KEY` to binary scanning tests

* moves scanner-results-reader to zebra-scan, adds zebrad bin to zebra-scan, adds a short sleep before trying to connect to gRPC server in test

* always include zebrad dependency in zebra-scan

* renames zebrad bin in zebra-scan to zebrad-for-scanner

* skip test when a cached state is missing

* remove scanning config from zebrad

* move `scan_task_commands` out of zebrad acceptance tests

* remove scanner config from zebrad

* do not panic if env var is not there

* re enable start_scan_where left, remove shielded-scan feature foir scanning tests in CI

* change app names

* uncomment test

* fix the scanner tests in CI by changing the test command

* add log needed by CI?

* move test

* rename trait, docs fix

---------

Co-authored-by: Arya <aryasolhi@gmail.com>
This commit is contained in:
Alfredo Garcia 2024-07-15 18:15:04 -03:00 committed by GitHub
parent f3c2e1971d
commit 1238ec0c63
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 398 additions and 306 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
.github/workflows/sub-ci-integration-tests-gcp.yml vendored
View File

@ -516,32 +516,31 @@ jobs:
zebra_state_dir: "zebrad-cache"
secrets: inherit
# TODO: Move this test once we have the new scanner binary.
# # Test that the scanner can continue scanning where it was left when zebrad restarts.
# #
# # Runs:
# # - after every PR is merged to `main`
# # - on every PR update
# #
# # If the state version has changed, waits for the new cached states to be created.
# # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
# scan-start-where-left-test:
# name: Scan starts where left
# needs: [test-full-sync, get-available-disks]
# uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
# if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
# with:
# app_name: zebrad
# test_id: scan-start-where-left
# test_description: Test that the scanner can continue scanning where it was left when zebrad restarts.
# test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCAN_START_WHERE_LEFT=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache"
# needs_zebra_state: true
# needs_lwd_state: false
# saves_to_disk: true
# disk_suffix: tip
# root_state_path: "/var/cache"
# zebra_state_dir: "zebrad-cache"
# secrets: inherit
# Test that the scanner can continue scanning where it was left when zebrad restarts.
#
# Runs:
# - after every PR is merged to `main`
# - on every PR update
#
# If the state version has changed, waits for the new cached states to be created.
# Otherwise, if the state rebuild was skipped, runs immediately after the build job.
scan-start-where-left-test:
name: Scan starts where left
needs: [test-full-sync, get-available-disks]
uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
with:
app_name: zebra-scan
test_id: scan-start-where-left
test_description: Test that the scanner can continue scanning where it was left when zebrad restarts.
test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCAN_START_WHERE_LEFT=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache"
needs_zebra_state: true
needs_lwd_state: false
saves_to_disk: true
disk_suffix: tip
root_state_path: "/var/cache"
zebra_state_dir: "zebrad-cache"
secrets: inherit
# Test that the scan task registers keys, deletes keys, and subscribes to results for keys while running.
#
@ -557,7 +556,7 @@ jobs:
uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
with:
app_name: zebrad
app_name: zebra-scan
test_id: scan-task-commands
test_description: Test that the scan task registers keys, deletes keys, and subscribes to results for keys while running.
test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCAN_TASK_COMMANDS=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache"
@ -589,7 +588,7 @@ jobs:
lightwalletd-grpc-test,
get-block-template-test,
submit-block-test,
# scan-start-where-left-test,
scan-start-where-left-test,
scan-task-commands-test,
]
# Only open tickets for failed scheduled jobs, manual workflow runs, or `main` branch merges.

8
Cargo.lock
View File

@ -6016,9 +6016,11 @@ dependencies = [
"ff",
"futures",
"group",
"hex",
"indexmap 2.2.6",
"insta",
"itertools 0.13.0",
"jsonrpc",
"jubjub",
"lazy_static",
"proptest",
@ -6029,7 +6031,10 @@ dependencies = [
"serde",
"serde_json",
"structopt",
"tempfile",
"tokio",
"toml 0.8.14",
"tonic 0.11.0",
"tower",
"tracing",
"tracing-subscriber",
@ -6044,6 +6049,7 @@ dependencies = [
"zebra-rpc",
"zebra-state",
"zebra-test",
"zebrad",
]
[[package]]
@ -6159,7 +6165,6 @@ dependencies = [
"zebra-chain",
"zebra-node-services",
"zebra-rpc",
"zebra-scan",
]
[[package]]
@ -6226,7 +6231,6 @@ dependencies = [
"zebra-network",
"zebra-node-services",
"zebra-rpc",
"zebra-scan",
"zebra-state",
"zebra-test",
"zebra-utils",

10
docker/entrypoint.sh
View File

@ -347,14 +347,14 @@ case "$1" in
run_cargo_test "${ENTRYPOINT_FEATURES}" "submit_block"
elif [[ "${TEST_SCAN_START_WHERE_LEFT}" -eq "1" ]]; then
# Test that the scanner can continue scanning where it was left when zebrad restarts.
# Test that the scanner can continue scanning where it was left when zebra-scanner restarts.
check_directory_files "${ZEBRA_CACHED_STATE_DIR}"
run_cargo_test "shielded-scan" "scan_start_where_left"
exec cargo test --locked --release --features "zebra-test" --package zebra-scan -- --nocapture --include-ignored scan_start_where_left
elif [[ "${TEST_SCAN_TASK_COMMANDS}" -eq "1" ]]; then
# Test that the scanner can continue scanning where it was left when zebrad restarts.
# Test that the scan task commands are working.
check_directory_files "${ZEBRA_CACHED_STATE_DIR}"
run_cargo_test "shielded-scan" "scan_task_commands"
exec cargo test --locked --release --features "zebra-test" --package zebra-scan -- --nocapture --include-ignored scan_task_commands
else
exec "$@"

23
zebra-scan/Cargo.toml
View File

@ -23,6 +23,15 @@ required-features = ["proptest-impl"]
name = "zebra-scanner"
path = "src/bin/scanner/main.rs"
[[bin]]
name = "scanning-results-reader"
path = "src/bin/scanning-results-reader/main.rs"
required-features = ["results-reader"]
[[bin]] # Bin to run zebrad, used in scanner tests
name = "zebrad-for-scanner"
path = "src/bin/zebrad-for-scanner/main.rs"
[features]
# Production features that activate extra dependencies, or extra features in dependencies
@ -44,7 +53,10 @@ proptest-impl = [
]
# Needed for the zebra-scanner binary.
shielded-scan = []
results-reader = [
"jsonrpc",
"hex"
]
[dependencies]
@ -91,6 +103,11 @@ structopt = "0.3.26"
lazy_static = "1.4.0"
serde_json = "1.0.117"
jsonrpc = { version = "0.18.0", optional = true }
hex = { version = "0.4.3", optional = true }
zebrad = { path = "../zebrad", version = "1.8.0" }
[dev-dependencies]
insta = { version = "1.39.0", features = ["ron", "redactions"] }
tokio = { version = "1.37.0", features = ["test-util"] }
@ -102,7 +119,11 @@ ff = "0.13.0"
group = "0.13.0"
jubjub = "0.10.0"
rand = "0.8.5"
tempfile = "3.10.1"
zcash_note_encryption = "0.4.0"
toml = "0.8.13"
tonic = "0.11.0"
zebra-state = { path = "../zebra-state", version = "1.0.0-beta.38", features = ["proptest-impl"] }
zebra-test = { path = "../zebra-test", version = "1.0.0-beta.38" }

0
zebra-utils/src/bin/scanning-results-reader/main.rs → zebra-scan/src/bin/scanning-results-reader/main.rs
View File

8
zebra-scan/src/bin/zebrad-for-scanner/main.rs Normal file
View File

@ -0,0 +1,8 @@
//! Main entry point for Zebrad, to be used in zebra-scan tests
use zebrad::application::{boot, APPLICATION};
/// Process entry point for `zebrad`
fn main() {
boot(&APPLICATION);
}

77
zebrad/tests/common/shielded_scan/scan_task_commands.rs → zebra-scan/tests/scan_task_commands.rs
View File

@ -5,12 +5,14 @@
//!
//! export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state"
//! cargo test scan_task_commands --features="shielded-scan" -- --ignored --nocapture
#![allow(dead_code)]
use std::{fs, time::Duration};
use color_eyre::{eyre::eyre, Result};
use tokio::sync::mpsc::error::TryRecvError;
use tower::{util::BoxService, Service};
use zebra_chain::{
block::Height,
chain_tip::ChainTip,
@ -20,13 +22,12 @@ use zebra_chain::{
use zebra_scan::{
service::ScanTask,
storage::{db::SCANNER_DATABASE_KIND, Storage},
tests::ZECPAGES_SAPLING_VIEWING_KEY,
};
use crate::common::{
cached_state::start_state_service_with_cache_dir, launch::can_spawn_zebrad_for_test_type,
test_type::TestType,
};
use zebra_state::{ChainTipChange, LatestChainTip};
pub type BoxStateService =
BoxService<zebra_state::Request, zebra_state::Response, zebra_state::BoxError>;
/// The minimum required tip height for the cached state in this test.
const REQUIRED_MIN_TIP_HEIGHT: Height = Height(1_000_000);
@ -38,6 +39,9 @@ const WAIT_FOR_RESULTS_DURATION: Duration = Duration::from_secs(60);
/// A block height where a scan result can be found with the [`ZECPAGES_SAPLING_VIEWING_KEY`]
const EXPECTED_RESULT_HEIGHT: Height = Height(780_532);
/// The extended Sapling viewing key of [ZECpages](https://zecpages.com/boardinfo)
const ZECPAGES_SAPLING_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz";
/// Initialize Zebra's state service with a cached state, then:
/// - Start the scan task,
/// - Add a new key,
@ -48,24 +52,23 @@ const EXPECTED_RESULT_HEIGHT: Height = Height(780_532);
pub(crate) async fn run() -> Result<()> {
let _init_guard = zebra_test::init();
let test_type = TestType::UpdateZebraCachedStateNoRpc;
let test_name = "scan_task_commands";
let network = Network::Mainnet;
// Skip the test unless the user specifically asked for it and there is a zebrad_state_path
if !can_spawn_zebrad_for_test_type(test_name, test_type, true) {
return Ok(());
}
// Logs the network as zebrad would as part of the metadata when starting up.
// This is currently needed for the 'Check startup logs' step in CI to pass.
tracing::info!("Zcash network: {network}");
tracing::info!(
?network,
?test_type,
"running scan_subscribe_results test using zebra state service",
);
let zebrad_state_path = match std::env::var_os("ZEBRA_CACHED_STATE_DIR") {
None => {
tracing::error!("ZEBRA_CACHED_STATE_DIR is not set");
return Ok(());
}
Some(path) => std::path::PathBuf::from(path),
};
let zebrad_state_path = test_type
.zebrad_state_path(test_name)
.expect("already checked that there is a cached state path");
// Remove the scan directory before starting.
let scan_db_path = zebrad_state_path.join(SCANNER_DATABASE_KIND);
fs::remove_dir_all(std::path::Path::new(&scan_db_path)).ok();
let mut scan_config = zebra_scan::Config::default();
scan_config
@ -73,14 +76,6 @@ pub(crate) async fn run() -> Result<()> {
.cache_dir
.clone_from(&zebrad_state_path);
// Logs the network as zebrad would as part of the metadata when starting up.
// This is currently needed for the 'Check startup logs' step in CI to pass.
tracing::info!("Zcash network: {network}");
// Remove the scan directory before starting.
let scan_db_path = zebrad_state_path.join(SCANNER_DATABASE_KIND);
fs::remove_dir_all(std::path::Path::new(&scan_db_path)).ok();
let (_state_service, _read_state_service, latest_chain_tip, chain_tip_change) =
start_state_service_with_cache_dir(&network, zebrad_state_path.clone()).await?;
@ -88,7 +83,8 @@ pub(crate) async fn run() -> Result<()> {
cache_dir: zebrad_state_path.clone(),
..zebra_state::Config::default()
};
let (read_state, _db, _) = zebra_state::init_read_only(state_config, &network);
let (read_state, _, _) = zebra_state::init_read_only(state_config.clone(), &network);
let chain_tip_height = latest_chain_tip
.best_tip_height()
@ -158,7 +154,6 @@ pub(crate) async fn run() -> Result<()> {
TryRecvError::Disconnected,
"any result senders should have been dropped"
);
break;
}
}
@ -166,3 +161,25 @@ pub(crate) async fn run() -> Result<()> {
Ok(())
}
pub async fn start_state_service_with_cache_dir(
network: &Network,
cache_dir: impl Into<std::path::PathBuf>,
) -> Result<(
BoxStateService,
impl Service<
zebra_state::ReadRequest,
Response = zebra_state::ReadResponse,
Error = zebra_state::BoxError,
>,
LatestChainTip,
ChainTipChange,
)> {
let config = zebra_state::Config {
cache_dir: cache_dir.into(),
..zebra_state::Config::default()
};
// These tests don't need UTXOs to be verified efficiently, because they use cached states.
Ok(zebra_state::init(config, network, Height::MAX, 0))
}

283
zebra-scan/tests/scanner.rs Normal file
View File

@ -0,0 +1,283 @@
//! `zebra-scanner` binary tests.
use tempfile::TempDir;
use zebra_grpc::scanner::{scanner_client::ScannerClient, Empty};
use zebra_test::{
args,
command::{Arguments, TestDirExt},
prelude::*,
};
use std::{io::Write, path::Path};
mod scan_task_commands;
/// The extended Sapling viewing key of [ZECpages](https://zecpages.com/boardinfo)
const ZECPAGES_SAPLING_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz";
/// Test the scanner binary with the `--help` flag.
#[test]
fn scanner_help() -> eyre::Result<()> {
let _init_guard = zebra_test::init();
let testdir = testdir()?;
let child = testdir.spawn_scanner_child(args!["--help"])?;
let output = child.wait_with_output()?;
let output = output.assert_success()?;
output.stdout_line_contains("USAGE:")?;
Ok(())
}
/// Test that the scanner binary starts the scan task.
///
/// This test creates a new zebrad cache dir by using a `zebrad` binary specified in the `CARGO_BIN_EXE_zebrad` env variable.
///
/// To run it locally, one way is:
///
/// ```
/// RUST_LOG=info cargo test scan_binary_starts -- --include-ignored --nocapture
/// ```
#[tokio::test]
#[cfg(not(target_os = "windows"))]
async fn scan_binary_starts() -> Result<()> {
use std::time::Duration;
let _init_guard = zebra_test::init();
// Create a directory to dump test data into it.
let test_dir = testdir()?;
// Create a config for zebrad with a cache directory inside the test directory.
let mut config = zebrad::config::ZebradConfig::default();
config.state.cache_dir = test_dir.path().join("zebra").to_path_buf();
let config_file = test_dir.path().join("zebrad.toml");
std::fs::File::create(config_file.clone())?.write_all(toml::to_string(&config)?.as_bytes())?;
// Start the node just to make sure a cache is created.
let mut zebrad =
test_dir.spawn_zebrad_child(args!["-c", config_file.clone().to_str().unwrap(), "start"])?;
zebrad.expect_stdout_line_matches("Opened Zebra state cache at .*")?;
// Kill the node now that we have a valid zebra node cache.
zebrad.kill(false)?;
let output = zebrad.wait_with_output()?;
// Make sure the command was killed
output.assert_was_killed()?;
// Create a new directory for the scanner
let test_dir = testdir()?;
// Create arguments for the scanner
let key = serde_json::json!({"key": ZECPAGES_SAPLING_VIEWING_KEY}).to_string();
let listen_addr = "127.0.0.1:8231";
let rpc_listen_addr = "127.0.0.1:8232";
let zebrad_cache_dir = config.state.cache_dir.clone();
let scanning_cache_dir = test_dir.path().join("scanner").to_path_buf();
let args = args![
"--zebrad-cache-dir",
zebrad_cache_dir.to_str().unwrap(),
"--scanning-cache-dir",
scanning_cache_dir.to_str().unwrap(),
"--sapling-keys-to-scan",
key,
"--listen-addr",
listen_addr,
"--zebra-rpc-listen-addr",
rpc_listen_addr
];
// Start the scaner using another test directory.
let mut zebra_scanner = test_dir.spawn_scanner_child(args)?;
// Check scanner was started.
zebra_scanner.expect_stdout_line_matches("loaded Zebra scanner cache")?;
// Wait for the scanner's gRPC server to start up
tokio::time::sleep(Duration::from_secs(1)).await;
// Check if the grpc server is up
let mut client = ScannerClient::connect(format!("http://{listen_addr}")).await?;
let request = tonic::Request::new(Empty {});
client.get_info(request).await?;
// Look for 2 scanner notices indicating we are below sapling activation.
zebra_scanner.expect_stdout_line_matches("scanner is waiting for Sapling activation. Current tip: [0-9]{1,4}, Sapling activation: 419200")?;
zebra_scanner.expect_stdout_line_matches("scanner is waiting for Sapling activation. Current tip: [0-9]{1,4}, Sapling activation: 419200")?;
// Kill the scanner.
zebra_scanner.kill(true)?;
// Check that scan task started and the scanning is done.
let output = zebra_scanner.wait_with_output()?;
// Make sure the command was killed
output.assert_was_killed()?;
output.assert_failure()?;
Ok(())
}
/// Test that the scanner can continue scanning where it was left when zebrad restarts.
///
/// Needs a cache state close to the tip. A possible way to run it locally is:
///
/// export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state"
/// RUST_LOG=info cargo test scan_start_where_left -- --ignored --nocapture
///
/// The test will run zebrad with a key to scan, scan the first few blocks after sapling and then stops.
/// Then it will restart zebrad and check that it resumes scanning where it was left.
#[ignore]
#[tokio::test]
#[cfg(not(target_os = "windows"))]
async fn scan_start_where_left() -> Result<()> {
use ZECPAGES_SAPLING_VIEWING_KEY;
let _init_guard = zebra_test::init();
let Ok(zebrad_cachedir) = std::env::var("ZEBRA_CACHED_STATE_DIR") else {
tracing::info!("skipping scan_start_where_left test due to missing cached state, \
please set a ZEBRA_CACHED_STATE_DIR env var with a populated and valid path to run this test");
return Ok(());
};
// Logs the network as zebrad would as part of the metadata when starting up.
// This is currently needed for the 'Check startup logs' step in CI to pass.
let network = zebra_chain::parameters::Network::Mainnet;
tracing::info!("Zcash network: {network}");
let scanning_cache_dir = testdir()?.path().join("scanner").to_path_buf();
// Create arguments for the scanner
let key = serde_json::json!({"key": ZECPAGES_SAPLING_VIEWING_KEY}).to_string();
let listen_addr = "127.0.0.1:18231";
let rpc_listen_addr = "127.0.0.1:18232";
let args = args![
"--zebrad-cache-dir",
zebrad_cachedir,
"--scanning-cache-dir",
scanning_cache_dir.to_str().unwrap(),
"--sapling-keys-to-scan",
key,
"--listen-addr",
listen_addr,
"--zebra-rpc-listen-addr",
rpc_listen_addr
];
// Start the scaner using another test directory.
let mut zebra_scanner: TestChild<TempDir> = testdir()?.spawn_scanner_child(args.clone())?;
// Check scanner was started.
zebra_scanner.expect_stdout_line_matches("loaded Zebra scanner cache")?;
// The first time
zebra_scanner.expect_stdout_line_matches(
r"Scanning the blockchain for key 0, started at block 419200, now at block 420000",
)?;
// Make sure scanner scans a few blocks.
zebra_scanner.expect_stdout_line_matches(
r"Scanning the blockchain for key 0, started at block 419200, now at block 430000",
)?;
zebra_scanner.expect_stdout_line_matches(
r"Scanning the blockchain for key 0, started at block 419200, now at block 440000",
)?;
// Kill the scanner.
zebra_scanner.kill(true)?;
// Check that scan task started and the first scanning is done.
let output = zebra_scanner.wait_with_output()?;
// Make sure the command was killed
output.assert_was_killed()?;
output.assert_failure()?;
// Start the node again with the same arguments.
let mut zebra_scanner: TestChild<TempDir> = testdir()?.spawn_scanner_child(args)?;
// Resuming message.
zebra_scanner.expect_stdout_line_matches(
"Last scanned height for key number 0 is 439000, resuming at 439001",
)?;
zebra_scanner.expect_stdout_line_matches("loaded Zebra scanner cache")?;
// Start scanning where it was left.
zebra_scanner.expect_stdout_line_matches(
r"Scanning the blockchain for key 0, started at block 439001, now at block 440000",
)?;
zebra_scanner.expect_stdout_line_matches(
r"Scanning the blockchain for key 0, started at block 439001, now at block 450000",
)?;
// Kill the scanner.
zebra_scanner.kill(true)?;
// Check that scan task started and the second scanning is done.
let output = zebra_scanner.wait_with_output()?;
// Make sure the command was killed
output.assert_was_killed()?;
output.assert_failure()?;
Ok(())
}
/// Tests successful:
/// - Registration of a new key,
/// - Subscription to scan results of new key, and
/// - Deletion of keys
/// in the scan task
/// See [`common::shielded_scan::scan_task_commands`] for more information.
///
/// Example of how to run the scan_task_commands test locally:
///
/// ```console
/// RUST_LOG=info ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state cargo test scan_task_commands -- --include-ignored --nocapture
/// ```
#[tokio::test]
#[ignore]
async fn scan_task_commands() -> Result<()> {
scan_task_commands::run().await
}
/// Create a temporary directory for testing `zebra-scanner`.
pub fn testdir() -> eyre::Result<TempDir> {
tempfile::Builder::new()
.prefix("zebra_scanner_tests")
.tempdir()
.map_err(Into::into)
}
/// Extension trait for methods on `tempfile::TempDir` for using it as a test
/// directory for `zebra-scanner`.
pub trait ScannerTestDirExt
where
Self: AsRef<Path> + Sized,
{
/// Spawn `zebra-scanner` with `args` as a child process in this test directory.
fn spawn_scanner_child(self, args: Arguments) -> Result<TestChild<Self>>;
/// Spawn `zebrad` with `args` as a child process in this test directory.
fn spawn_zebrad_child(self, args: Arguments) -> Result<TestChild<Self>>;
}
impl<T> ScannerTestDirExt for T
where
Self: TestDirExt + AsRef<Path> + Sized,
{
fn spawn_scanner_child(self, extra_args: Arguments) -> Result<TestChild<Self>> {
let mut args = Arguments::new();
args.merge_with(extra_args);
self.spawn_child_with_command(env!("CARGO_BIN_EXE_zebra-scanner"), args)
}
fn spawn_zebrad_child(self, extra_args: Arguments) -> Result<TestChild<Self>> {
let mut args = Arguments::new();
args.merge_with(extra_args);
self.spawn_child_with_command(env!("CARGO_BIN_EXE_zebrad-for-scanner"), args)
}
}

7
zebra-utils/Cargo.toml
View File

@ -36,11 +36,6 @@ name = "block-template-to-proposal"
path = "src/bin/block-template-to-proposal/main.rs"
required-features = ["getblocktemplate-rpcs"]
[[bin]]
name = "scanning-results-reader"
path = "src/bin/scanning-results-reader/main.rs"
required-features = ["shielded-scan"]
[[bin]]
name = "openapi-generator"
path = "src/bin/openapi-generator/main.rs"
@ -76,7 +71,6 @@ shielded-scan = [
"jsonrpc",
"zcash_primitives",
"zcash_client_backend",
"zebra-scan"
]
openapi-generator = [
@ -102,7 +96,6 @@ thiserror = "1.0.61"
zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.38" }
zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.38" }
zebra-scan = { path = "../zebra-scan", version = "0.1.0-alpha.7", optional = true }
# These crates are needed for the block-template-to-proposal binary
zebra-rpc = { path = "../zebra-rpc", version = "1.0.0-beta.38", optional = true }

8
zebrad/Cargo.toml
View File

@ -77,9 +77,6 @@ internal-miner = [
"zebra-rpc/getblocktemplate-rpcs",
]
# Experimental shielded blockchain scanning
shielded-scan = ["zebra-scan"]
# Experimental elasticsearch indexing
elasticsearch = [
"zebra-state/elasticsearch",
@ -127,7 +124,6 @@ proptest-impl = [
"zebra-state/proptest-impl",
"zebra-network/proptest-impl",
"zebra-chain/proptest-impl",
"zebra-scan?/proptest-impl",
]
# Build the zebra-checkpoints utility for checkpoint generation tests
@ -165,9 +161,6 @@ zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.3
zebra-rpc = { path = "../zebra-rpc", version = "1.0.0-beta.38" }
zebra-state = { path = "../zebra-state", version = "1.0.0-beta.38" }
# Experimental shielded-scan feature
zebra-scan = { path = "../zebra-scan", version = "0.1.0-alpha.7", optional = true }
# Required for crates.io publishing, but it's only used in tests
zebra-utils = { path = "../zebra-utils", version = "1.0.0-beta.38", optional = true }
@ -283,7 +276,6 @@ color-eyre = { version = "0.6.3" }
zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.38", features = ["proptest-impl"] }
zebra-consensus = { path = "../zebra-consensus", version = "1.0.0-beta.38", features = ["proptest-impl"] }
zebra-network = { path = "../zebra-network", version = "1.0.0-beta.38", features = ["proptest-impl"] }
zebra-scan = { path = "../zebra-scan", version = "0.1.0-alpha.7", features = ["proptest-impl"] }
zebra-state = { path = "../zebra-state", version = "1.0.0-beta.38", features = ["proptest-impl"] }
zebra-test = { path = "../zebra-test", version = "1.0.0-beta.38" }

4
zebrad/src/config.rs
View File

@ -54,8 +54,4 @@ pub struct ZebradConfig {
#[serde(skip_serializing_if = "zebra_rpc::config::mining::Config::skip_getblocktemplate")]
/// Mining configuration
pub mining: zebra_rpc::config::mining::Config,
#[cfg(feature = "shielded-scan")]
/// Scanner configuration
pub shielded_scan: zebra_scan::config::Config,
}

215
zebrad/tests/acceptance.rs
View File

@ -120,14 +120,6 @@
//!
//! Please refer to the documentation of each test for more information.
//!
//! ## Shielded scanning tests
//!
//! Example of how to run the scan_task_commands test:
//!
//! ```console
//! ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state cargo test scan_task_commands --features shielded-scan --release -- --ignored --nocapture
//! ```
//!
//! ## Checkpoint Generation Tests
//!
//! Generate checkpoints on mainnet and testnet using a cached state:
@ -2891,213 +2883,6 @@ async fn fully_synced_rpc_z_getsubtreesbyindex_snapshot_test() -> Result<()> {
Ok(())
}
// TODO: Move this test once we have the new scanner binary.
// /// Test that the scanner task gets started when the node starts.
// #[test]
// #[cfg(feature = "shielded-scan")]
// fn scan_task_starts() -> Result<()> {
// use indexmap::IndexMap;
// use zebra_scan::tests::ZECPAGES_SAPLING_VIEWING_KEY;
// let _init_guard = zebra_test::init();
// let test_type = TestType::LaunchWithEmptyState {
// launches_lightwalletd: false,
// };
// let mut config = default_test_config(&Mainnet)?;
// let mut keys = IndexMap::new();
// keys.insert(ZECPAGES_SAPLING_VIEWING_KEY.to_string(), 1);
// config.shielded_scan.sapling_keys_to_scan = keys;
// // Start zebra with the config.
// let mut zebrad = testdir()?
// .with_exact_config(&config)?
// .spawn_child(args!["start"])?
// .with_timeout(test_type.zebrad_timeout());
// // Check scanner was started.
// zebrad.expect_stdout_line_matches("loaded Zebra scanner cache")?;
// // Look for 2 scanner notices indicating we are below sapling activation.
// zebrad.expect_stdout_line_matches("scanner is waiting for Sapling activation. Current tip: [0-9]{1,4}, Sapling activation: 419200")?;
// zebrad.expect_stdout_line_matches("scanner is waiting for Sapling activation. Current tip: [0-9]{1,4}, Sapling activation: 419200")?;
// // Kill the node.
// zebrad.kill(false)?;
// // Check that scan task started and the first scanning is done.
// let output = zebrad.wait_with_output()?;
// // Make sure the command was killed
// output.assert_was_killed()?;
// output.assert_failure()?;
// Ok(())
// }
// TODO: Move this test once we have the new scanner binary.
// /// Test that the scanner gRPC server starts when the node starts.
// #[tokio::test]
// #[cfg(all(feature = "shielded-scan", not(target_os = "windows")))]
// async fn scan_rpc_server_starts() -> Result<()> {
// use zebra_grpc::scanner::{scanner_client::ScannerClient, Empty};
// let _init_guard = zebra_test::init();
// let test_type = TestType::LaunchWithEmptyState {
// launches_lightwalletd: false,
// };
// let port = random_known_port();
// let listen_addr = format!("127.0.0.1:{port}");
// let mut config = default_test_config(&Mainnet)?;
// config.shielded_scan.listen_addr = Some(listen_addr.parse()?);
// // Start zebra with the config.
// let mut zebrad = testdir()?
// .with_exact_config(&config)?
// .spawn_child(args!["start"])?
// .with_timeout(test_type.zebrad_timeout());
// // Wait until gRPC server is starting.
// tokio::time::sleep(LAUNCH_DELAY).await;
// zebrad.expect_stdout_line_matches("starting scan gRPC server")?;
// tokio::time::sleep(Duration::from_secs(1)).await;
// let mut client = ScannerClient::connect(format!("http://{listen_addr}")).await?;
// let request = tonic::Request::new(Empty {});
// client.get_info(request).await?;
// // Kill the node.
// zebrad.kill(false)?;
// // Check that scan task started and the first scanning is done.
// let output = zebrad.wait_with_output()?;
// // Make sure the command was killed
// output.assert_was_killed()?;
// output.assert_failure()?;
// Ok(())
// }
// TODO: Move this test once we have the new scanner binary.
// /// Test that the scanner can continue scanning where it was left when zebrad restarts.
// ///
// /// Needs a cache state close to the tip. A possible way to run it locally is:
// ///
// /// export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state"
// /// cargo test scan_start_where_left --features="shielded-scan" -- --ignored --nocapture
// ///
// /// The test will run zebrad with a key to scan, scan the first few blocks after sapling and then stops.
// /// Then it will restart zebrad and check that it resumes scanning where it was left.
// ///
// /// Note: This test will remove all the contents you may have in the ZEBRA_CACHED_STATE_DIR/private-scan directory
// /// so it can start with an empty scanning state.
// #[ignore]
// #[test]
// #[cfg(feature = "shielded-scan")]
// fn scan_start_where_left() -> Result<()> {
// use indexmap::IndexMap;
// use zebra_scan::{storage::db::SCANNER_DATABASE_KIND, tests::ZECPAGES_SAPLING_VIEWING_KEY};
// let _init_guard = zebra_test::init();
// // use `UpdateZebraCachedStateNoRpc` as the test type to make sure a zebrad cache state is available.
// let test_type = TestType::UpdateZebraCachedStateNoRpc;
// if let Some(cache_dir) = test_type.zebrad_state_path("scan test") {
// // Add a key to the config
// let mut config = default_test_config(&Mainnet)?;
// let mut keys = IndexMap::new();
// keys.insert(ZECPAGES_SAPLING_VIEWING_KEY.to_string(), 1);
// config.shielded_scan.sapling_keys_to_scan = keys;
// // Add the cache dir to shielded scan, make it the same as the zebrad cache state.
// config
// .shielded_scan
// .db_config_mut()
// .cache_dir
// .clone_from(&cache_dir);
// config.shielded_scan.db_config_mut().ephemeral = false;
// // Add the cache dir to state.
// config.state.cache_dir.clone_from(&cache_dir);
// config.state.ephemeral = false;
// // Remove the scan directory before starting.
// let scan_db_path = cache_dir.join(SCANNER_DATABASE_KIND);
// fs::remove_dir_all(std::path::Path::new(&scan_db_path)).ok();
// // Start zebra with the config.
// let mut zebrad = testdir()?
// .with_exact_config(&config)?
// .spawn_child(args!["start"])?
// .with_timeout(test_type.zebrad_timeout());
// // Check scanner was started.
// zebrad.expect_stdout_line_matches("loaded Zebra scanner cache")?;
// // The first time
// zebrad.expect_stdout_line_matches(
// r"Scanning the blockchain for key 0, started at block 419200, now at block 420000",
// )?;
// // Make sure scanner scans a few blocks.
// zebrad.expect_stdout_line_matches(
// r"Scanning the blockchain for key 0, started at block 419200, now at block 430000",
// )?;
// zebrad.expect_stdout_line_matches(
// r"Scanning the blockchain for key 0, started at block 419200, now at block 440000",
// )?;
// // Kill the node.
// zebrad.kill(false)?;
// let output = zebrad.wait_with_output()?;
// // Make sure the command was killed
// output.assert_was_killed()?;
// output.assert_failure()?;
// // Start the node again.
// let mut zebrad = testdir()?
// .with_exact_config(&config)?
// .spawn_child(args!["start"])?
// .with_timeout(test_type.zebrad_timeout());
// // Resuming message.
// zebrad.expect_stdout_line_matches(
// "Last scanned height for key number 0 is 439000, resuming at 439001",
// )?;
// zebrad.expect_stdout_line_matches("loaded Zebra scanner cache")?;
// // Start scanning where it was left.
// zebrad.expect_stdout_line_matches(
// r"Scanning the blockchain for key 0, started at block 439001, now at block 440000",
// )?;
// zebrad.expect_stdout_line_matches(
// r"Scanning the blockchain for key 0, started at block 439001, now at block 450000",
// )?;
// }
// Ok(())
// }
// TODO: Add this test to CI (#8236)
/// Tests successful:
/// - Registration of a new key,
/// - Subscription to scan results of new key, and
/// - Deletion of keys
/// in the scan task
/// See [`common::shielded_scan::scan_task_commands`] for more information.
#[tokio::test]
#[ignore]
#[cfg(feature = "shielded-scan")]
async fn scan_task_commands() -> Result<()> {
common::shielded_scan::scan_task_commands::run().await
}
/// Checks that the Regtest genesis block can be validated.
#[tokio::test]
async fn validate_regtest_genesis_block() {

3
zebrad/tests/common/mod.rs
View File

@ -26,6 +26,3 @@ pub mod get_block_template_rpcs;
#[cfg(feature = "getblocktemplate-rpcs")]
pub mod regtest;
#[cfg(feature = "shielded-scan")]
pub mod shielded_scan;

3
zebrad/tests/common/shielded_scan.rs
View File

@ -1,3 +0,0 @@
//! Acceptance tests for `shielded-scan`` feature in zebrad.
pub(crate) mod scan_task_commands;