zecfoundation
/
zebra
mirror of https://github.com/ZcashFoundation/zebra.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

change(ci): Generate mainnet checkpoints in CI (#6550) 

* Add extra test type modes to support zebra-checkpoints

* Add Mainnet and Testnet zebra-checkpoints test harnesses

* Add zebra-checkpoints to test docker images

* Add zebra-checkpoints test entrypoints

* Add Mainnet CI workflow for zebra-checkpoints

* Enable zebra-checkpoints feature in the test image

* Use the same features for (almost) all the docker tests

* Make workflow features match Docker features

* Add a feature note

* Add a zebra-checkpoints test feature to zebrad

* Remove the "no cached state" testnet code

* Log a startup message to standard error when launching zebra-checkpoints

* Rename tests to avoid partial name conflicts

* Fix log formatting

* Add sentry feature to experimental docker image build

* Explain what ENTRYPOINT_FEATURES is used for

* Use the correct zebra-checkpoints path

* Silence zebrad logs while generating checkpoints

* Fix zebra-checkpoints log handling

* Re-enable waiting for zebrad to fully sync

* Add documentation for how to run these tests individually

* Start generating checkpoints from the last compiled-in checkpoint

* Fix clippy lints

* Revert changes to TestType

* Wait for all the checkpoints before finishing

* Add more stderr debugging to zebra-checkpoints

* Fix an outdated module comment

* Add a workaround for zebra-checkpoints launch/run issues

* Use temp dir and log what it is

* Log extra metadata about the zebra-checkpoints binary

* Add note about unstable feature -Z bindeps

* Temporarily make the test run faster and with debug info

* Log the original test command name when showing stdout and stderr

* Try zebra-checkpoints in the system path first, then the cargo path

* Fix slow thread close bug in dual process test harness

* If the logs are shown, don't say they are hidden

* Run `zebra-checkpoints --help` to work out what's going on in CI

* Build `zebra-utils` binaries for `zebrad` integration tests

* Revert temporary debugging changes

* Revert changes that were moved to another PR
This commit is contained in:
teor 2023-04-27 14:39:43 +10:00 committed by GitHub
parent 0ffd31ec47
commit 1461c912f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 818 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/build-docker-image.yml vendored
View File

@ -37,12 +37,15 @@ on:
required: false
type: string
default: info
# keep these in sync with:
# https://github.com/ZcashFoundation/zebra/blob/main/docker/Dockerfile#L83
features:
required: false
default: "sentry"
type: string
test_features:
required: false
default: "lightwalletd-grpc-tests"
default: "lightwalletd-grpc-tests zebra-checkpoints"
type: string
rpc_port:
required: false

6
.github/workflows/continous-integration-docker.patch.yml vendored
View File

@ -90,6 +90,12 @@ jobs:
steps:
- run: 'echo "No build required"'
generate-checkpoints-mainnet:
name: Generate checkpoints mainnet / Run generate-checkpoints-mainnet test
runs-on: ubuntu-latest
steps:
- run: 'echo "No build required"'
lightwalletd-rpc-test:
name: Zebra tip JSON-RPC / Run fully-synced-rpc test
runs-on: ubuntu-latest

32
.github/workflows/continous-integration-docker.yml vendored
View File

@ -461,6 +461,38 @@ jobs:
height_grep_text: 'current_height.*=.*Height.*\('
secrets: inherit
# zebra checkpoint generation tests
# Test that Zebra can generate mainnet checkpoints after syncing to the chain tip,
# using a cached Zebra tip state,
#
# Runs:
# - after every PR is merged to `main`
# - on every PR update
#
# If the state version has changed, waits for the new cached state to be created.
# Otherwise, if the state rebuild was skipped, runs immediately after the build job.
generate-checkpoints-mainnet:
name: Generate checkpoints mainnet
needs: test-full-sync
uses: ./.github/workflows/deploy-gcp-tests.yml
if: ${{ !cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
with:
app_name: zebrad
test_id: generate-checkpoints-mainnet
test_description: Generate Zebra checkpoints on mainnet
test_variables: '-e GENERATE_CHECKPOINTS_MAINNET=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
needs_zebra_state: true
# update the disk on every PR, to increase CI speed
saves_to_disk: true
disk_suffix: tip
root_state_path: '/var/cache'
zebra_state_dir: 'zebrad-cache'
height_grep_text: 'current_height.*=.*Height.*\('
secrets: inherit
# TODO: testnet checkpoints, test-full-sync, test-update-sync, get-available-disks (+ reusable)
# lightwalletd cached tip state tests
# Test full sync of lightwalletd with a Zebra tip state

2
.github/workflows/release-binaries.yml vendored
View File

@ -46,7 +46,7 @@ jobs:
tag_suffix: .experimental
network: Testnet
rpc_port: '18232'
features: "getblocktemplate-rpcs"
features: "sentry getblocktemplate-rpcs"
test_features: ""
checkpoint_sync: true
rust_backtrace: '1'

1
Cargo.lock
View File

@ -6254,6 +6254,7 @@ dependencies = [
"zebra-rpc",
"zebra-state",
"zebra-test",
"zebra-utils",
]
[[package]]

16
docker/Dockerfile
View File

@ -80,8 +80,13 @@ ARG CHECKPOINT_SYNC
ENV CHECKPOINT_SYNC ${CHECKPOINT_SYNC:-true}
# Build zebrad with these features
ARG FEATURES
ARG TEST_FEATURES="lightwalletd-grpc-tests"
# Keep these in sync with:
# https://github.com/ZcashFoundation/zebra/blob/main/.github/workflows/build-docker-image.yml#L42
ARG FEATURES="sentry"
ARG TEST_FEATURES="lightwalletd-grpc-tests zebra-checkpoints"
# Use ENTRYPOINT_FEATURES to override the specific features used to run tests in entrypoint.sh,
# separately from the test and production image builds.
ENV ENTRYPOINT_FEATURES "$TEST_FEATURES $FEATURES"
ARG NETWORK
ENV NETWORK ${NETWORK:-Mainnet}
@ -104,11 +109,12 @@ COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/lightwalletd /opt/lightwalletd
# This is the caching Docker layer for Rust!
#
# TODO: is it faster to use --tests here?
RUN cargo chef cook --release --features "sentry ${TEST_FEATURES} ${FEATURES}" --workspace --recipe-path recipe.json
RUN cargo chef cook --release --features "${TEST_FEATURES} ${FEATURES}" --workspace --recipe-path recipe.json
COPY . .
RUN cargo test --locked --release --features "${TEST_FEATURES} ${FEATURES}" --workspace --no-run
RUN cp /opt/zebrad/target/release/zebrad /usr/local/bin
RUN cp /opt/zebrad/target/release/zebra-checkpoints /usr/local/bin
COPY ./docker/entrypoint.sh /
RUN chmod u+x /entrypoint.sh
@ -122,11 +128,11 @@ ENTRYPOINT [ "/entrypoint.sh" ]
# `test` stage. This step is a dependency for the `runtime` stage, which uses the resulting
# zebrad binary from this step.
FROM deps AS release
RUN cargo chef cook --release --features "sentry ${FEATURES}" --recipe-path recipe.json
RUN cargo chef cook --release --features "${FEATURES}" --recipe-path recipe.json
COPY . .
# Build zebra
RUN cargo build --locked --release --features "sentry ${FEATURES}" --package zebrad --bin zebrad
RUN cargo build --locked --release --features "${FEATURES}" --package zebrad --bin zebrad
# This stage is only used when deploying nodes or when only the resulting zebrad binary is needed
#

52
docker/entrypoint.sh
View File

@ -13,24 +13,25 @@ echo "ZEBRA_TEST_LIGHTWALLETD=$ZEBRA_TEST_LIGHTWALLETD"
echo "Hard-coded Zebra full sync directory: /zebrad-cache"
echo "ZEBRA_CACHED_STATE_DIR=$ZEBRA_CACHED_STATE_DIR"
echo "LIGHTWALLETD_DATA_DIR=$LIGHTWALLETD_DATA_DIR"
echo "ENTRYPOINT_FEATURES=$ENTRYPOINT_FEATURES"
case "$1" in
--* | -*)
exec zebrad "$@"
;;
*)
# For these tests, we activate the gRPC feature to avoid recompiling `zebrad`,
# but we might not actually run any gRPC tests.
# For these tests, we activate the test features to avoid recompiling `zebrad`,
# but we don't actually run any gRPC tests.
if [[ "$RUN_ALL_TESTS" -eq "1" ]]; then
# Run all the available tests for the current environment.
# If the lightwalletd environmental variables are set, we will also run those tests.
cargo test --locked --release --features lightwalletd-grpc-tests --workspace -- --nocapture --include-ignored
cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --workspace -- --nocapture --include-ignored
# For these tests, we activate the gRPC feature to avoid recompiling `zebrad`,
# but we don't actually run any gRPC tests.
elif [[ "$TEST_FULL_SYNC" -eq "1" ]]; then
# Run a Zebra full sync test.
cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored full_sync_mainnet
cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored full_sync_mainnet
# List directory generated by test
# TODO: replace with $ZEBRA_CACHED_STATE_DIR in Rust and workflows
ls -lh "/zebrad-cache"/*/* || (echo "No /zebrad-cache/*/*"; ls -lhR "/zebrad-cache" | head -50 || echo "No /zebrad-cache directory")
@ -38,7 +39,7 @@ case "$1" in
# Run a Zebra sync up to the mandatory checkpoint.
#
# TODO: use environmental variables instead of Rust features (part of #2995)
cargo test --locked --release --features "test_sync_to_mandatory_checkpoint_${NETWORK,,},lightwalletd-grpc-tests" --package zebrad --test acceptance -- --nocapture --include-ignored "sync_to_mandatory_checkpoint_${NETWORK,,}"
cargo test --locked --release --features "test_sync_to_mandatory_checkpoint_${NETWORK,,},$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored "sync_to_mandatory_checkpoint_${NETWORK,,}"
# TODO: replace with $ZEBRA_CACHED_STATE_DIR in Rust and workflows
ls -lh "/zebrad-cache"/*/* || (echo "No /zebrad-cache/*/*"; ls -lhR "/zebrad-cache" | head -50 || echo "No /zebrad-cache directory")
elif [[ "$TEST_UPDATE_SYNC" -eq "1" ]]; then
@ -46,7 +47,7 @@ case "$1" in
#
# List directory used by test
ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored zebrad_update_sync
cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored zebrad_update_sync
elif [[ "$TEST_CHECKPOINT_SYNC" -eq "1" ]]; then
# Run a Zebra sync starting at the cached mandatory checkpoint, and syncing past it.
#
@ -54,41 +55,64 @@ case "$1" in
# TODO: replace with $ZEBRA_CACHED_STATE_DIR in Rust and workflows
ls -lh "/zebrad-cache"/*/* || (echo "No /zebrad-cache/*/*"; ls -lhR "/zebrad-cache" | head -50 || echo "No /zebrad-cache directory")
# TODO: use environmental variables instead of Rust features (part of #2995)
cargo test --locked --release --features "test_sync_past_mandatory_checkpoint_${NETWORK,,},lightwalletd-grpc-tests" --package zebrad --test acceptance -- --nocapture --include-ignored "sync_past_mandatory_checkpoint_${NETWORK,,}"
cargo test --locked --release --features "test_sync_past_mandatory_checkpoint_${NETWORK,,},$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored "sync_past_mandatory_checkpoint_${NETWORK,,}"
elif [[ "$GENERATE_CHECKPOINTS_MAINNET" -eq "1" ]]; then
# Generate checkpoints after syncing Zebra from a cached state on mainnet.
#
# TODO: disable or filter out logs like:
# test generate_checkpoints_mainnet has been running for over 60 seconds
#
# List directory used by test
ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored generate_checkpoints_mainnet
elif [[ "$GENERATE_CHECKPOINTS_TESTNET" -eq "1" ]]; then
# Generate checkpoints after syncing Zebra on testnet.
#
# This test might fail if testnet is unstable.
#
# List directory used by test
ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored generate_checkpoints_testnet
elif [[ "$TEST_LWD_RPC_CALL" -eq "1" ]]; then
# Starting at a cached Zebra tip, test a JSON-RPC call to Zebra.
ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored fully_synced_rpc_test
cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored fully_synced_rpc_test
elif [[ "$TEST_LWD_FULL_SYNC" -eq "1" ]]; then
# Starting at a cached Zebra tip, run a lightwalletd sync to tip.
ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_full_sync
cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_full_sync
ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory")
elif [[ "$TEST_LWD_UPDATE_SYNC" -eq "1" ]]; then
# Starting with a cached Zebra and lightwalletd tip, run a quick update sync.
ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory")
cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_update_sync
cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_update_sync
# These tests actually use gRPC.
elif [[ "$TEST_LWD_GRPC" -eq "1" ]]; then
# Starting with a cached Zebra and lightwalletd tip, test all gRPC calls to lightwalletd, which calls Zebra.
ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory")
cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_wallet_grpc_tests
cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_wallet_grpc_tests
elif [[ "$TEST_LWD_TRANSACTIONS" -eq "1" ]]; then
# Starting with a cached Zebra and lightwalletd tip, test sending transactions gRPC call to lightwalletd, which calls Zebra.
ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory")
cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored sending_transactions_using_lightwalletd
cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored sending_transactions_using_lightwalletd
# These tests use mining code, but don't use gRPC.
# We add the mining feature here because our other code needs to pass tests without it.
elif [[ "$TEST_GET_BLOCK_TEMPLATE" -eq "1" ]]; then
# Starting with a cached Zebra tip, test getting a block template from Zebra's RPC server.
ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
cargo test --locked --release --features getblocktemplate-rpcs --package zebrad --test acceptance -- --nocapture --include-ignored get_block_template
cargo test --locked --release --features "getblocktemplate-rpcs,$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored get_block_template
elif [[ "$TEST_SUBMIT_BLOCK" -eq "1" ]]; then
# Starting with a cached Zebra tip, test sending a block to Zebra's RPC port.
ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
cargo test --locked --release --features getblocktemplate-rpcs --package zebrad --test acceptance -- --nocapture --include-ignored submit_block
cargo test --locked --release --features "getblocktemplate-rpcs,$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored submit_block
else
exec "$@"
fi

156
zebra-test/src/command.rs
View File

@ -52,8 +52,8 @@ pub trait CommandExt {
fn output2(&mut self) -> Result<TestOutput<NoDir>, Report>;
/// wrapper for `spawn` fn on `Command` that constructs informative error
/// reports
fn spawn2<T>(&mut self, dir: T) -> Result<TestChild<T>, Report>;
/// reports using the original `command_path`
fn spawn2<T>(&mut self, dir: T, command_path: impl ToString) -> Result<TestChild<T>, Report>;
}
impl CommandExt for Command {
@ -89,18 +89,19 @@ impl CommandExt for Command {
}
/// wrapper for `spawn` fn on `Command` that constructs informative error
/// reports
fn spawn2<T>(&mut self, dir: T) -> Result<TestChild<T>, Report> {
let cmd = format!("{self:?}");
/// reports using the original `command_path`
fn spawn2<T>(&mut self, dir: T, command_path: impl ToString) -> Result<TestChild<T>, Report> {
let command_and_args = format!("{self:?}");
let child = self.spawn();
let child = child
.wrap_err("failed to execute process")
.with_section(|| cmd.clone().header("Command:"))?;
.with_section(|| command_and_args.clone().header("Command:"))?;
Ok(TestChild {
dir: Some(dir),
cmd,
cmd: command_and_args,
command_path: command_path.to_string(),
child: Some(child),
stdout: None,
stderr: None,
@ -133,14 +134,18 @@ where
Self: AsRef<Path> + Sized,
{
#[allow(clippy::unwrap_in_result)]
fn spawn_child_with_command(self, cmd: &str, args: Arguments) -> Result<TestChild<Self>> {
let mut cmd = test_cmd(cmd, self.as_ref())?;
fn spawn_child_with_command(
self,
command_path: &str,
args: Arguments,
) -> Result<TestChild<Self>> {
let mut cmd = test_cmd(command_path, self.as_ref())?;
Ok(cmd
.args(args.into_arguments())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn2(self)
.spawn2(self, command_path)
.unwrap())
}
}
@ -183,9 +188,12 @@ pub struct TestChild<T> {
/// and its output has been taken.
pub dir: Option<T>,
/// The original command string.
/// The full command string, including arguments and working directory.
pub cmd: String,
/// The path of the command, as passed to spawn2().
pub command_path: String,
/// The child process itself.
///
/// `None` when the command has been waited on,
@ -533,7 +541,8 @@ impl<T> TestChild<T> {
//
// This checks for failure logs, and prevents some test hangs and deadlocks.
if self.child.is_some() || self.stdout.is_some() {
let wrote_lines = self.wait_for_stdout_line("\nChild Stdout:".to_string());
let wrote_lines =
self.wait_for_stdout_line(format!("\n{} Child Stdout:", self.command_path));
while self.wait_for_stdout_line(None) {}
@ -544,7 +553,8 @@ impl<T> TestChild<T> {
}
if self.child.is_some() || self.stderr.is_some() {
let wrote_lines = self.wait_for_stderr_line("\nChild Stderr:".to_string());
let wrote_lines =
self.wait_for_stderr_line(format!("\n{} Child Stderr:", self.command_path));
while self.wait_for_stderr_line(None) {}
@ -559,7 +569,7 @@ impl<T> TestChild<T> {
/// Waits until a line of standard output is available, then consumes it.
///
/// If there is a line, and `write_context` is `Some`, writes the context to the test logs.
/// Then writes the line to the test logs.
/// Always writes the line to the test logs.
///
/// Returns `true` if a line was available,
/// or `false` if the standard output has finished.
@ -592,7 +602,7 @@ impl<T> TestChild<T> {
/// Waits until a line of standard error is available, then consumes it.
///
/// If there is a line, and `write_context` is `Some`, writes the context to the test logs.
/// Then writes the line to the test logs.
/// Always writes the line to the test logs.
///
/// Returns `true` if a line was available,
/// or `false` if the standard error has finished.
@ -686,21 +696,21 @@ impl<T> TestChild<T> {
self
}
/// Configures testrunner to forward stdout and stderr to the true stdout,
/// Configures this test runner to forward stdout and stderr to the true stdout,
/// rather than the fakestdout used by cargo tests.
pub fn bypass_test_capture(mut self, cond: bool) -> Self {
self.bypass_test_capture = cond;
self
}
/// Checks each line of the child's stdout against `success_regex`, and returns Ok
/// if a line matches.
/// Checks each line of the child's stdout against `success_regex`,
/// and returns the first matching line. Prints all stdout lines.
///
/// Kills the child on error, or after the configured timeout has elapsed.
/// See [`Self::expect_line_matching_regex_set`] for details.
#[instrument(skip(self))]
#[allow(clippy::unwrap_in_result)]
pub fn expect_stdout_line_matches<R>(&mut self, success_regex: R) -> Result<&mut Self>
pub fn expect_stdout_line_matches<R>(&mut self, success_regex: R) -> Result<String>
where
R: ToRegex + Debug,
{
@ -711,11 +721,11 @@ impl<T> TestChild<T> {
.take()
.expect("child must capture stdout to call expect_stdout_line_matches, and it can't be called again after an error");
match self.expect_line_matching_regex_set(&mut lines, success_regex, "stdout") {
Ok(()) => {
match self.expect_line_matching_regex_set(&mut lines, success_regex, "stdout", true) {
Ok(line) => {
// Replace the log lines for the next check
self.stdout = Some(lines);
Ok(self)
Ok(line)
}
Err(report) => {
// Read all the log lines for error context
@ -725,14 +735,14 @@ impl<T> TestChild<T> {
}
}
/// Checks each line of the child's stderr against `success_regex`, and returns Ok
/// if a line matches.
/// Checks each line of the child's stderr against `success_regex`,
/// and returns the first matching line. Prints all stderr lines to stdout.
///
/// Kills the child on error, or after the configured timeout has elapsed.
/// See [`Self::expect_line_matching_regex_set`] for details.
#[instrument(skip(self))]
#[allow(clippy::unwrap_in_result)]
pub fn expect_stderr_line_matches<R>(&mut self, success_regex: R) -> Result<&mut Self>
pub fn expect_stderr_line_matches<R>(&mut self, success_regex: R) -> Result<String>
where
R: ToRegex + Debug,
{
@ -743,11 +753,75 @@ impl<T> TestChild<T> {
.take()
.expect("child must capture stderr to call expect_stderr_line_matches, and it can't be called again after an error");
match self.expect_line_matching_regex_set(&mut lines, success_regex, "stderr") {
Ok(()) => {
match self.expect_line_matching_regex_set(&mut lines, success_regex, "stderr", true) {
Ok(line) => {
// Replace the log lines for the next check
self.stderr = Some(lines);
Ok(self)
Ok(line)
}
Err(report) => {
// Read all the log lines for error context
self.stderr = Some(lines);
Err(report).context_from(self)
}
}
}
/// Checks each line of the child's stdout against `success_regex`,
/// and returns the first matching line. Does not print any output.
///
/// Kills the child on error, or after the configured timeout has elapsed.
/// See [`Self::expect_line_matching_regex_set`] for details.
#[instrument(skip(self))]
#[allow(clippy::unwrap_in_result)]
pub fn expect_stdout_line_matches_silent<R>(&mut self, success_regex: R) -> Result<String>
where
R: ToRegex + Debug,
{
self.apply_failure_regexes_to_outputs();
let mut lines = self
.stdout
.take()
.expect("child must capture stdout to call expect_stdout_line_matches, and it can't be called again after an error");
match self.expect_line_matching_regex_set(&mut lines, success_regex, "stdout", false) {
Ok(line) => {
// Replace the log lines for the next check
self.stdout = Some(lines);
Ok(line)
}
Err(report) => {
// Read all the log lines for error context
self.stdout = Some(lines);
Err(report).context_from(self)
}
}
}
/// Checks each line of the child's stderr against `success_regex`,
/// and returns the first matching line. Does not print any output.
///
/// Kills the child on error, or after the configured timeout has elapsed.
/// See [`Self::expect_line_matching_regex_set`] for details.
#[instrument(skip(self))]
#[allow(clippy::unwrap_in_result)]
pub fn expect_stderr_line_matches_silent<R>(&mut self, success_regex: R) -> Result<String>
where
R: ToRegex + Debug,
{
self.apply_failure_regexes_to_outputs();
let mut lines = self
.stderr
.take()
.expect("child must capture stderr to call expect_stderr_line_matches, and it can't be called again after an error");
match self.expect_line_matching_regex_set(&mut lines, success_regex, "stderr", false) {
Ok(line) => {
// Replace the log lines for the next check
self.stderr = Some(lines);
Ok(line)
}
Err(report) => {
// Read all the log lines for error context
@ -767,7 +841,8 @@ impl<T> TestChild<T> {
lines: &mut L,
success_regexes: R,
stream_name: &str,
) -> Result<()>
write_to_logs: bool,
) -> Result<String>
where
L: Iterator<Item = std::io::Result<String>>,
R: ToRegexSet,
@ -776,7 +851,7 @@ impl<T> TestChild<T> {
.to_regex_set()
.expect("regexes must be valid");
self.expect_line_matching_regexes(lines, success_regexes, stream_name)
self.expect_line_matching_regexes(lines, success_regexes, stream_name, write_to_logs)
}
/// Checks each line in `lines` against a regex set, and returns Ok if a line matches.
@ -788,7 +863,8 @@ impl<T> TestChild<T> {
lines: &mut L,
success_regexes: I,
stream_name: &str,
) -> Result<()>
write_to_logs: bool,
) -> Result<String>
where
L: Iterator<Item = std::io::Result<String>>,
I: CollectRegexSet,
@ -797,7 +873,7 @@ impl<T> TestChild<T> {
.collect_regex_set()
.expect("regexes must be valid");
self.expect_line_matching_regexes(lines, success_regexes, stream_name)
self.expect_line_matching_regexes(lines, success_regexes, stream_name, write_to_logs)
}
/// Checks each line in `lines` against `success_regexes`, and returns Ok if a line
@ -814,7 +890,8 @@ impl<T> TestChild<T> {
lines: &mut L,
success_regexes: RegexSet,
stream_name: &str,
) -> Result<()>
write_to_logs: bool,
) -> Result<String>
where
L: Iterator<Item = std::io::Result<String>>,
{
@ -831,11 +908,13 @@ impl<T> TestChild<T> {
break;
};
// Since we're about to discard this line write it to stdout.
Self::write_to_test_logs(&line, self.bypass_test_capture);
if write_to_logs {
// Since we're about to discard this line write it to stdout.
Self::write_to_test_logs(&line, self.bypass_test_capture);
}
if success_regexes.is_match(&line) {
return Ok(());
return Ok(line);
}
}
@ -1295,8 +1374,8 @@ impl<T> ContextFrom<&mut TestChild<T>> for Report {
}
}
self.section(stdout_buf.header("Unread Stdout:"))
.section(stderr_buf.header("Unread Stderr:"))
self.section(stdout_buf.header(format!("{} Unread Stdout:", source.command_path)))
.section(stderr_buf.header(format!("{} Unread Stderr:", source.command_path)))
}
}
@ -1313,6 +1392,7 @@ impl ContextFrom<&Output> for Report {
type Return = Report;
fn context_from(self, source: &Output) -> Self::Return {
// TODO: add TestChild.command_path before Stdout and Stderr header names
let stdout = || {
String::from_utf8_lossy(&source.stdout)
.into_owned()

7
zebra-utils/src/bin/zebra-checkpoints/main.rs
View File

@ -137,14 +137,19 @@ where
/// Process entry point for `zebra-checkpoints`
#[tokio::main]
#[allow(clippy::print_stdout)]
#[allow(clippy::print_stdout, clippy::print_stderr)]
async fn main() -> Result<()> {
eprintln!("zebra-checkpoints launched");
// initialise
init_tracing();
color_eyre::install()?;
let args = args::Args::from_args();
eprintln!("Command-line arguments: {args:?}");
eprintln!("Fetching block info and calculating checkpoints...\n\n");
// get the current block count
let get_block_chain_info = rpc_output(&args, "getblockchaininfo", None)
.await

17
zebra-utils/tests/build_utils_for_zebrad_tests.rs Normal file
View File

@ -0,0 +1,17 @@
//! # Dependency Workaround
//!
//! This empty integration test makes `cargo` build the `zebra-checkpoints` binary for the `zebrad`
//! integration tests:
//!
//! > Binary targets are automatically built if there is an integration test or benchmark being
//! > selected to test.
//!
//! <https://doc.rust-lang.org/cargo/commands/cargo-test.html#target-selection>
//!
//! Each utility binary will only be built if its corresponding Rust feature is activated.
//! <https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-required-features-field>
//!
//! # Unstable `cargo` Feature
//!
//! When `cargo -Z bindeps` is stabilised, add a binary dependency to `zebrad/Cargo.toml` instead:
//! https://github.com/rust-lang/cargo/issues/9096

16
zebrad/Cargo.toml
View File

@ -80,6 +80,11 @@ proptest-impl = [
"zebra-chain/proptest-impl",
]
# Build the zebra-checkpoints utility for checkpoint generation tests
zebra-checkpoints = [
"zebra-utils/zebra-checkpoints",
]
# The gRPC tests also need an installed lightwalletd binary
lightwalletd-grpc-tests = ["tonic-build"]
@ -219,3 +224,14 @@ zebra-state = { path = "../zebra-state", features = ["proptest-impl"] }
zebra-node-services = { path = "../zebra-node-services", features = ["rpc-client"] }
zebra-test = { path = "../zebra-test" }
# Used by the checkpoint generation tests via the zebra-checkpoints feature
# (the binaries in this crate won't be built unless their features are enabled).
#
# Currently, we use zebra-utils/tests/build_utils_for_zebrad_tests.rs as a workaround
# to build the zebra-checkpoints utility for the zebrad acceptance tests.
#
# When `-Z bindeps` is stabilised, enable this binary dependency instead:
# https://github.com/rust-lang/cargo/issues/9096
# zebra-utils { path = "../zebra-utils", artifact = "bin:zebra-checkpoints" }
zebra-utils = { path = "../zebra-utils" }

45
zebrad/tests/acceptance.rs
View File

@ -75,28 +75,28 @@
//! $ cargo test lightwalletd_integration -- --nocapture
//!
//! $ export ZEBRA_TEST_LIGHTWALLETD=true
//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/chain"
//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state"
//! $ export LIGHTWALLETD_DATA_DIR="/path/to/lightwalletd/database"
//! $ cargo test lightwalletd_update_sync -- --nocapture
//!
//! $ export ZEBRA_TEST_LIGHTWALLETD=true
//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/chain"
//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state"
//! $ cargo test lightwalletd_full_sync -- --ignored --nocapture
//!
//! $ export ZEBRA_TEST_LIGHTWALLETD=true
//! $ cargo test lightwalletd_test_suite -- --ignored --nocapture
//!
//! $ export ZEBRA_TEST_LIGHTWALLETD=true
//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/chain"
//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state"
//! $ cargo test fully_synced_rpc_test -- --ignored --nocapture
//!
//! $ export ZEBRA_TEST_LIGHTWALLETD=true
//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/chain"
//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state"
//! $ export LIGHTWALLETD_DATA_DIR="/path/to/lightwalletd/database"
//! $ cargo test sending_transactions_using_lightwalletd --features lightwalletd-grpc-tests -- --ignored --nocapture
//!
//! $ export ZEBRA_TEST_LIGHTWALLETD=true
//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/chain"
//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state"
//! $ export LIGHTWALLETD_DATA_DIR="/path/to/lightwalletd/database"
//! $ cargo test lightwalletd_wallet_grpc_tests --features lightwalletd-grpc-tests -- --ignored --nocapture
//! ```
@ -106,17 +106,25 @@
//! Example of how to run the get_block_template test:
//!
//! ```console
//! ZEBRA_CACHED_STATE_DIR=/path/to/zebra/chain cargo test get_block_template --features getblocktemplate-rpcs --release -- --ignored --nocapture
//! ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state cargo test get_block_template --features getblocktemplate-rpcs --release -- --ignored --nocapture
//! ```
//!
//! Example of how to run the submit_block test:
//!
//! ```console
//! ZEBRA_CACHED_STATE_DIR=/path/to/zebra/chain cargo test submit_block --features getblocktemplate-rpcs --release -- --ignored --nocapture
//! ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state cargo test submit_block --features getblocktemplate-rpcs --release -- --ignored --nocapture
//! ```
//!
//! Please refer to the documentation of each test for more information.
//!
//! ## Checkpoint Generation Tests
//!
//! Generate checkpoints on mainnet and testnet using a cached state:
//! ```console
//! GENERATE_CHECKPOINTS_MAINNET=1 ENTRYPOINT_FEATURES=zebra-checkpoints ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state docker/entrypoint.sh
//! GENERATE_CHECKPOINTS_TESTNET=1 ENTRYPOINT_FEATURES=zebra-checkpoints ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state docker/entrypoint.sh
//! ```
//!
//! ## Disk Space for Testing
//!
//! The full sync and lightwalletd tests with cached state expect a temporary directory with
@ -2205,3 +2213,26 @@ async fn get_block_template() -> Result<()> {
async fn submit_block() -> Result<()> {
common::get_block_template_rpcs::submit_block::run().await
}
/// Test `zebra-checkpoints` on mainnet.
///
/// If you want to run this test individually, see the module documentation.
/// See [`common::checkpoints`] for more information.
#[tokio::test]
#[ignore]
#[cfg(feature = "zebra-checkpoints")]
async fn generate_checkpoints_mainnet() -> Result<()> {
common::checkpoints::run(Mainnet).await
}
/// Test `zebra-checkpoints` on testnet.
/// This test might fail if testnet is unstable.
///
/// If you want to run this test individually, see the module documentation.
/// See [`common::checkpoints`] for more information.
#[tokio::test]
#[ignore]
#[cfg(feature = "zebra-checkpoints")]
async fn generate_checkpoints_testnet() -> Result<()> {
common::checkpoints::run(Testnet).await
}

489
zebrad/tests/common/checkpoints.rs Normal file
View File

@ -0,0 +1,489 @@
//! Test generating checkpoints using `zebra-checkpoints` directly connected to `zebrad`.
//!
//! This test requires a cached chain state that is synchronized close to the network chain tip
//! height. It will finish the sync and update the cached chain state.
use std::{
env, fs,
net::SocketAddr,
path::{Path, PathBuf},
sync::atomic::{AtomicBool, Ordering},
};
use color_eyre::eyre::Result;
use tempfile::TempDir;
use zebra_chain::{
block::{Height, HeightDiff, TryIntoHeight},
parameters::Network,
transparent::MIN_TRANSPARENT_COINBASE_MATURITY,
};
use zebra_consensus::MAX_CHECKPOINT_HEIGHT_GAP;
use zebra_node_services::rpc_client::RpcRequestClient;
use zebra_test::{
args,
command::{Arguments, TestDirExt, NO_MATCHES_REGEX_ITER},
prelude::TestChild,
};
use crate::common::{
launch::spawn_zebrad_for_rpc,
sync::{CHECKPOINT_VERIFIER_REGEX, SYNC_FINISHED_REGEX},
test_type::TestType::*,
};
use super::{
config::testdir,
failure_messages::{
PROCESS_FAILURE_MESSAGES, ZEBRA_CHECKPOINTS_FAILURE_MESSAGES, ZEBRA_FAILURE_MESSAGES,
},
launch::ZebradTestDirExt,
test_type::TestType,
};
/// The environmental variable used to activate zebrad logs in the checkpoint generation test.
///
/// We use a constant so the compiler detects typos.
pub const LOG_ZEBRAD_CHECKPOINTS: &str = "LOG_ZEBRAD_CHECKPOINTS";
/// The test entry point.
#[allow(clippy::print_stdout)]
pub async fn run(network: Network) -> Result<()> {
let _init_guard = zebra_test::init();
// We want a Zebra state dir, but we don't need `lightwalletd`.
let test_type = UpdateZebraCachedStateWithRpc;
let test_name = "zebra_checkpoints_test";
// Skip the test unless the user supplied the correct cached state env vars
let Some(zebrad_state_path) = test_type.zebrad_state_path(test_name) else {
return Ok(());
};
tracing::info!(
?network,
?test_type,
?zebrad_state_path,
"running zebra_checkpoints test, spawning zebrad...",
);
// Sync zebrad to the network chain tip
let (mut zebrad, zebra_rpc_address) = if let Some(zebrad_and_address) =
spawn_zebrad_for_rpc(network, test_name, test_type, true)?
{
zebrad_and_address
} else {
// Skip the test, we don't have the required cached state
return Ok(());
};
let zebra_rpc_address = zebra_rpc_address.expect("zebra_checkpoints test must have RPC port");
tracing::info!(
?network,
?zebra_rpc_address,
"spawned zebrad, waiting for it to load compiled-in checkpoints...",
);
let last_checkpoint = zebrad.expect_stdout_line_matches(CHECKPOINT_VERIFIER_REGEX)?;
// TODO: do this with a regex?
let (_prefix, last_checkpoint) = last_checkpoint
.split_once("max_checkpoint_height")
.expect("just checked log format");
let (_prefix, last_checkpoint) = last_checkpoint
.split_once('(')
.expect("unexpected log format");
let (last_checkpoint, _suffix) = last_checkpoint
.split_once(')')
.expect("unexpected log format");
tracing::info!(
?network,
?zebra_rpc_address,
?last_checkpoint,
"found zebrad's current last checkpoint",
);
tracing::info!(
?network,
?zebra_rpc_address,
"waiting for zebrad to open its RPC port...",
);
zebrad.expect_stdout_line_matches(&format!("Opened RPC endpoint at {zebra_rpc_address}"))?;
tracing::info!(
?network,
?zebra_rpc_address,
"zebrad opened its RPC port, waiting for it to sync...",
);
zebrad.expect_stdout_line_matches(SYNC_FINISHED_REGEX)?;
let zebra_tip_height = zebrad_tip_height(zebra_rpc_address).await?;
tracing::info!(
?network,
?zebra_rpc_address,
?zebra_tip_height,
?last_checkpoint,
"zebrad synced to the tip, launching zebra-checkpoints...",
);
let zebra_checkpoints =
spawn_zebra_checkpoints_direct(network, test_type, zebra_rpc_address, last_checkpoint)?;
let show_zebrad_logs = env::var(LOG_ZEBRAD_CHECKPOINTS).is_ok();
if !show_zebrad_logs {
tracing::info!(
"zebrad logs are hidden, show them using {LOG_ZEBRAD_CHECKPOINTS}=1 and RUST_LOG=debug"
);
}
tracing::info!(
?network,
?zebra_rpc_address,
?zebra_tip_height,
?last_checkpoint,
"spawned zebra-checkpoints connected to zebrad, checkpoints should appear here...",
);
println!("\n\n");
let (_zebra_checkpoints, _zebrad) = wait_for_zebra_checkpoints_generation(
zebra_checkpoints,
zebrad,
zebra_tip_height,
test_type,
show_zebrad_logs,
)?;
println!("\n\n");
tracing::info!(
?network,
?zebra_tip_height,
?last_checkpoint,
"finished generating Zebra checkpoints",
);
Ok(())
}
/// Spawns a `zebra-checkpoints` instance on `network`, connected to `zebrad_rpc_address`.
///
/// Returns:
/// - `Ok(zebra_checkpoints)` on success,
/// - `Err(_)` if spawning `zebra-checkpoints` fails.
#[tracing::instrument]
pub fn spawn_zebra_checkpoints_direct(
network: Network,
test_type: TestType,
zebrad_rpc_address: SocketAddr,
last_checkpoint: &str,
) -> Result<TestChild<TempDir>> {
let zebrad_rpc_address = zebrad_rpc_address.to_string();
let arguments = args![
"--addr": zebrad_rpc_address,
"--last-checkpoint": last_checkpoint,
];
// TODO: add logs for different kinds of zebra_checkpoints failures
let zebra_checkpoints_failure_messages = PROCESS_FAILURE_MESSAGES
.iter()
.chain(ZEBRA_FAILURE_MESSAGES)
.chain(ZEBRA_CHECKPOINTS_FAILURE_MESSAGES)
.cloned();
let zebra_checkpoints_ignore_messages = NO_MATCHES_REGEX_ITER.iter().cloned();
// Currently unused, but we might put a copy of the checkpoints file in it later
let zebra_checkpoints_dir = testdir()?;
let mut zebra_checkpoints = zebra_checkpoints_dir
.spawn_zebra_checkpoints_child(arguments)?
.with_timeout(test_type.zebrad_timeout())
.with_failure_regex_iter(
zebra_checkpoints_failure_messages,
zebra_checkpoints_ignore_messages,
);
// zebra-checkpoints logs to stderr when it launches.
//
// This log happens very quickly, so it is ok to block for a short while here.
zebra_checkpoints.expect_stderr_line_matches(regex::escape("calculating checkpoints"))?;
Ok(zebra_checkpoints)
}
/// Extension trait for methods on `tempfile::TempDir` for using it as a test
/// directory for `zebra-checkpoints`.
pub trait ZebraCheckpointsTestDirExt: ZebradTestDirExt
where
Self: AsRef<Path> + Sized,
{
/// Spawn `zebra-checkpoints` with `extra_args`, as a child process in this test directory,
/// potentially taking ownership of the tempdir for the duration of the child process.
///
/// By default, launch an instance that connects directly to `zebrad`.
fn spawn_zebra_checkpoints_child(self, extra_args: Arguments) -> Result<TestChild<Self>>;
}
impl ZebraCheckpointsTestDirExt for TempDir {
#[allow(clippy::unwrap_in_result)]
fn spawn_zebra_checkpoints_child(mut self, extra_args: Arguments) -> Result<TestChild<Self>> {
// By default, launch an instance that connects directly to `zebrad`.
let mut args = Arguments::new();
args.set_parameter("--transport", "direct");
// Apply user provided arguments
args.merge_with(extra_args);
// Create debugging info
let temp_dir = self.as_ref().display().to_string();
// Try searching the system $PATH first, that's what the test Docker image uses
let zebra_checkpoints_path = "zebra-checkpoints";
// Make sure we have the right zebra-checkpoints binary.
//
// When we were creating this test, we spent a lot of time debugging a build issue where
// `zebra-checkpoints` had an empty `main()` function. This check makes sure that doesn't
// happen again.
let debug_checkpoints = env::var(LOG_ZEBRAD_CHECKPOINTS).is_ok();
if debug_checkpoints {
let mut args = Arguments::new();
args.set_argument("--help");
let help_dir = testdir()?;
tracing::info!(
?zebra_checkpoints_path,
?args,
?help_dir,
system_path = ?env::var("PATH"),
// TODO: disable when the tests are working well
usr_local_zebra_checkpoints_info = ?fs::metadata("/usr/local/bin/zebra-checkpoints"),
"Trying to launch `zebra-checkpoints --help` by searching system $PATH...",
);
let zebra_checkpoints = help_dir.spawn_child_with_command(zebra_checkpoints_path, args);
if let Err(help_error) = zebra_checkpoints {
tracing::info!(?help_error, "Failed to launch `zebra-checkpoints --help`");
} else {
tracing::info!("Launched `zebra-checkpoints --help`, output is:");
let mut zebra_checkpoints = zebra_checkpoints.unwrap();
let mut output_is_empty = true;
// Get the help output
while zebra_checkpoints.wait_for_stdout_line(None) {
output_is_empty = false;
}
while zebra_checkpoints.wait_for_stderr_line(None) {
output_is_empty = false;
}
if output_is_empty {
tracing::info!(
"`zebra-checkpoints --help` did not log any output. \
Is the binary being built during tests? Are its required-features active?"
);
}
}
}
// Try the `zebra-checkpoints` binary the Docker image copied just after it built the tests.
tracing::info!(
?zebra_checkpoints_path,
?args,
?temp_dir,
system_path = ?env::var("PATH"),
// TODO: disable when the tests are working well
usr_local_zebra_checkpoints_info = ?fs::metadata("/usr/local/bin/zebra-checkpoints"),
"Trying to launch zebra-checkpoints by searching system $PATH...",
);
let zebra_checkpoints = self.spawn_child_with_command(zebra_checkpoints_path, args.clone());
let Err(system_path_error) = zebra_checkpoints else {
return zebra_checkpoints;
};
// Fall back to assuming zebra-checkpoints is in the same directory as zebrad.
let mut zebra_checkpoints_path: PathBuf = env!("CARGO_BIN_EXE_zebrad").into();
assert!(
zebra_checkpoints_path.pop(),
"must have at least one path component",
);
zebra_checkpoints_path.push("zebra-checkpoints");
if zebra_checkpoints_path.exists() {
// Create a new temporary directory, because the old one has been used up.
//
// TODO: instead, return the TempDir from spawn_child_with_command() on error.
self = testdir()?;
// Create debugging info
let temp_dir = self.as_ref().display().to_string();
tracing::info!(
?zebra_checkpoints_path,
?args,
?temp_dir,
?system_path_error,
// TODO: disable when the tests are working well
zebra_checkpoints_info = ?fs::metadata(&zebra_checkpoints_path),
"Launching from system $PATH failed, \
trying to launch zebra-checkpoints from cargo path...",
);
self.spawn_child_with_command(
zebra_checkpoints_path.to_str().expect(
"internal test harness error: path is not UTF-8 \
TODO: change spawn child methods to take &OsStr not &str",
),
args,
)
} else {
tracing::info!(
cargo_path = ?zebra_checkpoints_path,
?system_path_error,
// TODO: disable when the tests are working well
cargo_path_info = ?fs::metadata(&zebra_checkpoints_path),
"Launching from system $PATH failed, \
and zebra-checkpoints cargo path does not exist...",
);
// Return the original error
Err(system_path_error)
}
}
}
/// Wait for `zebra-checkpoints` to generate checkpoints, clearing Zebra's logs at the same time.
#[tracing::instrument]
pub fn wait_for_zebra_checkpoints_generation<
P: ZebradTestDirExt + std::fmt::Debug + std::marker::Send + 'static,
>(
mut zebra_checkpoints: TestChild<TempDir>,
mut zebrad: TestChild<P>,
zebra_tip_height: Height,
test_type: TestType,
show_zebrad_logs: bool,
) -> Result<(TestChild<TempDir>, TestChild<P>)> {
let last_checkpoint_gap = HeightDiff::try_from(MIN_TRANSPARENT_COINBASE_MATURITY)
.expect("constant fits in HeightDiff")
+ HeightDiff::try_from(MAX_CHECKPOINT_HEIGHT_GAP).expect("constant fits in HeightDiff");
let expected_final_checkpoint_height =
(zebra_tip_height - last_checkpoint_gap).expect("network tip is high enough");
let is_zebra_checkpoints_finished = AtomicBool::new(false);
let is_zebra_checkpoints_finished = &is_zebra_checkpoints_finished;
// Check Zebra's logs for errors.
//
// Checkpoint generation can take a long time, so we need to check `zebrad` for errors
// in parallel.
let zebrad_mut = &mut zebrad;
let zebrad_wait_fn = || -> Result<_> {
tracing::debug!(
?test_type,
"zebrad is waiting for zebra-checkpoints to generate checkpoints...",
);
while !is_zebra_checkpoints_finished.load(Ordering::SeqCst) {
// Just keep silently checking the Zebra logs for errors,
// so the checkpoint list can be copied from the output.
//
// Make sure the sync is still finished, this is logged every minute or so.
if env::var(LOG_ZEBRAD_CHECKPOINTS).is_ok() {
zebrad_mut.expect_stdout_line_matches(SYNC_FINISHED_REGEX)?;
} else {
zebrad_mut.expect_stdout_line_matches_silent(SYNC_FINISHED_REGEX)?;
}
}
Ok(zebrad_mut)
};
// Wait until `zebra-checkpoints` has generated a full set of checkpoints.
// Also checks `zebra-checkpoints` logs for errors.
//
// Checkpoints generation can take a long time, so we need to run it in parallel with `zebrad`.
let zebra_checkpoints_mut = &mut zebra_checkpoints;
let zebra_checkpoints_wait_fn = || -> Result<_> {
tracing::debug!(
?test_type,
"waiting for zebra_checkpoints to generate checkpoints...",
);
// zebra-checkpoints does not log anything when it finishes, it just prints checkpoints.
//
// We know that checkpoints are always less than 1000 blocks apart, but they can happen
// anywhere in that range due to block sizes. So we ignore the last 3 digits of the height.
let expected_final_checkpoint_prefix = expected_final_checkpoint_height.0 / 1000;
// Mainnet and testnet checkpoints always have at least one leading zero in their hash.
let expected_final_checkpoint =
format!("{expected_final_checkpoint_prefix}[0-9][0-9][0-9] 0");
zebra_checkpoints_mut.expect_stdout_line_matches(&expected_final_checkpoint)?;
// Write the rest of the checkpoints: there can be 0-2 more checkpoints.
while zebra_checkpoints_mut.wait_for_stdout_line(None) {}
// Tell the other thread that `zebra_checkpoints` has finished
is_zebra_checkpoints_finished.store(true, Ordering::SeqCst);
Ok(zebra_checkpoints_mut)
};
// Run both threads in parallel, automatically propagating any panics to this thread.
std::thread::scope(|s| {
// Launch the sync-waiting threads
let zebrad_thread = s.spawn(|| {
zebrad_wait_fn().expect("test failed while waiting for zebrad to sync");
});
let zebra_checkpoints_thread = s.spawn(|| {
let zebra_checkpoints_result = zebra_checkpoints_wait_fn();
is_zebra_checkpoints_finished.store(true, Ordering::SeqCst);
zebra_checkpoints_result
.expect("test failed while waiting for zebra_checkpoints to sync.");
});
// Mark the sync-waiting threads as finished if they fail or panic.
// This tells the other thread that it can exit.
//
// TODO: use `panic::catch_unwind()` instead,
// when `&mut zebra_test::command::TestChild<TempDir>` is unwind-safe
s.spawn(|| {
let zebrad_result = zebrad_thread.join();
zebrad_result.expect("test panicked or failed while waiting for zebrad to sync");
});
s.spawn(|| {
let zebra_checkpoints_result = zebra_checkpoints_thread.join();
is_zebra_checkpoints_finished.store(true, Ordering::SeqCst);
zebra_checkpoints_result
.expect("test panicked or failed while waiting for zebra_checkpoints to sync");
});
});
Ok((zebra_checkpoints, zebrad))
}
/// Returns an approximate `zebrad` tip height, using JSON-RPC.
#[tracing::instrument]
pub async fn zebrad_tip_height(zebra_rpc_address: SocketAddr) -> Result<Height> {
let client = RpcRequestClient::new(zebra_rpc_address);
let zebrad_blockchain_info = client
.text_from_call("getblockchaininfo", "[]".to_string())
.await?;
let zebrad_blockchain_info: serde_json::Value = serde_json::from_str(&zebrad_blockchain_info)?;
let zebrad_tip_height = zebrad_blockchain_info["result"]["blocks"]
.try_into_height()
.expect("unexpected block height: invalid Height value");
Ok(zebrad_tip_height)
}

23
zebrad/tests/common/failure_messages.rs
View File

@ -118,3 +118,26 @@ pub const LIGHTWALLETD_EMPTY_ZEBRA_STATE_IGNORE_MESSAGES: &[&str] = &[
// but we expect Zebra to start with an empty state.
r#"No Chain tip available yet","level":"warning","msg":"error with getblockchaininfo rpc, retrying"#,
];
/// Failure log messages from `zebra-checkpoints`.
///
/// These `zebra-checkpoints` messages show that checkpoint generation has failed.
/// So when we see them in the logs, we make the test fail.
#[cfg(feature = "zebra-checkpoints")]
pub const ZEBRA_CHECKPOINTS_FAILURE_MESSAGES: &[&str] = &[
// Rust-specific panics
"The application panicked",
// RPC port errors
"Unable to start RPC server",
// RPC argument errors: parsing and data
//
// These logs are produced by jsonrpc_core inside Zebra,
// but it doesn't log them yet.
//
// TODO: log these errors in Zebra, and check for them in the Zebra logs?
"Invalid params",
"Method not found",
// Incorrect command-line arguments
"USAGE",
"Invalid value",
];

2
zebrad/tests/common/lightwalletd.rs
View File

@ -148,7 +148,7 @@ pub fn can_spawn_lightwalletd_for_rpc<S: AsRef<str> + std::fmt::Debug>(
}
/// Extension trait for methods on `tempfile::TempDir` for using it as a test
/// directory for `zebrad`.
/// directory for `lightwalletd`.
pub trait LightWalletdTestDirExt: ZebradTestDirExt
where
Self: AsRef<Path> + Sized,

5
zebrad/tests/common/lightwalletd/send_transaction_test.rs
View File

@ -52,11 +52,6 @@ fn max_sent_transactions() -> usize {
const MAX_NUM_FUTURE_BLOCKS: u32 = 50;
/// The test entry point.
//
// TODO:
// - check output of zebrad and lightwalletd in different threads,
// to avoid test hangs due to full output pipes
// (see lightwalletd_integration_test for an example)
pub async fn run() -> Result<()> {
let _init_guard = zebra_test::init();

10
zebrad/tests/common/lightwalletd/sync.rs
View File

@ -132,11 +132,17 @@ pub fn wait_for_zebrad_and_lightwalletd_sync<
std::thread::scope(|s| {
// Launch the sync-waiting threads
let zebrad_thread = s.spawn(|| {
zebrad_wait_fn().expect("test failed while waiting for zebrad to sync");
let zebrad_result = zebrad_wait_fn();
is_zebrad_finished.store(true, Ordering::SeqCst);
zebrad_result.expect("test failed while waiting for zebrad to sync");
});
let lightwalletd_thread = s.spawn(|| {
lightwalletd_wait_fn().expect("test failed while waiting for lightwalletd to sync.");
let lightwalletd_result = lightwalletd_wait_fn();
is_lightwalletd_finished.store(true, Ordering::SeqCst);
lightwalletd_result.expect("test failed while waiting for lightwalletd to sync.");
});
// Mark the sync-waiting threads as finished if they fail or panic.

5
zebrad/tests/common/lightwalletd/wallet_grpc_test.rs
View File

@ -59,11 +59,6 @@ use crate::common::{
};
/// The test entry point.
//
// TODO:
// - check output of zebrad and lightwalletd in different threads,
// to avoid test hangs due to full output pipes
// (see lightwalletd_integration_test for an example)
pub async fn run() -> Result<()> {
let _init_guard = zebra_test::init();

3
zebrad/tests/common/mod.rs
View File

@ -18,5 +18,8 @@ pub mod lightwalletd;
pub mod sync;
pub mod test_type;
#[cfg(feature = "zebra-checkpoints")]
pub mod checkpoints;
#[cfg(feature = "getblocktemplate-rpcs")]
pub mod get_block_template_rpcs;

9
zebrad/tests/common/sync.rs
View File

@ -46,11 +46,14 @@ pub const SYNC_FINISHED_REGEX: &str =
r"finished initial sync to chain tip, using gossiped blocks .*sync_percent.*=.*100\.";
/// The text that should be logged every time Zebra checks the sync progress.
//
// This is only used with `--feature lightwalletd-grpc-tests`
#[allow(dead_code)]
#[cfg(feature = "lightwalletd-grpc-tests")]
pub const SYNC_PROGRESS_REGEX: &str = r"sync_percent";
/// The text that should be logged when Zebra loads its compiled-in checkpoints.
#[cfg(feature = "zebra-checkpoints")]
pub const CHECKPOINT_VERIFIER_REGEX: &str =
r"initializing chain verifier.*max_checkpoint_height.*=.*Height";
/// The maximum amount of time Zebra should take to reload after shutting down.
///
/// This should only take a second, but sometimes CI VMs or RocksDB can be slow.