From 375a997d2f4a0d96b661399dc9c7ec75ae3945ea Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 19 Nov 2021 11:55:38 +1000 Subject: [PATCH] Stop downloading unnecessary blocks in Zebra acceptance tests (#3072) * Implement graceful shutdown for the peer set * Use the minimum lookahead limit in acceptance tests * Enable a doctest that compiles with newly public modules --- zebrad/src/components.rs | 4 +++ zebrad/src/components/inbound.rs | 11 +++++++ zebrad/src/components/mempool.rs | 2 +- zebrad/src/components/mempool/crawler.rs | 2 +- zebrad/src/components/sync.rs | 11 ++++++- zebrad/src/components/tracing.rs | 2 ++ zebrad/src/config.rs | 5 +-- zebrad/src/lib.rs | 3 +- zebrad/tests/acceptance.rs | 39 ++++++++++++++++++++---- 9 files changed, 66 insertions(+), 13 deletions(-) diff --git a/zebrad/src/components.rs b/zebrad/src/components.rs index 59c62691d..2e0752b0a 100644 --- a/zebrad/src/components.rs +++ b/zebrad/src/components.rs @@ -6,10 +6,14 @@ //! don't fit the async context well. mod inbound; +#[allow(missing_docs)] pub mod mempool; pub mod metrics; +#[allow(missing_docs)] pub mod sync; +#[allow(missing_docs)] pub mod tokio; +#[allow(missing_docs)] pub mod tracing; pub use inbound::Inbound; diff --git a/zebrad/src/components/inbound.rs b/zebrad/src/components/inbound.rs index 6b02ec3cc..20150b0be 100644 --- a/zebrad/src/components/inbound.rs +++ b/zebrad/src/components/inbound.rs @@ -1,3 +1,10 @@ +//! The inbound service handles requests from Zebra's peers. +//! +//! It downloads and verifies gossiped blocks and mempool transactions, +//! when Zebra is close to the chain tip. +//! +//! It also responds to peer requests for blocks, transactions, and peer addresses. + use std::{ future::Future, pin::Pin, @@ -128,6 +135,10 @@ pub struct Inbound { } impl Inbound { + /// Create a new inbound service. + /// + /// The address book and peer set use the newly created inbound service. + /// So they are sent via the `network_setup` channel after initialization. pub fn new( network_setup: oneshot::Receiver, state: State, diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index ae60bb5b9..ce5ad83ed 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -42,7 +42,7 @@ use zebra_state::{ChainTipChange, TipAction}; use crate::components::sync::SyncStatus; -mod config; +pub mod config; mod crawler; pub mod downloads; mod error; diff --git a/zebrad/src/components/mempool/crawler.rs b/zebrad/src/components/mempool/crawler.rs index 275e784de..cc8109ab9 100644 --- a/zebrad/src/components/mempool/crawler.rs +++ b/zebrad/src/components/mempool/crawler.rs @@ -13,7 +13,7 @@ //! //! # Example //! -//! ```compile_fail +//! ``` //! use zebrad::components::mempool; //! # //! # use zebra_chain::parameters::Network; diff --git a/zebrad/src/components/sync.rs b/zebrad/src/components/sync.rs index 140a282c4..a5a852243 100644 --- a/zebrad/src/components/sync.rs +++ b/zebrad/src/components/sync.rs @@ -1,3 +1,7 @@ +//! The syncer downloads and verifies large numbers of blocks from peers to Zebra. +//! +//! It is used when Zebra is a long way behind the current chain tip. + use std::{collections::HashSet, pin::Pin, sync::Arc, time::Duration}; use color_eyre::eyre::{eyre, Report}; @@ -69,7 +73,12 @@ const BLOCK_DOWNLOAD_RETRY_LIMIT: usize = 2; /// Once these malicious blocks start failing validation, the syncer will cancel all /// the pending download and verify tasks, drop all the blocks, and start a new /// ObtainTips with a new set of peers. -const MIN_LOOKAHEAD_LIMIT: usize = zebra_consensus::MAX_CHECKPOINT_HEIGHT_GAP * 2; +pub const MIN_LOOKAHEAD_LIMIT: usize = zebra_consensus::MAX_CHECKPOINT_HEIGHT_GAP * 2; + +/// The default for the user-specified lookahead limit. +/// +/// See [`MIN_LOOKAHEAD_LIMIT`] for details. +pub const DEFAULT_LOOKAHEAD_LIMIT: usize = zebra_consensus::MAX_CHECKPOINT_HEIGHT_GAP * 5; /// Controls how long we wait for a tips response to return. /// diff --git a/zebrad/src/components/tracing.rs b/zebrad/src/components/tracing.rs index c9605e30d..804b54739 100644 --- a/zebrad/src/components/tracing.rs +++ b/zebrad/src/components/tracing.rs @@ -1,3 +1,5 @@ +//! Tracing and logging infrastructure for Zebra. + mod component; mod endpoint; mod flame; diff --git a/zebrad/src/config.rs b/zebrad/src/config.rs index 25a483f15..2920a5667 100644 --- a/zebrad/src/config.rs +++ b/zebrad/src/config.rs @@ -8,11 +8,12 @@ use std::{net::SocketAddr, path::PathBuf}; use serde::{Deserialize, Serialize}; -use crate::components::mempool::Config as MempoolSection; use zebra_consensus::Config as ConsensusSection; use zebra_network::Config as NetworkSection; use zebra_state::Config as StateSection; +use crate::components::{mempool::Config as MempoolSection, sync}; + /// Configuration for `zebrad`. /// /// The `zebrad` config is a TOML-encoded version of this structure. The meaning @@ -180,7 +181,7 @@ impl Default for SyncSection { fn default() -> Self { Self { max_concurrent_block_requests: 50, - lookahead_limit: 2_000, + lookahead_limit: sync::DEFAULT_LOOKAHEAD_LIMIT, } } } diff --git a/zebrad/src/lib.rs b/zebrad/src/lib.rs index 38ae0eac6..81df250cc 100644 --- a/zebrad/src/lib.rs +++ b/zebrad/src/lib.rs @@ -36,11 +36,10 @@ extern crate tracing; /// parameterized by 'a), *not* that the object itself has 'static lifetime. pub type BoxError = Box; -mod components; - pub mod application; pub mod async_ext; pub mod commands; +pub mod components; pub mod config; pub mod prelude; pub mod sentry; diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 989d45837..6f90a809f 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -46,7 +46,10 @@ use zebra_test::{ net::random_known_port, prelude::*, }; -use zebrad::config::ZebradConfig; +use zebrad::{ + components::{mempool, sync}, + config::{SyncSection, ZebradConfig}, +}; /// The amount of time we wait after launching `zebrad`. /// @@ -54,18 +57,41 @@ use zebrad::config::ZebradConfig; /// metrics or tracing test failures in Windows CI. const LAUNCH_DELAY: Duration = Duration::from_secs(10); +/// Returns a config with: +/// - a Zcash listener on an unused port on IPv4 localhost, and +/// - an ephemeral state, +/// - the minimum syncer lookahead limit, and +/// - shorter task intervals, to improve test coverage. fn default_test_config() -> Result { - let auto_port_ipv4_local = zebra_network::Config { + const TEST_DURATION: Duration = Duration::from_secs(30); + + let network = zebra_network::Config { + // The OS automatically chooses an unused port. listen_addr: "127.0.0.1:0".parse()?, - crawl_new_peer_interval: Duration::from_secs(30), + crawl_new_peer_interval: TEST_DURATION, ..zebra_network::Config::default() }; - let local_ephemeral = ZebradConfig { + + let sync = SyncSection { + // Avoid downloading unnecessary blocks. + lookahead_limit: sync::MIN_LOOKAHEAD_LIMIT, + ..SyncSection::default() + }; + + let mempool = mempool::Config { + eviction_memory_time: TEST_DURATION, + ..mempool::Config::default() + }; + + let config = ZebradConfig { + network, state: zebra_state::Config::ephemeral(), - network: auto_port_ipv4_local, + sync, + mempool, ..ZebradConfig::default() }; - Ok(local_ephemeral) + + Ok(config) } fn persistent_test_config() -> Result { @@ -949,6 +975,7 @@ fn sync_until( fn cached_mandatory_checkpoint_test_config() -> Result { let mut config = persistent_test_config()?; config.state.cache_dir = "/zebrad-cache".into(); + config.sync.lookahead_limit = sync::DEFAULT_LOOKAHEAD_LIMIT; Ok(config) }