zecfoundation
/
zebra
mirror of https://github.com/ZcashFoundation/zebra.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add hints to port conflict and lock file panics (#1535) 

* add hint for port error
* add issue filter for port panic
* add lock file hint
* add metrics endpoint port conflict hint
* add hint for tracing endpoint port conflict
* add acceptance test for resource conflics
* Split out common conflict test code into a function
* Add state, metrics, and tracing conflict tests

* Add a full set of stderr acceptance test functions

This change makes the stdout and stderr acceptance test interfaces
identical.

* move Zcash listener opening
* add todo about hint for disk full
* add constant for lock file
* match path in state cache
* don't match windows cache path

* Use Display for state path logs

Avoids weird escaping on Windows when using Debug

* Add Windows conflict error messages

* Turn PORT_IN_USE_ERROR into a regex

And add another alternative Windows-specific port error

Co-authored-by: teor <teor@riseup.net>
Co-authored-by: Jane Lusby <jane@zfnd.org>
This commit is contained in:
Alfredo Garcia 2021-01-29 09:36:33 -03:00 committed by GitHub
parent 24f1b9bad1
commit 4b34482264
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 424 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.lock generated
View File

@ -3941,11 +3941,13 @@ dependencies = [
"futures",
"hex",
"indexmap",
"lazy_static",
"metrics",
"pin-project 0.4.27",
"proptest",
"proptest-derive",
"rand 0.7.3",
"regex",
"serde",
"thiserror",
"tokio 0.3.6",
@ -3991,6 +3993,7 @@ dependencies = [
"primitive-types",
"proptest",
"proptest-derive",
"regex",
"rlimit",
"rocksdb",
"serde",

2
zebra-network/Cargo.toml
View File

@ -16,8 +16,10 @@ hex = "0.4"
# indexmap has rayon support for parallel iteration,
# which we don't use, so disable it to drop the dependencies.
indexmap = { version = "1.6", default-features = false }
lazy_static = "1.4.0"
pin-project = "0.4"
rand = "0.7"
regex = "1"
serde = { version = "1", features = ["serde_derive"] }
thiserror = "1"

13
zebra-network/src/constants.rs
View File

@ -2,6 +2,9 @@
use std::time::Duration;
use lazy_static::lazy_static;
use regex::Regex;
// XXX should these constants be split into protocol also?
use crate::protocol::external::types::*;
@ -95,6 +98,16 @@ pub const EWMA_DEFAULT_RTT: Duration = Duration::from_secs(20 + 1);
/// better peers when we restart the sync.
pub const EWMA_DECAY_TIME: Duration = Duration::from_secs(200);
lazy_static! {
/// OS-specific error when the port attempting to be opened is already in use.
pub static ref PORT_IN_USE_ERROR: Regex = if cfg!(unix) {
#[allow(clippy::trivial_regex)]
Regex::new("already in use")
} else {
Regex::new("(access a socket in a way forbidden by its access permissions)|(Only one usage of each socket address)")
}.expect("regex is valid");
}
/// Magic numbers used to identify different Zcash networks.
pub mod magics {
use super::*;

2
zebra-network/src/lib.rs
View File

@ -66,7 +66,7 @@ pub type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
mod address_book;
mod config;
mod constants;
pub mod constants;
mod isolated;
mod meta_addr;
mod peer;

36
zebra-network/src/peer_set/initialize.rs
View File

@ -115,16 +115,7 @@ where
);
let peer_set = Buffer::new(BoxService::new(peer_set), constants::PEERSET_BUFFER_SIZE);
// Connect the tx end to the 3 peer sources:
// 1. Initial peers, specified in the config.
let add_guard = tokio::spawn(add_initial_peers(
config.initial_peers(),
connector.clone(),
peerset_tx.clone(),
));
// 2. Incoming peer connections, via a listener.
// 1. Incoming peer connections, via a listener.
// Warn if we're configured using the wrong network port.
// TODO: use the right port if the port is unspecified
@ -144,6 +135,18 @@ where
let listen_guard = tokio::spawn(listen(config.listen_addr, listener, peerset_tx.clone()));
let initial_peers_fut = {
let initial_peers = config.initial_peers();
let connector = connector.clone();
let tx = peerset_tx.clone();
// Connect the tx end to the 3 peer sources:
add_initial_peers(initial_peers, connector, tx)
};
// 2. Initial peers, specified in the config.
let add_guard = tokio::spawn(initial_peers_fut);
// 3. Outgoing peers we connect to in response to load.
let mut candidates = CandidateSet::new(address_book.clone(), peer_set.clone());
@ -211,7 +214,18 @@ where
S: Service<(TcpStream, SocketAddr), Response = peer::Client, Error = BoxError> + Clone,
S::Future: Send + 'static,
{
let listener = TcpListener::bind(addr).await?;
let listener_result = TcpListener::bind(addr).await;
let listener = match listener_result {
Ok(l) => l,
Err(e) => panic!(
"Opening Zcash network protocol listener {:?} failed: {:?}. \
Hint: Check if another zebrad or zcashd process is running. \
Try changing the network listen_addr in the Zebra config.",
addr, e,
),
};
let local_addr = listener.local_addr()?;
info!("Opened Zcash protocol endpoint at {}", local_addr);
loop {

1
zebra-state/Cargo.toml
View File

@ -13,6 +13,7 @@ zebra-chain = { path = "../zebra-chain" }
dirs = "3.0.1"
hex = "0.4.2"
lazy_static = "1.4.0"
regex = "1"
serde = { version = "1", features = ["serde_derive"] }
futures = "0.3.12"

10
zebra-state/src/constants.rs
View File

@ -1,3 +1,5 @@
//! Definitions of constants.
/// The maturity threshold for transparent coinbase outputs.
///
/// A transaction MUST NOT spend a transparent output of a coinbase transaction
@ -13,3 +15,11 @@ pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRANSPARENT_COINBASE_MATURITY - 1;
/// The database format version, incremented each time the database format changes.
pub const DATABASE_FORMAT_VERSION: u32 = 4;
use lazy_static::lazy_static;
use regex::Regex;
lazy_static! {
/// Regex that matches the RocksDB error when its lock file is already open.
pub static ref LOCK_FILE_ERROR: Regex = Regex::new("(lock file).*(temporarily unavailable)|(in use)|(being used by another process)").expect("regex is valid");
}

2
zebra-state/src/lib.rs
View File

@ -16,7 +16,7 @@
#![allow(clippy::unnecessary_wraps)]
mod config;
mod constants;
pub mod constants;
mod error;
mod request;
mod response;

17
zebra-state/src/service/finalized_state.rs
View File

@ -45,8 +45,21 @@ impl FinalizedState {
rocksdb::ColumnFamilyDescriptor::new("sprout_nullifiers", db_options.clone()),
rocksdb::ColumnFamilyDescriptor::new("sapling_nullifiers", db_options.clone()),
];
let db = rocksdb::DB::open_cf_descriptors(&db_options, path, column_families)
.expect("database path and options are valid");
let db_result = rocksdb::DB::open_cf_descriptors(&db_options, &path, column_families);
let db = match db_result {
Ok(d) => {
tracing::info!("Opened Zebra state cache at {}", path.display());
d
}
// TODO: provide a different hint if the disk is full, see #1623
Err(e) => panic!(
"Opening database {:?} failed: {:?}. \
Hint: Check if another zebrad process is running. \
Try changing the state cache_dir in the Zebra config.",
path, e,
),
};
let new_state = Self {
queued_by_prev_hash: HashMap::new(),

146
zebra-test/src/command.rs
View File

@ -12,7 +12,7 @@ use std::{
io::BufRead,
io::{BufReader, Lines, Read},
path::Path,
process::{Child, ChildStdout, Command, ExitStatus, Output, Stdio},
process::{Child, ChildStderr, ChildStdout, Command, ExitStatus, Output, Stdio},
time::{Duration, Instant},
};
@ -86,6 +86,7 @@ impl CommandExt for Command {
dir,
deadline: None,
stdout: None,
stderr: None,
bypass_test_capture: false,
})
}
@ -152,6 +153,7 @@ pub struct TestChild<T> {
pub cmd: String,
pub child: Child,
pub stdout: Option<Lines<BufReader<ChildStdout>>>,
pub stderr: Option<Lines<BufReader<ChildStderr>>>,
pub deadline: Option<Instant>,
bypass_test_capture: bool,
}
@ -184,7 +186,7 @@ impl<T> TestChild<T> {
})
}
/// Set a timeout for `expect_stdout`.
/// Set a timeout for `expect_stdout` or `expect_stderr`.
///
/// Does not apply to `wait_with_output`.
pub fn with_timeout(mut self, timeout: Duration) -> Self {
@ -192,17 +194,18 @@ impl<T> TestChild<T> {
self
}
/// Configures testrunner to forward stdout to the true stdout rather than
/// fakestdout used by cargo tests.
/// Configures testrunner to forward stdout and stderr to the true stdout,
/// rather than the fakestdout used by cargo tests.
pub fn bypass_test_capture(mut self, cond: bool) -> Self {
self.bypass_test_capture = cond;
self
}
/// Checks each line of the child's stdout against `regex`, and returns matching lines.
/// Checks each line of the child's stdout against `regex`, and returns Ok
/// if a line matches.
///
/// Kills the child after the configured timeout has elapsed.
/// Note: the timeout is only checked after each line.
/// See `expect_line_matching` for details.
#[instrument(skip(self))]
pub fn expect_stdout(&mut self, regex: &str) -> Result<&mut Self> {
if self.stdout.is_none() {
@ -214,12 +217,67 @@ impl<T> TestChild<T> {
.map(BufRead::lines)
}
let re = regex::Regex::new(regex).expect("regex must be valid");
let mut lines = self
.stdout
.take()
.expect("child must capture stdout to call expect_stdout");
match self.expect_line_matching(&mut lines, regex, "stdout") {
Ok(()) => {
self.stdout = Some(lines);
Ok(self)
}
Err(report) => Err(report),
}
}
/// Checks each line of the child's stderr against `regex`, and returns Ok
/// if a line matches.
///
/// Kills the child after the configured timeout has elapsed.
/// See `expect_line_matching` for details.
#[instrument(skip(self))]
pub fn expect_stderr(&mut self, regex: &str) -> Result<&mut Self> {
if self.stderr.is_none() {
self.stderr = self
.child
.stderr
.take()
.map(BufReader::new)
.map(BufRead::lines)
}
let mut lines = self
.stderr
.take()
.expect("child must capture stderr to call expect_stderr");
match self.expect_line_matching(&mut lines, regex, "stderr") {
Ok(()) => {
self.stderr = Some(lines);
Ok(self)
}
Err(report) => Err(report),
}
}
/// Checks each line in `lines` against `regex`, and returns Ok if a line
/// matches. Uses `stream_name` as the name for `lines` in error reports.
///
/// Kills the child after the configured timeout has elapsed.
/// Note: the timeout is only checked after each full line is received from
/// the child.
#[instrument(skip(self, lines))]
pub fn expect_line_matching<L>(
&mut self,
lines: &mut L,
regex: &str,
stream_name: &str,
) -> Result<()>
where
L: Iterator<Item = std::io::Result<String>>,
{
let re = regex::Regex::new(regex).expect("regex must be valid");
while !self.past_deadline() && self.is_running() {
let line = if let Some(line) = lines.next() {
line?
@ -227,20 +285,21 @@ impl<T> TestChild<T> {
break;
};
// since we're about to discard this line write it to stdout so our
// test runner can capture it and display if the test fails, may
// cause weird reordering for stdout / stderr
if !self.bypass_test_capture {
println!("{}", line);
} else {
// Since we're about to discard this line write it to stdout, so it
// can be preserved. May cause weird reordering for stdout / stderr.
// Uses stdout even if the original lines were from stderr.
if self.bypass_test_capture {
// send lines to the terminal (or process stdout file redirect)
use std::io::Write;
#[allow(clippy::explicit_write)]
writeln!(std::io::stdout(), "{}", line).unwrap();
} else {
// if the test fails, the test runner captures and displays it
println!("{}", line);
}
if re.is_match(&line) {
self.stdout = Some(lines);
return Ok(self);
return Ok(());
}
}
@ -251,7 +310,10 @@ impl<T> TestChild<T> {
self.kill()?;
}
let report = eyre!("stdout of command did not contain any matches for the given regex")
let report = eyre!(
"{} of command did not contain any matches for the given regex",
stream_name
)
.context_from(self)
.with_section(|| format!("{:?}", regex).header("Match Regex:"));
@ -340,6 +402,51 @@ impl<T> TestOutput<T> {
.with_section(|| format!("{:?}", regex).header("Match Regex:"))
}
#[instrument(skip(self))]
pub fn stderr_contains(&self, regex: &str) -> Result<&Self> {
let re = regex::Regex::new(regex)?;
let stderr = String::from_utf8_lossy(&self.output.stderr);
for line in stderr.lines() {
if re.is_match(line) {
return Ok(self);
}
}
Err(eyre!(
"stderr of command did not contain any matches for the given regex"
))
.context_from(self)
.with_section(|| format!("{:?}", regex).header("Match Regex:"))
}
#[instrument(skip(self))]
pub fn stderr_equals(&self, s: &str) -> Result<&Self> {
let stderr = String::from_utf8_lossy(&self.output.stderr);
if stderr == s {
return Ok(self);
}
Err(eyre!("stderr of command is not equal the given string"))
.context_from(self)
.with_section(|| format!("{:?}", s).header("Match String:"))
}
#[instrument(skip(self))]
pub fn stderr_matches(&self, regex: &str) -> Result<&Self> {
let re = regex::Regex::new(regex)?;
let stderr = String::from_utf8_lossy(&self.output.stderr);
if re.is_match(&stderr) {
return Ok(self);
}
Err(eyre!("stderr of command is not equal to the given regex"))
.context_from(self)
.with_section(|| format!("{:?}", regex).header("Match Regex:"))
}
/// Returns Ok if the program was killed, Err(Report) if exit was by another
/// reason.
pub fn assert_was_killed(&self) -> Result<()> {
@ -423,7 +530,12 @@ impl<T> ContextFrom<&mut TestChild<T>> for Report {
let _ = stdout.read_to_string(&mut stdout_buf);
}
if let Some(stderr) = &mut source.child.stderr {
if let Some(stderr) = &mut source.stderr {
for line in stderr {
let line = if let Ok(line) = line { line } else { break };
let _ = writeln!(&mut stderr_buf, "{}", line);
}
} else if let Some(stderr) = &mut source.child.stderr {
let _ = stderr.read_to_string(&mut stderr_buf);
}

19
zebrad/src/application.rs
View File

@ -12,6 +12,9 @@ use abscissa_core::{
use application::fatal_error;
use std::process;
use zebra_network::constants::PORT_IN_USE_ERROR;
use zebra_state::constants::LOCK_FILE_ERROR;
/// Application state
pub static APPLICATION: AppCell<ZebradApp> = AppCell::new();
@ -171,7 +174,21 @@ impl Application for ZebradApp {
.panic_section(metadata_section)
.issue_url(concat!(env!("CARGO_PKG_REPOSITORY"), "/issues/new"))
.issue_filter(|kind| match kind {
color_eyre::ErrorKind::NonRecoverable(_) => true,
color_eyre::ErrorKind::NonRecoverable(error) => {
let error_str = match error.downcast_ref::<String>() {
Some(as_string) => as_string,
None => return true,
};
// listener port conflicts
if PORT_IN_USE_ERROR.is_match(error_str) {
return false;
}
// RocksDB lock file conflicts
if LOCK_FILE_ERROR.is_match(error_str) {
return false;
}
true
}
color_eyre::ErrorKind::Recoverable(error) => {
// type checks should be faster than string conversions
if error.is::<tower::timeout::error::Elapsed>()

18
zebrad/src/components/metrics.rs
View File

@ -12,11 +12,21 @@ impl MetricsEndpoint {
/// Create the component.
pub fn new(config: &ZebradConfig) -> Result<Self, FrameworkError> {
if let Some(addr) = config.metrics.endpoint_addr {
info!("Initializing metrics endpoint at {}", addr);
metrics_exporter_prometheus::PrometheusBuilder::new()
let endpoint_result = metrics_exporter_prometheus::PrometheusBuilder::new()
.listen_address(addr)
.install()
.expect("FIXME ERROR CONVERSION");
.install();
match endpoint_result {
Ok(endpoint) => {
info!("Opened metrics endpoint at {}", addr);
endpoint
}
Err(e) => panic!(
"Opening metrics endpoint listener {:?} failed: {:?}. \
Hint: Check if another zebrad or zcashd process is running. \
Try changing the metrics endpoint_addr in the Zebra config.",
addr, e,
),
}
}
Ok(Self {})
}

15
zebrad/src/components/tracing/endpoint.rs
View File

@ -41,7 +41,6 @@ impl TracingEndpoint {
} else {
return Ok(());
};
info!("Initializing tracing endpoint at {}", addr);
let service =
make_service_fn(|_| async { Ok::<_, hyper::Error>(service_fn(request_handler)) });
@ -54,12 +53,16 @@ impl TracingEndpoint {
// try_bind uses the tokio runtime, so we
// need to construct it inside the task.
let server = match Server::try_bind(&addr) {
Ok(s) => s,
Err(e) => {
error!("Could not open tracing endpoint listener");
error!("Error: {}", e);
return;
Ok(s) => {
info!("Opened tracing endpoint at {}", addr);
s
}
Err(e) => panic!(
"Opening tracing endpoint listener {:?} failed: {:?}. \
Hint: Check if another zebrad or zcashd process is running. \
Try changing the tracing endpoint_addr in the Zebra config.",
addr, e,
),
}
.serve(service);

183
zebrad/tests/acceptance.rs
View File

@ -2,6 +2,7 @@
//! output for given argument combinations matches what is expected.
//!
//! ### Note on port conflict
//!
//! If the test child has a cache or port conflict with another test, or a
//! running zebrad or zcashd, then it will panic. But the acceptance tests
//! expect it to run until it is killed.
@ -29,6 +30,8 @@ use zebra_chain::{
NetworkUpgrade,
},
};
use zebra_network::constants::PORT_IN_USE_ERROR;
use zebra_state::constants::LOCK_FILE_ERROR;
use zebra_test::{command::TestDirExt, prelude::*};
use zebrad::config::ZebradConfig;
@ -974,7 +977,7 @@ async fn metrics_endpoint() -> Result<()> {
let output = output.assert_failure()?;
// Make sure metrics was started
output.stdout_contains(format!(r"Initializing metrics endpoint at {}", endpoint).as_str())?;
output.stdout_contains(format!(r"Opened metrics endpoint at {}", endpoint).as_str())?;
// [Note on port conflict](#Note on port conflict)
output
@ -1041,7 +1044,7 @@ async fn tracing_endpoint() -> Result<()> {
let output = output.assert_failure()?;
// Make sure tracing endpoint was started
output.stdout_contains(format!(r"Initializing tracing endpoint at {}", endpoint).as_str())?;
output.stdout_contains(format!(r"Opened tracing endpoint at {}", endpoint).as_str())?;
// Todo: Match some trace level messages from output
// [Note on port conflict](#Note on port conflict)
@ -1051,3 +1054,179 @@ async fn tracing_endpoint() -> Result<()> {
Ok(())
}
/// Test will start 2 zebrad nodes one after the other using the same Zcash listener.
/// It is expected that the first node spawned will get exclusive use of the port.
/// The second node will panic with the Zcash listener conflict hint added in #1535.
#[test]
fn zcash_listener_conflict() -> Result<()> {
zebra_test::init();
// [Note on port conflict](#Note on port conflict)
let port = random_known_port();
let listen_addr = format!("127.0.0.1:{}", port);
// Write a configuration that has our created network listen_addr
let mut config = default_test_config()?;
config.network.listen_addr = listen_addr.parse().unwrap();
let dir1 = TempDir::new("zebrad_tests")?.with_config(&mut config)?;
let regex1 = format!(r"Opened Zcash protocol endpoint at {}", listen_addr);
// From another folder create a configuration with the same listener.
// `network.listen_addr` will be the same in the 2 nodes.
// (But since the config is ephemeral, they will have different state paths.)
let dir2 = TempDir::new("zebrad_tests")?.with_config(&mut config)?;
check_config_conflict(dir1, regex1.as_str(), dir2, PORT_IN_USE_ERROR.as_str())?;
Ok(())
}
/// Start 2 zebrad nodes using the same metrics listener port, but different
/// state directories and Zcash listener ports. The first node should get
/// exclusive use of the port. The second node will panic with the Zcash metrics
/// conflict hint added in #1535.
#[test]
fn zcash_metrics_conflict() -> Result<()> {
zebra_test::init();
// [Note on port conflict](#Note on port conflict)
let port = random_known_port();
let listen_addr = format!("127.0.0.1:{}", port);
// Write a configuration that has our created metrics endpoint_addr
let mut config = default_test_config()?;
config.metrics.endpoint_addr = Some(listen_addr.parse().unwrap());
let dir1 = TempDir::new("zebrad_tests")?.with_config(&mut config)?;
let regex1 = format!(r"Opened metrics endpoint at {}", listen_addr);
// From another folder create a configuration with the same endpoint.
// `metrics.endpoint_addr` will be the same in the 2 nodes.
// But they will have different Zcash listeners (auto port) and states (ephemeral)
let dir2 = TempDir::new("zebrad_tests")?.with_config(&mut config)?;
check_config_conflict(dir1, regex1.as_str(), dir2, PORT_IN_USE_ERROR.as_str())?;
Ok(())
}
/// Start 2 zebrad nodes using the same tracing listener port, but different
/// state directories and Zcash listener ports. The first node should get
/// exclusive use of the port. The second node will panic with the Zcash tracing
/// conflict hint added in #1535.
#[test]
fn zcash_tracing_conflict() -> Result<()> {
zebra_test::init();
// [Note on port conflict](#Note on port conflict)
let port = random_known_port();
let listen_addr = format!("127.0.0.1:{}", port);
// Write a configuration that has our created tracing endpoint_addr
let mut config = default_test_config()?;
config.tracing.endpoint_addr = Some(listen_addr.parse().unwrap());
let dir1 = TempDir::new("zebrad_tests")?.with_config(&mut config)?;
let regex1 = format!(r"Opened tracing endpoint at {}", listen_addr);
// From another folder create a configuration with the same endpoint.
// `tracing.endpoint_addr` will be the same in the 2 nodes.
// But they will have different Zcash listeners (auto port) and states (ephemeral)
let dir2 = TempDir::new("zebrad_tests")?.with_config(&mut config)?;
check_config_conflict(dir1, regex1.as_str(), dir2, PORT_IN_USE_ERROR.as_str())?;
Ok(())
}
/// Start 2 zebrad nodes using the same state directory, but different Zcash
/// listener ports. The first node should get exclusive access to the database.
/// The second node will panic with the Zcash state conflict hint added in #1535.
#[test]
fn zcash_state_conflict() -> Result<()> {
zebra_test::init();
// A persistent config has a fixed temp state directory, but asks the OS to
// automatically choose an unused port
let mut config = persistent_test_config()?;
let dir_conflict = TempDir::new("zebrad_tests")?.with_config(&mut config)?;
// Windows problems with this match will be worked on at #1654
// We are matching the whole opened path only for unix by now.
let regex = if cfg!(unix) {
let mut dir_conflict_full = PathBuf::new();
dir_conflict_full.push(dir_conflict.path());
dir_conflict_full.push("state");
dir_conflict_full.push("state");
dir_conflict_full.push(format!(
"v{}",
zebra_state::constants::DATABASE_FORMAT_VERSION
));
dir_conflict_full.push(config.network.network.to_string().to_lowercase());
format!(
"Opened Zebra state cache at {}",
dir_conflict_full.display()
)
} else {
String::from("Opened Zebra state cache at ")
};
check_config_conflict(
dir_conflict.path(),
regex.as_str(),
dir_conflict.path(),
LOCK_FILE_ERROR.as_str(),
)?;
Ok(())
}
/// Launch a node in `first_dir`, wait a few seconds, then launch a node in
/// `second_dir`. Check that the first node's stdout contains
/// `first_stdout_regex`, and the second node's stderr contains
/// `second_stderr_regex`.
fn check_config_conflict<T, U>(
first_dir: T,
first_stdout_regex: &str,
second_dir: U,
second_stderr_regex: &str,
) -> Result<()>
where
T: ZebradTestDirExt,
U: ZebradTestDirExt,
{
// By DNS issues we want to skip all port conflict tests on macOS by now.
// Follow up at #1631
if cfg!(target_os = "macos") {
return Ok(());
}
// Start the first node
let mut node1 = first_dir.spawn_child(&["start"])?;
// Wait a bit to spawn the second node, we want the first fully started.
std::thread::sleep(LAUNCH_DELAY);
// Spawn the second node
let node2 = second_dir.spawn_child(&["start"])?;
// Wait a few seconds and kill first node.
// Second node is terminated by panic, no need to kill.
std::thread::sleep(LAUNCH_DELAY);
node1.kill()?;
// In node1 we want to check for the success regex
let output1 = node1.wait_with_output()?;
output1.stdout_contains(first_stdout_regex)?;
output1
.assert_was_killed()
.wrap_err("Possible port conflict. Are there other acceptance tests running?")?;
// In the second node we look for the conflict regex
let output2 = node2.wait_with_output()?;
output2.stderr_contains(second_stderr_regex)?;
output2
.assert_was_not_killed()
.wrap_err("Possible port conflict. Are there other acceptance tests running?")?;
Ok(())
}