Add hints to port conflict and lock file panics (#1535)

* add hint for port error
* add issue filter for port panic
* add lock file hint
* add metrics endpoint port conflict hint
* add hint for tracing endpoint port conflict
* add acceptance test for resource conflics
* Split out common conflict test code into a function
* Add state, metrics, and tracing conflict tests

* Add a full set of stderr acceptance test functions

This change makes the stdout and stderr acceptance test interfaces
identical.

* move Zcash listener opening
* add todo about hint for disk full
* add constant for lock file
* match path in state cache
* don't match windows cache path

* Use Display for state path logs

Avoids weird escaping on Windows when using Debug

* Add Windows conflict error messages

* Turn PORT_IN_USE_ERROR into a regex

And add another alternative Windows-specific port error

Co-authored-by: teor <teor@riseup.net>
Co-authored-by: Jane Lusby <jane@zfnd.org>

Cargo.lock

@@ -3941,11 +3941,13 @@ dependencies = [
  "futures",
  "hex",
  "indexmap",
+ "lazy_static",
  "metrics",
  "pin-project 0.4.27",
  "proptest",
  "proptest-derive",
  "rand 0.7.3",
+ "regex",
  "serde",
  "thiserror",
  "tokio 0.3.6",
@@ -3991,6 +3993,7 @@ dependencies = [
  "primitive-types",
  "proptest",
  "proptest-derive",
+ "regex",
  "rlimit",
  "rocksdb",
  "serde",

zebra-network/Cargo.toml

@@ -16,8 +16,10 @@ hex = "0.4"
 # indexmap has rayon support for parallel iteration,
 # which we don't use, so disable it to drop the dependencies.
 indexmap = { version = "1.6", default-features = false }
+lazy_static = "1.4.0"
 pin-project = "0.4"
 rand = "0.7"
+regex = "1"
 serde = { version = "1", features = ["serde_derive"] }
 thiserror = "1"
 

zebra-network/src/constants.rs

@@ -2,6 +2,9 @@
 
 use std::time::Duration;
 
+use lazy_static::lazy_static;
+use regex::Regex;
+
 // XXX should these constants be split into protocol also?
 use crate::protocol::external::types::*;
 
@@ -95,6 +98,16 @@ pub const EWMA_DEFAULT_RTT: Duration = Duration::from_secs(20 + 1);
 /// better peers when we restart the sync.
 pub const EWMA_DECAY_TIME: Duration = Duration::from_secs(200);
 
+lazy_static! {
+    /// OS-specific error when the port attempting to be opened is already in use.
+    pub static ref PORT_IN_USE_ERROR: Regex = if cfg!(unix) {
+        #[allow(clippy::trivial_regex)]
+        Regex::new("already in use")
+    } else {
+        Regex::new("(access a socket in a way forbidden by its access permissions)|(Only one usage of each socket address)")
+    }.expect("regex is valid");
+}
+
 /// Magic numbers used to identify different Zcash networks.
 pub mod magics {
     use super::*;

zebra-network/src/lib.rs

@@ -66,7 +66,7 @@ pub type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
 
 mod address_book;
 mod config;
-mod constants;
+pub mod constants;
 mod isolated;
 mod meta_addr;
 mod peer;

zebra-network/src/peer_set/initialize.rs

@@ -115,16 +115,7 @@ where
     );
     let peer_set = Buffer::new(BoxService::new(peer_set), constants::PEERSET_BUFFER_SIZE);
 
-    // Connect the tx end to the 3 peer sources:
+    // 1. Incoming peer connections, via a listener.
-
-    // 1. Initial peers, specified in the config.
-    let add_guard = tokio::spawn(add_initial_peers(
-        config.initial_peers(),
-        connector.clone(),
-        peerset_tx.clone(),
-    ));
-
-    // 2. Incoming peer connections, via a listener.
 
     // Warn if we're configured using the wrong network port.
     // TODO: use the right port if the port is unspecified
@@ -144,6 +135,18 @@ where
 
     let listen_guard = tokio::spawn(listen(config.listen_addr, listener, peerset_tx.clone()));
 
+    let initial_peers_fut = {
+        let initial_peers = config.initial_peers();
+        let connector = connector.clone();
+        let tx = peerset_tx.clone();
+
+        // Connect the tx end to the 3 peer sources:
+        add_initial_peers(initial_peers, connector, tx)
+    };
+
+    // 2. Initial peers, specified in the config.
+    let add_guard = tokio::spawn(initial_peers_fut);
+
     // 3. Outgoing peers we connect to in response to load.
     let mut candidates = CandidateSet::new(address_book.clone(), peer_set.clone());
 
@@ -211,7 +214,18 @@ where
     S: Service<(TcpStream, SocketAddr), Response = peer::Client, Error = BoxError> + Clone,
     S::Future: Send + 'static,
 {
-    let listener = TcpListener::bind(addr).await?;
+    let listener_result = TcpListener::bind(addr).await;
+
+    let listener = match listener_result {
+        Ok(l) => l,
+        Err(e) => panic!(
+            "Opening Zcash network protocol listener {:?} failed: {:?}. \
+             Hint: Check if another zebrad or zcashd process is running. \
+             Try changing the network listen_addr in the Zebra config.",
+            addr, e,
+        ),
+    };
+
     let local_addr = listener.local_addr()?;
     info!("Opened Zcash protocol endpoint at {}", local_addr);
     loop {

zebra-state/Cargo.toml

@@ -13,6 +13,7 @@ zebra-chain = { path = "../zebra-chain" }
 dirs = "3.0.1"
 hex = "0.4.2"
 lazy_static = "1.4.0"
+regex = "1"
 serde = { version = "1", features = ["serde_derive"] }
 
 futures = "0.3.12"

zebra-state/src/constants.rs

@@ -1,3 +1,5 @@
+//! Definitions of constants.
+
 /// The maturity threshold for transparent coinbase outputs.
 ///
 /// A transaction MUST NOT spend a transparent output of a coinbase transaction
@@ -13,3 +15,11 @@ pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRANSPARENT_COINBASE_MATURITY - 1;
 
 /// The database format version, incremented each time the database format changes.
 pub const DATABASE_FORMAT_VERSION: u32 = 4;
+
+use lazy_static::lazy_static;
+use regex::Regex;
+
+lazy_static! {
+    /// Regex that matches the RocksDB error when its lock file is already open.
+    pub static ref LOCK_FILE_ERROR: Regex = Regex::new("(lock file).*(temporarily unavailable)|(in use)|(being used by another process)").expect("regex is valid");
+}

zebra-state/src/lib.rs

@@ -16,7 +16,7 @@
 #![allow(clippy::unnecessary_wraps)]
 
 mod config;
-mod constants;
+pub mod constants;
 mod error;
 mod request;
 mod response;

zebra-state/src/service/finalized_state.rs

@@ -45,8 +45,21 @@ impl FinalizedState {
             rocksdb::ColumnFamilyDescriptor::new("sprout_nullifiers", db_options.clone()),
             rocksdb::ColumnFamilyDescriptor::new("sapling_nullifiers", db_options.clone()),
         ];
-        let db = rocksdb::DB::open_cf_descriptors(&db_options, path, column_families)
+        let db_result = rocksdb::DB::open_cf_descriptors(&db_options, &path, column_families);
-            .expect("database path and options are valid");
+
+        let db = match db_result {
+            Ok(d) => {
+                tracing::info!("Opened Zebra state cache at {}", path.display());
+                d
+            }
+            // TODO: provide a different hint if the disk is full, see #1623
+            Err(e) => panic!(
+                "Opening database {:?} failed: {:?}. \
+                 Hint: Check if another zebrad process is running. \
+                 Try changing the state cache_dir in the Zebra config.",
+                path, e,
+            ),
+        };
 
         let new_state = Self {
             queued_by_prev_hash: HashMap::new(),

zebra-test/src/command.rs

@@ -12,7 +12,7 @@ use std::{
     io::BufRead,
     io::{BufReader, Lines, Read},
     path::Path,
-    process::{Child, ChildStdout, Command, ExitStatus, Output, Stdio},
+    process::{Child, ChildStderr, ChildStdout, Command, ExitStatus, Output, Stdio},
     time::{Duration, Instant},
 };
 
@@ -86,6 +86,7 @@ impl CommandExt for Command {
             dir,
             deadline: None,
             stdout: None,
+            stderr: None,
             bypass_test_capture: false,
         })
     }
@@ -152,6 +153,7 @@ pub struct TestChild<T> {
     pub cmd: String,
     pub child: Child,
     pub stdout: Option<Lines<BufReader<ChildStdout>>>,
+    pub stderr: Option<Lines<BufReader<ChildStderr>>>,
     pub deadline: Option<Instant>,
     bypass_test_capture: bool,
 }
@@ -184,7 +186,7 @@ impl<T> TestChild<T> {
         })
     }
 
-    /// Set a timeout for `expect_stdout`.
+    /// Set a timeout for `expect_stdout` or `expect_stderr`.
     ///
     /// Does not apply to `wait_with_output`.
     pub fn with_timeout(mut self, timeout: Duration) -> Self {
@@ -192,17 +194,18 @@ impl<T> TestChild<T> {
         self
     }
 
-    /// Configures testrunner to forward stdout to the true stdout rather than
+    /// Configures testrunner to forward stdout and stderr to the true stdout,
-    /// fakestdout used by cargo tests.
+    /// rather than the fakestdout used by cargo tests.
     pub fn bypass_test_capture(mut self, cond: bool) -> Self {
         self.bypass_test_capture = cond;
         self
     }
 
-    /// Checks each line of the child's stdout against `regex`, and returns matching lines.
+    /// Checks each line of the child's stdout against `regex`, and returns Ok
+    /// if a line matches.
     ///
     /// Kills the child after the configured timeout has elapsed.
-    /// Note: the timeout is only checked after each line.
+    /// See `expect_line_matching` for details.
     #[instrument(skip(self))]
     pub fn expect_stdout(&mut self, regex: &str) -> Result<&mut Self> {
         if self.stdout.is_none() {
@@ -214,12 +217,67 @@ impl<T> TestChild<T> {
                 .map(BufRead::lines)
         }
 
-        let re = regex::Regex::new(regex).expect("regex must be valid");
         let mut lines = self
             .stdout
             .take()
             .expect("child must capture stdout to call expect_stdout");
 
+        match self.expect_line_matching(&mut lines, regex, "stdout") {
+            Ok(()) => {
+                self.stdout = Some(lines);
+                Ok(self)
+            }
+            Err(report) => Err(report),
+        }
+    }
+
+    /// Checks each line of the child's stderr against `regex`, and returns Ok
+    /// if a line matches.
+    ///
+    /// Kills the child after the configured timeout has elapsed.
+    /// See `expect_line_matching` for details.
+    #[instrument(skip(self))]
+    pub fn expect_stderr(&mut self, regex: &str) -> Result<&mut Self> {
+        if self.stderr.is_none() {
+            self.stderr = self
+                .child
+                .stderr
+                .take()
+                .map(BufReader::new)
+                .map(BufRead::lines)
+        }
+
+        let mut lines = self
+            .stderr
+            .take()
+            .expect("child must capture stderr to call expect_stderr");
+
+        match self.expect_line_matching(&mut lines, regex, "stderr") {
+            Ok(()) => {
+                self.stderr = Some(lines);
+                Ok(self)
+            }
+            Err(report) => Err(report),
+        }
+    }
+
+    /// Checks each line in `lines` against `regex`, and returns Ok if a line
+    /// matches. Uses `stream_name` as the name for `lines` in error reports.
+    ///
+    /// Kills the child after the configured timeout has elapsed.
+    /// Note: the timeout is only checked after each full line is received from
+    /// the child.
+    #[instrument(skip(self, lines))]
+    pub fn expect_line_matching<L>(
+        &mut self,
+        lines: &mut L,
+        regex: &str,
+        stream_name: &str,
+    ) -> Result<()>
+    where
+        L: Iterator<Item = std::io::Result<String>>,
+    {
+        let re = regex::Regex::new(regex).expect("regex must be valid");
         while !self.past_deadline() && self.is_running() {
             let line = if let Some(line) = lines.next() {
                 line?
@@ -227,20 +285,21 @@ impl<T> TestChild<T> {
                 break;
             };
 
-            // since we're about to discard this line write it to stdout so our
+            // Since we're about to discard this line write it to stdout, so it
-            // test runner can capture it and display if the test fails, may
+            // can be preserved. May cause weird reordering for stdout / stderr.
-            // cause weird reordering for stdout / stderr
+            // Uses stdout even if the original lines were from stderr.
-            if !self.bypass_test_capture {
+            if self.bypass_test_capture {
-                println!("{}", line);
+                // send lines to the terminal (or process stdout file redirect)
-            } else {
                 use std::io::Write;
                 #[allow(clippy::explicit_write)]
                 writeln!(std::io::stdout(), "{}", line).unwrap();
+            } else {
+                // if the test fails, the test runner captures and displays it
+                println!("{}", line);
             }
 
             if re.is_match(&line) {
-                self.stdout = Some(lines);
+                return Ok(());
-                return Ok(self);
             }
         }
 
@@ -251,7 +310,10 @@ impl<T> TestChild<T> {
             self.kill()?;
         }
 
-        let report = eyre!("stdout of command did not contain any matches for the given regex")
+        let report = eyre!(
+            "{} of command did not contain any matches for the given regex",
+            stream_name
+        )
         .context_from(self)
         .with_section(|| format!("{:?}", regex).header("Match Regex:"));
 
@@ -340,6 +402,51 @@ impl<T> TestOutput<T> {
             .with_section(|| format!("{:?}", regex).header("Match Regex:"))
     }
 
+    #[instrument(skip(self))]
+    pub fn stderr_contains(&self, regex: &str) -> Result<&Self> {
+        let re = regex::Regex::new(regex)?;
+        let stderr = String::from_utf8_lossy(&self.output.stderr);
+
+        for line in stderr.lines() {
+            if re.is_match(line) {
+                return Ok(self);
+            }
+        }
+
+        Err(eyre!(
+            "stderr of command did not contain any matches for the given regex"
+        ))
+        .context_from(self)
+        .with_section(|| format!("{:?}", regex).header("Match Regex:"))
+    }
+
+    #[instrument(skip(self))]
+    pub fn stderr_equals(&self, s: &str) -> Result<&Self> {
+        let stderr = String::from_utf8_lossy(&self.output.stderr);
+
+        if stderr == s {
+            return Ok(self);
+        }
+
+        Err(eyre!("stderr of command is not equal the given string"))
+            .context_from(self)
+            .with_section(|| format!("{:?}", s).header("Match String:"))
+    }
+
+    #[instrument(skip(self))]
+    pub fn stderr_matches(&self, regex: &str) -> Result<&Self> {
+        let re = regex::Regex::new(regex)?;
+        let stderr = String::from_utf8_lossy(&self.output.stderr);
+
+        if re.is_match(&stderr) {
+            return Ok(self);
+        }
+
+        Err(eyre!("stderr of command is not equal to the given regex"))
+            .context_from(self)
+            .with_section(|| format!("{:?}", regex).header("Match Regex:"))
+    }
+
     /// Returns Ok if the program was killed, Err(Report) if exit was by another
     /// reason.
     pub fn assert_was_killed(&self) -> Result<()> {
@@ -423,7 +530,12 @@ impl<T> ContextFrom<&mut TestChild<T>> for Report {
             let _ = stdout.read_to_string(&mut stdout_buf);
         }
 
-        if let Some(stderr) = &mut source.child.stderr {
+        if let Some(stderr) = &mut source.stderr {
+            for line in stderr {
+                let line = if let Ok(line) = line { line } else { break };
+                let _ = writeln!(&mut stderr_buf, "{}", line);
+            }
+        } else if let Some(stderr) = &mut source.child.stderr {
             let _ = stderr.read_to_string(&mut stderr_buf);
         }
 

zebrad/src/application.rs

@@ -12,6 +12,9 @@ use abscissa_core::{
 use application::fatal_error;
 use std::process;
 
+use zebra_network::constants::PORT_IN_USE_ERROR;
+use zebra_state::constants::LOCK_FILE_ERROR;
+
 /// Application state
 pub static APPLICATION: AppCell<ZebradApp> = AppCell::new();
 
@@ -171,7 +174,21 @@ impl Application for ZebradApp {
             .panic_section(metadata_section)
             .issue_url(concat!(env!("CARGO_PKG_REPOSITORY"), "/issues/new"))
             .issue_filter(|kind| match kind {
-                color_eyre::ErrorKind::NonRecoverable(_) => true,
+                color_eyre::ErrorKind::NonRecoverable(error) => {
+                    let error_str = match error.downcast_ref::<String>() {
+                        Some(as_string) => as_string,
+                        None => return true,
+                    };
+                    // listener port conflicts
+                    if PORT_IN_USE_ERROR.is_match(error_str) {
+                        return false;
+                    }
+                    // RocksDB lock file conflicts
+                    if LOCK_FILE_ERROR.is_match(error_str) {
+                        return false;
+                    }
+                    true
+                }
                 color_eyre::ErrorKind::Recoverable(error) => {
                     // type checks should be faster than string conversions
                     if error.is::<tower::timeout::error::Elapsed>()

zebrad/src/components/metrics.rs

@@ -12,11 +12,21 @@ impl MetricsEndpoint {
     /// Create the component.
     pub fn new(config: &ZebradConfig) -> Result<Self, FrameworkError> {
         if let Some(addr) = config.metrics.endpoint_addr {
-            info!("Initializing metrics endpoint at {}", addr);
+            let endpoint_result = metrics_exporter_prometheus::PrometheusBuilder::new()
-            metrics_exporter_prometheus::PrometheusBuilder::new()
                 .listen_address(addr)
-                .install()
+                .install();
-                .expect("FIXME ERROR CONVERSION");
+            match endpoint_result {
+                Ok(endpoint) => {
+                    info!("Opened metrics endpoint at {}", addr);
+                    endpoint
+                }
+                Err(e) => panic!(
+                    "Opening metrics endpoint listener {:?} failed: {:?}. \
+                     Hint: Check if another zebrad or zcashd process is running. \
+                     Try changing the metrics endpoint_addr in the Zebra config.",
+                    addr, e,
+                ),
+            }
         }
         Ok(Self {})
     }

zebrad/src/components/tracing/endpoint.rs

@@ -41,7 +41,6 @@ impl TracingEndpoint {
         } else {
             return Ok(());
         };
-        info!("Initializing tracing endpoint at {}", addr);
 
         let service =
             make_service_fn(|_| async { Ok::<_, hyper::Error>(service_fn(request_handler)) });
@@ -54,12 +53,16 @@ impl TracingEndpoint {
                 // try_bind uses the tokio runtime, so we
                 // need to construct it inside the task.
                 let server = match Server::try_bind(&addr) {
-                    Ok(s) => s,
+                    Ok(s) => {
-                    Err(e) => {
+                        info!("Opened tracing endpoint at {}", addr);
-                        error!("Could not open tracing endpoint listener");
+                        s
-                        error!("Error: {}", e);
-                        return;
                     }
+                    Err(e) => panic!(
+                        "Opening tracing endpoint listener {:?} failed: {:?}. \
+                         Hint: Check if another zebrad or zcashd process is running. \
+                         Try changing the tracing endpoint_addr in the Zebra config.",
+                        addr, e,
+                    ),
                 }
                 .serve(service);
 

zebrad/tests/acceptance.rs

@@ -2,6 +2,7 @@
 //! output for given argument combinations matches what is expected.
 //!
 //! ### Note on port conflict
+//!
 //! If the test child has a cache or port conflict with another test, or a
 //! running zebrad or zcashd, then it will panic. But the acceptance tests
 //! expect it to run until it is killed.
@@ -29,6 +30,8 @@ use zebra_chain::{
         NetworkUpgrade,
     },
 };
+use zebra_network::constants::PORT_IN_USE_ERROR;
+use zebra_state::constants::LOCK_FILE_ERROR;
 use zebra_test::{command::TestDirExt, prelude::*};
 use zebrad::config::ZebradConfig;
 
@@ -974,7 +977,7 @@ async fn metrics_endpoint() -> Result<()> {
     let output = output.assert_failure()?;
 
     // Make sure metrics was started
-    output.stdout_contains(format!(r"Initializing metrics endpoint at {}", endpoint).as_str())?;
+    output.stdout_contains(format!(r"Opened metrics endpoint at {}", endpoint).as_str())?;
 
     // [Note on port conflict](#Note on port conflict)
     output
@@ -1041,7 +1044,7 @@ async fn tracing_endpoint() -> Result<()> {
     let output = output.assert_failure()?;
 
     // Make sure tracing endpoint was started
-    output.stdout_contains(format!(r"Initializing tracing endpoint at {}", endpoint).as_str())?;
+    output.stdout_contains(format!(r"Opened tracing endpoint at {}", endpoint).as_str())?;
     // Todo: Match some trace level messages from output
 
     // [Note on port conflict](#Note on port conflict)
@@ -1051,3 +1054,179 @@ async fn tracing_endpoint() -> Result<()> {
 
     Ok(())
 }
+
+/// Test will start 2 zebrad nodes one after the other using the same Zcash listener.
+/// It is expected that the first node spawned will get exclusive use of the port.
+/// The second node will panic with the Zcash listener conflict hint added in #1535.
+#[test]
+fn zcash_listener_conflict() -> Result<()> {
+    zebra_test::init();
+
+    // [Note on port conflict](#Note on port conflict)
+    let port = random_known_port();
+    let listen_addr = format!("127.0.0.1:{}", port);
+
+    // Write a configuration that has our created network listen_addr
+    let mut config = default_test_config()?;
+    config.network.listen_addr = listen_addr.parse().unwrap();
+    let dir1 = TempDir::new("zebrad_tests")?.with_config(&mut config)?;
+    let regex1 = format!(r"Opened Zcash protocol endpoint at {}", listen_addr);
+
+    // From another folder create a configuration with the same listener.
+    // `network.listen_addr` will be the same in the 2 nodes.
+    // (But since the config is ephemeral, they will have different state paths.)
+    let dir2 = TempDir::new("zebrad_tests")?.with_config(&mut config)?;
+
+    check_config_conflict(dir1, regex1.as_str(), dir2, PORT_IN_USE_ERROR.as_str())?;
+
+    Ok(())
+}
+
+/// Start 2 zebrad nodes using the same metrics listener port, but different
+/// state directories and Zcash listener ports. The first node should get
+/// exclusive use of the port. The second node will panic with the Zcash metrics
+/// conflict hint added in #1535.
+#[test]
+fn zcash_metrics_conflict() -> Result<()> {
+    zebra_test::init();
+
+    // [Note on port conflict](#Note on port conflict)
+    let port = random_known_port();
+    let listen_addr = format!("127.0.0.1:{}", port);
+
+    // Write a configuration that has our created metrics endpoint_addr
+    let mut config = default_test_config()?;
+    config.metrics.endpoint_addr = Some(listen_addr.parse().unwrap());
+    let dir1 = TempDir::new("zebrad_tests")?.with_config(&mut config)?;
+    let regex1 = format!(r"Opened metrics endpoint at {}", listen_addr);
+
+    // From another folder create a configuration with the same endpoint.
+    // `metrics.endpoint_addr` will be the same in the 2 nodes.
+    // But they will have different Zcash listeners (auto port) and states (ephemeral)
+    let dir2 = TempDir::new("zebrad_tests")?.with_config(&mut config)?;
+
+    check_config_conflict(dir1, regex1.as_str(), dir2, PORT_IN_USE_ERROR.as_str())?;
+
+    Ok(())
+}
+
+/// Start 2 zebrad nodes using the same tracing listener port, but different
+/// state directories and Zcash listener ports. The first node should get
+/// exclusive use of the port. The second node will panic with the Zcash tracing
+/// conflict hint added in #1535.
+#[test]
+fn zcash_tracing_conflict() -> Result<()> {
+    zebra_test::init();
+
+    // [Note on port conflict](#Note on port conflict)
+    let port = random_known_port();
+    let listen_addr = format!("127.0.0.1:{}", port);
+
+    // Write a configuration that has our created tracing endpoint_addr
+    let mut config = default_test_config()?;
+    config.tracing.endpoint_addr = Some(listen_addr.parse().unwrap());
+    let dir1 = TempDir::new("zebrad_tests")?.with_config(&mut config)?;
+    let regex1 = format!(r"Opened tracing endpoint at {}", listen_addr);
+
+    // From another folder create a configuration with the same endpoint.
+    // `tracing.endpoint_addr` will be the same in the 2 nodes.
+    // But they will have different Zcash listeners (auto port) and states (ephemeral)
+    let dir2 = TempDir::new("zebrad_tests")?.with_config(&mut config)?;
+
+    check_config_conflict(dir1, regex1.as_str(), dir2, PORT_IN_USE_ERROR.as_str())?;
+
+    Ok(())
+}
+
+/// Start 2 zebrad nodes using the same state directory, but different Zcash
+/// listener ports. The first node should get exclusive access to the database.
+/// The second node will panic with the Zcash state conflict hint added in #1535.
+#[test]
+fn zcash_state_conflict() -> Result<()> {
+    zebra_test::init();
+
+    // A persistent config has a fixed temp state directory, but asks the OS to
+    // automatically choose an unused port
+    let mut config = persistent_test_config()?;
+    let dir_conflict = TempDir::new("zebrad_tests")?.with_config(&mut config)?;
+
+    // Windows problems with this match will be worked on at #1654
+    // We are matching the whole opened path only for unix by now.
+    let regex = if cfg!(unix) {
+        let mut dir_conflict_full = PathBuf::new();
+        dir_conflict_full.push(dir_conflict.path());
+        dir_conflict_full.push("state");
+        dir_conflict_full.push("state");
+        dir_conflict_full.push(format!(
+            "v{}",
+            zebra_state::constants::DATABASE_FORMAT_VERSION
+        ));
+        dir_conflict_full.push(config.network.network.to_string().to_lowercase());
+        format!(
+            "Opened Zebra state cache at {}",
+            dir_conflict_full.display()
+        )
+    } else {
+        String::from("Opened Zebra state cache at ")
+    };
+
+    check_config_conflict(
+        dir_conflict.path(),
+        regex.as_str(),
+        dir_conflict.path(),
+        LOCK_FILE_ERROR.as_str(),
+    )?;
+
+    Ok(())
+}
+
+/// Launch a node in `first_dir`, wait a few seconds, then launch a node in
+/// `second_dir`. Check that the first node's stdout contains
+/// `first_stdout_regex`, and the second node's stderr contains
+/// `second_stderr_regex`.
+fn check_config_conflict<T, U>(
+    first_dir: T,
+    first_stdout_regex: &str,
+    second_dir: U,
+    second_stderr_regex: &str,
+) -> Result<()>
+where
+    T: ZebradTestDirExt,
+    U: ZebradTestDirExt,
+{
+    // By DNS issues we want to skip all port conflict tests on macOS by now.
+    // Follow up at #1631
+    if cfg!(target_os = "macos") {
+        return Ok(());
+    }
+
+    // Start the first node
+    let mut node1 = first_dir.spawn_child(&["start"])?;
+
+    // Wait a bit to spawn the second node, we want the first fully started.
+    std::thread::sleep(LAUNCH_DELAY);
+
+    // Spawn the second node
+    let node2 = second_dir.spawn_child(&["start"])?;
+
+    // Wait a few seconds and kill first node.
+    // Second node is terminated by panic, no need to kill.
+    std::thread::sleep(LAUNCH_DELAY);
+    node1.kill()?;
+
+    // In node1 we want to check for the success regex
+    let output1 = node1.wait_with_output()?;
+    output1.stdout_contains(first_stdout_regex)?;
+    output1
+        .assert_was_killed()
+        .wrap_err("Possible port conflict. Are there other acceptance tests running?")?;
+
+    // In the second node we look for the conflict regex
+    let output2 = node2.wait_with_output()?;
+    output2.stderr_contains(second_stderr_regex)?;
+    output2
+        .assert_was_not_killed()
+        .wrap_err("Possible port conflict. Are there other acceptance tests running?")?;
+
+    Ok(())
+}