add(network): Adds a network magic field to `testnet::Parameters` and the config (#8524)

* Adds a `target_difficulty_limit` field on `testnet::Parameters`

* updates test to increment block nonce until finding a block that's below the difficulty threshold

* increment the nonce while the difficulty is invalid instead of while the difficulty threshold is invalid

* Adds comments

* moves network Magic type from zebra-network to zebra-chain

* Adds `network_magic` field to `testnet::Parameters` and uses the regtest network magic from zcashd

* Add a network magic config field for custom testnets

* Adds/updates tests

* Update zebra-chain/src/parameters/network/testnet.rs

* Adds a link to the Regtest network magic in zcashd

---------

Co-authored-by: Alfredo Garcia <oxarbitrage@gmail.com>

zebra-chain/src/parameters.rs

@@ -22,7 +22,7 @@ mod transaction;
 pub mod arbitrary;
 
 pub use genesis::*;
-pub use network::{testnet, Network, NetworkKind};
+pub use network::{magic::Magic, testnet, Network, NetworkKind};
 pub use network_upgrade::*;
 pub use transaction::*;
 

zebra-chain/src/parameters/constants.rs

@@ -15,3 +15,15 @@ pub const SLOW_START_INTERVAL: Height = Height(20_000);
 ///
 /// This calculation is exact, because `SLOW_START_INTERVAL` is divisible by 2.
 pub const SLOW_START_SHIFT: Height = Height(SLOW_START_INTERVAL.0 / 2);
+
+/// Magic numbers used to identify different Zcash networks.
+pub mod magics {
+    use crate::parameters::network::magic::Magic;
+
+    /// The production mainnet.
+    pub const MAINNET: Magic = Magic([0x24, 0xe9, 0x27, 0x64]);
+    /// The testnet.
+    pub const TESTNET: Magic = Magic([0xfa, 0x1a, 0xf9, 0xbf]);
+    /// The regtest, see <https://github.com/zcash/zcash/blob/master/src/chainparams.cpp#L716-L719>
+    pub const REGTEST: Magic = Magic([0xaa, 0xe8, 0x3f, 0x5f]);
+}

zebra-chain/src/parameters/network.rs

@@ -9,6 +9,7 @@ use crate::{
     parameters::NetworkUpgrade,
 };
 
+pub mod magic;
 pub mod testnet;
 
 #[cfg(test)]
@@ -75,7 +76,7 @@ impl From<Network> for NetworkKind {
 }
 
 /// An enum describing the possible network choices.
-#[derive(Clone, Debug, Default, Eq, PartialEq, Hash, Serialize)]
+#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)]
 #[serde(into = "NetworkKind")]
 pub enum Network {
     /// The production mainnet.

zebra-chain/src/parameters/network/magic.rs

@@ -0,0 +1,56 @@
+//! Network `Magic` type and implementation.
+
+use std::fmt;
+
+use crate::parameters::{constants::magics, Network};
+
+#[cfg(any(test, feature = "proptest-impl"))]
+use proptest_derive::Arbitrary;
+
+/// A magic number identifying the network.
+#[derive(Copy, Clone, Eq, PartialEq)]
+#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))]
+pub struct Magic(pub [u8; 4]);
+
+impl fmt::Debug for Magic {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.debug_tuple("Magic").field(&hex::encode(self.0)).finish()
+    }
+}
+
+impl Network {
+    /// Get the magic value associated to this `Network`.
+    pub fn magic(&self) -> Magic {
+        match self {
+            Network::Mainnet => magics::MAINNET,
+            Network::Testnet(params) => params.network_magic(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod proptest {
+
+    use proptest::prelude::*;
+
+    use super::{magics, Magic};
+
+    #[test]
+    fn magic_debug() {
+        let _init_guard = zebra_test::init();
+
+        assert_eq!(format!("{:?}", magics::MAINNET), "Magic(\"24e92764\")");
+        assert_eq!(format!("{:?}", magics::TESTNET), "Magic(\"fa1af9bf\")");
+        assert_eq!(format!("{:?}", magics::REGTEST), "Magic(\"aae83f5f\")");
+    }
+
+    proptest! {
+
+        #[test]
+        fn proptest_magic_from_array(data in any::<[u8; 4]>()) {
+            let _init_guard = zebra_test::init();
+
+            assert_eq!(format!("{:?}", Magic(data)), format!("Magic({:x?})", hex::encode(data)));
+        }
+    }
+}

zebra-chain/src/parameters/network/testnet.rs

@@ -6,13 +6,15 @@ use zcash_primitives::constants as zp_constants;
 use crate::{
     block::{self, Height},
     parameters::{
-        constants::{SLOW_START_INTERVAL, SLOW_START_SHIFT},
+        constants::{magics, SLOW_START_INTERVAL, SLOW_START_SHIFT},
         network_upgrade::TESTNET_ACTIVATION_HEIGHTS,
         Network, NetworkUpgrade, NETWORK_UPGRADES_IN_ORDER,
     },
     work::difficulty::{ExpandedDifficulty, U256},
 };
 
+use super::magic::Magic;
+
 /// The Regtest NU5 activation height in tests
 // TODO: Serialize testnet parameters in Config then remove this and use a configured NU5 activation height.
 #[cfg(any(test, feature = "proptest-impl"))]
@@ -64,10 +66,12 @@ pub struct ConfiguredActivationHeights {
 }
 
 /// Builder for the [`Parameters`] struct.
-#[derive(Clone, Debug, Eq, PartialEq, Hash)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct ParametersBuilder {
     /// The name of this network to be used by the `Display` trait impl.
     network_name: String,
+    /// The network magic, acts as an identifier for the network.
+    network_magic: Magic,
     /// The genesis block hash
     genesis_hash: block::Hash,
     /// The network upgrade activation heights for this network, see [`Parameters::activation_heights`] for more details.
@@ -91,6 +95,7 @@ impl Default for ParametersBuilder {
     fn default() -> Self {
         Self {
             network_name: "UnknownTestnet".to_string(),
+            network_magic: magics::TESTNET,
             // # Correctness
             //
             // `Genesis` network upgrade activation height must always be 0
@@ -148,6 +153,20 @@ impl ParametersBuilder {
         self
     }
 
+    /// Sets the network name to be used in the [`Parameters`] being built.
+    pub fn with_network_magic(mut self, network_magic: Magic) -> Self {
+        assert!(
+            [magics::MAINNET, magics::REGTEST]
+                .into_iter()
+                .all(|reserved_magic| network_magic != reserved_magic),
+            "network magic should be distinct from reserved network magics"
+        );
+
+        self.network_magic = network_magic;
+
+        self
+    }
+
     /// Checks that the provided Sapling human-readable prefixes (HRPs) are valid and unique, then
     /// sets the Sapling HRPs to be used in the [`Parameters`] being built.
     pub fn with_sapling_hrps(
@@ -283,6 +302,7 @@ impl ParametersBuilder {
     pub fn finish(self) -> Parameters {
         let Self {
             network_name,
+            network_magic,
             genesis_hash,
             activation_heights,
             hrp_sapling_extended_spending_key,
@@ -294,6 +314,7 @@ impl ParametersBuilder {
         } = self;
         Parameters {
             network_name,
+            network_magic,
             genesis_hash,
             activation_heights,
             hrp_sapling_extended_spending_key,
@@ -313,10 +334,12 @@ impl ParametersBuilder {
 }
 
 /// Network consensus parameters for test networks such as Regtest and the default Testnet.
-#[derive(Clone, Debug, Eq, PartialEq, Hash)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct Parameters {
     /// The name of this network to be used by the `Display` trait impl.
     network_name: String,
+    /// The network magic, acts as an identifier for the network.
+    network_magic: Magic,
     /// The genesis block hash
     genesis_hash: block::Hash,
     /// The network upgrade activation heights for this network.
@@ -366,6 +389,7 @@ impl Parameters {
 
         Self {
             network_name: "Regtest".to_string(),
+            network_magic: magics::REGTEST,
             ..Self::build()
                 .with_genesis_hash(REGTEST_GENESIS_HASH)
                 // This value is chosen to match zcashd, see: <https://github.com/zcash/zcash/blob/master/src/chainparams.cpp#L654>
@@ -397,6 +421,7 @@ impl Parameters {
     pub fn is_regtest(&self) -> bool {
         let Self {
             network_name,
+            network_magic,
             genesis_hash,
             // Activation heights are configurable on Regtest
             activation_heights: _,
@@ -410,6 +435,7 @@ impl Parameters {
         } = Self::new_regtest(None);
 
         self.network_name == network_name
+            && self.network_magic == network_magic
             && self.genesis_hash == genesis_hash
             && self.hrp_sapling_extended_spending_key == hrp_sapling_extended_spending_key
             && self.hrp_sapling_extended_full_viewing_key == hrp_sapling_extended_full_viewing_key
@@ -425,6 +451,11 @@ impl Parameters {
         &self.network_name
     }
 
+    /// Returns the network magic
+    pub fn network_magic(&self) -> Magic {
+        self.network_magic
+    }
+
     /// Returns the genesis hash
     pub fn genesis_hash(&self) -> block::Hash {
         self.genesis_hash

zebra-network/src/config.rs

@@ -16,7 +16,7 @@ use tracing::Span;
 
 use zebra_chain::parameters::{
     testnet::{self, ConfiguredActivationHeights},
-    Network, NetworkKind,
+    Magic, Network, NetworkKind,
 };
 
 use crate::{
@@ -636,6 +636,7 @@ impl<'de> Deserialize<'de> for Config {
     {
         #[derive(Deserialize)]
         struct DTestnetParameters {
+            network_magic: Option<[u8; 4]>,
             network_name: Option<String>,
             activation_heights: Option<ConfiguredActivationHeights>,
         }
@@ -718,26 +719,34 @@ impl<'de> Deserialize<'de> for Config {
                 NetworkKind::Testnet,
                 Some(DTestnetParameters {
                     network_name,
+                    network_magic,
                     activation_heights,
                 }),
             ) => {
                 let mut params_builder = testnet::Parameters::build();
+                let should_avoid_default_peers =
+                    network_magic.is_some() || activation_heights.is_some();
+
+                // Return an error if the initial testnet peers includes any of the default initial Mainnet or Testnet
+                // peers while activation heights or a custom network magic is configured.
+                if should_avoid_default_peers
+                    && contains_default_initial_peers(&initial_testnet_peers)
+                {
+                    return Err(de::Error::custom(
+                        "cannot use default initials peers with incompatible testnet",
+                    ));
+                }
 
                 if let Some(network_name) = network_name {
                     params_builder = params_builder.with_network_name(network_name)
                 }
 
+                if let Some(network_magic) = network_magic {
+                    params_builder = params_builder.with_network_magic(Magic(network_magic));
+                }
+
                 // Retain default Testnet activation heights unless there's an empty [testnet_parameters.activation_heights] section.
                 if let Some(activation_heights) = activation_heights {
-                    // Return an error if the initial testnet peers includes any of the default initial Mainnet or Testnet
-                    // peers while activation heights are configured.
-                    // TODO: Check that the network magic is different from the default Mainnet/Testnet network magic too?
-                    if contains_default_initial_peers(&initial_testnet_peers) {
-                        return Err(de::Error::custom(
-                            "cannot use default initial testnet peers with configured activation heights",
-                        ));
-                    }
-
                     params_builder = params_builder.with_activation_heights(activation_heights)
                 }
 

zebra-network/src/constants.rs

@@ -422,15 +422,6 @@ lazy_static! {
 /// [6.1.3.3  Efficient Resource Usage] <https://tools.ietf.org/rfcmarkup?doc=1123#page-77>
 pub const DNS_LOOKUP_TIMEOUT: Duration = Duration::from_secs(5);
 
-/// Magic numbers used to identify different Zcash networks.
-pub mod magics {
-    use super::*;
-    /// The production mainnet.
-    pub const MAINNET: Magic = Magic([0x24, 0xe9, 0x27, 0x64]);
-    /// The testnet.
-    pub const TESTNET: Magic = Magic([0xfa, 0x1a, 0xf9, 0xbf]);
-}
-
 #[cfg(test)]
 mod tests {
     use zebra_chain::parameters::POST_BLOSSOM_POW_TARGET_SPACING;

zebra-network/src/protocol/external/codec.rs

@@ -13,7 +13,7 @@ use tokio_util::codec::{Decoder, Encoder};
 
 use zebra_chain::{
     block::{self, Block},
-    parameters::Network,
+    parameters::{Magic, Network},
     serialization::{
         sha256d, zcash_deserialize_bytes_external_count, zcash_deserialize_string_external_count,
         CompactSizeMessage, FakeWriter, ReadZcashExt, SerializationError as Error,
@@ -163,7 +163,7 @@ impl Encoder<Message> for Codec {
         let start_len = dst.len();
         {
             let dst = &mut dst.writer();
-            dst.write_all(&self.builder.network.magic_value().0[..])?;
+            dst.write_all(&self.builder.network.magic().0[..])?;
             dst.write_all(command)?;
             dst.write_u32::<LittleEndian>(body_length as u32)?;
 
@@ -389,7 +389,7 @@ impl Decoder for Codec {
                     "read header from src buffer"
                 );
 
-                if magic != self.builder.network.magic_value() {
+                if magic != self.builder.network.magic() {
                     return Err(Parse("supplied magic did not meet expectations"));
                 }
                 if body_len > self.builder.max_len {

zebra-network/src/protocol/external/codec/tests/vectors.rs

@@ -587,3 +587,23 @@ fn reject_command_and_reason_size_limits() {
         };
     }
 }
+
+/// Check that the version test vector deserialization fails when there's a network magic mismatch.
+#[test]
+fn message_with_wrong_network_magic_returns_error() {
+    let _init_guard = zebra_test::init();
+    let mut codec = Codec::builder().finish();
+    let mut bytes = BytesMut::new();
+
+    codec
+        .encode(VERSION_TEST_VECTOR.clone(), &mut bytes)
+        .expect("encoding should succeed");
+
+    let mut codec = Codec::builder()
+        .for_network(&Network::new_default_testnet())
+        .finish();
+
+    codec
+        .decode(&mut bytes)
+        .expect_err("decoding message with mismatching network magic should return an error");
+}

zebra-network/src/protocol/external/types.rs

@@ -8,35 +8,11 @@ use zebra_chain::{
     },
 };
 
-use crate::constants::{self, magics, CURRENT_NETWORK_PROTOCOL_VERSION};
+use crate::constants::{self, CURRENT_NETWORK_PROTOCOL_VERSION};
 
 #[cfg(any(test, feature = "proptest-impl"))]
 use proptest_derive::Arbitrary;
 
-/// A magic number identifying the network.
-#[derive(Copy, Clone, Eq, PartialEq)]
-#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))]
-pub struct Magic(pub [u8; 4]);
-
-impl fmt::Debug for Magic {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        f.debug_tuple("Magic").field(&hex::encode(self.0)).finish()
-    }
-}
-pub(crate) trait ParameterMagic {
-    fn magic_value(&self) -> Magic;
-}
-impl ParameterMagic for Network {
-    /// Get the magic value associated to this `Network`.
-    fn magic_value(&self) -> Magic {
-        match self {
-            Network::Mainnet => magics::MAINNET,
-            // TODO: Move `Magic` struct definition to `zebra-chain`, add it as a field in `testnet::Parameters`, and return it here.
-            Network::Testnet(_params) => magics::TESTNET,
-        }
-    }
-}
-
 /// A protocol version number.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd)]
 pub struct Version(pub u32);
@@ -168,34 +144,6 @@ impl Default for Tweak {
 #[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))]
 pub struct Filter(pub Vec<u8>);
 
-#[cfg(test)]
-mod proptest {
-
-    use proptest::prelude::*;
-
-    use super::Magic;
-
-    use crate::constants::magics;
-
-    #[test]
-    fn magic_debug() {
-        let _init_guard = zebra_test::init();
-
-        assert_eq!(format!("{:?}", magics::MAINNET), "Magic(\"24e92764\")");
-        assert_eq!(format!("{:?}", magics::TESTNET), "Magic(\"fa1af9bf\")");
-    }
-
-    proptest! {
-
-        #[test]
-        fn proptest_magic_from_array(data in any::<[u8; 4]>()) {
-            let _init_guard = zebra_test::init();
-
-            assert_eq!(format!("{:?}", Magic(data)), format!("Magic({:x?})", hex::encode(data)));
-        }
-    }
-}
-
 #[cfg(test)]
 mod test {
     use super::*;

zebra-state/src/service/read/address/utxo.rs

@@ -33,7 +33,7 @@ pub const ADDRESS_HEIGHTS_FULL_RANGE: RangeInclusive<Height> = Height(1)..=Heigh
 
 /// A convenience wrapper that efficiently stores unspent transparent outputs,
 /// and the corresponding transaction IDs.
-#[derive(Clone, Debug, Default, Eq, PartialEq, Hash)]
+#[derive(Clone, Debug, Default, Eq, PartialEq)]
 pub struct AddressUtxos {
     /// A set of unspent transparent outputs.
     utxos: BTreeMap<OutputLocation, transparent::Output>,

zebrad/tests/common/configs/v1.8.0.toml

@@ -0,0 +1,91 @@
+# Default configuration for zebrad.
+#
+# This file can be used as a skeleton for custom configs.
+#
+# Unspecified fields use default values. Optional fields are Some(field) if the
+# field is present and None if it is absent.
+#
+# This file is generated as an example using zebrad's current defaults.
+# You should set only the config options you want to keep, and delete the rest.
+# Only a subset of fields are present in the skeleton, since optional values
+# whose default is None are omitted.
+#
+# The config format (including a complete list of sections and fields) is
+# documented here:
+# https://docs.rs/zebrad/latest/zebrad/config/struct.ZebradConfig.html
+#
+# zebrad attempts to load configs in the following order:
+#
+# 1. The -c flag on the command line, e.g., `zebrad -c myconfig.toml start`;
+# 2. The file `zebrad.toml` in the users's preference directory (platform-dependent);
+# 3. The default config.
+#
+# The user's preference directory and the default path to the `zebrad` config are platform dependent,
+# based on `dirs::preference_dir`, see https://docs.rs/dirs/latest/dirs/fn.preference_dir.html :
+#
+# | Platform | Value                                 | Example                                        |
+# | -------- | ------------------------------------- | ---------------------------------------------- |
+# | Linux    | `$XDG_CONFIG_HOME` or `$HOME/.config` | `/home/alice/.config/zebrad.toml`              |
+# | macOS    | `$HOME/Library/Preferences`           | `/Users/Alice/Library/Preferences/zebrad.toml` |
+# | Windows  | `{FOLDERID_RoamingAppData}`           | `C:\Users\Alice\AppData\Local\zebrad.toml`     |
+
+[consensus]
+checkpoint_sync = true
+
+[mempool]
+eviction_memory_time = "1h"
+tx_cost_limit = 80000000
+
+[metrics]
+
+[mining]
+debug_like_zcashd = true
+
+[network]
+cache_dir = true
+crawl_new_peer_interval = "1m 1s"
+initial_mainnet_peers = [
+    "dnsseed.z.cash:8233",
+    "dnsseed.str4d.xyz:8233",
+    "mainnet.seeder.zfnd.org:8233",
+    "mainnet.is.yolo.money:8233",
+]
+initial_testnet_peers = []
+listen_addr = "0.0.0.0:8233"
+max_connections_per_ip = 1
+network = "Testnet"
+peerset_initial_target_size = 25
+
+[network.testnet_parameters]
+network_name = "ConfiguredTestnet_1"
+network_magic = [0, 0, 0, 0]
+
+[network.testnet_parameters.activation_heights]
+BeforeOverwinter = 1
+Overwinter = 207_500
+Sapling = 280_000
+Blossom = 584_000
+Heartwood = 903_800
+Canopy = 1_028_500
+NU5 = 1_842_420
+
+[rpc]
+debug_force_finished_sync = false
+parallel_cpu_threads = 0
+
+[state]
+cache_dir = "cache_dir"
+delete_old_database = true
+ephemeral = false
+
+[sync]
+checkpoint_verify_concurrency_limit = 1000
+download_concurrency_limit = 50
+full_verify_concurrency_limit = 20
+parallel_cpu_threads = 0
+
+[tracing]
+buffer_limit = 128000
+force_use_color = false
+use_color = true
+use_journald = false