change(consensus): Allow configurable NU5 activation height on Regtest (#8505)

* Allow configurable Nu5 activation height on Regtest

* Fixes test

* removes outdated TODO

* Adds `current_with_activation_height()` method and uses it in `Commitment::from_bytes()`

zebra-chain/src/block/commitment.rs

@@ -109,25 +109,34 @@ impl Commitment {
         use Commitment::*;
         use CommitmentError::*;
 
-        match NetworkUpgrade::current(network, height) {
+        match NetworkUpgrade::current_with_activation_height(network, height) {
-            Genesis | BeforeOverwinter | Overwinter => Ok(PreSaplingReserved(bytes)),
+            (Genesis | BeforeOverwinter | Overwinter, _) => Ok(PreSaplingReserved(bytes)),
-            Sapling | Blossom => match sapling::tree::Root::try_from(bytes) {
+            (Sapling | Blossom, _) => match sapling::tree::Root::try_from(bytes) {
                 Ok(root) => Ok(FinalSaplingRoot(root)),
                 _ => Err(InvalidSapingRootBytes),
             },
-            // NetworkUpgrade::current() returns the latest network upgrade that's activated at the provided height, so
+            (Heartwood, activation_height) if height == activation_height => {
-            // on Regtest for heights above height 0, it returns NU5, and it's possible for the current network upgrade
-            // to be NU5 (or Canopy, or any network upgrade above Heartwood) at the Heartwood activation height.
-            // TODO: Check Canopy too once Zebra can construct Canopy block templates.
-            Heartwood | Nu5 if Some(height) == Heartwood.activation_height(network) => {
                 if bytes == CHAIN_HISTORY_ACTIVATION_RESERVED {
                     Ok(ChainHistoryActivationReserved)
                 } else {
                     Err(InvalidChainHistoryActivationReserved { actual: bytes })
                 }
             }
-            Heartwood | Canopy => Ok(ChainHistoryRoot(ChainHistoryMmrRootHash(bytes))),
+            // NetworkUpgrade::current() returns the latest network upgrade that's activated at the provided height, so
-            Nu5 => Ok(ChainHistoryBlockTxAuthCommitment(
+            // on Regtest for heights above height 0, it returns NU5, and it's possible for the current network upgrade
+            // to be NU5 (or Canopy, or any network upgrade above Heartwood) at the Heartwood activation height.
+            (Canopy | Nu5, activation_height)
+                if height == activation_height
+                    && Some(height) == Heartwood.activation_height(network) =>
+            {
+                if bytes == CHAIN_HISTORY_ACTIVATION_RESERVED {
+                    Ok(ChainHistoryActivationReserved)
+                } else {
+                    Err(InvalidChainHistoryActivationReserved { actual: bytes })
+                }
+            }
+            (Heartwood | Canopy, _) => Ok(ChainHistoryRoot(ChainHistoryMmrRootHash(bytes))),
+            (Nu5, _) => Ok(ChainHistoryBlockTxAuthCommitment(
                 ChainHistoryBlockTxAuthCommitmentHash(bytes),
             )),
         }

zebra-chain/src/parameters/network.rs

@@ -167,8 +167,8 @@ impl Network {
     }
 
     /// Creates a new [`Network::Testnet`] with `Regtest` parameters and the provided network upgrade activation heights.
-    pub fn new_regtest() -> Self {
+    pub fn new_regtest(nu5_activation_height: Option<u32>) -> Self {
-        Self::new_configured_testnet(testnet::Parameters::new_regtest())
+        Self::new_configured_testnet(testnet::Parameters::new_regtest(nu5_activation_height))
     }
 
     /// Returns true if the network is the default Testnet, or false otherwise.

zebra-chain/src/parameters/network/testnet.rs

@@ -11,6 +11,11 @@ use crate::{
     },
 };
 
+/// The Regtest NU5 activation height in tests
+// TODO: Serialize testnet parameters in Config then remove this and use a configured NU5 activation height.
+#[cfg(any(test, feature = "proptest-impl"))]
+pub const REGTEST_NU5_ACTIVATION_HEIGHT: u32 = 100;
+
 /// Reserved network names that should not be allowed for configured Testnets.
 pub const RESERVED_NETWORK_NAMES: [&str; 6] = [
     "Mainnet",
@@ -175,9 +180,7 @@ impl ParametersBuilder {
     pub fn with_activation_heights(
         mut self,
         ConfiguredActivationHeights {
-            // TODO: Find out if `BeforeOverwinter` is required at Height(1), allow for
+            before_overwinter,
-            //       configuring its activation height if it's not required to be at Height(1)
-            before_overwinter: _,
             overwinter,
             sapling,
             blossom,
@@ -192,9 +195,10 @@ impl ParametersBuilder {
         //
         // These must be in order so that later network upgrades overwrite prior ones
         // if multiple network upgrades are configured with the same activation height.
-        let activation_heights: BTreeMap<_, _> = overwinter
+        let activation_heights: BTreeMap<_, _> = before_overwinter
             .into_iter()
-            .map(|h| (h, Overwinter))
+            .map(|h| (h, BeforeOverwinter))
+            .chain(overwinter.into_iter().map(|h| (h, Overwinter)))
             .chain(sapling.into_iter().map(|h| (h, Sapling)))
             .chain(blossom.into_iter().map(|h| (h, Blossom)))
             .chain(heartwood.into_iter().map(|h| (h, Heartwood)))
@@ -227,8 +231,7 @@ impl ParametersBuilder {
         // # Correctness
         //
         // Height(0) must be reserved for the `NetworkUpgrade::Genesis`.
-        // TODO: Find out if `BeforeOverwinter` must always be active at Height(1), remove it here if it's not required.
+        self.activation_heights.split_off(&Height(1));
-        self.activation_heights.split_off(&Height(2));
         self.activation_heights.extend(activation_heights);
 
         self
@@ -310,7 +313,10 @@ impl Parameters {
     /// Accepts a [`ConfiguredActivationHeights`].
     ///
     /// Creates an instance of [`Parameters`] with `Regtest` values.
-    pub fn new_regtest() -> Self {
+    pub fn new_regtest(nu5_activation_height: Option<u32>) -> Self {
+        #[cfg(any(test, feature = "proptest-impl"))]
+        let nu5_activation_height = nu5_activation_height.or(Some(100));
+
         Self {
             network_name: "Regtest".to_string(),
             ..Self::build()
@@ -324,7 +330,8 @@ impl Parameters {
                 // Removes default Testnet activation heights if not configured,
                 // most network upgrades are disabled by default for Regtest in zcashd
                 .with_activation_heights(ConfiguredActivationHeights {
-                    nu5: Some(1),
+                    canopy: Some(1),
+                    nu5: nu5_activation_height,
                     ..Default::default()
                 })
                 .finish()
@@ -347,7 +354,7 @@ impl Parameters {
             hrp_sapling_extended_full_viewing_key,
             hrp_sapling_payment_address,
             disable_pow,
-        } = Self::new_regtest();
+        } = Self::new_regtest(None);
 
         self.network_name == network_name
             && self.genesis_hash == genesis_hash

zebra-chain/src/parameters/network/tests/vectors.rs

@@ -135,14 +135,16 @@ fn activates_network_upgrades_correctly() {
 
     let expected_default_regtest_activation_heights = &[
         (Height(0), NetworkUpgrade::Genesis),
-        (Height(1), NetworkUpgrade::Nu5),
+        (Height(1), NetworkUpgrade::Canopy),
+        // TODO: Remove this once the testnet parameters are being serialized.
+        (Height(100), NetworkUpgrade::Nu5),
     ];
 
     for (network, expected_activation_heights) in [
         (Network::Mainnet, MAINNET_ACTIVATION_HEIGHTS),
         (Network::new_default_testnet(), TESTNET_ACTIVATION_HEIGHTS),
         (
-            Network::new_regtest(),
+            Network::new_regtest(None),
             expected_default_regtest_activation_heights,
         ),
     ] {
@@ -193,7 +195,7 @@ fn check_configured_network_name() {
         "Mainnet should be displayed as 'Mainnet'"
     );
     assert_eq!(
-        Network::new_regtest().to_string(),
+        Network::new_regtest(None).to_string(),
         "Regtest",
         "Regtest should be displayed as 'Regtest'"
     );
@@ -238,6 +240,7 @@ fn check_configured_sapling_hrps() {
         .expect_err("should panic when setting Sapling HRPs that are too long or contain non-alphanumeric characters (except '-')");
     }
 
+    // Restore the regular panic hook for any unexpected panics
     drop(std::panic::take_hook());
 
     // Check that Sapling HRPs can contain lowercase ascii characters and dashes.
@@ -317,6 +320,7 @@ fn check_network_name() {
         .expect_err("should panic when setting network name that's too long or contains non-alphanumeric characters (except '_')");
     }
 
+    // Restore the regular panic hook for any unexpected panics
     drop(std::panic::take_hook());
 
     // Checks that network names are displayed correctly

zebra-chain/src/parameters/network_upgrade.rs

@@ -280,6 +280,19 @@ impl Network {
 }
 
 impl NetworkUpgrade {
+    /// Returns the current network upgrade and its activation height for `network` and `height`.
+    pub fn current_with_activation_height(
+        network: &Network,
+        height: block::Height,
+    ) -> (NetworkUpgrade, block::Height) {
+        network
+            .activation_list()
+            .range(..=height)
+            .map(|(&h, &nu)| (nu, h))
+            .next_back()
+            .expect("every height has a current network upgrade")
+    }
+
     /// Returns the current network upgrade for `network` and `height`.
     pub fn current(network: &Network, height: block::Height) -> NetworkUpgrade {
         network

zebra-consensus/src/checkpoint/list/tests.rs

@@ -237,7 +237,7 @@ fn checkpoint_list_load_hard_coded() -> Result<(), BoxError> {
 
     let _ = Mainnet.checkpoint_list();
     let _ = Network::new_default_testnet().checkpoint_list();
-    let _ = Network::new_regtest().checkpoint_list();
+    let _ = Network::new_regtest(None).checkpoint_list();
 
     Ok(())
 }

zebra-network/src/config.rs

@@ -707,7 +707,13 @@ impl<'de> Deserialize<'de> for Config {
         let network = match (network_kind, testnet_parameters) {
             (NetworkKind::Mainnet, _) => Network::Mainnet,
             (NetworkKind::Testnet, None) => Network::new_default_testnet(),
-            (NetworkKind::Regtest, _) => Network::new_regtest(),
+            (NetworkKind::Regtest, testnet_parameters) => {
+                let nu5_activation_height = testnet_parameters
+                    .and_then(|params| params.activation_heights)
+                    .and_then(|activation_height| activation_height.nu5);
+
+                Network::new_regtest(nu5_activation_height)
+            }
             (
                 NetworkKind::Testnet,
                 Some(DTestnetParameters {

zebra-network/src/constants.rs

@@ -399,7 +399,7 @@ lazy_static! {
 
         hash_map.insert(NetworkKind::Mainnet, Version::min_specified_for_upgrade(&Mainnet, Nu5));
         hash_map.insert(NetworkKind::Testnet, Version::min_specified_for_upgrade(&Network::new_default_testnet(), Nu5));
-        hash_map.insert(NetworkKind::Regtest, Version::min_specified_for_upgrade(&Network::new_regtest(), Nu5));
+        hash_map.insert(NetworkKind::Regtest, Version::min_specified_for_upgrade(&Network::new_regtest(None), Nu5));
 
         hash_map
     };

zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/proposal.rs

@@ -6,6 +6,7 @@ use std::{num::ParseIntError, str::FromStr, sync::Arc};
 
 use zebra_chain::{
     block::{self, Block, Height},
+    parameters::NetworkUpgrade,
     serialization::{DateTime32, SerializationError, ZcashDeserializeInto},
     work::equihash::Solution,
 };
@@ -167,6 +168,7 @@ impl FromStr for TimeSource {
 pub fn proposal_block_from_template(
     template: &GetBlockTemplate,
     time_source: impl Into<Option<TimeSource>>,
+    network_upgrade: NetworkUpgrade,
 ) -> Result<Block, SerializationError> {
     let GetBlockTemplate {
         version,
@@ -176,6 +178,7 @@ pub fn proposal_block_from_template(
             DefaultRoots {
                 merkle_root,
                 block_commitments_hash,
+                chain_history_root,
                 ..
             },
         bits: difficulty_threshold,
@@ -201,12 +204,23 @@ pub fn proposal_block_from_template(
         transactions.push(tx_template.data.as_ref().zcash_deserialize_into()?);
     }
 
+    let commitment_bytes = match network_upgrade {
+        NetworkUpgrade::Genesis
+        | NetworkUpgrade::BeforeOverwinter
+        | NetworkUpgrade::Overwinter
+        | NetworkUpgrade::Sapling
+        | NetworkUpgrade::Blossom
+        | NetworkUpgrade::Heartwood => panic!("pre-Canopy block templates not supported"),
+        NetworkUpgrade::Canopy => chain_history_root.bytes_in_serialized_order().into(),
+        NetworkUpgrade::Nu5 => block_commitments_hash.bytes_in_serialized_order().into(),
+    };
+
     Ok(Block {
         header: Arc::new(block::Header {
             version,
             previous_block_hash,
             merkle_root,
-            commitment_bytes: block_commitments_hash.bytes_in_serialized_order().into(),
+            commitment_bytes,
             time: time.into(),
             difficulty_threshold,
             nonce: [0; 32].into(),

zebra-utils/src/bin/block-template-to-proposal/main.rs

@@ -10,7 +10,10 @@ use color_eyre::eyre::Result;
 use serde_json::Value;
 use structopt::StructOpt;
 
-use zebra_chain::serialization::{DateTime32, ZcashSerialize};
+use zebra_chain::{
+    parameters::NetworkUpgrade,
+    serialization::{DateTime32, ZcashSerialize},
+};
 use zebra_rpc::methods::get_block_template_rpcs::{
     get_block_template::proposal_block_from_template,
     types::{get_block_template::GetBlockTemplate, long_poll::LONG_POLL_ID_LENGTH},
@@ -96,7 +99,7 @@ fn main() -> Result<()> {
     let template: GetBlockTemplate = serde_json::from_value(template)?;
 
     // generate proposal according to arguments
-    let proposal = proposal_block_from_template(&template, time_source)?;
+    let proposal = proposal_block_from_template(&template, time_source, NetworkUpgrade::Nu5)?;
     eprintln!("{proposal:#?}");
 
     let proposal = proposal

zebrad/src/components/miner.rs

@@ -20,6 +20,7 @@ use zebra_chain::{
     chain_sync_status::ChainSyncStatus,
     chain_tip::ChainTip,
     diagnostic::task::WaitForPanics,
+    parameters::NetworkUpgrade,
     serialization::{AtLeastOne, ZcashSerialize},
     shutdown::is_shutting_down,
     work::equihash::{Solution, SolverCancelled},
@@ -290,7 +291,8 @@ where
         // Tell the next get_block_template() call to wait until the template has changed.
         parameters.long_poll_id = Some(template.long_poll_id);
 
-        let block = proposal_block_from_template(&template, TimeSource::CurTime)
+        let block =
+            proposal_block_from_template(&template, TimeSource::CurTime, NetworkUpgrade::Nu5)
                 .expect("unexpected invalid block template");
 
         // If the template has actually changed, send an updated template.

zebrad/tests/acceptance.rs

@@ -3108,7 +3108,7 @@ async fn scan_task_commands() -> Result<()> {
 async fn validate_regtest_genesis_block() {
     let _init_guard = zebra_test::init();
 
-    let network = Network::new_regtest();
+    let network = Network::new_regtest(None);
     let state = zebra_state::init_test(&network);
     let (
         block_verifier_router,

zebrad/tests/common/get_block_template_rpcs/get_block_template.rs

@@ -11,7 +11,10 @@ use color_eyre::eyre::{eyre, Context, Result};
 
 use futures::FutureExt;
 
-use zebra_chain::{parameters::Network, serialization::ZcashSerialize};
+use zebra_chain::{
+    parameters::{Network, NetworkUpgrade},
+    serialization::ZcashSerialize,
+};
 use zebra_node_services::rpc_client::RpcRequestClient;
 use zebra_rpc::methods::get_block_template_rpcs::{
     get_block_template::{
@@ -214,7 +217,11 @@ async fn try_validate_block_template(client: &RpcRequestClient) -> Result<()> {
             // Propose a new block with an empty solution and nonce field
 
             let raw_proposal_block = hex::encode(
-                proposal_block_from_template(&response_json_result, time_source)?
+                proposal_block_from_template(
+                    &response_json_result,
+                    time_source,
+                    NetworkUpgrade::Nu5,
+                )?
                 .zcash_serialize_to_vec()?,
             );
             let template = response_json_result.clone();

zebrad/tests/common/regtest.rs

@@ -8,7 +8,10 @@ use std::{net::SocketAddr, time::Duration};
 use color_eyre::eyre::{Context, Result};
 use tracing::*;
 
-use zebra_chain::{parameters::Network, serialization::ZcashSerialize};
+use zebra_chain::{
+    parameters::{testnet::REGTEST_NU5_ACTIVATION_HEIGHT, Network, NetworkUpgrade},
+    serialization::ZcashSerialize,
+};
 use zebra_node_services::rpc_client::RpcRequestClient;
 use zebra_rpc::methods::get_block_template_rpcs::get_block_template::{
     proposal::TimeSource, proposal_block_from_template, GetBlockTemplate,
@@ -27,7 +30,7 @@ pub(crate) async fn submit_blocks_test() -> Result<()> {
     let _init_guard = zebra_test::init();
     info!("starting regtest submit_blocks test");
 
-    let network = Network::new_regtest();
+    let network = Network::new_regtest(None);
     let mut config = random_known_rpc_port_config(false, &network)?;
     config.mempool.debug_enable_at_height = Some(0);
     let rpc_address = config.rpc.listen_addr.unwrap();
@@ -58,14 +61,20 @@ pub(crate) async fn submit_blocks_test() -> Result<()> {
 async fn submit_blocks(rpc_address: SocketAddr) -> Result<()> {
     let client = RpcRequestClient::new(rpc_address);
 
-    for _ in 0..NUM_BLOCKS_TO_SUBMIT {
+    for height in 1..=NUM_BLOCKS_TO_SUBMIT {
         let block_template: GetBlockTemplate = client
             .json_result_from_call("getblocktemplate", "[]".to_string())
             .await
             .expect("response should be success output with a serialized `GetBlockTemplate`");
 
+        let network_upgrade = if height < REGTEST_NU5_ACTIVATION_HEIGHT.try_into().unwrap() {
+            NetworkUpgrade::Canopy
+        } else {
+            NetworkUpgrade::Nu5
+        };
+
         let block_data = hex::encode(
-            proposal_block_from_template(&block_template, TimeSource::default())?
+            proposal_block_from_template(&block_template, TimeSource::default(), network_upgrade)?
                 .zcash_serialize_to_vec()?,
         );
 
@@ -75,7 +84,14 @@ async fn submit_blocks(rpc_address: SocketAddr) -> Result<()> {
 
         let was_submission_successful = submit_block_response.contains(r#""result":null"#);
 
-        info!(was_submission_successful, "submitted block");
+        if height % 40 == 0 {
+            info!(
+                was_submission_successful,
+                ?block_template,
+                ?network_upgrade,
+                "submitted block"
+            );
+        }
 
         // Check that the block was validated and committed.
         assert!(