add(consensus): Adds block construction and validation for NU5 from block height 1 for Regtest (#8475)

* Always activate Canopy at Height(1) on Regtest

* Ignores the zip 212 grace period on configured Testnets and Regtest

* - Returns early when there is no Heartwood activation height when creating or updating HistoryTree
- Skips call to `check::legacy_chain()` when no NU5 activation height is set (it would return immediately anyway)
- Replaces `.map()` with `.filter_map()` in `NetworkUpgrade::target_spacings()`
- Removes outdated TODO

* - When proof of work is disabled, skips checking if Zebra is synced to the network tip in the getblocktemplate method

* Sets full_verifier_utxo_lookahead to Height::MIN instead of panicking

* When network is regtest, skips starting sync task and commits the genesis block if it's missing in the state

* updates/fixes test config

* Adds test for committing Canopy blocks on Regtest

* - Updates median time past and difficulty checks to use fewer than 11/17/28 blocks

* uses SLOW_START_INTERVAL of 0 if PoW is disabled, adds TODOs

* Update getblocktemplate method to return reserved chain history activation root hash, uses Nu5 at height 1

Test passes.

* Updates test to expect NU5 as the default nu activation at Height(1)

* Removes invalid difficulty snapshot

* fixes tests

* removes regtest NU5 activation height config field

* Apply suggestions from code review

* Update zebra-state/src/service/check/difficulty.rs

Co-authored-by: Marek <mail@marek.onl>

* Update zebra-state/src/service/check/difficulty.rs

* Updates docs/comments, renames an argument

---------

Co-authored-by: Marek <mail@marek.onl>

zebra-chain/src/block/commitment.rs

@@ -115,7 +115,11 @@ impl Commitment {
                 Ok(root) => Ok(FinalSaplingRoot(root)),
                 _ => Err(InvalidSapingRootBytes),
             },
-            Heartwood if Some(height) == Heartwood.activation_height(network) => {
+            // NetworkUpgrade::current() returns the latest network upgrade that's activated at the provided height, so
+            // on Regtest for heights above height 0, it returns NU5, and it's possible for the current network upgrade
+            // to be NU5 (or Canopy, or any network upgrade above Heartwood) at the Heartwood activation height.
+            // TODO: Check Canopy too once Zebra can construct Canopy block templates.
+            Heartwood | Nu5 if Some(height) == Heartwood.activation_height(network) => {
                 if bytes == CHAIN_HISTORY_ACTIVATION_RESERVED {
                     Ok(ChainHistoryActivationReserved)
                 } else {

zebra-chain/src/history_tree.rs

@@ -428,9 +428,11 @@ impl HistoryTree {
         sapling_root: &sapling::tree::Root,
         orchard_root: &orchard::tree::Root,
     ) -> Result<Self, HistoryTreeError> {
-        let heartwood_height = NetworkUpgrade::Heartwood
+        let Some(heartwood_height) = NetworkUpgrade::Heartwood.activation_height(network) else {
-            .activation_height(network)
+            // Return early if there is no Heartwood activation height.
-            .expect("Heartwood height is known");
+            return Ok(HistoryTree(None));
+        };
+
         match block
             .coinbase_height()
             .expect("must have height")

zebra-chain/src/parameters/network.rs

@@ -9,8 +9,6 @@ use crate::{
     parameters::NetworkUpgrade,
 };
 
-use self::testnet::ConfiguredActivationHeights;
-
 pub mod testnet;
 
 #[cfg(test)]
@@ -169,8 +167,8 @@ impl Network {
     }
 
     /// Creates a new [`Network::Testnet`] with `Regtest` parameters and the provided network upgrade activation heights.
-    pub fn new_regtest(activation_heights: ConfiguredActivationHeights) -> Self {
+    pub fn new_regtest() -> Self {
-        Self::new_configured_testnet(testnet::Parameters::new_regtest(activation_heights))
+        Self::new_configured_testnet(testnet::Parameters::new_regtest())
     }
 
     /// Returns true if the network is the default Testnet, or false otherwise.
@@ -203,14 +201,13 @@ impl Network {
     pub fn kind(&self) -> NetworkKind {
         match self {
             Network::Mainnet => NetworkKind::Mainnet,
-            // TODO: Return `NetworkKind::Regtest` if the parameters match the default Regtest params
+            Network::Testnet(params) if params.is_regtest() => NetworkKind::Regtest,
             Network::Testnet(_) => NetworkKind::Testnet,
         }
     }
 
     /// Returns an iterator over [`Network`] variants.
     pub fn iter() -> impl Iterator<Item = Self> {
-        // TODO: Use default values of `Testnet` variant when adding fields for #7845.
         [Self::Mainnet, Self::new_default_testnet()].into_iter()
     }
 
@@ -244,6 +241,12 @@ impl Network {
     /// Mandatory checkpoints are a Zebra-specific feature.
     /// If a Zcash consensus rule only applies before the mandatory checkpoint,
     /// Zebra can skip validation of that rule.
+    ///
+    /// ZIP-212 grace period is only applied to default networks.
+    // TODO:
+    // - Support constructing pre-Canopy coinbase tx and block templates and return `Height::MAX` instead of panicking
+    //   when Canopy activation height is `None` (#8434)
+    // - Add semantic block validation during the ZIP-212 grace period and update this method to always apply the ZIP-212 grace period
     pub fn mandatory_checkpoint_height(&self) -> Height {
         // Currently this is after the ZIP-212 grace period.
         //
@@ -253,8 +256,19 @@ impl Network {
             .activation_height(self)
             .expect("Canopy activation height must be present for both networks");
 
-        (canopy_activation + ZIP_212_GRACE_PERIOD_DURATION)
+        let is_a_default_network = match self {
-            .expect("ZIP-212 grace period ends at a valid block height")
+            Network::Mainnet => true,
+            Network::Testnet(params) => params.is_default_testnet(),
+        };
+
+        if is_a_default_network {
+            (canopy_activation + ZIP_212_GRACE_PERIOD_DURATION)
+                .expect("ZIP-212 grace period ends at a valid block height")
+        } else {
+            canopy_activation
+                .previous()
+                .expect("Canopy activation must be above Genesis height")
+        }
     }
 
     /// Return the network name as defined in

zebra-chain/src/parameters/network/testnet.rs

@@ -310,7 +310,7 @@ impl Parameters {
     /// Accepts a [`ConfiguredActivationHeights`].
     ///
     /// Creates an instance of [`Parameters`] with `Regtest` values.
-    pub fn new_regtest(activation_heights: ConfiguredActivationHeights) -> Self {
+    pub fn new_regtest() -> Self {
         Self {
             network_name: "Regtest".to_string(),
             ..Self::build()
@@ -323,7 +323,10 @@ impl Parameters {
                 )
                 // Removes default Testnet activation heights if not configured,
                 // most network upgrades are disabled by default for Regtest in zcashd
-                .with_activation_heights(activation_heights)
+                .with_activation_heights(ConfiguredActivationHeights {
+                    nu5: Some(1),
+                    ..Default::default()
+                })
                 .finish()
         }
     }
@@ -344,7 +347,7 @@ impl Parameters {
             hrp_sapling_extended_full_viewing_key,
             hrp_sapling_payment_address,
             disable_pow,
-        } = Self::new_regtest(ConfiguredActivationHeights::default());
+        } = Self::new_regtest();
 
         self.network_name == network_name
             && self.genesis_hash == genesis_hash

zebra-chain/src/parameters/network/tests/vectors.rs

@@ -135,14 +135,14 @@ fn activates_network_upgrades_correctly() {
 
     let expected_default_regtest_activation_heights = &[
         (Height(0), NetworkUpgrade::Genesis),
-        (Height(1), NetworkUpgrade::BeforeOverwinter),
+        (Height(1), NetworkUpgrade::Nu5),
     ];
 
     for (network, expected_activation_heights) in [
         (Network::Mainnet, MAINNET_ACTIVATION_HEIGHTS),
         (Network::new_default_testnet(), TESTNET_ACTIVATION_HEIGHTS),
         (
-            Network::new_regtest(Default::default()),
+            Network::new_regtest(),
             expected_default_regtest_activation_heights,
         ),
     ] {
@@ -193,7 +193,7 @@ fn check_configured_network_name() {
         "Mainnet should be displayed as 'Mainnet'"
     );
     assert_eq!(
-        Network::new_regtest(Default::default()).to_string(),
+        Network::new_regtest().to_string(),
         "Regtest",
         "Regtest should be displayed as 'Regtest'"
     );

zebra-chain/src/parameters/network_upgrade.rs

@@ -402,14 +402,10 @@ impl NetworkUpgrade {
             ),
         ]
         .into_iter()
-        .map(move |(upgrade, spacing_seconds)| {
+        .filter_map(move |(upgrade, spacing_seconds)| {
-            let activation_height = upgrade
+            let activation_height = upgrade.activation_height(network)?;
-                .activation_height(network)
-                .expect("missing activation height for target spacing change");
-
             let target_spacing = Duration::seconds(spacing_seconds);
-
+            Some((activation_height, target_spacing))
-            (activation_height, target_spacing)
         })
     }
 

zebra-chain/src/work/difficulty.rs

@@ -699,7 +699,7 @@ impl ParameterDifficulty for Network {
             /* 2^243 - 1 */
             Network::Mainnet => (U256::one() << 243) - 1,
             /* 2^251 - 1 */
-            // TODO: Add a `target_difficulty_limit` field to `testnet::Parameters` to return here.
+            // TODO: Add a `target_difficulty_limit` field to `testnet::Parameters` to return here. (`U256::from_big_endian(&[0x0f].repeat(8))` for Regtest)
             Network::Testnet(_params) => (U256::one() << 251) - 1,
         };
 

zebra-consensus/src/block/check.rs

@@ -157,10 +157,18 @@ pub fn subsidy_is_valid(block: &Block, network: &Network) -> Result<(), BlockErr
         .activation_height(network)
         .expect("Canopy activation height is known");
 
-    if height < SLOW_START_INTERVAL {
+    // TODO: Add this as a field on `testnet::Parameters` instead of checking `disable_pow()`, this is 0 for Regtest in zcashd,
+    //       see <https://github.com/zcash/zcash/blob/master/src/chainparams.cpp#L640>
+    let slow_start_interval = if network.disable_pow() {
+        Height(0)
+    } else {
+        SLOW_START_INTERVAL
+    };
+
+    if height < slow_start_interval {
         unreachable!(
             "unsupported block height: callers should handle blocks below {:?}",
-            SLOW_START_INTERVAL
+            slow_start_interval
         )
     } else if halving_div.count_ones() != 1 {
         unreachable!("invalid halving divisor: the halving divisor must be a non-zero power of two")

zebra-consensus/src/block/subsidy/general.rs

@@ -25,12 +25,20 @@ pub fn halving_divisor(height: Height, network: &Network) -> Option<u64> {
         .activation_height(network)
         .expect("blossom activation height should be available");
 
-    if height < SLOW_START_SHIFT {
+    // TODO: Add this as a field on `testnet::Parameters` instead of checking `disable_pow()`, this is 0 for Regtest in zcashd,
+    //       see <https://github.com/zcash/zcash/blob/master/src/chainparams.cpp#L640>
+    let slow_start_shift = if network.disable_pow() {
+        Height(0)
+    } else {
+        SLOW_START_SHIFT
+    };
+
+    if height < slow_start_shift {
         unreachable!(
-            "unsupported block height {height:?}: checkpoints should handle blocks below {SLOW_START_SHIFT:?}",
+            "unsupported block height {height:?}: checkpoints should handle blocks below {slow_start_shift:?}",
         )
     } else if height < blossom_height {
-        let pre_blossom_height = height - SLOW_START_SHIFT;
+        let pre_blossom_height = height - slow_start_shift;
         let halving_shift = pre_blossom_height / PRE_BLOSSOM_HALVING_INTERVAL;
 
         let halving_div = 1u64
@@ -43,7 +51,7 @@ pub fn halving_divisor(height: Height, network: &Network) -> Option<u64> {
 
         Some(halving_div)
     } else {
-        let pre_blossom_height = blossom_height - SLOW_START_SHIFT;
+        let pre_blossom_height = blossom_height - slow_start_shift;
         let scaled_pre_blossom_height =
             pre_blossom_height * HeightDiff::from(BLOSSOM_POW_TARGET_SPACING_RATIO);
 
@@ -77,7 +85,9 @@ pub fn block_subsidy(height: Height, network: &Network) -> Result<Amount<NonNega
         return Ok(Amount::zero());
     };
 
-    if height < SLOW_START_INTERVAL {
+    // TODO: Add this as a field on `testnet::Parameters` instead of checking `disable_pow()`, this is 0 for Regtest in zcashd,
+    //       see <https://github.com/zcash/zcash/blob/master/src/chainparams.cpp#L640>
+    if height < SLOW_START_INTERVAL && !network.disable_pow() {
         unreachable!(
             "unsupported block height {height:?}: callers should handle blocks below {SLOW_START_INTERVAL:?}",
         )

zebra-consensus/src/checkpoint/list/tests.rs

@@ -237,7 +237,7 @@ fn checkpoint_list_load_hard_coded() -> Result<(), BoxError> {
 
     let _ = Mainnet.checkpoint_list();
     let _ = Network::new_default_testnet().checkpoint_list();
-    let _ = Network::new_regtest(Default::default()).checkpoint_list();
+    let _ = Network::new_regtest().checkpoint_list();
 
     Ok(())
 }

zebra-consensus/src/parameters/subsidy.rs

@@ -109,12 +109,11 @@ lazy_static! {
         let mut hash_map = HashMap::new();
         hash_map.insert(NetworkKind::Mainnet, Height(1_046_400)..Height(2_726_400));
         hash_map.insert(NetworkKind::Testnet, Height(1_028_500)..Height(2_796_000));
+        hash_map.insert(NetworkKind::Regtest, Height(1_028_500)..Height(2_796_000));
         hash_map
     };
 
     /// Convenient storage for all addresses, for all receivers and networks
-    // TODO: Move the value here to a field on `testnet::Parameters` (#8367)
-    //       There are no funding stream addresses on Regtest in zcashd, zebrad should do the same for compatibility.
     pub static ref FUNDING_STREAM_ADDRESSES: HashMap<NetworkKind, HashMap<FundingStreamReceiver, Vec<String>>> = {
         let mut addresses_by_network = HashMap::with_capacity(2);
 
@@ -132,6 +131,16 @@ lazy_static! {
         testnet_addresses.insert(FundingStreamReceiver::MajorGrants, FUNDING_STREAM_MG_ADDRESSES_TESTNET.iter().map(|a| a.to_string()).collect());
         addresses_by_network.insert(NetworkKind::Testnet, testnet_addresses);
 
+
+        // Regtest addresses
+        // TODO: Move the value here to a field on `testnet::Parameters` (#8367)
+        //       There are no funding stream addresses on Regtest in zcashd, zebrad should do the same for compatibility.
+        let mut regtest_addresses = HashMap::with_capacity(3);
+        regtest_addresses.insert(FundingStreamReceiver::Ecc, FUNDING_STREAM_ECC_ADDRESSES_TESTNET.iter().map(|a| a.to_string()).collect());
+        regtest_addresses.insert(FundingStreamReceiver::ZcashFoundation, FUNDING_STREAM_ZF_ADDRESSES_TESTNET.iter().map(|a| a.to_string()).collect());
+        regtest_addresses.insert(FundingStreamReceiver::MajorGrants, FUNDING_STREAM_MG_ADDRESSES_TESTNET.iter().map(|a| a.to_string()).collect());
+        addresses_by_network.insert(NetworkKind::Testnet, regtest_addresses);
+
         addresses_by_network
     };
 }

zebra-consensus/src/transaction/check.rs

@@ -356,7 +356,6 @@ pub fn coinbase_expiry_height(
 ) -> Result<(), TransactionError> {
     let expiry_height = coinbase.expiry_height();
 
-    // TODO: replace `if let` with `expect` after NU5 mainnet activation
     if let Some(nu5_activation_height) = NetworkUpgrade::Nu5.activation_height(network) {
         // # Consensus
         //

zebra-network/src/config.rs

@@ -230,9 +230,7 @@ impl Config {
             Network::Testnet(params) if params.is_default_testnet() => {
                 self.initial_testnet_peers.clone()
             }
-            // TODO: Check if the network is an incompatible custom testnet (_not_ Regtest), then panic if `initial_testnet_peers`
+            // TODO: Add a `disable_peers` field to `Network` to check instead of `is_default_testnet()` (#8361)
-            //       contains any of the default testnet peers, or return `initial_testnet_peers` otherwise. See:
-            //       <https://github.com/ZcashFoundation/zebra/pull/7924#discussion_r1385881828>
             Network::Testnet(_params) => IndexSet::new(),
         }
     }
@@ -639,7 +637,6 @@ impl<'de> Deserialize<'de> for Config {
             listen_addr: String,
             network: NetworkKind,
             testnet_parameters: Option<DTestnetParameters>,
-            regtest_activation_heights: ConfiguredActivationHeights,
             initial_mainnet_peers: IndexSet<String>,
             initial_testnet_peers: IndexSet<String>,
             cache_dir: CacheDir,
@@ -656,7 +653,6 @@ impl<'de> Deserialize<'de> for Config {
                     listen_addr: "0.0.0.0".to_string(),
                     network: Default::default(),
                     testnet_parameters: None,
-                    regtest_activation_heights: ConfiguredActivationHeights::default(),
                     initial_mainnet_peers: config.initial_mainnet_peers,
                     initial_testnet_peers: config.initial_testnet_peers,
                     cache_dir: config.cache_dir,
@@ -671,7 +667,6 @@ impl<'de> Deserialize<'de> for Config {
             listen_addr,
             network: network_kind,
             testnet_parameters,
-            regtest_activation_heights,
             initial_mainnet_peers,
             initial_testnet_peers,
             cache_dir,
@@ -700,7 +695,7 @@ impl<'de> Deserialize<'de> for Config {
         let network = match (network_kind, testnet_parameters) {
             (NetworkKind::Mainnet, _) => Network::Mainnet,
             (NetworkKind::Testnet, None) => Network::new_default_testnet(),
-            (NetworkKind::Regtest, _) => Network::new_regtest(regtest_activation_heights),
+            (NetworkKind::Regtest, _) => Network::new_regtest(),
             (
                 NetworkKind::Testnet,
                 Some(DTestnetParameters {

zebra-network/src/constants.rs

@@ -399,6 +399,7 @@ lazy_static! {
 
         hash_map.insert(NetworkKind::Mainnet, Version::min_specified_for_upgrade(&Mainnet, Nu5));
         hash_map.insert(NetworkKind::Testnet, Version::min_specified_for_upgrade(&Network::new_default_testnet(), Nu5));
+        hash_map.insert(NetworkKind::Regtest, Version::min_specified_for_upgrade(&Network::new_regtest(), Nu5));
 
         hash_map
     };

zebra-rpc/src/methods/get_block_template_rpcs.rs

@@ -638,8 +638,12 @@ where
                 //
                 // Optional TODO:
                 // - add `async changed()` method to ChainSyncStatus (like `ChainTip`)
-                check_synced_to_tip(&network, latest_chain_tip.clone(), sync_status.clone())?;
+                // TODO:
-
+                // - Add a `disable_peers` field to `Network` to check instead of `disable_pow()` (#8361)
+                // - Check the field in `sync_status` so it applies to the mempool as well.
+                if !network.disable_pow() {
+                    check_synced_to_tip(&network, latest_chain_tip.clone(), sync_status.clone())?;
+                }
                 // TODO: return an error if we have no peers, like `zcashd` does,
                 //       and add a developer config that mines regardless of how many peers we have.
                 // https://github.com/zcash/zcash/blob/6fdd9f1b81d3b228326c9826fa10696fc516444b/src/miner.cpp#L865-L880

zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs

@@ -10,11 +10,11 @@ use zebra_chain::{
     block::{
         self,
         merkle::{self, AuthDataRoot},
-        Block, ChainHistoryBlockTxAuthCommitmentHash, Height,
+        Block, ChainHistoryBlockTxAuthCommitmentHash, ChainHistoryMmrRootHash, Height,
     },
     chain_sync_status::ChainSyncStatus,
     chain_tip::ChainTip,
-    parameters::Network,
+    parameters::{Network, NetworkUpgrade},
     serialization::ZcashDeserializeInto,
     transaction::{Transaction, UnminedTx, VerifiedUnminedTx},
     transparent,
@@ -284,7 +284,7 @@ where
 /// in the `getblocktemplate` RPC.
 pub fn generate_coinbase_and_roots(
     network: &Network,
-    height: Height,
+    block_template_height: Height,
     miner_address: &transparent::Address,
     mempool_txs: &[VerifiedUnminedTx],
     history_tree: Arc<zebra_chain::history_tree::HistoryTree>,
@@ -295,7 +295,7 @@ pub fn generate_coinbase_and_roots(
     let miner_fee = calculate_miner_fee(mempool_txs);
     let coinbase_txn = generate_coinbase_transaction(
         network,
-        height,
+        block_template_height,
         miner_address,
         miner_fee,
         like_zcashd,
@@ -305,7 +305,15 @@ pub fn generate_coinbase_and_roots(
     // Calculate block default roots
     //
     // TODO: move expensive root, hash, and tree cryptography to a rayon thread?
-    let default_roots = calculate_default_root_hashes(&coinbase_txn, mempool_txs, history_tree);
+    let chain_history_root = history_tree
+        .hash()
+        .or_else(|| {
+            (NetworkUpgrade::Heartwood.activation_height(network) == Some(block_template_height))
+                .then_some([0; 32].into())
+        })
+        .expect("history tree can't be empty");
+    let default_roots =
+        calculate_default_root_hashes(&coinbase_txn, mempool_txs, chain_history_root);
 
     let coinbase_txn = TransactionTemplate::from_coinbase(&coinbase_txn, miner_fee);
 
@@ -434,16 +442,18 @@ fn combine_coinbase_outputs(
 pub fn calculate_default_root_hashes(
     coinbase_txn: &UnminedTx,
     mempool_txs: &[VerifiedUnminedTx],
-    history_tree: Arc<zebra_chain::history_tree::HistoryTree>,
+    chain_history_root: ChainHistoryMmrRootHash,
 ) -> DefaultRoots {
     let (merkle_root, auth_data_root) = calculate_transaction_roots(coinbase_txn, mempool_txs);
 
-    let chain_history_root = history_tree.hash().expect("history tree can't be empty");
+    let block_commitments_hash = if chain_history_root == [0; 32].into() {
-
+        [0; 32].into()
-    let block_commitments_hash = ChainHistoryBlockTxAuthCommitmentHash::from_commitments(
+    } else {
-        &chain_history_root,
+        ChainHistoryBlockTxAuthCommitmentHash::from_commitments(
-        &auth_data_root,
+            &chain_history_root,
-    );
+            &auth_data_root,
+        )
+    };
 
     DefaultRoots {
         merkle_root,

zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs

@@ -467,11 +467,6 @@ pub async fn test_responses<State, ReadState>(
 
     snapshot_rpc_getdifficulty_valid("mock", mock_get_difficulty, &settings);
 
-    // Now use the populated state
-
-    let populated_get_difficulty = get_block_template_rpc.get_difficulty().await;
-    snapshot_rpc_getdifficulty_invalid("populated", populated_get_difficulty, &settings);
-
     // `z_listunifiedreceivers`
 
     let ua1 = String::from("u1l8xunezsvhq8fgzfl7404m450nwnd76zshscn6nfys7vyz2ywyh4cc5daaq0c7q2su5lqfh23sp7fkf3kt27ve5948mzpfdvckzaect2jtte308mkwlycj2u0eac077wu70vqcetkxf");
@@ -607,17 +602,6 @@ fn snapshot_rpc_getdifficulty_valid(
     });
 }
 
-/// Snapshot invalid `getdifficulty` response, using `cargo insta` and JSON serialization.
-fn snapshot_rpc_getdifficulty_invalid(
-    variant: &'static str,
-    difficulty: Result<f64>,
-    settings: &insta::Settings,
-) {
-    settings.bind(|| {
-        insta::assert_json_snapshot!(format!("get_difficulty_invalid_{variant}"), difficulty)
-    });
-}
-
 /// Snapshot `snapshot_rpc_z_listunifiedreceivers` response, using `cargo insta` and JSON serialization.
 fn snapshot_rpc_z_listunifiedreceivers(
     variant: &'static str,

zebra-rpc/src/methods/tests/snapshot/snapshots/get_difficulty_invalid_populated@testnet_10.snap

@@ -1,10 +0,0 @@
----
-source: zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs
-expression: difficulty
----
-{
-  "Err": {
-    "code": 0,
-    "message": "Zebra's state only has a few blocks, wait until it syncs to the chain tip"
-  }
-}

zebra-state/src/service.rs

@@ -395,8 +395,7 @@ impl StateService {
             - HeightDiff::try_from(checkpoint_verify_concurrency_limit)
                 .expect("fits in HeightDiff");
         let full_verifier_utxo_lookahead =
-            full_verifier_utxo_lookahead.expect("unexpected negative height");
+            full_verifier_utxo_lookahead.unwrap_or(block::Height::MIN);
-
         let non_finalized_state_queued_blocks = QueuedBlocks::default();
         let pending_utxos = PendingUtxos::default();
 
@@ -422,11 +421,10 @@ impl StateService {
         tracing::info!("starting legacy chain check");
         let timer = CodeTimer::start();
 
-        if let Some(tip) = state.best_tip() {
+        if let (Some(tip), Some(nu5_activation_height)) = (
-            let nu5_activation_height = NetworkUpgrade::Nu5
+            state.best_tip(),
-                .activation_height(network)
+            NetworkUpgrade::Nu5.activation_height(network),
-                .expect("NU5 activation height is set");
+        ) {
-
             if let Err(error) = check::legacy_chain(
                 nu5_activation_height,
                 any_ancestor_blocks(

zebra-state/src/service/check.rs

@@ -46,10 +46,6 @@ pub(crate) use difficulty::AdjustedDifficulty;
 ///
 /// The relevant chain is an iterator over the ancestors of `block`, starting
 /// with its parent block.
-///
-/// # Panics
-///
-/// If the state contains less than 28 ([`POW_ADJUSTMENT_BLOCK_SPAN`]) blocks.
 #[tracing::instrument(skip(semantically_verified, finalized_tip_height, relevant_chain))]
 pub(crate) fn block_is_valid_for_recent_chain<C>(
     semantically_verified: &SemanticallyVerifiedBlock,
@@ -80,23 +76,35 @@ where
         .expect("valid blocks have a coinbase height");
     check::height_one_more_than_parent_height(parent_height, semantically_verified.height)?;
 
+    // skip this check during tests if we don't have enough blocks in the chain
+    // process_queued also checks the chain length, so we can skip this assertion during testing
+    // (tests that want to check this code should use the correct number of blocks)
+    //
+    // TODO: accept a NotReadyToBeCommitted error in those tests instead
+    #[cfg(test)]
     if relevant_chain.len() < POW_ADJUSTMENT_BLOCK_SPAN {
-        // skip this check during tests if we don't have enough blocks in the chain
-        // process_queued also checks the chain length, so we can skip this assertion during testing
-        // (tests that want to check this code should use the correct number of blocks)
-        //
-        // TODO: accept a NotReadyToBeCommitted error in those tests instead
-        #[cfg(test)]
         return Ok(());
+    }
 
-        // In production, blocks without enough context are invalid.
+    // In production, blocks without enough context are invalid.
-        //
+    //
-        // The BlockVerifierRouter makes sure that the first 1 million blocks (or more) are
+    // The BlockVerifierRouter makes sure that the first 1 million blocks (or more) are
-        // checkpoint verified. The state queues and block write task make sure that blocks are
+    // checkpoint verified. The state queues and block write task make sure that blocks are
-        // committed in strict height order. But this function is only called on semantically
+    // committed in strict height order. But this function is only called on semantically
-        // verified blocks, so there will be at least 1 million blocks in the state when it is
+    // verified blocks, so there will be at least 1 million blocks in the state when it is
-        // called. So this error should never happen.
+    // called. So this error should never happen on Mainnet or the default Testnet.
-        #[cfg(not(test))]
+    //
+    // It's okay to use a relevant chain of fewer than `POW_ADJUSTMENT_BLOCK_SPAN` blocks, because
+    // the MedianTime function uses height 0 if passed a negative height by the ActualTimespan function:
+    // > ActualTimespan(height : N) := MedianTime(height) − MedianTime(height − PoWAveragingWindow)
+    // > MedianTime(height : N) := median([[ nTime(𝑖) for 𝑖 from max(0, height − PoWMedianBlockSpan) up to height − 1 ]])
+    // and the MeanTarget function only requires the past `PoWAveragingWindow` (17) blocks for heights above 17,
+    // > PoWLimit, if height ≤ PoWAveragingWindow
+    // > ([ToTarget(nBits(𝑖)) for 𝑖 from height−PoWAveragingWindow up to height−1]) otherwise
+    //
+    // See the 'Difficulty Adjustment' section (page 132) in the Zcash specification.
+    #[cfg(not(test))]
+    if relevant_chain.is_empty() {
         return Err(ValidateContextError::NotReadyToBeCommitted);
     }
 

zebra-state/src/service/check/difficulty.rs

@@ -26,7 +26,8 @@ pub const POW_MEDIAN_BLOCK_SPAN: usize = 11;
 
 /// The overall block span used for adjusting Zcash block difficulty.
 ///
-/// `PoWAveragingWindow + PoWMedianBlockSpan` in the Zcash specification.
+/// `PoWAveragingWindow + PoWMedianBlockSpan` in the Zcash specification based on
+/// > ActualTimespan(height : N) := MedianTime(height) − MedianTime(height − PoWAveragingWindow)
 pub const POW_ADJUSTMENT_BLOCK_SPAN: usize = POW_AVERAGING_WINDOW + POW_MEDIAN_BLOCK_SPAN;
 
 /// The damping factor for median timespan variance.
@@ -64,14 +65,14 @@ pub(crate) struct AdjustedDifficulty {
     /// The `header.difficulty_threshold`s from the previous
     /// `PoWAveragingWindow + PoWMedianBlockSpan` (28) blocks, in reverse height
     /// order.
-    relevant_difficulty_thresholds: [CompactDifficulty; POW_ADJUSTMENT_BLOCK_SPAN],
+    relevant_difficulty_thresholds: Vec<CompactDifficulty>,
     /// The `header.time`s from the previous
     /// `PoWAveragingWindow + PoWMedianBlockSpan` (28) blocks, in reverse height
     /// order.
     ///
     /// Only the first and last `PoWMedianBlockSpan` times are used. Times
     /// `11..=16` are ignored.
-    relevant_times: [DateTime<Utc>; POW_ADJUSTMENT_BLOCK_SPAN],
+    relevant_times: Vec<DateTime<Utc>>,
 }
 
 impl AdjustedDifficulty {
@@ -138,13 +139,6 @@ impl AdjustedDifficulty {
             .take(POW_ADJUSTMENT_BLOCK_SPAN)
             .unzip::<_, _, Vec<_>, Vec<_>>();
 
-        let relevant_difficulty_thresholds = relevant_difficulty_thresholds
-            .try_into()
-            .expect("not enough context: difficulty adjustment needs at least 28 (PoWAveragingWindow + PoWMedianBlockSpan) headers");
-        let relevant_times = relevant_times
-            .try_into()
-            .expect("not enough context: difficulty adjustment needs at least 28 (PoWAveragingWindow + PoWMedianBlockSpan) headers");
-
         AdjustedDifficulty {
             candidate_time: candidate_header_time,
             candidate_height,
@@ -226,7 +220,11 @@ impl AdjustedDifficulty {
         // specification is unreachable.
 
         let averaging_window_thresholds =
-            &self.relevant_difficulty_thresholds[0..POW_AVERAGING_WINDOW];
+            if self.relevant_difficulty_thresholds.len() >= POW_AVERAGING_WINDOW {
+                &self.relevant_difficulty_thresholds[0..POW_AVERAGING_WINDOW]
+            } else {
+                return self.network.target_difficulty_limit();
+            };
 
         // Since the PoWLimits are `2^251 − 1` for Testnet, and `2^243 − 1` for
         // Mainnet, the sum of 17 `ExpandedDifficulty` will be less than or equal
@@ -297,10 +295,14 @@ impl AdjustedDifficulty {
     fn median_timespan(&self) -> Duration {
         let newer_median = self.median_time_past();
 
-        let older_times: [DateTime<Utc>; POW_MEDIAN_BLOCK_SPAN] = self.relevant_times
+        let older_times: Vec<_> = self
-            [POW_AVERAGING_WINDOW..]
+            .relevant_times
-            .try_into()
+            .iter()
-            .expect("relevant times is the correct length");
+            .rev()
+            .cloned()
+            .take(POW_MEDIAN_BLOCK_SPAN)
+            .collect();
+
         let older_median = AdjustedDifficulty::median_time(older_times);
 
         // `ActualTimespan` in the Zcash specification
@@ -314,22 +316,30 @@ impl AdjustedDifficulty {
     /// Zcash specification. (These functions are identical, but they are
     /// specified in slightly different ways.)
     pub fn median_time_past(&self) -> DateTime<Utc> {
-        let median_times: [DateTime<Utc>; POW_MEDIAN_BLOCK_SPAN] = self.relevant_times
+        let median_times: Vec<DateTime<Utc>> = self
-            [0..POW_MEDIAN_BLOCK_SPAN]
+            .relevant_times
-            .try_into()
+            .iter()
-            .expect("relevant times is the correct length");
+            .take(POW_MEDIAN_BLOCK_SPAN)
+            .cloned()
+            .collect();
 
         AdjustedDifficulty::median_time(median_times)
     }
 
     /// Calculate the median of the `median_block_span_times`: the `time`s from a
-    /// slice of `PoWMedianBlockSpan` (11) blocks in the relevant chain.
+    /// Vec of `PoWMedianBlockSpan` (11) or fewer blocks in the relevant chain.
     ///
     /// Implements `MedianTime` from the Zcash specification.
-    pub(crate) fn median_time(
+    ///
-        mut median_block_span_times: [DateTime<Utc>; POW_MEDIAN_BLOCK_SPAN],
+    /// # Panics
-    ) -> DateTime<Utc> {
+    ///
+    /// If provided an empty Vec
+    pub(crate) fn median_time(mut median_block_span_times: Vec<DateTime<Utc>>) -> DateTime<Utc> {
         median_block_span_times.sort_unstable();
-        median_block_span_times[POW_MEDIAN_BLOCK_SPAN / 2]
+
+        // > median(𝑆) := sorted(𝑆)_{ceiling((length(𝑆)+1)/2)}
+        // <https://zips.z.cash/protocol/protocol.pdf>, section 7.7.3, Difficulty Adjustment (p. 132)
+        let median_idx = median_block_span_times.len() / 2;
+        median_block_span_times[median_idx]
     }
 }

zebra-state/src/service/read/difficulty.rs

@@ -152,15 +152,7 @@ pub fn solution_rate(
 fn best_relevant_chain_and_history_tree(
     non_finalized_state: &NonFinalizedState,
     db: &ZebraDb,
-) -> Result<
+) -> Result<(Height, block::Hash, Vec<Arc<Block>>, Arc<HistoryTree>), BoxError> {
-    (
-        Height,
-        block::Hash,
-        [Arc<Block>; POW_ADJUSTMENT_BLOCK_SPAN],
-        Arc<HistoryTree>,
-    ),
-    BoxError,
-> {
     let state_tip_before_queries = read::best_tip(non_finalized_state, db).ok_or_else(|| {
         BoxError::from("Zebra's state is empty, wait until it syncs to the chain tip")
     })?;
@@ -171,9 +163,10 @@ fn best_relevant_chain_and_history_tree(
         .into_iter()
         .take(POW_ADJUSTMENT_BLOCK_SPAN)
         .collect();
-    let best_relevant_chain = best_relevant_chain.try_into().map_err(|_error| {
+
-        "Zebra's state only has a few blocks, wait until it syncs to the chain tip"
+    if best_relevant_chain.is_empty() {
-    })?;
+        return Err("missing genesis block, wait until it is committed".into());
+    };
 
     let history_tree = history_tree(
         non_finalized_state.best_chain(),
@@ -206,7 +199,7 @@ fn best_relevant_chain_and_history_tree(
 ///
 /// See [`get_block_template_chain_info()`] for details.
 fn difficulty_time_and_history_tree(
-    relevant_chain: [Arc<Block>; POW_ADJUSTMENT_BLOCK_SPAN],
+    relevant_chain: Vec<Arc<Block>>,
     tip_height: Height,
     tip_hash: block::Hash,
     network: &Network,
@@ -223,10 +216,11 @@ fn difficulty_time_and_history_tree(
     // > the median-time-past of that block.
     // https://zips.z.cash/protocol/protocol.pdf#blockheader
     let median_time_past = calculate_median_time_past(
-        relevant_chain[0..POW_MEDIAN_BLOCK_SPAN]
+        relevant_chain
-            .to_vec()
+            .iter()
-            .try_into()
+            .take(POW_MEDIAN_BLOCK_SPAN)
-            .expect("slice is correct size"),
+            .cloned()
+            .collect(),
     );
 
     let min_time = median_time_past
@@ -307,9 +301,13 @@ fn adjust_difficulty_and_time_for_testnet(
         .try_into()
         .expect("valid blocks have in-range times");
 
-    let minimum_difficulty_spacing =
+    let Some(minimum_difficulty_spacing) =
         NetworkUpgrade::minimum_difficulty_spacing_for_height(network, previous_block_height)
-            .expect("just checked testnet, and the RPC returns an error for low heights");
+    else {
+        // Returns early if the testnet minimum difficulty consensus rule is not active
+        return;
+    };
+
     let minimum_difficulty_spacing: Duration32 = minimum_difficulty_spacing
         .try_into()
         .expect("small positive values are in-range");

zebra-state/src/service/read/find.rs

@@ -596,7 +596,7 @@ pub fn next_median_time_past(
 fn best_relevant_chain(
     non_finalized_state: &NonFinalizedState,
     db: &ZebraDb,
-) -> Result<[Arc<Block>; POW_MEDIAN_BLOCK_SPAN], BoxError> {
+) -> Result<Vec<Arc<Block>>, BoxError> {
     let state_tip_before_queries = read::best_tip(non_finalized_state, db).ok_or_else(|| {
         BoxError::from("Zebra's state is empty, wait until it syncs to the chain tip")
     })?;
@@ -607,9 +607,10 @@ fn best_relevant_chain(
         .into_iter()
         .take(POW_MEDIAN_BLOCK_SPAN)
         .collect();
-    let best_relevant_chain = best_relevant_chain.try_into().map_err(|_error| {
+
-        "Zebra's state only has a few blocks, wait until it syncs to the chain tip"
+    if best_relevant_chain.is_empty() {
-    })?;
+        return Err("missing genesis block, wait until it is committed".into());
+    };
 
     let state_tip_after_queries =
         read::best_tip(non_finalized_state, db).expect("already checked for an empty tip");
@@ -628,9 +629,7 @@ fn best_relevant_chain(
 /// The `relevant_chain` has blocks in reverse height order.
 ///
 /// See [`next_median_time_past()`] for details.
-pub(crate) fn calculate_median_time_past(
+pub(crate) fn calculate_median_time_past(relevant_chain: Vec<Arc<Block>>) -> DateTime32 {
-    relevant_chain: [Arc<Block>; POW_MEDIAN_BLOCK_SPAN],
-) -> DateTime32 {
     let relevant_data: Vec<DateTime<Utc>> = relevant_chain
         .iter()
         .map(|block| block.header.time)
@@ -640,11 +639,7 @@ pub(crate) fn calculate_median_time_past(
     // > preceding PoWMedianBlockSpan blocks (or all preceding blocks if there are fewer than
     // > PoWMedianBlockSpan). The median-time-past of a genesis block is not defined.
     // https://zips.z.cash/protocol/protocol.pdf#blockheader
-    let median_time_past = AdjustedDifficulty::median_time(
+    let median_time_past = AdjustedDifficulty::median_time(relevant_data);
-        relevant_data
-            .try_into()
-            .expect("always has the correct length due to function argument type"),
-    );
 
     DateTime32::try_from(median_time_past).expect("valid blocks have in-range times")
 }

zebrad/src/commands/start.rs

@@ -82,10 +82,11 @@ use abscissa_core::{config, Command, FrameworkError};
 use color_eyre::eyre::{eyre, Report};
 use futures::FutureExt;
 use tokio::{pin, select, sync::oneshot};
-use tower::{builder::ServiceBuilder, util::BoxService};
+use tower::{builder::ServiceBuilder, util::BoxService, ServiceExt};
 use tracing_futures::Instrument;
 
-use zebra_consensus::router::BackgroundTaskHandles;
+use zebra_chain::block::genesis::regtest_genesis_block;
+use zebra_consensus::{router::BackgroundTaskHandles, ParameterCheckpoint};
 use zebra_rpc::server::RpcServer;
 
 use crate::{
@@ -177,7 +178,7 @@ impl StartCmd {
             .await;
 
         info!("initializing syncer");
-        let (syncer, sync_status) = ChainSync::new(
+        let (mut syncer, sync_status) = ChainSync::new(
             &config,
             max_checkpoint_height,
             peer_set.clone(),
@@ -300,7 +301,28 @@ impl StartCmd {
         );
 
         info!("spawning syncer task");
-        let syncer_task_handle = tokio::spawn(syncer.sync().in_current_span());
+        let syncer_task_handle = if config.network.network.is_regtest() {
+            if !syncer
+                .state_contains(config.network.network.genesis_hash())
+                .await?
+            {
+                let genesis_hash = block_verifier_router
+                    .clone()
+                    .oneshot(zebra_consensus::Request::Commit(regtest_genesis_block()))
+                    .await
+                    .expect("should validate Regtest genesis block");
+
+                assert_eq!(
+                    genesis_hash,
+                    config.network.network.genesis_hash(),
+                    "validated block hash should match network genesis hash"
+                )
+            }
+
+            tokio::spawn(std::future::pending().in_current_span())
+        } else {
+            tokio::spawn(syncer.sync().in_current_span())
+        };
 
         #[cfg(feature = "shielded-scan")]
         // Spawn never ending scan task only if we have keys to scan for.

zebrad/src/components/sync.rs

@@ -1136,7 +1136,7 @@ where
 
     /// Returns `true` if the hash is present in the state, and `false`
     /// if the hash is not present in the state.
-    async fn state_contains(&mut self, hash: block::Hash) -> Result<bool, Report> {
+    pub(crate) async fn state_contains(&mut self, hash: block::Hash) -> Result<bool, Report> {
         match self
             .state
             .ready()

zebrad/tests/acceptance.rs

@@ -3107,7 +3107,7 @@ async fn scan_task_commands() -> Result<()> {
 async fn validate_regtest_genesis_block() {
     let _init_guard = zebra_test::init();
 
-    let network = Network::new_regtest(Default::default());
+    let network = Network::new_regtest();
     let state = zebra_state::init_test(&network);
     let (
         block_verifier_router,
@@ -3127,3 +3127,14 @@ async fn validate_regtest_genesis_block() {
         "validated block hash should match network genesis hash"
     )
 }
+
+/// Test successful `getblocktemplate` and `submitblock` RPC calls on Regtest on Canopy.
+///
+/// See [`common::regtest::submit_blocks`] for more information.
+// TODO: Test this with an NU5 activation height too once config can be serialized.
+#[tokio::test]
+#[cfg(feature = "getblocktemplate-rpcs")]
+async fn regtest_submit_blocks() -> Result<()> {
+    common::regtest::submit_blocks_test().await?;
+    Ok(())
+}

zebrad/tests/common/configs/v1.7.0.toml

@@ -72,15 +72,6 @@ Heartwood = 903_800
 Canopy = 1_028_500
 NU5 = 1_842_420
 
-[network.regtest_activation_heights]
-BeforeOverwinter = 1
-Overwinter = 207_500
-Sapling = 280_000
-Blossom = 584_000
-Heartwood = 903_800
-Canopy = 1_028_500
-NU5 = 1_842_420
-
 [rpc]
 debug_force_finished_sync = false
 parallel_cpu_threads = 0

zebrad/tests/common/mod.rs

@@ -24,5 +24,8 @@ pub mod checkpoints;
 #[cfg(feature = "getblocktemplate-rpcs")]
 pub mod get_block_template_rpcs;
 
+#[cfg(feature = "getblocktemplate-rpcs")]
+pub mod regtest;
+
 #[cfg(feature = "shielded-scan")]
 pub mod shielded_scan;

zebrad/tests/common/regtest.rs

@@ -0,0 +1,88 @@
+//! Test submitblock RPC method on Regtest.
+//!
+//! This test will get block templates via the `getblocktemplate` RPC method and submit them as new blocks
+//! via the `submitblock` RPC method on Regtest.
+
+use std::{net::SocketAddr, time::Duration};
+
+use color_eyre::eyre::{Context, Result};
+use tracing::*;
+
+use zebra_chain::{parameters::Network, serialization::ZcashSerialize};
+use zebra_node_services::rpc_client::RpcRequestClient;
+use zebra_rpc::methods::get_block_template_rpcs::get_block_template::{
+    proposal::TimeSource, proposal_block_from_template, GetBlockTemplate,
+};
+use zebra_test::args;
+
+use crate::common::{
+    config::{random_known_rpc_port_config, testdir},
+    launch::ZebradTestDirExt,
+};
+
+/// Number of blocks that should be submitted before the test is considered successful.
+const NUM_BLOCKS_TO_SUBMIT: usize = 200;
+
+pub(crate) async fn submit_blocks_test() -> Result<()> {
+    let _init_guard = zebra_test::init();
+    info!("starting regtest submit_blocks test");
+
+    let network = Network::new_regtest();
+    let mut config = random_known_rpc_port_config(false, &network)?;
+    config.mempool.debug_enable_at_height = Some(0);
+    let rpc_address = config.rpc.listen_addr.unwrap();
+
+    let mut zebrad = testdir()?
+        .with_config(&mut config)?
+        .spawn_child(args!["start"])?;
+
+    info!("waiting for zebrad to start");
+
+    tokio::time::sleep(Duration::from_secs(30)).await;
+
+    info!("attempting to submit blocks");
+    submit_blocks(rpc_address).await?;
+
+    zebrad.kill(false)?;
+
+    let output = zebrad.wait_with_output()?;
+    let output = output.assert_failure()?;
+
+    // [Note on port conflict](#Note on port conflict)
+    output
+        .assert_was_killed()
+        .wrap_err("Possible port conflict. Are there other acceptance tests running?")
+}
+
+/// Get block templates and submit blocks
+async fn submit_blocks(rpc_address: SocketAddr) -> Result<()> {
+    let client = RpcRequestClient::new(rpc_address);
+
+    for _ in 0..NUM_BLOCKS_TO_SUBMIT {
+        let block_template: GetBlockTemplate = client
+            .json_result_from_call("getblocktemplate", "[]".to_string())
+            .await
+            .expect("response should be success output with a serialized `GetBlockTemplate`");
+
+        let block_data = hex::encode(
+            proposal_block_from_template(&block_template, TimeSource::default())?
+                .zcash_serialize_to_vec()?,
+        );
+
+        let submit_block_response = client
+            .text_from_call("submitblock", format!(r#"["{block_data}"]"#))
+            .await?;
+
+        let was_submission_successful = submit_block_response.contains(r#""result":null"#);
+
+        info!(was_submission_successful, "submitted block");
+
+        // Check that the block was validated and committed.
+        assert!(
+            submit_block_response.contains(r#""result":null"#),
+            "unexpected response from submitblock RPC, should be null, was: {submit_block_response}"
+        );
+    }
+
+    Ok(())
+}