change(test): Create test harness for calling getblocktemplate in proposal mode, but don't use it yet (#5884)

* adds ValidateBlock request to state

* adds `Request` enum in block verifier

skips solution check for BlockProposal requests

calls CheckBlockValidity instead of Commit block for BlockProposal requests

* uses new Request in references to chain verifier

* adds getblocktemplate proposal mode response type

* makes getblocktemplate-rpcs feature in zebra-consensus select getblocktemplate-rpcs in zebra-state

* Adds PR review revisions

* adds info log in CheckBlockProposalValidity

* Reverts replacement of match statement

* adds `GetBlockTemplate::capabilities` fn

* conditions calling checkpoint verifier on !request.is_proposal

* updates references to validate_and_commit_non_finalized

* adds snapshot test, updates test vectors

* adds `should_count_metrics` to NonFinalizedState

* Returns an error from chain verifier for block proposal requests below checkpoint height

adds feature flags

* adds "proposal" to GET_BLOCK_TEMPLATE_CAPABILITIES_FIELD

* adds back block::Request to zebra-consensus lib

* updates snapshots

* Removes unnecessary network arg

* skips req in tracing intstrument for read state

* Moves out block proposal validation to its own fn

* corrects `difficulty_threshold_is_valid` docs

adds/fixes some comments, adds TODOs

general cleanup from a self-review.

* Update zebra-state/src/service.rs

* Apply suggestions from code review

Co-authored-by: teor <teor@riseup.net>

* Update zebra-rpc/src/methods/get_block_template_rpcs.rs

Co-authored-by: teor <teor@riseup.net>

* check best chain tip

* Update zebra-state/src/service.rs

Co-authored-by: teor <teor@riseup.net>

* Applies cleanup suggestions from code review

* updates gbt acceptance test to make a block proposal

* fixes json parsing mistake

* adds retries

* returns reject reason if there are no retries left

* moves result deserialization to RPCRequestClient method, adds docs, moves jsonrpc_core to dev-dependencies

* moves sleep(EXPECTED_TX_TIME) out of loop

* updates/adds info logs in retry loop

* Revert "moves sleep(EXPECTED_TX_TIME) out of loop"

This reverts commit f7f0926f4050519687a79afc16656c3f345c004b.

* adds `allow(dead_code)`

* tests with curtime, mintime, & maxtime

* Fixes doc comment

* Logs error responses from chain_verifier CheckProposal requests

* Removes retry loop, adds num_txs log

* removes verbose info log

* sorts mempool_txs before generating merkle root

* Make imports conditional on a feature

* Disable new CI tests until bugs are fixed

Co-authored-by: teor <teor@riseup.net>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Cargo.lock

@@ -5653,6 +5653,7 @@ dependencies = [
  "hyper",
  "indexmap",
  "inferno",
+ "jsonrpc-core",
  "lazy_static",
  "log",
  "metrics",

zebra-chain/src/work/equihash.rs

@@ -59,6 +59,12 @@ impl Solution {
 
         Ok(())
     }
+
+    #[cfg(feature = "getblocktemplate-rpcs")]
+    /// Returns a [`Solution`] of `[0; SOLUTION_SIZE]` to be used in block proposals.
+    pub fn for_proposal() -> Self {
+        Self([0; SOLUTION_SIZE])
+    }
 }
 
 impl PartialEq<Solution> for Solution {

zebra-rpc/src/methods.rs

@@ -1193,7 +1193,7 @@ pub enum GetBlock {
 ///
 /// Also see the notes for the [`Rpc::get_best_block_hash`] and `get_block_hash` methods.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
-pub struct GetBlockHash(#[serde(with = "hex")] block::Hash);
+pub struct GetBlockHash(#[serde(with = "hex")] pub block::Hash);
 
 /// Response to a `z_gettreestate` RPC request.
 ///

zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs

@@ -108,7 +108,7 @@ where
         + Sync
         + 'static,
 {
-    let Ok(block) = block_proposal_bytes.zcash_deserialize_into() else {
+    let Ok(block) = block_proposal_bytes.zcash_deserialize_into::<block::Block>() else {
          return Ok(ProposalRejectReason::Rejected.into())
     };
 
@@ -125,7 +125,14 @@ where
 
     Ok(chain_verifier_response
         .map(|_hash| ProposalResponse::Valid)
-        .unwrap_or_else(|_| ProposalRejectReason::Rejected.into())
+        .unwrap_or_else(|verify_chain_error| {
+            tracing::info!(
+                verify_chain_error,
+                "Got error response from chain_verifier CheckProposal request"
+            );
+
+            ProposalRejectReason::Rejected.into()
+        })
         .into())
 }
 

zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs

@@ -1,4 +1,5 @@
-//! The `GetBlockTempate` type is the output of the `getblocktemplate` RPC method.
+//! The `GetBlockTempate` type is the output of the `getblocktemplate` RPC method in the
+//! default 'template' mode. See [`ProposalResponse`] for the output in 'proposal' mode.
 
 use zebra_chain::{
     amount,
@@ -27,8 +28,10 @@ use crate::methods::{
 };
 
 pub mod parameters;
+pub mod proposal;
 
 pub use parameters::*;
+pub use proposal::*;
 
 /// A serialized `getblocktemplate` RPC response in template mode.
 #[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
@@ -283,33 +286,6 @@ impl GetBlockTemplate {
     }
 }
 
-/// Error response to a `getblocktemplate` RPC request in proposal mode.
-#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
-#[serde(rename_all = "kebab-case")]
-pub enum ProposalRejectReason {
-    /// Block proposal rejected as invalid.
-    Rejected,
-}
-
-/// Response to a `getblocktemplate` RPC request in proposal mode.
-///
-/// See <https://en.bitcoin.it/wiki/BIP_0023#Block_Proposal>
-#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
-#[serde(untagged, rename_all = "kebab-case")]
-pub enum ProposalResponse {
-    /// Block proposal was rejected as invalid, returns `reject-reason` and server `capabilities`.
-    ErrorResponse {
-        /// Reason the proposal was invalid as-is.
-        reject_reason: ProposalRejectReason,
-
-        /// The getblocktemplate RPC capabilities supported by Zebra.
-        capabilities: Vec<String>,
-    },
-
-    /// Block proposal was successfully validated, returns null.
-    Valid,
-}
-
 #[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
 #[serde(untagged)]
 /// A `getblocktemplate` RPC response.
@@ -320,30 +296,3 @@ pub enum Response {
     /// `getblocktemplate` RPC request in proposal mode.
     ProposalMode(ProposalResponse),
 }
-
-impl From<ProposalRejectReason> for ProposalResponse {
-    fn from(reject_reason: ProposalRejectReason) -> Self {
-        Self::ErrorResponse {
-            reject_reason,
-            capabilities: GetBlockTemplate::capabilities(),
-        }
-    }
-}
-
-impl From<ProposalRejectReason> for Response {
-    fn from(error_response: ProposalRejectReason) -> Self {
-        Self::ProposalMode(ProposalResponse::from(error_response))
-    }
-}
-
-impl From<ProposalResponse> for Response {
-    fn from(proposal_response: ProposalResponse) -> Self {
-        Self::ProposalMode(proposal_response)
-    }
-}
-
-impl From<GetBlockTemplate> for Response {
-    fn from(template: GetBlockTemplate) -> Self {
-        Self::TemplateMode(Box::new(template))
-    }
-}

zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/proposal.rs

@@ -0,0 +1,59 @@
+//! `ProposalResponse` is the output of the `getblocktemplate` RPC method in 'proposal' mode.
+
+use super::{GetBlockTemplate, Response};
+
+/// Error response to a `getblocktemplate` RPC request in proposal mode.
+///
+/// See <https://en.bitcoin.it/wiki/BIP_0022#Appendix:_Example_Rejection_Reasons>
+#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub enum ProposalRejectReason {
+    /// Block proposal rejected as invalid.
+    Rejected,
+}
+
+/// Response to a `getblocktemplate` RPC request in proposal mode.
+///
+/// See <https://en.bitcoin.it/wiki/BIP_0023#Block_Proposal>
+#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+#[serde(untagged, rename_all = "kebab-case")]
+pub enum ProposalResponse {
+    /// Block proposal was rejected as invalid, returns `reject-reason` and server `capabilities`.
+    ErrorResponse {
+        /// Reason the proposal was invalid as-is.
+        reject_reason: ProposalRejectReason,
+
+        /// The getblocktemplate RPC capabilities supported by Zebra.
+        capabilities: Vec<String>,
+    },
+
+    /// Block proposal was successfully validated, returns null.
+    Valid,
+}
+
+impl From<ProposalRejectReason> for ProposalResponse {
+    fn from(reject_reason: ProposalRejectReason) -> Self {
+        Self::ErrorResponse {
+            reject_reason,
+            capabilities: GetBlockTemplate::capabilities(),
+        }
+    }
+}
+
+impl From<ProposalRejectReason> for Response {
+    fn from(error_response: ProposalRejectReason) -> Self {
+        Self::ProposalMode(ProposalResponse::from(error_response))
+    }
+}
+
+impl From<ProposalResponse> for Response {
+    fn from(proposal_response: ProposalResponse) -> Self {
+        Self::ProposalMode(proposal_response)
+    }
+}
+
+impl From<GetBlockTemplate> for Response {
+    fn from(template: GetBlockTemplate) -> Self {
+        Self::TemplateMode(Box::new(template))
+    }
+}

zebra-rpc/src/methods/get_block_template_rpcs/types/transaction.rs

@@ -16,7 +16,7 @@ where
 {
     /// The hex-encoded serialized data for this transaction.
     #[serde(with = "hex")]
-    pub(crate) data: SerializedTransaction,
+    pub data: SerializedTransaction,
 
     /// The transaction ID of this transaction.
     #[serde(with = "hex")]

zebrad/Cargo.toml

@@ -174,6 +174,7 @@ tonic-build = { version = "0.8.0", optional = true }
 [dev-dependencies]
 abscissa_core = { version = "0.5", features = ["testing"] }
 hex = "0.4.3"
+jsonrpc-core = "18.0.0"
 once_cell = "1.17.0"
 regex = "1.7.1"
 semver = "1.0.16"

zebrad/tests/common/get_block_template_rpcs/get_block_template.rs

@@ -5,11 +5,23 @@
 //!
 //! After finishing the sync, it will call getblocktemplate.
 
-use std::time::Duration;
+use std::{sync::Arc, time::Duration};
 
 use color_eyre::eyre::{eyre, Context, Result};
 
-use zebra_chain::parameters::Network;
+use zebra_chain::{
+    block::{self, Block, Height},
+    parameters::Network,
+    serialization::{ZcashDeserializeInto, ZcashSerialize},
+    work::equihash::Solution,
+};
+use zebra_rpc::methods::{
+    get_block_template_rpcs::{
+        get_block_template::{GetBlockTemplate, ProposalResponse},
+        types::default_roots::DefaultRoots,
+    },
+    GetBlockHash,
+};
 
 use crate::common::{
     launch::{can_spawn_zebrad_for_rpc, spawn_zebrad_for_rpc},
@@ -58,11 +70,13 @@ pub(crate) async fn run() -> Result<()> {
         true,
     )?;
 
+    let client = RPCRequestClient::new(rpc_address);
+
     tracing::info!(
         "calling getblocktemplate RPC method at {rpc_address}, \
          with a mempool that is likely empty...",
     );
-    let getblocktemplate_response = RPCRequestClient::new(rpc_address)
+    let getblocktemplate_response = client
         .call(
             "getblocktemplate",
             // test that unknown capabilities are parsed as valid input
@@ -84,25 +98,21 @@ pub(crate) async fn run() -> Result<()> {
         "waiting {EXPECTED_MEMPOOL_TRANSACTION_TIME:?} for the mempool \
          to download and verify some transactions...",
     );
+
     tokio::time::sleep(EXPECTED_MEMPOOL_TRANSACTION_TIME).await;
 
+    /* TODO: activate this test after #5925 and #5953 have merged,
+             and we've checked for any other bugs using #5944.
     tracing::info!(
         "calling getblocktemplate RPC method at {rpc_address}, \
-         with a mempool that likely has transactions...",
-    );
-    let getblocktemplate_response = RPCRequestClient::new(rpc_address)
-        .call("getblocktemplate", "[]".to_string())
-        .await?;
-
-    let is_response_success = getblocktemplate_response.status().is_success();
-    let response_text = getblocktemplate_response.text().await?;
-
-    tracing::info!(
-        response_text,
-        "got getblocktemplate response, hopefully with transactions"
+             with a mempool that likely has transactions and attempting \
+             to validate response result as a block proposal",
     );
 
-    assert!(is_response_success);
+    try_validate_block_template(&client)
+        .await
+        .expect("block proposal validation failed");
+     */
 
     zebrad.kill(false)?;
 
@@ -112,7 +122,112 @@ pub(crate) async fn run() -> Result<()> {
     // [Note on port conflict](#Note on port conflict)
     output
         .assert_was_killed()
-        .wrap_err("Possible port conflict. Are there other acceptance tests running?")?;
+        .wrap_err("Possible port conflict. Are there other acceptance tests running?")
+}
+
+/// Accepts an [`RPCRequestClient`], calls getblocktemplate in template mode,
+/// deserializes and transforms the block template in the response into block proposal data,
+/// then calls getblocktemplate RPC in proposal mode with the serialized and hex-encoded data.
+///
+/// Returns an error if it fails to transform template to block proposal or serialize the block proposal
+/// Returns `Ok(())` if the block proposal is valid or an error with the reject-reason if the result is
+/// an `ErrorResponse`.
+///
+/// ## Panics
+///
+/// If an RPC call returns a failure
+/// If the response result cannot be deserialized to `GetBlockTemplate` in 'template' mode
+/// or `ProposalResponse` in 'proposal' mode.
+#[allow(dead_code)]
+async fn try_validate_block_template(client: &RPCRequestClient) -> Result<()> {
+    let response_json_result = client
+        .json_result_from_call("getblocktemplate", "[]".to_string())
+        .await
+        .expect("response should be success output with with a serialized `GetBlockTemplate`");
+
+    tracing::info!(
+        ?response_json_result,
+        "got getblocktemplate response, hopefully with transactions"
+    );
+
+    // Propose a new block with an empty solution and nonce field
+    tracing::info!("calling getblocktemplate with a block proposal...",);
+
+    for proposal_block in proposal_block_from_template(response_json_result)? {
+        let raw_proposal_block = hex::encode(proposal_block.zcash_serialize_to_vec()?);
+
+        let json_result = client
+            .json_result_from_call(
+                "getblocktemplate",
+                format!(r#"[{{"mode":"proposal","data":"{raw_proposal_block}"}}]"#),
+            )
+            .await
+            .expect("response should be success output with with a serialized `ProposalResponse`");
+
+        tracing::info!(
+            ?json_result,
+            ?proposal_block.header.time,
+            "got getblocktemplate proposal response"
+        );
+
+        if let ProposalResponse::ErrorResponse { reject_reason, .. } = json_result {
+            Err(eyre!(
+                "unsuccessful block proposal validation, reason: {reject_reason:?}"
+            ))?;
+        } else {
+            assert_eq!(ProposalResponse::Valid, json_result);
+        }
+    }
 
     Ok(())
 }
+
+/// Make block proposals from [`GetBlockTemplate`]
+///
+/// Returns an array of 3 block proposals using `curtime`, `mintime`, and `maxtime`
+/// for their `block.header.time` fields.
+#[allow(dead_code)]
+fn proposal_block_from_template(
+    GetBlockTemplate {
+        version,
+        height,
+        previous_block_hash: GetBlockHash(previous_block_hash),
+        default_roots:
+            DefaultRoots {
+                merkle_root,
+                block_commitments_hash,
+                ..
+            },
+        bits: difficulty_threshold,
+        coinbase_txn,
+        transactions: tx_templates,
+        cur_time,
+        min_time,
+        max_time,
+        ..
+    }: GetBlockTemplate,
+) -> Result<[Block; 3]> {
+    if Height(height) > Height::MAX {
+        Err(eyre!("height field must be lower than Height::MAX"))?;
+    };
+
+    let mut transactions = vec![coinbase_txn.data.as_ref().zcash_deserialize_into()?];
+
+    for tx_template in tx_templates {
+        transactions.push(tx_template.data.as_ref().zcash_deserialize_into()?);
+    }
+
+    Ok([cur_time, min_time, max_time].map(|time| Block {
+        header: Arc::new(block::Header {
+            version,
+            previous_block_hash,
+            merkle_root,
+            commitment_bytes: block_commitments_hash.into(),
+            time: time.into(),
+            difficulty_threshold,
+            nonce: [0; 32],
+            solution: Solution::for_proposal(),
+        }),
+        transactions: transactions.clone(),
+    }))
+}

zebrad/tests/common/rpc_client.rs

@@ -4,6 +4,9 @@ use std::net::SocketAddr;
 
 use reqwest::Client;
 
+#[cfg(feature = "getblocktemplate-rpcs")]
+use color_eyre::{eyre::eyre, Result};
+
 /// An http client for making Json-RPC requests
 pub struct RPCRequestClient {
     client: Client,
@@ -44,4 +47,33 @@ impl RPCRequestClient {
     ) -> reqwest::Result<String> {
         self.call(method, params).await?.text().await
     }
+
+    /// Builds an RPC request, awaits its response, and attempts to deserialize
+    /// it to the expected result type.
+    ///
+    /// Returns Ok with json result from response if successful.
+    /// Returns an error if the call or result deserialization fail.
+    #[cfg(feature = "getblocktemplate-rpcs")]
+    pub async fn json_result_from_call<T: serde::de::DeserializeOwned>(
+        &self,
+        method: &'static str,
+        params: impl Into<String>,
+    ) -> Result<T> {
+        Self::json_result_from_response_text(&self.text_from_call(method, params).await?)
+    }
+
+    /// Accepts response text from an RPC call
+    /// Returns `Ok` with a deserialized `result` value in the expected type, or an error report.
+    #[cfg(feature = "getblocktemplate-rpcs")]
+    fn json_result_from_response_text<T: serde::de::DeserializeOwned>(
+        response_text: &str,
+    ) -> Result<T> {
+        use jsonrpc_core::Output;
+
+        let output: Output = serde_json::from_str(response_text)?;
+        match output {
+            Output::Success(success) => Ok(serde_json::from_value(success.result)?),
+            Output::Failure(failure) => Err(eyre!("RPC call failed with: {failure:?}")),
+        }
+    }
 }