fix(security): Stop panicking on state RPC or block requests with very large heights (#6699)

* fix panic in height

* Update security comments and turn literals into constants

* Add a test-only assertion that the maximum output index can't ever be reached

* Fix a MISSING_BLOCK_ERROR_CODE that was mistakenly 0

* Add production RPC tests with excessive heights

* Add and update snapshots for production RPCs

* Add excessive height tests for most getblocktemplate RPCs

* Use correct snapshot names by running `cargo insta review`

---------

Co-authored-by: Alfredo Garcia <oxarbitrage@gmail.com>

zebra-rpc/src/methods.rs

@@ -977,7 +977,7 @@ where
                 zebra_state::ReadResponse::Block(Some(block)) => block,
                 zebra_state::ReadResponse::Block(None) => {
                     return Err(Error {
-                        code: ErrorCode::ServerError(0),
+                        code: MISSING_BLOCK_ERROR_CODE,
                         message: "the requested block was not found".to_string(),
                         data: None,
                     })

zebra-rpc/src/methods/tests/snapshot.rs

@@ -14,6 +14,7 @@ use zebra_chain::{
     parameters::Network::{Mainnet, Testnet},
     serialization::ZcashDeserializeInto,
 };
+use zebra_state::MAX_ON_DISK_HEIGHT;
 use zebra_test::mock_service::MockService;
 
 use super::super::*;
@@ -21,6 +22,10 @@ use super::super::*;
 #[cfg(feature = "getblocktemplate-rpcs")]
 mod get_block_template_rpcs;
 
+/// The first block height in the state that can never be stored in the database,
+/// due to optimisations in the disk format.
+pub const EXCESSIVE_BLOCK_HEIGHT: u32 = MAX_ON_DISK_HEIGHT.0 + 1;
+
 /// Snapshot test for RPC methods responses.
 #[tokio::test(flavor = "multi_thread")]
 async fn test_rpc_response_data() {
@@ -104,7 +109,9 @@ async fn test_rpc_response_data_for_network(network: Network) {
     snapshot_rpc_getaddressbalance(get_address_balance, &settings);
 
     // `getblock` variants
+    // A valid block height in the populated state
     const BLOCK_HEIGHT: u32 = 1;
+
     let block_hash = blocks[BLOCK_HEIGHT as usize].hash();
 
     // `getblock`, verbosity=0, height
@@ -119,6 +126,11 @@ async fn test_rpc_response_data_for_network(network: Network) {
         &settings,
     );
 
+    let get_block = rpc
+        .get_block(EXCESSIVE_BLOCK_HEIGHT.to_string(), Some(0u8))
+        .await;
+    snapshot_rpc_getblock_invalid("excessive_height_verbosity_0", get_block, &settings);
+
     // `getblock`, verbosity=0, hash
     let get_block = rpc
         .get_block(block_hash.to_string(), Some(0u8))
@@ -138,6 +150,11 @@ async fn test_rpc_response_data_for_network(network: Network) {
         .expect("We should have a GetBlock struct");
     snapshot_rpc_getblock_verbose("height_verbosity_1", get_block, &settings);
 
+    let get_block = rpc
+        .get_block(EXCESSIVE_BLOCK_HEIGHT.to_string(), Some(1u8))
+        .await;
+    snapshot_rpc_getblock_invalid("excessive_height_verbosity_1", get_block, &settings);
+
     // `getblock`, verbosity=1, hash
     let get_block = rpc
         .get_block(block_hash.to_string(), Some(1u8))
@@ -152,6 +169,11 @@ async fn test_rpc_response_data_for_network(network: Network) {
         .expect("We should have a GetBlock struct");
     snapshot_rpc_getblock_verbose("height_verbosity_default", get_block, &settings);
 
+    let get_block = rpc
+        .get_block(EXCESSIVE_BLOCK_HEIGHT.to_string(), None)
+        .await;
+    snapshot_rpc_getblock_invalid("excessive_height_verbosity_default", get_block, &settings);
+
     // `getblock`, no verbosity - defaults to 1, hash
     let get_block = rpc
         .get_block(block_hash.to_string(), None)
@@ -202,7 +224,12 @@ async fn test_rpc_response_data_for_network(network: Network) {
         .z_get_treestate(BLOCK_HEIGHT.to_string())
         .await
         .expect("We should have a GetTreestate struct");
-    snapshot_rpc_z_gettreestate(tree_state, &settings);
+    snapshot_rpc_z_gettreestate_valid(tree_state, &settings);
+
+    let tree_state = rpc
+        .z_get_treestate(EXCESSIVE_BLOCK_HEIGHT.to_string())
+        .await;
+    snapshot_rpc_z_gettreestate_invalid("excessive_height", tree_state, &settings);
 
     // `getrawtransaction` verbosity=0
     //
@@ -250,7 +277,35 @@ async fn test_rpc_response_data_for_network(network: Network) {
         })
         .await
         .expect("We should have a vector of strings");
-    snapshot_rpc_getaddresstxids(get_address_tx_ids, &settings);
+    snapshot_rpc_getaddresstxids_valid("multi_block", get_address_tx_ids, &settings);
+
+    let get_address_tx_ids = rpc
+        .get_address_tx_ids(GetAddressTxIdsRequest {
+            addresses: addresses.clone(),
+            start: 2,
+            end: 2,
+        })
+        .await
+        .expect("We should have a vector of strings");
+    snapshot_rpc_getaddresstxids_valid("single_block", get_address_tx_ids, &settings);
+
+    let get_address_tx_ids = rpc
+        .get_address_tx_ids(GetAddressTxIdsRequest {
+            addresses: addresses.clone(),
+            start: 3,
+            end: EXCESSIVE_BLOCK_HEIGHT,
+        })
+        .await;
+    snapshot_rpc_getaddresstxids_invalid("excessive_end", get_address_tx_ids, &settings);
+
+    let get_address_tx_ids = rpc
+        .get_address_tx_ids(GetAddressTxIdsRequest {
+            addresses: addresses.clone(),
+            start: EXCESSIVE_BLOCK_HEIGHT,
+            end: EXCESSIVE_BLOCK_HEIGHT + 1,
+        })
+        .await;
+    snapshot_rpc_getaddresstxids_invalid("excessive_start", get_address_tx_ids, &settings);
 
     // `getaddressutxos`
     let get_address_utxos = rpc
@@ -293,22 +348,23 @@ fn snapshot_rpc_getaddressbalance(address_balance: AddressBalance, settings: &in
     settings.bind(|| insta::assert_json_snapshot!("get_address_balance", address_balance));
 }
 
-/// Check `getblock` response, using `cargo insta`, JSON serialization, and block test vectors.
+/// Check valid `getblock` data response with verbosity=0, using `cargo insta`, JSON serialization,
+/// and block test vectors.
 ///
 /// The snapshot file does not contain any data, but it does enforce the response format.
 fn snapshot_rpc_getblock_data(
     variant: &'static str,
     block: GetBlock,
-    block_data: &[u8],
+    expected_block_data: &[u8],
     settings: &insta::Settings,
 ) {
-    let block_data = hex::encode(block_data);
+    let expected_block_data = hex::encode(expected_block_data);
 
     settings.bind(|| {
         insta::assert_json_snapshot!(format!("get_block_data_{variant}"), block, {
             "." => dynamic_redaction(move |value, _path| {
                 // assert that the block data matches, without creating a 1.5 kB snapshot file
-                assert_eq!(value.as_str().unwrap(), block_data);
+                assert_eq!(value.as_str().unwrap(), expected_block_data);
                 // replace with:
                 "[BlockData]"
             }),
@@ -316,7 +372,7 @@ fn snapshot_rpc_getblock_data(
     });
 }
 
-/// Check `getblock` response with verbosity=1, using `cargo insta` and JSON serialization.
+/// Check valid `getblock` response with verbosity=1, using `cargo insta` and JSON serialization.
 fn snapshot_rpc_getblock_verbose(
     variant: &'static str,
     block: GetBlock,
@@ -325,6 +381,16 @@ fn snapshot_rpc_getblock_verbose(
     settings.bind(|| insta::assert_json_snapshot!(format!("get_block_verbose_{variant}"), block));
 }
 
+/// Check invalid height `getblock` response using `cargo insta`.
+fn snapshot_rpc_getblock_invalid(
+    variant: &'static str,
+    response: Result<GetBlock>,
+    settings: &insta::Settings,
+) {
+    settings
+        .bind(|| insta::assert_json_snapshot!(format!("get_block_invalid_{variant}"), response));
+}
+
 /// Snapshot `getbestblockhash` response, using `cargo insta` and JSON serialization.
 fn snapshot_rpc_getbestblockhash(tip_hash: GetBlockHash, settings: &insta::Settings) {
     settings.bind(|| insta::assert_json_snapshot!("get_best_block_hash", tip_hash));
@@ -335,9 +401,20 @@ fn snapshot_rpc_getrawmempool(raw_mempool: Vec<String>, settings: &insta::Settin
     settings.bind(|| insta::assert_json_snapshot!("get_raw_mempool", raw_mempool));
 }
 
-/// Snapshot `z_gettreestate` response, using `cargo insta` and JSON serialization.
+/// Snapshot a valid `z_gettreestate` response, using `cargo insta` and JSON serialization.
-fn snapshot_rpc_z_gettreestate(tree_state: GetTreestate, settings: &insta::Settings) {
+fn snapshot_rpc_z_gettreestate_valid(tree_state: GetTreestate, settings: &insta::Settings) {
-    settings.bind(|| insta::assert_json_snapshot!("z_get_treestate", tree_state));
+    settings.bind(|| insta::assert_json_snapshot!(format!("z_get_treestate_valid"), tree_state));
+}
+
+/// Snapshot an invalid `z_gettreestate` response, using `cargo insta` and JSON serialization.
+fn snapshot_rpc_z_gettreestate_invalid(
+    variant: &'static str,
+    tree_state: Result<GetTreestate>,
+    settings: &insta::Settings,
+) {
+    settings.bind(|| {
+        insta::assert_json_snapshot!(format!("z_get_treestate_invalid_{variant}"), tree_state)
+    });
 }
 
 /// Snapshot `getrawtransaction` response, using `cargo insta` and JSON serialization.
@@ -351,9 +428,29 @@ fn snapshot_rpc_getrawtransaction(
     });
 }
 
-/// Snapshot `getaddressbalance` response, using `cargo insta` and JSON serialization.
+/// Snapshot valid `getaddressbalance` response, using `cargo insta` and JSON serialization.
-fn snapshot_rpc_getaddresstxids(transactions: Vec<String>, settings: &insta::Settings) {
+fn snapshot_rpc_getaddresstxids_valid(
-    settings.bind(|| insta::assert_json_snapshot!("get_address_tx_ids", transactions));
+    variant: &'static str,
+    transactions: Vec<String>,
+    settings: &insta::Settings,
+) {
+    settings.bind(|| {
+        insta::assert_json_snapshot!(format!("get_address_tx_ids_valid_{variant}"), transactions)
+    });
+}
+
+/// Snapshot invalid `getaddressbalance` response, using `cargo insta` and JSON serialization.
+fn snapshot_rpc_getaddresstxids_invalid(
+    variant: &'static str,
+    transactions: Result<Vec<String>>,
+    settings: &insta::Settings,
+) {
+    settings.bind(|| {
+        insta::assert_json_snapshot!(
+            format!("get_address_tx_ids_invalid_{variant}"),
+            transactions
+        )
+    });
 }
 
 /// Snapshot `getaddressutxos` response, using `cargo insta` and JSON serialization.

zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs

@@ -9,6 +9,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 
 use hex::FromHex;
 use insta::Settings;
+use jsonrpc_core::Result;
 use tower::{buffer::Buffer, Service};
 
 use zebra_chain::{
@@ -45,7 +46,7 @@ use crate::methods::{
             unified_address, validate_address, z_validate_address,
         },
     },
-    tests::utils::fake_history_tree,
+    tests::{snapshot::EXCESSIVE_BLOCK_HEIGHT, utils::fake_history_tree},
     GetBlockHash, GetBlockTemplateRpc, GetBlockTemplateRpcImpl,
 };
 
@@ -155,11 +156,21 @@ pub async fn test_responses<State, ReadState>(
 
     // `getblockhash`
     const BLOCK_HEIGHT10: i32 = 10;
+
     let get_block_hash = get_block_template_rpc
         .get_block_hash(BLOCK_HEIGHT10)
         .await
         .expect("We should have a GetBlockHash struct");
-    snapshot_rpc_getblockhash(get_block_hash, &settings);
+    snapshot_rpc_getblockhash_valid(get_block_hash, &settings);
+
+    let get_block_hash = get_block_template_rpc
+        .get_block_hash(
+            EXCESSIVE_BLOCK_HEIGHT
+                .try_into()
+                .expect("constant fits in i32"),
+        )
+        .await;
+    snapshot_rpc_getblockhash_invalid("excessive_height", get_block_hash, &settings);
 
     // `getmininginfo`
     let get_mining_info = get_block_template_rpc
@@ -174,7 +185,19 @@ pub async fn test_responses<State, ReadState>(
         .get_block_subsidy(Some(fake_future_block_height))
         .await
         .expect("We should have a success response");
-    snapshot_rpc_getblocksubsidy(get_block_subsidy, &settings);
+    snapshot_rpc_getblocksubsidy("future_height", get_block_subsidy, &settings);
+
+    let get_block_subsidy = get_block_template_rpc
+        .get_block_subsidy(None)
+        .await
+        .expect("We should have a success response");
+    snapshot_rpc_getblocksubsidy("tip_height", get_block_subsidy, &settings);
+
+    let get_block_subsidy = get_block_template_rpc
+        .get_block_subsidy(Some(EXCESSIVE_BLOCK_HEIGHT))
+        .await
+        .expect("We should have a success response");
+    snapshot_rpc_getblocksubsidy("excessive_height", get_block_subsidy, &settings);
 
     // `getpeerinfo`
     let get_peer_info = get_block_template_rpc
@@ -184,6 +207,9 @@ pub async fn test_responses<State, ReadState>(
     snapshot_rpc_getpeerinfo(get_peer_info, &settings);
 
     // `getnetworksolps` (and `getnetworkhashps`)
+    //
+    // TODO: add tests for excessive num_blocks and height (#6688)
+    //       add the same tests for get_network_hash_ps
     let get_network_sol_ps = get_block_template_rpc
         .get_network_sol_ps(None, None)
         .await
@@ -454,9 +480,20 @@ fn snapshot_rpc_getblockcount(block_count: u32, settings: &insta::Settings) {
     settings.bind(|| insta::assert_json_snapshot!("get_block_count", block_count));
 }
 
-/// Snapshot `getblockhash` response, using `cargo insta` and JSON serialization.
+/// Snapshot valid `getblockhash` response, using `cargo insta` and JSON serialization.
-fn snapshot_rpc_getblockhash(block_hash: GetBlockHash, settings: &insta::Settings) {
+fn snapshot_rpc_getblockhash_valid(block_hash: GetBlockHash, settings: &insta::Settings) {
-    settings.bind(|| insta::assert_json_snapshot!("get_block_hash", block_hash));
+    settings.bind(|| insta::assert_json_snapshot!("get_block_hash_valid", block_hash));
+}
+
+/// Snapshot invalid `getblockhash` response, using `cargo insta` and JSON serialization.
+fn snapshot_rpc_getblockhash_invalid(
+    variant: &'static str,
+    block_hash: Result<GetBlockHash>,
+    settings: &insta::Settings,
+) {
+    settings.bind(|| {
+        insta::assert_json_snapshot!(format!("get_block_hash_invalid_{variant}"), block_hash)
+    });
 }
 
 /// Snapshot `getblocktemplate` response, using `cargo insta` and JSON serialization.
@@ -499,8 +536,14 @@ fn snapshot_rpc_getmininginfo(
 }
 
 /// Snapshot `getblocksubsidy` response, using `cargo insta` and JSON serialization.
-fn snapshot_rpc_getblocksubsidy(get_block_subsidy: BlockSubsidy, settings: &insta::Settings) {
+fn snapshot_rpc_getblocksubsidy(
-    settings.bind(|| insta::assert_json_snapshot!("get_block_subsidy", get_block_subsidy));
+    variant: &'static str,
+    get_block_subsidy: BlockSubsidy,
+    settings: &insta::Settings,
+) {
+    settings.bind(|| {
+        insta::assert_json_snapshot!(format!("get_block_subsidy_{variant}"), get_block_subsidy)
+    });
 }
 
 /// Snapshot `getpeerinfo` response, using `cargo insta` and JSON serialization.

zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_hash_invalid_excessive_height@mainnet_10.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs
+expression: block_hash
+---
+{
+  "Err": {
+    "code": -32602,
+    "message": "Provided index is greater than the current tip"
+  }
+}

zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_hash_invalid_excessive_height@testnet_10.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs
+expression: block_hash
+---
+{
+  "Err": {
+    "code": -32602,
+    "message": "Provided index is greater than the current tip"
+  }
+}

zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_subsidy_excessive_height@mainnet_10.snap

@@ -0,0 +1,9 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs
+expression: get_block_subsidy
+---
+{
+  "fundingstreams": [],
+  "miner": 0.00610351,
+  "founders": 0.0
+}

zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_subsidy_excessive_height@testnet_10.snap

@@ -0,0 +1,9 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs
+expression: get_block_subsidy
+---
+{
+  "fundingstreams": [],
+  "miner": 0.00610351,
+  "founders": 0.0
+}

zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_subsidy_tip_height@mainnet_10.snap

@@ -0,0 +1,31 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs
+expression: get_block_subsidy
+---
+{
+  "fundingstreams": [
+    {
+      "recipient": "Electric Coin Company",
+      "specification": "https://zips.z.cash/zip-0214",
+      "value": 0.21875,
+      "valueZat": 21875000,
+      "address": "t3UCKW2LrHFqPMQFEbZn6FpjqnhAAbfpMYR"
+    },
+    {
+      "recipient": "Zcash Foundation",
+      "specification": "https://zips.z.cash/zip-0214",
+      "value": 0.15625,
+      "valueZat": 15625000,
+      "address": "t3dvVE3SQEi7kqNzwrfNePxZ1d4hUyztBA1"
+    },
+    {
+      "recipient": "Major Grants",
+      "specification": "https://zips.z.cash/zip-0214",
+      "value": 0.25,
+      "valueZat": 25000000,
+      "address": "t3XyYW8yBFRuMnfvm5KLGFbEVz25kckZXym"
+    }
+  ],
+  "miner": 2.5,
+  "founders": 0.0
+}

zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_subsidy_tip_height@testnet_10.snap

@@ -0,0 +1,31 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs
+expression: get_block_subsidy
+---
+{
+  "fundingstreams": [
+    {
+      "recipient": "Electric Coin Company",
+      "specification": "https://zips.z.cash/zip-0214",
+      "value": 0.21875,
+      "valueZat": 21875000,
+      "address": "t2DgYDXMJTYLwNcxighQ9RCgPxMVATRcUdC"
+    },
+    {
+      "recipient": "Zcash Foundation",
+      "specification": "https://zips.z.cash/zip-0214",
+      "value": 0.15625,
+      "valueZat": 15625000,
+      "address": "t27eWDgjFYJGVXmzrXeVjnb5J3uXDM9xH9v"
+    },
+    {
+      "recipient": "Major Grants",
+      "specification": "https://zips.z.cash/zip-0214",
+      "value": 0.25,
+      "valueZat": 25000000,
+      "address": "t2Gvxv2uNM7hbbACjNox4H6DjByoKZ2Fa3P"
+    }
+  ],
+  "miner": 2.5,
+  "founders": 0.0
+}

zebra-rpc/src/methods/tests/snapshots/get_address_tx_ids_invalid_excessive_end@mainnet_10.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot.rs
+expression: transactions
+---
+{
+  "Err": {
+    "code": -32602,
+    "message": "start Height(3) and end Height(16777216) must both be less than or equal to the chain tip Height(10)"
+  }
+}

zebra-rpc/src/methods/tests/snapshots/get_address_tx_ids_invalid_excessive_end@testnet_10.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot.rs
+expression: transactions
+---
+{
+  "Err": {
+    "code": -32602,
+    "message": "start Height(3) and end Height(16777216) must both be less than or equal to the chain tip Height(10)"
+  }
+}

zebra-rpc/src/methods/tests/snapshots/get_address_tx_ids_invalid_excessive_start@mainnet_10.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot.rs
+expression: transactions
+---
+{
+  "Err": {
+    "code": -32602,
+    "message": "start Height(16777216) and end Height(16777217) must both be less than or equal to the chain tip Height(10)"
+  }
+}

zebra-rpc/src/methods/tests/snapshots/get_address_tx_ids_invalid_excessive_start@testnet_10.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot.rs
+expression: transactions
+---
+{
+  "Err": {
+    "code": -32602,
+    "message": "start Height(16777216) and end Height(16777217) must both be less than or equal to the chain tip Height(10)"
+  }
+}

zebra-rpc/src/methods/tests/snapshots/get_address_tx_ids_valid_multi_block@mainnet_10.snap

@@ -1,6 +1,5 @@
 ---
 source: zebra-rpc/src/methods/tests/snapshot.rs
-assertion_line: 227
 expression: transactions
 ---
 [

zebra-rpc/src/methods/tests/snapshots/get_address_tx_ids_valid_multi_block@testnet_10.snap

@@ -1,6 +1,5 @@
 ---
 source: zebra-rpc/src/methods/tests/snapshot.rs
-assertion_line: 227
 expression: transactions
 ---
 [

zebra-rpc/src/methods/tests/snapshots/get_address_tx_ids_valid_single_block@mainnet_10.snap

@@ -0,0 +1,7 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot.rs
+expression: transactions
+---
+[
+  "8974d08d1c5f9c860d8b629d582a56659a4a1dcb2b5f98a25a5afcc2a784b0f4"
+]

zebra-rpc/src/methods/tests/snapshots/get_address_tx_ids_valid_single_block@testnet_10.snap

@@ -0,0 +1,7 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot.rs
+expression: transactions
+---
+[
+  "5822c0532da8a008259ac39933d3210e508c17e3ba21d2b2c428785efdccb3d5"
+]

zebra-rpc/src/methods/tests/snapshots/get_block_invalid_excessive_height_verbosity_0@mainnet_10.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot.rs
+expression: response
+---
+{
+  "Err": {
+    "code": -8,
+    "message": "Block not found"
+  }
+}

zebra-rpc/src/methods/tests/snapshots/get_block_invalid_excessive_height_verbosity_0@testnet_10.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot.rs
+expression: response
+---
+{
+  "Err": {
+    "code": -8,
+    "message": "Block not found"
+  }
+}

zebra-rpc/src/methods/tests/snapshots/get_block_invalid_excessive_height_verbosity_1@mainnet_10.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot.rs
+expression: response
+---
+{
+  "Err": {
+    "code": -8,
+    "message": "block height not in best chain"
+  }
+}

zebra-rpc/src/methods/tests/snapshots/get_block_invalid_excessive_height_verbosity_1@testnet_10.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot.rs
+expression: response
+---
+{
+  "Err": {
+    "code": -8,
+    "message": "block height not in best chain"
+  }
+}

zebra-rpc/src/methods/tests/snapshots/get_block_invalid_excessive_height_verbosity_default@mainnet_10.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot.rs
+expression: response
+---
+{
+  "Err": {
+    "code": -8,
+    "message": "block height not in best chain"
+  }
+}

zebra-rpc/src/methods/tests/snapshots/get_block_invalid_excessive_height_verbosity_default@testnet_10.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot.rs
+expression: response
+---
+{
+  "Err": {
+    "code": -8,
+    "message": "block height not in best chain"
+  }
+}

zebra-rpc/src/methods/tests/snapshots/z_get_treestate_invalid_excessive_height@mainnet_10.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot.rs
+expression: tree_state
+---
+{
+  "Err": {
+    "code": -8,
+    "message": "the requested block was not found"
+  }
+}

zebra-rpc/src/methods/tests/snapshots/z_get_treestate_invalid_excessive_height@testnet_10.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-rpc/src/methods/tests/snapshot.rs
+expression: tree_state
+---
+{
+  "Err": {
+    "code": -8,
+    "message": "the requested block was not found"
+  }
+}

zebra-rpc/src/methods/tests/snapshots/z_get_treestate_valid@mainnet_10.snap

@@ -1,6 +1,5 @@
 ---
 source: zebra-rpc/src/methods/tests/snapshot.rs
-assertion_line: 226
 expression: tree_state
 ---
 {

zebra-rpc/src/methods/tests/snapshots/z_get_treestate_valid@testnet_10.snap

@@ -1,6 +1,5 @@
 ---
 source: zebra-rpc/src/methods/tests/snapshot.rs
-assertion_line: 226
 expression: tree_state
 ---
 {

zebra-state/src/lib.rs

@@ -50,7 +50,7 @@ pub use response::GetBlockTemplateChainInfo;
 pub use service::{
     arbitrary::{populated_state, CHAIN_TIP_UPDATE_WAIT_LIMIT},
     chain_tip::{ChainTipBlock, ChainTipSender},
-    finalized_state::{DiskWriteBatch, WriteDisk},
+    finalized_state::{DiskWriteBatch, WriteDisk, MAX_ON_DISK_HEIGHT},
     init_test, init_test_services, ReadStateService,
 };
 

zebra-state/src/service/finalized_state.rs

@@ -38,7 +38,7 @@ mod arbitrary;
 #[cfg(test)]
 mod tests;
 
-pub use disk_format::{OutputIndex, OutputLocation, TransactionLocation};
+pub use disk_format::{OutputIndex, OutputLocation, TransactionLocation, MAX_ON_DISK_HEIGHT};
 
 pub(super) use zebra_db::ZebraDb;
 

zebra-state/src/service/finalized_state/disk_format.rs

@@ -15,7 +15,7 @@ pub mod transparent;
 #[cfg(test)]
 mod tests;
 
-pub use block::{TransactionIndex, TransactionLocation};
+pub use block::{TransactionIndex, TransactionLocation, MAX_ON_DISK_HEIGHT};
 pub use transparent::{OutputIndex, OutputLocation};
 
 /// Helper type for writing types to disk as raw bytes.
@@ -114,31 +114,30 @@ impl FromDisk for () {
 /// Truncates `mem_bytes` to `disk_len`, by removing zero bytes from the start of the slice.
 /// Used to discard unused zero bytes during serialization.
 ///
+/// Return `None` if any of the truncated bytes are non-zero
+///
 /// # Panics
 ///
-/// - if `mem_bytes` is shorter than `disk_len`
+/// - if `mem_bytes` is shorter than `disk_len`.
-/// - if any of the truncated bytes are non-zero
+pub fn truncate_zero_be_bytes(mem_bytes: &[u8], disk_len: usize) -> Option<&[u8]> {
-pub fn truncate_zero_be_bytes(mem_bytes: &[u8], disk_len: usize) -> &[u8] {
+    let discarded_bytes = mem_bytes
-    let truncated_bytes = mem_bytes
         .len()
         .checked_sub(disk_len)
         .expect("unexpected `mem_bytes` length: must be at least `disk_len`");
 
-    if truncated_bytes == 0 {
+    if discarded_bytes == 0 {
-        return mem_bytes;
+        return Some(mem_bytes);
     }
 
-    let (discarded, truncated) = mem_bytes.split_at(truncated_bytes);
+    let (discarded, truncated) = mem_bytes.split_at(discarded_bytes);
 
-    assert!(
+    if !discarded.iter().all(|&byte| byte == 0) {
-        discarded.iter().all(|&byte| byte == 0),
+        return None;
-        "unexpected `mem_bytes` content: non-zero discarded bytes: {discarded:?}\n\
+    }
-         truncated: {truncated:?}",
-    );
 
     assert_eq!(truncated.len(), disk_len);
 
-    truncated
+    Some(truncated)
 }
 
 /// Expands `disk_bytes` to `mem_len`, by adding zero bytes at the start of the slice.

zebra-state/src/service/finalized_state/disk_format/block.rs

@@ -186,7 +186,24 @@ impl IntoDisk for Height {
 
         let disk_bytes = truncate_zero_be_bytes(&mem_bytes, HEIGHT_DISK_BYTES);
 
-        disk_bytes.try_into().unwrap()
+        match disk_bytes {
+            Some(b) => b.try_into().unwrap(),
+
+            // # Security
+            //
+            // The RPC method or state query was given a block height that is ridiculously high.
+            // But to save space in database indexes, we don't support heights 2^24 and above.
+            //
+            // Instead we return the biggest valid database Height to the lookup code.
+            // So RPC methods and queued block checks will return an error or None.
+            //
+            // At the current block production rate, these heights can't be inserted into the
+            // database until at least 2050. (Blocks are verified in strict height order.)
+            None => truncate_zero_be_bytes(&MAX_ON_DISK_HEIGHT.0.to_be_bytes(), HEIGHT_DISK_BYTES)
+                .expect("max on disk height is valid")
+                .try_into()
+                .unwrap(),
+        }
     }
 }
 

zebra-state/src/service/finalized_state/disk_format/transparent.rs

@@ -36,6 +36,20 @@ pub const BALANCE_DISK_BYTES: usize = 8;
 /// This reduces database size and increases lookup performance.
 pub const OUTPUT_INDEX_DISK_BYTES: usize = 3;
 
+/// The maximum value of an on-disk serialized [`OutputIndex`].
+///
+/// This allows us to store [`OutputLocation`](crate::OutputLocation)s in
+/// 8 bytes, which makes database searches more efficient.
+///
+/// # Consensus
+///
+/// This output index is impossible with the current 2 MB block size limit.
+///
+/// Since Zebra only stores fully verified blocks on disk, blocks with larger indexes
+/// are rejected before reaching the database.
+pub const MAX_ON_DISK_OUTPUT_INDEX: OutputIndex =
+    OutputIndex((1 << (OUTPUT_INDEX_DISK_BYTES * 8)) - 1);
+
 /// [`OutputLocation`]s are stored as a 3 byte height, 2 byte transaction index,
 /// and 3 byte output index on disk.
 ///
@@ -541,8 +555,6 @@ impl FromDisk for Amount<NonNegative> {
     }
 }
 
-// TODO: serialize the index into a smaller number of bytes (#3953)
-//       serialize the index in big-endian order (#3953)
 impl IntoDisk for OutputIndex {
     type Bytes = [u8; OUTPUT_INDEX_DISK_BYTES];
 
@@ -551,7 +563,37 @@ impl IntoDisk for OutputIndex {
 
         let disk_bytes = truncate_zero_be_bytes(&mem_bytes, OUTPUT_INDEX_DISK_BYTES);
 
-        disk_bytes.try_into().unwrap()
+        match disk_bytes {
+            Some(b) => b.try_into().unwrap(),
+            // # Security
+            //
+            // The RPC method or state query was given a transparent output index that is
+            // impossible with the current block size limit of 2 MB. To save space in database
+            // indexes, we don't support output indexes 2^24 and above.
+            //
+            // Instead,  we return an invalid database output index to the lookup code,
+            // which can never be inserted into the database as part of a valid block.
+            // So RPC methods will return an error or None.
+            None => {
+                #[cfg(test)]
+                {
+                    use zebra_chain::serialization::TrustedPreallocate;
+                    assert!(
+                        u64::from(MAX_ON_DISK_OUTPUT_INDEX.0)
+                            > zebra_chain::transparent::Output::max_allocation(),
+                        "increased block size requires database output index format change",
+                    );
+                }
+
+                truncate_zero_be_bytes(
+                    &MAX_ON_DISK_OUTPUT_INDEX.0.to_be_bytes(),
+                    OUTPUT_INDEX_DISK_BYTES,
+                )
+                .expect("max on disk output index is valid")
+                .try_into()
+                .unwrap()
+            }
+        }
     }
 }