fix(state): return non-finalized UTXOs and tx IDs in address queries (#4356)

* Assert that address TxIDs and UTXOs are in chain order

* Always output colour by default in zcash-rpc-diff

* Cross-check getaddressutxos and getaddressbalance in zcash-rpc-diff

* Make balances with no UTXOs match in zcash-rpc-diff

* Add some TODOs

* Display the actual connected node software in zcash-rpc-diff

* Log address UTXOs request summaries

* Log address count for address UTXO requests

* Simplify zcash-rpc-diff node names

* Log chain address UTXOs request processing

* Stop ignoring all non-finalized UTXOs in address queries

* Make zcash-rpc-diff node names more consistent

* Downgrade logs to debug level

* Stop ignoring all non-finalized tx IDs in address queries

zebra-rpc/src/methods.rs

@@ -28,7 +28,7 @@ use zebra_chain::{
 };
 use zebra_network::constants::USER_AGENT;
 use zebra_node_services::{mempool, BoxError};
-use zebra_state::OutputIndex;
+use zebra_state::{OutputIndex, OutputLocation, TransactionLocation};
 
 use crate::queue::Queue;
 
@@ -699,7 +699,24 @@ where
 
             let hashes = match response {
                 zebra_state::ReadResponse::AddressesTransactionIds(hashes) => {
-                    hashes.values().map(|tx_id| tx_id.to_string()).collect()
+                    let mut last_tx_location = TransactionLocation::from_usize(Height(0), 0);
+
+                    hashes
+                        .iter()
+                        .map(|(tx_loc, tx_id)| {
+                            // TODO: downgrade to debug, because there's nothing the user can do
+                            assert!(
+                                *tx_loc > last_tx_location,
+                                "Transactions were not in chain order:\n\
+                                 {tx_loc:?} {tx_id:?} was after:\n\
+                                 {last_tx_location:?}",
+                            );
+
+                            last_tx_location = *tx_loc;
+
+                            tx_id.to_string()
+                        })
+                        .collect()
                 }
                 _ => unreachable!("unmatched response to a TransactionsByAddresses request"),
             };
@@ -735,6 +752,8 @@ where
                 _ => unreachable!("unmatched response to a UtxosByAddresses request"),
             };
 
+            let mut last_output_location = OutputLocation::from_usize(Height(0), 0, 0);
+
             for utxo_data in utxos.utxos() {
                 let address = utxo_data.0;
                 let txid = *utxo_data.1;
@@ -743,6 +762,15 @@ where
                 let script = utxo_data.3.lock_script.clone();
                 let satoshis = u64::from(utxo_data.3.value);
 
+                let output_location = *utxo_data.2;
+                // TODO: downgrade to debug, because there's nothing the user can do
+                assert!(
+                    output_location > last_output_location,
+                    "UTXOs were not in chain order:\n\
+                     {output_location:?} {address:?} {txid:?} was after:\n\
+                     {last_output_location:?}",
+                );
+
                 let entry = GetAddressUtxos {
                     address,
                     txid,
@@ -752,6 +780,8 @@ where
                     height,
                 };
                 response_utxos.push(entry);
+
+                last_output_location = output_location;
             }
 
             Ok(response_utxos)

zebra-state/src/service/read.rs

@@ -235,26 +235,63 @@ where
     C: AsRef<Chain>,
 {
     let mut utxo_error = None;
+    let address_count = addresses.len();
 
     // Retry the finalized UTXO query if it was interrupted by a finalizing block,
     // and the non-finalized chain doesn't overlap the changed heights.
-    for _ in 0..=FINALIZED_ADDRESS_INDEX_RETRIES {
+    for attempt in 0..=FINALIZED_ADDRESS_INDEX_RETRIES {
+        debug!(?attempt, ?address_count, "starting address UTXO query");
+
         let (finalized_utxos, finalized_tip_range) = finalized_transparent_utxos(db, &addresses);
 
+        debug!(
+            finalized_utxo_count = ?finalized_utxos.len(),
+            ?finalized_tip_range,
+            ?address_count,
+            ?attempt,
+            "finalized address UTXO response",
+        );
+
         // Apply the non-finalized UTXO changes.
         let chain_utxo_changes =
             chain_transparent_utxo_changes(chain.as_ref(), &addresses, finalized_tip_range);
 
         // If the UTXOs are valid, return them, otherwise, retry or return an error.
         match chain_utxo_changes {
-            Ok(chain_utxo_changes) => {
+            Ok((created_chain_utxos, spent_chain_utxos)) => {
-                let utxos = apply_utxo_changes(finalized_utxos, chain_utxo_changes);
+                debug!(
+                    chain_utxo_count = ?created_chain_utxos.len(),
+                    chain_utxo_spent = ?spent_chain_utxos.len(),
+                    ?address_count,
+                    ?attempt,
+                    "chain address UTXO response",
+                );
+
+                let utxos =
+                    apply_utxo_changes(finalized_utxos, created_chain_utxos, spent_chain_utxos);
                 let tx_ids = lookup_tx_ids_for_utxos(chain, db, &addresses, &utxos);
 
+                debug!(
+                    full_utxo_count = ?utxos.len(),
+                    tx_id_count = ?tx_ids.len(),
+                    ?address_count,
+                    ?attempt,
+                    "full address UTXO response",
+                );
+
                 return Ok(AddressUtxos::new(network, utxos, tx_ids));
             }
 
-            Err(error) => utxo_error = Some(Err(error)),
+            Err(chain_utxo_error) => {
+                debug!(
+                    ?chain_utxo_error,
+                    ?address_count,
+                    ?attempt,
+                    "chain address UTXO response",
+                );
+
+                utxo_error = Some(Err(chain_utxo_error))
+            }
         }
     }
 
@@ -317,6 +354,8 @@ fn chain_transparent_utxo_changes<C>(
 where
     C: AsRef<Chain>,
 {
+    let address_count = addresses.len();
+
     let finalized_tip_range = match finalized_tip_range {
         Some(finalized_tip_range) => finalized_tip_range,
         None => {
@@ -325,7 +364,12 @@ where
                 "unexpected non-finalized chain when finalized state is empty"
             );
 
-            // Empty chains don't contain any changes.
+            debug!(
+                ?finalized_tip_range,
+                ?address_count,
+                "chain address UTXO query: state is empty, no UTXOs available",
+            );
+
             return Ok(Default::default());
         }
     };
@@ -338,51 +382,112 @@ where
     // But we can compensate for deleted UTXOs by applying the overlapping non-finalized UTXO changes.
 
     // Check if the finalized and non-finalized states match or overlap
-    let required_min_chain_root = finalized_tip_range.start().0 + 1;
+    let required_min_non_finalized_root = finalized_tip_range.start().0 + 1;
-    let mut required_chain_overlap = required_min_chain_root..=finalized_tip_range.end().0;
+
+    // Work out if we need to compensate for finalized query results from multiple heights:
+    // - Ok contains the finalized tip height (no need to compensate)
+    // - Err contains the required non-finalized chain overlap
+    let finalized_tip_status = required_min_non_finalized_root..=finalized_tip_range.end().0;
+    let finalized_tip_status = if finalized_tip_status.is_empty() {
+        let finalized_tip_height = *finalized_tip_range.end();
+        Ok(finalized_tip_height)
+    } else {
+        let required_non_finalized_overlap = finalized_tip_status;
+        Err(required_non_finalized_overlap)
+    };
 
     if chain.is_none() {
-        if required_chain_overlap.is_empty() {
+        if finalized_tip_status.is_ok() {
-            // The non-finalized chain is empty, and we don't need it.
+            debug!(
+                ?finalized_tip_status,
+                ?required_min_non_finalized_root,
+                ?finalized_tip_range,
+                ?address_count,
+                "chain address UTXO query: \
+                 finalized chain is consistent, and non-finalized chain is empty",
+            );
+
             return Ok(Default::default());
         } else {
             // We can't compensate for inconsistent database queries,
             // because the non-finalized chain is empty.
-            return Err("unable to get UTXOs: state was committing a block, and non-finalized chain is empty".into());
+            debug!(
+                ?finalized_tip_status,
+                ?required_min_non_finalized_root,
+                ?finalized_tip_range,
+                ?address_count,
+                "chain address UTXO query: \
+                 finalized tip query was inconsistent, but non-finalized chain is empty",
+            );
+
+            return Err("unable to get UTXOs: \
+                        state was committing a block, and non-finalized chain is empty"
+                .into());
         }
     }
 
     let chain = chain.unwrap();
     let chain = chain.as_ref();
 
-    let chain_root = chain.non_finalized_root_height().0;
+    let non_finalized_root = chain.non_finalized_root_height();
-    let chain_tip = chain.non_finalized_tip_height().0;
+    let non_finalized_tip = chain.non_finalized_tip_height();
 
     assert!(
-        chain_root <= required_min_chain_root,
+        non_finalized_root.0 <= required_min_non_finalized_root,
-        "unexpected chain gap: the best chain is updated after its previous root is finalized"
+        "unexpected chain gap: the best chain is updated after its previous root is finalized",
     );
 
-    // If we've already committed this entire chain, ignore its UTXO changes.
+    match finalized_tip_status {
-    // This is more likely if the non-finalized state is just getting started.
+        Ok(finalized_tip_height) => {
-    if chain_tip > *required_chain_overlap.end() {
+            // If we've already committed this entire chain, ignore its UTXO changes.
-        if required_chain_overlap.is_empty() {
+            // This is more likely if the non-finalized state is just getting started.
-            // The non-finalized chain has been committed, and we don't need it.
+            if finalized_tip_height >= non_finalized_tip {
-            return Ok(Default::default());
+                debug!(
-        } else {
+                    ?non_finalized_root,
+                    ?non_finalized_tip,
+                    ?finalized_tip_status,
+                    ?finalized_tip_range,
+                    ?address_count,
+                    "chain address UTXO query: \
+                     non-finalized blocks have all been finalized, no new UTXO changes",
+                );
+
+                return Ok(Default::default());
+            }
+        }
+
+        Err(ref required_non_finalized_overlap) => {
             // We can't compensate for inconsistent database queries,
             // because the non-finalized chain is below the inconsistent query range.
-            return Err("unable to get UTXOs: state was committing a block, and non-finalized chain has been committed".into());
+            if *required_non_finalized_overlap.end() > non_finalized_tip.0 {
+                debug!(
+                    ?non_finalized_root,
+                    ?non_finalized_tip,
+                    ?finalized_tip_status,
+                    ?finalized_tip_range,
+                    ?address_count,
+                    "chain address UTXO query: \
+                     finalized tip query was inconsistent, \
+                     and some inconsistent blocks are missing from the non-finalized chain",
+                );
+
+                return Err("unable to get UTXOs: \
+                            state was committing a block, \
+                            that is missing from the non-finalized chain"
+                    .into());
+            }
+
+            // Correctness: some finalized UTXOs might have duplicate creates or spends,
+            // but we've just checked they can be corrected by applying the non-finalized UTXO changes.
+            assert!(
+                required_non_finalized_overlap
+                    .clone()
+                    .all(|height| chain.blocks.contains_key(&Height(height))),
+                "UTXO query inconsistency: chain must contain required overlap blocks",
+            );
         }
     }
 
-    // Correctness: some finalized UTXOs might have duplicate creates or spends,
-    // but we've just checked they can be corrected by applying the non-finalized UTXO changes.
-    assert!(
-        required_chain_overlap.all(|height| chain.blocks.contains_key(&Height(height))),
-        "UTXO query inconsistency: chain must contain required overlap blocks",
-    );
-
     Ok(chain.partial_transparent_utxo_changes(addresses))
 }
 
@@ -390,10 +495,8 @@ where
 /// removes the spent UTXOs, and returns the result.
 fn apply_utxo_changes(
     finalized_utxos: BTreeMap<OutputLocation, transparent::Output>,
-    (created_chain_utxos, spent_chain_utxos): (
+    created_chain_utxos: BTreeMap<OutputLocation, transparent::Output>,
-        BTreeMap<OutputLocation, transparent::Output>,
+    spent_chain_utxos: BTreeSet<OutputLocation>,
-        BTreeSet<OutputLocation>,
-    ),
 ) -> BTreeMap<OutputLocation, transparent::Output> {
     // Correctness: combine the created UTXOs, then remove spent UTXOs,
     // to compensate for overlapping finalized and non-finalized blocks.
@@ -547,6 +650,8 @@ fn chain_transparent_tx_id_changes<C>(
 where
     C: AsRef<Chain>,
 {
+    let address_count = addresses.len();
+
     let finalized_tip_range = match finalized_tip_range {
         Some(finalized_tip_range) => finalized_tip_range,
         None => {
@@ -555,7 +660,12 @@ where
                 "unexpected non-finalized chain when finalized state is empty"
             );
 
-            // Empty chains don't contain any tx IDs.
+            debug!(
+                ?finalized_tip_range,
+                ?address_count,
+                "chain address tx ID query: state is empty, no tx IDs available",
+            );
+
             return Ok(Default::default());
         }
     };
@@ -573,51 +683,117 @@ where
     // and they are queried in chain order.
 
     // Check if the finalized and non-finalized states match or overlap
-    let required_min_chain_root = finalized_tip_range.start().0 + 1;
+    let required_min_non_finalized_root = finalized_tip_range.start().0 + 1;
-    let mut required_chain_overlap = required_min_chain_root..=finalized_tip_range.end().0;
+
+    // Work out if we need to compensate for finalized query results from multiple heights:
+    // - Ok contains the finalized tip height (no need to compensate)
+    // - Err contains the required non-finalized chain overlap
+    let finalized_tip_status = required_min_non_finalized_root..=finalized_tip_range.end().0;
+    let finalized_tip_status = if finalized_tip_status.is_empty() {
+        let finalized_tip_height = *finalized_tip_range.end();
+        Ok(finalized_tip_height)
+    } else {
+        let required_non_finalized_overlap = finalized_tip_status;
+        Err(required_non_finalized_overlap)
+    };
 
     if chain.is_none() {
-        if required_chain_overlap.is_empty() || addresses.len() <= 1 {
+        if address_count <= 1 || finalized_tip_status.is_ok() {
-            // The non-finalized chain is empty, and we don't need it.
+            debug!(
+                ?finalized_tip_status,
+                ?required_min_non_finalized_root,
+                ?finalized_tip_range,
+                ?address_count,
+                "chain address tx ID query: \
+                 finalized chain is consistent, and non-finalized chain is empty",
+            );
+
             return Ok(Default::default());
         } else {
             // We can't compensate for inconsistent database queries,
             // because the non-finalized chain is empty.
-            return Err("unable to get tx IDs: state was committing a block, and non-finalized chain is empty".into());
+            debug!(
+                ?finalized_tip_status,
+                ?required_min_non_finalized_root,
+                ?finalized_tip_range,
+                ?address_count,
+                "chain address tx ID query: \
+                 finalized tip query was inconsistent, but non-finalized chain is empty",
+            );
+
+            return Err("unable to get tx IDs: \
+                        state was committing a block, and non-finalized chain is empty"
+                .into());
         }
     }
 
     let chain = chain.unwrap();
     let chain = chain.as_ref();
 
-    let chain_root = chain.non_finalized_root_height().0;
+    let non_finalized_root = chain.non_finalized_root_height();
-    let chain_tip = chain.non_finalized_tip_height().0;
+    let non_finalized_tip = chain.non_finalized_tip_height();
 
     assert!(
-        chain_root <= required_min_chain_root,
+        non_finalized_root.0 <= required_min_non_finalized_root,
-        "unexpected chain gap: the best chain is updated after its previous root is finalized"
+        "unexpected chain gap: the best chain is updated after its previous root is finalized",
     );
 
-    // If we've already committed this entire chain, ignore its UTXO changes.
+    match finalized_tip_status {
-    // This is more likely if the non-finalized state is just getting started.
+        Ok(finalized_tip_height) => {
-    if chain_tip > *required_chain_overlap.end() {
+            // If we've already committed this entire chain, ignore its UTXO changes.
-        if required_chain_overlap.is_empty() || addresses.len() <= 1 {
+            // This is more likely if the non-finalized state is just getting started.
-            // The non-finalized chain has been committed, and we don't need it.
+            if finalized_tip_height >= non_finalized_tip {
-            return Ok(Default::default());
+                debug!(
-        } else {
+                    ?non_finalized_root,
+                    ?non_finalized_tip,
+                    ?finalized_tip_status,
+                    ?finalized_tip_range,
+                    ?address_count,
+                    "chain address tx ID query: \
+                     non-finalized blocks have all been finalized, no new UTXO changes",
+                );
+
+                return Ok(Default::default());
+            }
+        }
+
+        Err(ref required_non_finalized_overlap) => {
             // We can't compensate for inconsistent database queries,
             // because the non-finalized chain is below the inconsistent query range.
-            return Err("unable to get tx IDs: state was committing a block, and non-finalized chain has been committed".into());
+            if address_count > 1 && *required_non_finalized_overlap.end() > non_finalized_tip.0 {
+                debug!(
+                    ?non_finalized_root,
+                    ?non_finalized_tip,
+                    ?finalized_tip_status,
+                    ?finalized_tip_range,
+                    ?address_count,
+                    "chain address tx ID query: \
+                     finalized tip query was inconsistent, \
+                     some inconsistent blocks are missing from the non-finalized chain, \
+                     and the query has multiple addresses",
+                );
+
+                return Err("unable to get tx IDs: \
+                            state was committing a block, \
+                            that is missing from the non-finalized chain, \
+                            and the query has multiple addresses"
+                    .into());
+            }
+
+            // Correctness: some finalized UTXOs might have duplicate creates or spends,
+            // but we've just checked they can be corrected by applying the non-finalized UTXO changes.
+            assert!(
+                address_count <= 1
+                    || required_non_finalized_overlap
+                        .clone()
+                        .all(|height| chain.blocks.contains_key(&Height(height))),
+                "tx ID query inconsistency: \
+                 chain must contain required overlap blocks \
+                 or query must only have one address",
+            );
         }
     }
 
-    // Correctness: some finalized tx IDs might have come from different blocks for different addresses,
-    // but we've just checked they can be corrected by applying the non-finalized UTXO changes.
-    assert!(
-        required_chain_overlap.all(|height| chain.blocks.contains_key(&Height(height))) || addresses.len() <= 1,
-        "tx ID query inconsistency: chain must contain required overlap blocks if there are multiple addresses",
-    );
-
     Ok(chain.partial_transparent_tx_ids(addresses, query_height_range))
 }
 

zebra-utils/README.md

@@ -83,6 +83,7 @@ The script:
 2. sends the RPC request to both of them using `zcash-cli`
 3. compares the responses using `diff`
 4. leaves the full responses in files in a temporary directory, so you can check them in detail
+5. if possible, compares different RPC methods for consistency
 
 Assuming `zebrad`'s RPC port is 28232, you should be able to run:
 ```sh
@@ -131,3 +132,4 @@ so you can compare two `zcashd` or `zebrad` nodes if you want.
 You can override the binaries the script calls using these environmental variables:
 - `$ZCASH_CLI`
 - `$DIFF`
+- `$JQ`

zebra-utils/zcash-rpc-diff

@@ -16,7 +16,8 @@ function usage()
 
 # Override the commands used by this script using these environmental variables:
 ZCASH_CLI="${ZCASH_CLI:-zcash-cli}"
-DIFF="${DIFF:-diff --unified --color}"
+DIFF="${DIFF:-diff --unified --color=always}"
+JQ="${JQ:-jq}"
 
 if [ $# -lt 2 ]; then
     usage
@@ -28,16 +29,35 @@ shift
 
 ZCASH_RPC_TMP_DIR=$(mktemp -d)
 
-ZEBRAD_BLOCKCHAIN_INFO="$ZCASH_RPC_TMP_DIR/zebrad-check-getblockchaininfo.json"
+ZEBRAD_RELEASE_INFO="$ZCASH_RPC_TMP_DIR/first-check-getinfo.json"
-ZCASHD_BLOCKCHAIN_INFO="$ZCASH_RPC_TMP_DIR/zcashd-check-getblockchaininfo.json"
+ZCASHD_RELEASE_INFO="$ZCASH_RPC_TMP_DIR/second-check-getinfo.json"
 
-echo "Checking zebrad network and tip height..."
+echo "Checking first node release info..."
+$ZCASH_CLI -rpcport="$ZEBRAD_RPC_PORT" getinfo > "$ZEBRAD_RELEASE_INFO"
+
+ZEBRAD=$(cat "$ZEBRAD_RELEASE_INFO" | grep '"subversion"' | cut -d: -f2 | cut -d/ -f2 | \
+             tr 'A-Z' 'a-z' | sed 's/magicbean/zcashd/ ; s/zebra$/zebrad/')
+
+echo "Checking second node release info..."
+$ZCASH_CLI getinfo > "$ZCASHD_RELEASE_INFO"
+
+ZCASHD=$(cat "$ZCASHD_RELEASE_INFO" | grep '"subversion"' | cut -d: -f2 | cut -d/ -f2 | \
+             tr 'A-Z' 'a-z' | sed 's/magicbean/zcashd/ ; s/zebra$/zebrad/')
+
+echo "Connected to $ZEBRAD (port $ZEBRAD_RPC_PORT) and $ZCASHD ($ZCASH_CLI zcash.conf port)."
+
+echo
+
+ZEBRAD_BLOCKCHAIN_INFO="$ZCASH_RPC_TMP_DIR/$ZEBRAD-check-getblockchaininfo.json"
+ZCASHD_BLOCKCHAIN_INFO="$ZCASH_RPC_TMP_DIR/$ZCASHD-check-getblockchaininfo.json"
+
+echo "Checking $ZEBRAD network and tip height..."
 $ZCASH_CLI -rpcport="$ZEBRAD_RPC_PORT" getblockchaininfo > "$ZEBRAD_BLOCKCHAIN_INFO"
 
 ZEBRAD_NET=$(cat "$ZEBRAD_BLOCKCHAIN_INFO" | grep '"chain"' | cut -d: -f2 | tr -d ' ,"')
 ZEBRAD_HEIGHT=$(cat "$ZEBRAD_BLOCKCHAIN_INFO" | grep '"blocks"' | cut -d: -f2 | tr -d ' ,"')
 
-echo "Checking zcashd network and tip height..."
+echo "Checking $ZCASHD network and tip height..."
 $ZCASH_CLI getblockchaininfo > "$ZCASHD_BLOCKCHAIN_INFO"
 
 ZCASHD_NET=$(cat "$ZCASHD_BLOCKCHAIN_INFO" | grep '"chain"' | cut -d: -f2 | tr -d ' ,"')
@@ -47,34 +67,35 @@ echo
 
 if [ "$ZEBRAD_NET" != "$ZCASHD_NET" ]; then
     echo "WARNING: comparing RPC responses from different networks:"
-    echo "zcashd is on: $ZCASHD_NET"
+    echo "$ZCASHD is on: $ZCASHD_NET"
-    echo "zebrad is on: $ZEBRAD_NET"
+    echo "$ZEBRAD is on: $ZEBRAD_NET"
     echo
 fi
 
 if [ "$ZEBRAD_HEIGHT" -ne "$ZCASHD_HEIGHT" ]; then
     echo "WARNING: comparing RPC responses from different heights:"
-    echo "zcashd is at: $ZCASHD_HEIGHT"
+    echo "$ZCASHD is at: $ZCASHD_HEIGHT"
-    echo "zebrad is at: $ZEBRAD_HEIGHT"
+    echo "$ZEBRAD is at: $ZEBRAD_HEIGHT"
     echo
 fi
 
-ZEBRAD_RESPONSE="$ZCASH_RPC_TMP_DIR/zebrad-$ZEBRAD_NET-$ZEBRAD_HEIGHT-$1.json"
+ZEBRAD_RESPONSE="$ZCASH_RPC_TMP_DIR/$ZEBRAD-$ZEBRAD_NET-$ZEBRAD_HEIGHT-$1.json"
-ZCASHD_RESPONSE="$ZCASH_RPC_TMP_DIR/zcashd-$ZCASHD_NET-$ZCASHD_HEIGHT-$1.json"
+ZCASHD_RESPONSE="$ZCASH_RPC_TMP_DIR/$ZCASHD-$ZCASHD_NET-$ZCASHD_HEIGHT-$1.json"
 
 echo "Request:"
 echo "$@"
 echo
 
-echo "Querying zebrad $ZEBRAD_NET chain at height $ZEBRAD_HEIGHT..."
+echo "Querying $ZEBRAD $ZEBRAD_NET chain at height >=$ZEBRAD_HEIGHT..."
 $ZCASH_CLI -rpcport="$ZEBRAD_RPC_PORT" "$@" > "$ZEBRAD_RESPONSE"
 
-echo "Querying zcashd $ZCASHD_NET chain at height $ZCASHD_HEIGHT..."
+echo "Querying $ZCASHD $ZCASHD_NET chain at height >=$ZCASHD_HEIGHT..."
 $ZCASH_CLI "$@" > "$ZCASHD_RESPONSE"
 
 echo
 
-echo "Response diff (between zcashd port and port $ZEBRAD_RPC_PORT):"
+echo "Response diff between $ZCASHD and $ZEBRAD:"
+
 $DIFF "$ZEBRAD_RESPONSE" "$ZCASHD_RESPONSE" \
     && ( \
         echo "RPC responses were identical"; \
@@ -82,3 +103,93 @@ $DIFF "$ZEBRAD_RESPONSE" "$ZCASHD_RESPONSE" \
         echo "$ZEBRAD_RESPONSE:"; \
         cat "$ZEBRAD_RESPONSE"; \
         )
+
+EXIT_STATUS=$?
+
+# Consistency checks between RPCs
+#
+# TODO:
+# - sum of getaddressutxos.satoshis equals getaddressbalance
+# - set of getaddressutxos.txid is a subset of getaddresstxids <addresses> 1 <max height>
+# - getblockchaininfo.bestblockhash equals getbestblockhash
+
+if [ "$1" == "getaddressutxos" ]; then
+    set "getaddressbalance" "$2"
+else
+    exit $EXIT_STATUS
+fi
+
+ZEBRAD_CHECK_RESPONSE="$ZCASH_RPC_TMP_DIR/$ZEBRAD-$ZEBRAD_NET-$ZEBRAD_HEIGHT-$1.json"
+ZCASHD_CHECK_RESPONSE="$ZCASH_RPC_TMP_DIR/$ZCASHD-$ZCASHD_NET-$ZCASHD_HEIGHT-$1.json"
+
+echo
+
+echo "Cross-checking request:"
+echo "$@"
+echo
+
+echo "Querying $ZEBRAD $ZEBRAD_NET chain at height >=$ZEBRAD_HEIGHT..."
+$ZCASH_CLI -rpcport="$ZEBRAD_RPC_PORT" "$@" > "$ZEBRAD_CHECK_RESPONSE"
+
+echo "Querying $ZCASHD $ZCASHD_NET chain at height >=$ZCASHD_HEIGHT..."
+$ZCASH_CLI "$@" > "$ZCASHD_CHECK_RESPONSE"
+
+echo
+
+echo "$1 diff between $ZCASHD and $ZEBRAD:"
+
+$DIFF "$ZEBRAD_CHECK_RESPONSE" "$ZCASHD_CHECK_RESPONSE" \
+    && ( \
+        echo "RPC check responses were identical"; \
+        echo ; \
+        echo "$ZEBRAD_CHECK_RESPONSE:"; \
+        cat "$ZEBRAD_CHECK_RESPONSE"; \
+        )
+
+CHECK_EXIT_STATUS=$?
+
+if [ "$1" == "getaddressbalance" ]; then
+    echo
+
+    echo "Extracting getaddressbalance.balance..."
+
+    ZEBRAD_NUM_RESPONSE="$ZCASH_RPC_TMP_DIR/$ZEBRAD-$ZEBRAD_NET-$ZEBRAD_HEIGHT-getaddressbalance-num.txt"
+    ZCASHD_NUM_RESPONSE="$ZCASH_RPC_TMP_DIR/$ZCASHD-$ZCASHD_NET-$ZCASHD_HEIGHT-getaddressbalance-num.txt"
+
+    cat "$ZEBRAD_CHECK_RESPONSE" | $JQ '.balance' > "$ZEBRAD_NUM_RESPONSE"
+    cat "$ZCASHD_CHECK_RESPONSE" | $JQ '.balance' > "$ZCASHD_NUM_RESPONSE"
+
+    echo "Summing getaddressutxos.satoshis..."
+
+    ZEBRAD_SUM_RESPONSE="$ZCASH_RPC_TMP_DIR/$ZEBRAD-$ZEBRAD_NET-$ZEBRAD_HEIGHT-getaddressutxos-sum.txt"
+    ZCASHD_SUM_RESPONSE="$ZCASH_RPC_TMP_DIR/$ZCASHD-$ZCASHD_NET-$ZCASHD_HEIGHT-getaddressutxos-sum.txt"
+
+    cat "$ZEBRAD_RESPONSE" | $JQ 'map(.satoshis) | add // 0' > "$ZEBRAD_SUM_RESPONSE"
+    cat "$ZCASHD_RESPONSE" | $JQ 'map(.satoshis) | add // 0' > "$ZCASHD_SUM_RESPONSE"
+
+    echo
+
+    echo "Balance diff between $ZCASHD and $ZEBRAD:"
+    echo "(for both getaddressbalance and getaddressutxos)"
+
+    $DIFF --from-file="$ZEBRAD_NUM_RESPONSE" "$ZCASHD_NUM_RESPONSE" \
+          "$ZEBRAD_SUM_RESPONSE" "$ZCASHD_SUM_RESPONSE" \
+        && ( \
+             echo "RPC balances were identical"; \
+             echo ; \
+             echo "$ZEBRAD_NUM_RESPONSE:"; \
+             cat "$ZEBRAD_NUM_RESPONSE"; \
+        )
+
+    COMPARE_EXIT_STATUS=$?
+
+    if [ $COMPARE_EXIT_STATUS -ne 0 ]; then
+        exit $COMPARE_EXIT_STATUS
+    fi
+fi
+
+if [ $EXIT_STATUS -ne 0 ]; then
+    exit $EXIT_STATUS
+else
+    exit $CHECK_EXIT_STATUS
+fi