Parse `MSG_WTX` inventory type (part of ZIP-239) (#2446)

* Rename constant to `MIN_INVENTORY_HASH_SIZE`

Because the size is not constant anymore, since the `MSG_WTX` inventory
type is larger.

* Add `InventoryHash::smallest_types_strategy`

A method for a proptest strategy that generates the `InventoryHash`
variants that have the smallest serialized size.

* Update proptest to use only smallest inventories

In order to properly test the maximum allocation.

* Add intra-doc links in some method documentation

Make it easier to navigate from the documentation of the proptest
strategies to the variants they generate.

* Parse `MSG_WTX` inventory type

Avoid returning an error if a received `GetData` or `Inv` message
contains a `MSG_WTX` inventory type. This prevents Zebra from
disconnecting from peers that announce V5 transactions.

* Fix inventory hash size proptest

The serialized size now depends on what type of `InventoryHash` is being
tested.

* Implement serialization of `InventoryHash::Wtx`

For now it just copies the stored bytes, in order to allow the tests to
run correctly.

* Test if `MSG_WTX` inventory is parsed correctly

Create some mock input bytes representing a serialized `MSG_WTX`
inventory item, and check that it can be deserialized successfully.

* Generate arbitrary `InventoryHash::Wtx` for tests

Create a strategy that only generates `InventoryHash::Wtx` instances,
and also update the `Arbitrary` implementation for `InventoryHash` to
also generate `Wtx` variants.

* Test `InventoryHash` serialization roundtrip

Given an arbitrary `InventoryHash`, check that it does not change after
being serialized and deserialized.

Currently, `InventoryHash::Wtx` can't be serialized, so this test will
is expected to panic for now, but it will fail once the serialization
code is implemented, and then the `should_panic` should be removed.

* Test deserialize `InventoryHash` from random bytes

Create an random input vector of bytes, and try to deserialize an
`InventoryHash` from it. This should either succeed or fail in an
expected way.

* Remove redundant attribute

The attribute is redundant because the `arbitrary` module already has
that attribute.

* Implement `Message::inv_strategy()`

A method to return a proptest strategy that creates `Message::Inv`
instances.

* Implement `Message::get_data_strategy()`

A method that returns a proptest strategy that creates
`Message::GetData` instances.

* Test encode/decode roundtrip of some `Message`s

Create a `Message` instance, encode it and then decode it using a
`Codec` instance and check that the result is the same as the initial
`Message`.

For now, this only tests `Message::Inv` and `Message::GetData`, because
these are the variants that are related to the scope of the current set
of changes to support parsing the `MSG_WTX` inventory type.

Even so, the test relies on being able to serialize an
`InventoryHash::Wtx`, which is currently not implemented. Therefore the
test was marked as `should_panic` until the serialization code is
implemented.

zebra-network/proptest-regressions/protocol/external/tests/prop.txt

@@ -0,0 +1,7 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc 62951c0bf7f003f29184881befbd1e8144493b3b14a6dd738ecef9e4c8c06148 # shrinks to inventory_hash = Wtx

zebra-network/src/protocol/external/arbitrary.rs

@@ -1,16 +1,18 @@
-use proptest::{arbitrary::any, arbitrary::Arbitrary, prelude::*};
+use std::convert::TryInto;
 
-use super::{types::PeerServices, InventoryHash};
+use proptest::{arbitrary::any, arbitrary::Arbitrary, collection::vec, prelude::*};
+
+use super::{types::PeerServices, InventoryHash, Message};
 
 use zebra_chain::{block, transaction};
 
 impl InventoryHash {
-    /// Generate a proptest strategy for Inv Errors
+    /// Generate a proptest strategy for [`InventoryHash::Error`]s.
     pub fn error_strategy() -> BoxedStrategy<Self> {
         Just(InventoryHash::Error).boxed()
     }
 
-    /// Generate a proptest strategy for Inv Tx hashes
+    /// Generate a proptest strategy for [`InventoryHash::Tx`] hashes.
     pub fn tx_strategy() -> BoxedStrategy<Self> {
         // using any::<transaction::Hash> causes a trait impl error
         // when building the zebra-network crate separately
@@ -20,7 +22,7 @@ impl InventoryHash {
             .boxed()
     }
 
-    /// Generate a proptest strategy for Inv Block hashes
+    /// Generate a proptest strategy for [`InventotryHash::Block`] hashes.
     pub fn block_strategy() -> BoxedStrategy<Self> {
         (any::<[u8; 32]>())
             .prop_map(block::Hash)
@@ -28,13 +30,36 @@ impl InventoryHash {
             .boxed()
     }
 
-    /// Generate a proptest strategy for Inv FilteredBlock hashes
+    /// Generate a proptest strategy for [`InventoryHash::FilteredBlock`] hashes.
     pub fn filtered_block_strategy() -> BoxedStrategy<Self> {
         (any::<[u8; 32]>())
             .prop_map(block::Hash)
             .prop_map(InventoryHash::FilteredBlock)
             .boxed()
     }
+
+    /// Generate a proptest strategy for [`InventoryHash::Wtx`] hashes.
+    pub fn wtx_strategy() -> BoxedStrategy<Self> {
+        vec(any::<u8>(), 64)
+            .prop_map(|bytes| InventoryHash::Wtx(bytes.try_into().unwrap()))
+            .boxed()
+    }
+
+    /// Generate a proptest strategy for [`InventoryHash`] variants of the smallest serialized size.
+    pub fn smallest_types_strategy() -> BoxedStrategy<Self> {
+        InventoryHash::arbitrary()
+            .prop_filter(
+                "inventory type is not one of the smallest",
+                |inventory_hash| match inventory_hash {
+                    InventoryHash::Error
+                    | InventoryHash::Tx(_)
+                    | InventoryHash::Block(_)
+                    | InventoryHash::FilteredBlock(_) => true,
+                    InventoryHash::Wtx(_) => false,
+                },
+            )
+            .boxed()
+    }
 }
 
 impl Arbitrary for InventoryHash {
@@ -46,6 +71,7 @@ impl Arbitrary for InventoryHash {
             Self::tx_strategy(),
             Self::block_strategy(),
             Self::filtered_block_strategy(),
+            Self::wtx_strategy(),
         ]
         .boxed()
     }
@@ -53,7 +79,6 @@ impl Arbitrary for InventoryHash {
     type Strategy = BoxedStrategy<Self>;
 }
 
-#[cfg(any(test, feature = "proptest-impl"))]
 impl Arbitrary for PeerServices {
     type Parameters = ();
 
@@ -65,3 +90,17 @@ impl Arbitrary for PeerServices {
 
     type Strategy = BoxedStrategy<Self>;
 }
+
+impl Message {
+    /// Create a strategy that only generates [`Message::Inv`] messages.
+    pub fn inv_strategy() -> BoxedStrategy<Self> {
+        any::<Vec<InventoryHash>>().prop_map(Message::Inv).boxed()
+    }
+
+    /// Create a strategy that only generates [`Message::GetData`] messages.
+    pub fn get_data_strategy() -> BoxedStrategy<Self> {
+        any::<Vec<InventoryHash>>()
+            .prop_map(Message::GetData)
+            .boxed()
+    }
+}

zebra-network/src/protocol/external/inv.rs

@@ -38,6 +38,15 @@ pub enum InventoryHash {
     /// rather than a block message; this only works if a bloom filter has been
     /// set.
     FilteredBlock(block::Hash),
+    /// A pair with the hash of a V5 transaction and the [Authorizing Data Commitment][auth_digest].
+    ///
+    /// Introduced by [ZIP-239][zip239], which is analogous to Bitcoin's [BIP-339][bip339].
+    ///
+    /// [auth_digest]: https://zips.z.cash/zip-0244#authorizing-data-commitment
+    /// [zip239]: https://zips.z.cash/zip-0239
+    /// [bip339]: https://github.com/bitcoin/bips/blob/master/bip-0339.mediawiki
+    // TODO: Actually handle this variant once the mempool is implemented
+    Wtx([u8; 64]),
 }
 
 impl From<transaction::Hash> for InventoryHash {
@@ -56,14 +65,15 @@ impl From<block::Hash> for InventoryHash {
 
 impl ZcashSerialize for InventoryHash {
     fn zcash_serialize<W: Write>(&self, mut writer: W) -> Result<(), std::io::Error> {
-        let (code, bytes) = match *self {
-            InventoryHash::Error => (0, [0; 32]),
-            InventoryHash::Tx(hash) => (1, hash.0),
-            InventoryHash::Block(hash) => (2, hash.0),
-            InventoryHash::FilteredBlock(hash) => (3, hash.0),
+        let (code, bytes): (_, &[u8]) = match self {
+            InventoryHash::Error => (0, &[0; 32]),
+            InventoryHash::Tx(hash) => (1, &hash.0),
+            InventoryHash::Block(hash) => (2, &hash.0),
+            InventoryHash::FilteredBlock(hash) => (3, &hash.0),
+            InventoryHash::Wtx(bytes) => (5, bytes),
         };
         writer.write_u32::<LittleEndian>(code)?;
-        writer.write_all(&bytes)?;
+        writer.write_all(bytes)?;
         Ok(())
     }
 }
@@ -77,18 +87,28 @@ impl ZcashDeserialize for InventoryHash {
             1 => Ok(InventoryHash::Tx(transaction::Hash(bytes))),
             2 => Ok(InventoryHash::Block(block::Hash(bytes))),
             3 => Ok(InventoryHash::FilteredBlock(block::Hash(bytes))),
+            5 => {
+                let auth_digest = reader.read_32_bytes()?;
+
+                let mut wtx_bytes = [0u8; 64];
+                wtx_bytes[..32].copy_from_slice(&bytes);
+                wtx_bytes[32..].copy_from_slice(&auth_digest);
+
+                Ok(InventoryHash::Wtx(wtx_bytes))
+            }
             _ => Err(SerializationError::Parse("invalid inventory code")),
         }
     }
 }
 
-/// The serialized size of an [`InventoryHash`].
-pub(crate) const INV_HASH_SIZE: usize = 36;
+/// The minimum serialized size of an [`InventoryHash`].
+pub(crate) const MIN_INV_HASH_SIZE: usize = 36;
 
 impl TrustedPreallocate for InventoryHash {
     fn max_allocation() -> u64 {
-        // An Inventory hash takes 36 bytes, and we reserve at least one byte for the Vector length
-        // so we can never receive more than ((MAX_PROTOCOL_MESSAGE_LEN - 1) / 36) in a single message
-        ((MAX_PROTOCOL_MESSAGE_LEN - 1) / INV_HASH_SIZE) as u64
+        // An Inventory hash takes at least 36 bytes, and we reserve at least one byte for the
+        // Vector length so we can never receive more than ((MAX_PROTOCOL_MESSAGE_LEN - 1) / 36) in
+        // a single message
+        ((MAX_PROTOCOL_MESSAGE_LEN - 1) / MIN_INV_HASH_SIZE) as u64
     }
 }

zebra-network/src/protocol/external/tests.rs

@@ -1 +1,3 @@
 mod preallocate;
+mod prop;
+mod vectors;

zebra-network/src/protocol/external/tests/preallocate.rs

@@ -1,6 +1,6 @@
 //! Tests for trusted preallocation during deserialization.
 
-use super::super::inv::{InventoryHash, INV_HASH_SIZE};
+use super::super::inv::InventoryHash;
 
 use zebra_chain::serialization::{TrustedPreallocate, ZcashSerialize, MAX_PROTOCOL_MESSAGE_LEN};
 
@@ -8,21 +8,30 @@ use proptest::prelude::*;
 use std::convert::TryInto;
 
 proptest! {
-    /// Confirm that each InventoryHash takes exactly INV_HASH_SIZE bytes when serialized.
-    /// This verifies that our calculated [`TrustedPreallocate::max_allocation`] is indeed an upper bound.
+    /// Confirm that each InventoryHash takes the expected size in bytes when serialized.
     #[test]
-    fn inv_hash_size_is_correct(inv in InventoryHash::arbitrary()) {
+    fn inv_hash_size_is_correct(inv in any::<InventoryHash>()) {
         let serialized_inv = inv
             .zcash_serialize_to_vec()
             .expect("Serialization to vec must succeed");
-        assert!(serialized_inv.len() == INV_HASH_SIZE);
+
+        let expected_size = match inv {
+            InventoryHash::Error
+            | InventoryHash::Tx(_)
+            | InventoryHash::Block(_)
+            | InventoryHash::FilteredBlock(_) => 32 + 4,
+
+            InventoryHash::Wtx(_) => 32 + 32 + 4,
+        };
+
+        assert_eq!(serialized_inv.len(), expected_size);
     }
 
     /// Verifies that...
     /// 1. The smallest disallowed vector of `InventoryHash`s is too large to fit in a legal Zcash message
     /// 2. The largest allowed vector is small enough to fit in a legal Zcash message
     #[test]
-    fn inv_hash_max_allocation_is_correct(inv in InventoryHash::arbitrary()) {
+    fn inv_hash_max_allocation_is_correct(inv in InventoryHash::smallest_types_strategy()) {
         let max_allocation: usize = InventoryHash::max_allocation().try_into().unwrap();
         let mut smallest_disallowed_vec = Vec::with_capacity(max_allocation + 1);
         for _ in 0..(InventoryHash::max_allocation() + 1) {

zebra-network/src/protocol/external/tests/prop.rs

@@ -0,0 +1,92 @@
+use bytes::BytesMut;
+use proptest::{collection::vec, prelude::*};
+use tokio_util::codec::{Decoder, Encoder};
+
+use zebra_chain::serialization::{
+    SerializationError, ZcashDeserializeInto, ZcashSerialize, MAX_PROTOCOL_MESSAGE_LEN,
+};
+
+use super::super::{Codec, InventoryHash, Message};
+
+/// Maximum number of random input bytes to try to deserialize an [`InventoryHash`] from.
+///
+/// This is two bytes larger than the maximum [`InventoryHash`] size.
+const MAX_INVENTORY_HASH_BYTES: usize = 70;
+
+proptest! {
+    /// Test if [`InventoryHash`] is not changed after serializing and deserializing it.
+    #[test]
+    fn inventory_hash_roundtrip(inventory_hash in any::<InventoryHash>()) {
+        let mut bytes = Vec::new();
+
+        let serialization_result = inventory_hash.zcash_serialize(&mut bytes);
+
+        prop_assert!(serialization_result.is_ok());
+        prop_assert!(bytes.len() < MAX_INVENTORY_HASH_BYTES);
+
+        let deserialized: Result<InventoryHash, _> = bytes.zcash_deserialize_into();
+
+        prop_assert!(deserialized.is_ok());
+        prop_assert_eq!(deserialized.unwrap(), inventory_hash);
+    }
+
+    /// Test attempting to deserialize an [`InventoryHash`] from random bytes.
+    #[test]
+    fn inventory_hash_from_random_bytes(input in vec(any::<u8>(), 0..MAX_INVENTORY_HASH_BYTES)) {
+        let deserialized: Result<InventoryHash, _> = input.zcash_deserialize_into();
+
+        if input.len() < 36 {
+            // Not enough bytes for any inventory hash
+            prop_assert!(deserialized.is_err());
+            prop_assert_eq!(
+                deserialized.unwrap_err().to_string(),
+                "io error: failed to fill whole buffer"
+            );
+        } else if input[1..4] != [0u8; 3] || input[0] > 5 || input[0] == 4 {
+            // Invalid inventory code
+            prop_assert!(matches!(
+                deserialized,
+                Err(SerializationError::Parse("invalid inventory code"))
+            ));
+        } else if input[0] == 5 && input.len() < 68 {
+            // Not enough bytes for a WTX inventory hash
+            prop_assert!(deserialized.is_err());
+            prop_assert_eq!(
+                deserialized.unwrap_err().to_string(),
+                "io error: failed to fill whole buffer"
+            );
+        } else {
+            // Deserialization should have succeeded
+            prop_assert!(deserialized.is_ok());
+
+            // Reserialize inventory hash
+            let mut bytes = Vec::new();
+            let serialization_result = deserialized.unwrap().zcash_serialize(&mut bytes);
+
+            prop_assert!(serialization_result.is_ok());
+
+            // Check that the reserialization produces the same bytes as the input
+            prop_assert!(bytes.len() <= input.len());
+            prop_assert_eq!(&bytes, &input[..bytes.len()]);
+        }
+    }
+
+    /// Test if a [`Message::{Inv, GetData}`] is not changed after encoding and decoding it.
+    // TODO: Update this test to cover all `Message` variants.
+    #[test]
+    fn inv_and_getdata_message_roundtrip(
+        message in prop_oneof!(Message::inv_strategy(), Message::get_data_strategy()),
+    ) {
+        let mut codec = Codec::builder().finish();
+        let mut bytes = BytesMut::with_capacity(MAX_PROTOCOL_MESSAGE_LEN);
+
+        let encoding_result = codec.encode(message.clone(), &mut bytes);
+
+        prop_assert!(encoding_result.is_ok());
+
+        let decoded: Result<Option<Message>, _> = codec.decode(&mut bytes);
+
+        prop_assert!(decoded.is_ok());
+        prop_assert_eq!(decoded.unwrap(), Some(message));
+    }
+}

zebra-network/src/protocol/external/tests/vectors.rs

@@ -0,0 +1,26 @@
+use std::io::Write;
+
+use byteorder::{LittleEndian, WriteBytesExt};
+
+use zebra_chain::serialization::ZcashDeserializeInto;
+
+use super::super::InventoryHash;
+
+/// Test if deserializing [`InventoryHash::Wtx`] does not produce an error.
+#[test]
+fn parses_msg_wtx_inventory_type() {
+    let mut input = Vec::new();
+
+    input
+        .write_u32::<LittleEndian>(5)
+        .expect("Failed to write MSG_WTX code");
+    input
+        .write_all(&[0u8; 64])
+        .expect("Failed to write dummy inventory data");
+
+    let deserialized: InventoryHash = input
+        .zcash_deserialize_into()
+        .expect("Failed to deserialize dummy `InventoryHash::Wtx`");
+
+    assert_eq!(deserialized, InventoryHash::Wtx([0; 64]));
+}