Add ZIP-0244 TxId Digest support (#2129)

* Add ZIP-0244 TxId Digest support

* Apply suggestions from code review

Co-authored-by: Janito Vaqueiro Ferreira Filho <janito.vff@gmail.com>
Co-authored-by: Deirdre Connolly <deirdre@zfnd.org>

Cargo.lock

@@ -4492,6 +4492,7 @@ dependencies = [
  "bitvec",
  "blake2b_simd",
  "blake2s_simd",
+ "bls12_381",
  "bs58",
  "byteorder",
  "chrono",

zebra-chain/Cargo.toml

@@ -9,7 +9,7 @@ edition = "2018"
 
 [features]
 default = []
-proptest-impl = ["proptest", "proptest-derive", "itertools", "zebra-test"]
+proptest-impl = ["proptest", "proptest-derive", "itertools", "zebra-test", "rand", "rand_chacha"]
 bench = ["zebra-test"]
 
 [dependencies]
@@ -45,10 +45,13 @@ zcash_history = { git = "https://github.com/zcash/librustzcash.git", rev = "0c3e
 zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "0c3ed159985affa774e44d10172d4471d798a85a" }
 bigint = "4"
 uint = "0.9.1"
+bls12_381 = "0.5.0"
 
 proptest = { version = "0.10", optional = true }
 proptest-derive = { version = "0.3.0", optional = true }
 itertools = { version = "0.10.1", optional = true }
+rand = { version = "0.8", optional = true }
+rand_chacha = { version = "0.3", optional = true }
 
 # ZF deps
 ed25519-zebra = "2"

zebra-chain/src/orchard/arbitrary.rs

@@ -1,8 +1,8 @@
 use group::prime::PrimeCurveAffine;
-use halo2::pasta::pallas;
+use halo2::{arithmetic::FieldExt, pasta::pallas};
 use proptest::{arbitrary::any, array, collection::vec, prelude::*};
 
-use crate::primitives::redpallas::{Signature, SpendAuth, VerificationKeyBytes};
+use crate::primitives::redpallas::{Signature, SpendAuth, VerificationKey, VerificationKeyBytes};
 
 use super::{keys, note, tree, Action, AuthorizedAction, Flags, NoteCommitment, ValueCommitment};
 
@@ -17,21 +17,19 @@ impl Arbitrary for Action {
     fn arbitrary_with(_args: Self::Parameters) -> Self::Strategy {
         (
             any::<note::Nullifier>(),
-            array::uniform32(any::<u8>()),
+            any::<VerificationKeyBytes<SpendAuth>>(),
             any::<note::EncryptedNote>(),
             any::<note::WrappedNoteKey>(),
         )
-            .prop_map(
+            .prop_map(|(nullifier, rk, enc_ciphertext, out_ciphertext)| Self {
-                |(nullifier, rpk_bytes, enc_ciphertext, out_ciphertext)| Self {
+                cv: ValueCommitment(pallas::Affine::identity()),
-                    cv: ValueCommitment(pallas::Affine::identity()),
+                nullifier,
-                    nullifier,
+                rk,
-                    rk: VerificationKeyBytes::from(rpk_bytes),
+                cm_x: NoteCommitment(pallas::Affine::identity()).extract_x(),
-                    cm_x: NoteCommitment(pallas::Affine::identity()).extract_x(),
+                ephemeral_key: keys::EphemeralPublicKey(pallas::Affine::identity()),
-                    ephemeral_key: keys::EphemeralPublicKey(pallas::Affine::identity()),
+                enc_ciphertext,
-                    enc_ciphertext,
+                out_ciphertext,
-                    out_ciphertext,
+            })
-                },
-            )
             .boxed()
     }
 
@@ -42,8 +40,6 @@ impl Arbitrary for note::Nullifier {
     type Parameters = ();
 
     fn arbitrary_with(_args: Self::Parameters) -> Self::Strategy {
-        use halo2::arithmetic::FieldExt;
-
         (vec(any::<u8>(), 64))
             .prop_map(|bytes| {
                 let bytes = bytes.try_into().expect("vec is the correct length");
@@ -87,6 +83,23 @@ impl Arbitrary for Signature<SpendAuth> {
     type Strategy = BoxedStrategy<Self>;
 }
 
+impl Arbitrary for VerificationKeyBytes<SpendAuth> {
+    type Parameters = ();
+
+    fn arbitrary_with(_args: Self::Parameters) -> Self::Strategy {
+        (vec(any::<u8>(), 64))
+            .prop_map(|bytes| {
+                let bytes = bytes.try_into().expect("vec is the correct length");
+                let sk = pallas::Scalar::from_bytes_wide(&bytes);
+                let pk = VerificationKey::from_scalar(&sk);
+                pk.into()
+            })
+            .boxed()
+    }
+
+    type Strategy = BoxedStrategy<Self>;
+}
+
 impl Arbitrary for Flags {
     type Parameters = ();
 
@@ -101,8 +114,6 @@ impl Arbitrary for tree::Root {
     type Parameters = ();
 
     fn arbitrary_with(_args: Self::Parameters) -> Self::Strategy {
-        use halo2::arithmetic::FieldExt;
-
         (vec(any::<u8>(), 64))
             .prop_map(|bytes| {
                 let bytes = bytes.try_into().expect("vec is the correct length");

zebra-chain/src/primitives.rs

@@ -15,3 +15,4 @@ pub use x25519_dalek as x25519;
 pub use proofs::{Bctv14Proof, Groth16Proof, Halo2Proof, ZkSnarkProof};
 
 pub mod zcash_history;
+mod zcash_primitives;

zebra-chain/src/primitives/zcash_primitives.rs

@@ -0,0 +1,46 @@
+//! Contains code that interfaces with the zcash_primitives crate from
+//! librustzcash.
+
+use std::{
+    convert::{TryFrom, TryInto},
+    io,
+};
+
+use crate::{serialization::ZcashSerialize, transaction::Transaction};
+
+impl TryFrom<&Transaction> for zcash_primitives::transaction::Transaction {
+    type Error = io::Error;
+
+    /// Convert a Zebra transaction into a librustzcash one.
+    ///
+    /// # Panics
+    ///
+    /// If the transaction is not V5. (Currently there is no need for this
+    /// conversion for other versions.)
+    fn try_from(trans: &Transaction) -> Result<Self, Self::Error> {
+        let network_upgrade = match trans {
+            Transaction::V5 {
+                network_upgrade, ..
+            } => network_upgrade,
+            Transaction::V1 { .. }
+            | Transaction::V2 { .. }
+            | Transaction::V3 { .. }
+            | Transaction::V4 { .. } => panic!("Zebra only uses librustzcash for V5 transactions"),
+        };
+
+        let serialized_tx = trans.zcash_serialize_to_vec()?;
+        // The `read` method currently ignores the BranchId for V5 transactions;
+        // but we use the correct BranchId anyway.
+        let branch_id: u32 = network_upgrade
+            .branch_id()
+            .expect("Network upgrade must have a Branch ID")
+            .into();
+        // We've already parsed this transaction, so its network upgrade must be valid.
+        let branch_id: zcash_primitives::consensus::BranchId = branch_id
+            .try_into()
+            .expect("zcash_primitives and Zebra have the same branch ids");
+        let alt_tx =
+            zcash_primitives::transaction::Transaction::read(&serialized_tx[..], branch_id)?;
+        Ok(alt_tx)
+    }
+}

zebra-chain/src/sapling/arbitrary.rs

@@ -1,5 +1,9 @@
+use std::convert::TryInto;
+
 use jubjub::AffinePoint;
-use proptest::{arbitrary::any, array, collection::vec, prelude::*};
+use proptest::{arbitrary::any, collection::vec, prelude::*};
+use rand::SeedableRng;
+use rand_chacha::ChaChaRng;
 
 use crate::primitives::Groth16Proof;
 
@@ -15,24 +19,22 @@ impl Arbitrary for Spend<PerSpendAnchor> {
         (
             any::<tree::Root>(),
             any::<note::Nullifier>(),
-            array::uniform32(any::<u8>()),
+            spendauth_verification_key_bytes(),
             any::<Groth16Proof>(),
             vec(any::<u8>(), 64),
         )
-            .prop_map(
+            .prop_map(|(per_spend_anchor, nullifier, rk, proof, sig_bytes)| Self {
-                |(per_spend_anchor, nullifier, rpk_bytes, proof, sig_bytes)| Self {
+                per_spend_anchor,
-                    per_spend_anchor,
+                cv: ValueCommitment(AffinePoint::identity()),
-                    cv: ValueCommitment(AffinePoint::identity()),
+                nullifier,
-                    nullifier,
+                rk,
-                    rk: redjubjub::VerificationKeyBytes::from(rpk_bytes),
+                zkproof: proof,
-                    zkproof: proof,
+                spend_auth_sig: redjubjub::Signature::from({
-                    spend_auth_sig: redjubjub::Signature::from({
+                    let mut b = [0u8; 64];
-                        let mut b = [0u8; 64];
+                    b.copy_from_slice(sig_bytes.as_slice());
-                        b.copy_from_slice(sig_bytes.as_slice());
+                    b
-                        b
+                }),
-                    }),
+            })
-                },
-            )
             .boxed()
     }
 
@@ -45,15 +47,15 @@ impl Arbitrary for Spend<SharedAnchor> {
     fn arbitrary_with(_args: Self::Parameters) -> Self::Strategy {
         (
             any::<note::Nullifier>(),
-            array::uniform32(any::<u8>()),
+            spendauth_verification_key_bytes(),
             any::<Groth16Proof>(),
             vec(any::<u8>(), 64),
         )
-            .prop_map(|(nullifier, rpk_bytes, proof, sig_bytes)| Self {
+            .prop_map(|(nullifier, rk, proof, sig_bytes)| Self {
                 per_spend_anchor: FieldNotPresent,
                 cv: ValueCommitment(AffinePoint::identity()),
                 nullifier,
-                rk: redjubjub::VerificationKeyBytes::from(rpk_bytes),
+                rk,
                 zkproof: proof,
                 spend_auth_sig: redjubjub::Signature::from({
                     let mut b = [0u8; 64];
@@ -99,3 +101,29 @@ impl Arbitrary for OutputInTransactionV4 {
 
     type Strategy = BoxedStrategy<Self>;
 }
+
+/// Creates Strategy for generation VerificationKeyBytes, since the `redjubjub`
+/// crate does not provide an Arbitrary implementation for it.
+fn spendauth_verification_key_bytes(
+) -> impl Strategy<Value = redjubjub::VerificationKeyBytes<redjubjub::SpendAuth>> {
+    prop::array::uniform32(any::<u8>()).prop_map(|bytes| {
+        let mut rng = ChaChaRng::from_seed(bytes);
+        let sk = redjubjub::SigningKey::<redjubjub::SpendAuth>::new(&mut rng);
+        redjubjub::VerificationKey::<redjubjub::SpendAuth>::from(&sk).into()
+    })
+}
+
+impl Arbitrary for tree::Root {
+    type Parameters = ();
+
+    fn arbitrary_with(_args: Self::Parameters) -> Self::Strategy {
+        (vec(any::<u8>(), 64))
+            .prop_map(|bytes| {
+                let bytes = bytes.try_into().expect("vec is the correct length");
+                jubjub::Fq::from_bytes_wide(&bytes).to_bytes().into()
+            })
+            .boxed()
+    }
+
+    type Strategy = BoxedStrategy<Self>;
+}

zebra-chain/src/sapling/tree.rs

@@ -15,12 +15,9 @@
 
 use std::{collections::VecDeque, fmt};
 
+use super::commitment::{pedersen_hashes::pedersen_hash, NoteCommitment};
 use bitvec::prelude::*;
 use lazy_static::lazy_static;
-#[cfg(any(test, feature = "proptest-impl"))]
-use proptest_derive::Arbitrary;
-
-use super::commitment::{pedersen_hashes::pedersen_hash, NoteCommitment};
 
 const MERKLE_DEPTH: usize = 32;
 
@@ -75,7 +72,6 @@ pub struct Position(pub(crate) u64);
 /// this block. A root of a note commitment tree is associated with
 /// each treestate.
 #[derive(Clone, Copy, Default, Eq, PartialEq, Serialize, Deserialize, Hash)]
-#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))]
 pub struct Root(pub [u8; 32]);
 
 impl fmt::Debug for Root {

zebra-chain/src/transaction.rs

@@ -8,6 +8,7 @@ mod lock_time;
 mod memo;
 mod serialize;
 mod sighash;
+mod txid;
 
 #[cfg(any(test, feature = "proptest-impl"))]
 pub mod arbitrary;

zebra-chain/src/transaction/hash.rs

@@ -5,9 +5,9 @@ use std::fmt;
 use proptest_derive::Arbitrary;
 use serde::{Deserialize, Serialize};
 
-use crate::serialization::{sha256d, SerializationError, ZcashSerialize};
+use crate::serialization::SerializationError;
 
-use super::Transaction;
+use super::{txid::TxIdBuilder, Transaction};
 
 /// A transaction hash.
 ///
@@ -19,11 +19,10 @@ pub struct Hash(pub [u8; 32]);
 
 impl<'a> From<&'a Transaction> for Hash {
     fn from(transaction: &'a Transaction) -> Self {
-        let mut hash_writer = sha256d::Writer::default();
+        let hasher = TxIdBuilder::new(&transaction);
-        transaction
+        hasher
-            .zcash_serialize(&mut hash_writer)
+            .txid()
-            .expect("Transactions must serialize into the hash.");
+            .expect("zcash_primitives and Zebra transaction formats must be compatible")
-        Self(hash_writer.finish())
     }
 }
 

zebra-chain/src/transaction/tests/vectors.rs

@@ -1,12 +1,29 @@
-use super::super::*;
+use std::convert::TryInto;
 
+use color_eyre::eyre::Result;
+use lazy_static::lazy_static;
+
+use zebra_test::zip0244;
+
+use super::super::*;
 use crate::{
     block::{Block, Height, MAX_BLOCK_BYTES},
     parameters::{Network, NetworkUpgrade},
     serialization::{SerializationError, ZcashDeserialize, ZcashDeserializeInto, ZcashSerialize},
+    transaction::txid::TxIdBuilder,
 };
 
-use std::convert::TryInto;
+lazy_static! {
+    pub static ref EMPTY_V5_TX: Transaction = Transaction::V5 {
+        network_upgrade: NetworkUpgrade::Nu5,
+        lock_time: LockTime::min_lock_time(),
+        expiry_height: block::Height(0),
+        inputs: Vec::new(),
+        outputs: Vec::new(),
+        sapling_shielded_data: None,
+        orchard_shielded_data: None,
+    };
+}
 
 #[test]
 fn librustzcash_tx_deserialize_and_round_trip() {
@@ -133,18 +150,10 @@ fn zip243_deserialize_and_round_trip() {
 fn empty_v5_round_trip() {
     zebra_test::init();
 
-    let tx = Transaction::V5 {
+    let tx: &Transaction = &*EMPTY_V5_TX;
-        network_upgrade: NetworkUpgrade::Nu5,
-        lock_time: LockTime::min_lock_time(),
-        expiry_height: block::Height(0),
-        inputs: Vec::new(),
-        outputs: Vec::new(),
-        sapling_shielded_data: None,
-        orchard_shielded_data: None,
-    };
 
     let data = tx.zcash_serialize_to_vec().expect("tx should serialize");
-    let tx2 = data
+    let tx2: &Transaction = &data
         .zcash_deserialize_into()
         .expect("tx should deserialize");
 
@@ -188,6 +197,18 @@ fn empty_v4_round_trip() {
     assert_eq!(data, data2, "data must be equal if structs are equal");
 }
 
+/// Check if an empty V5 transaction can be deserialized by librustzcash too.
+#[test]
+fn empty_v5_librustzcash_round_trip() {
+    zebra_test::init();
+
+    let tx: &Transaction = &*EMPTY_V5_TX;
+    let _alt_tx: zcash_primitives::transaction::Transaction = tx.try_into().expect(
+        "librustzcash deserialization might work for empty zebra serialized transactions. \
+        Hint: if empty transactions fail, but other transactions work, delete this test",
+    );
+}
+
 /// Do a round-trip test on fake v5 transactions created from v4 transactions
 /// in the block test vectors.
 ///
@@ -207,21 +228,20 @@ fn fake_v5_round_trip_for_network(network: Network) {
         Network::Testnet => zebra_test::vectors::TESTNET_BLOCKS.iter(),
     };
 
-    for (height, original_bytes) in block_iter {
+    let overwinter_activation_height = NetworkUpgrade::Overwinter
+        .activation_height(network)
+        .expect("a valid height")
+        .0;
+
+    // skip blocks that are before overwinter as they will not have a valid consensus branch id
+    let blocks_after_overwinter =
+        block_iter.skip_while(|(height, _)| **height < overwinter_activation_height);
+
+    for (height, original_bytes) in blocks_after_overwinter {
         let original_block = original_bytes
             .zcash_deserialize_into::<Block>()
             .expect("block is structurally valid");
 
-        // skip blocks that are before overwinter as they will not have a valid consensus branch id
-        if *height
-            < NetworkUpgrade::Overwinter
-                .activation_height(network)
-                .expect("a valid height")
-                .0
-        {
-            continue;
-        }
-
         // skip this block if it only contains v5 transactions,
         // the block round-trip test covers it already
         if original_block
@@ -341,3 +361,86 @@ fn invalid_orchard_nullifier() {
         SerializationError::Parse("Invalid pallas::Base value for orchard Nullifier").to_string()
     );
 }
+
+/// Do a round-trip test via librustzcash on fake v5 transactions created from v4 transactions
+/// in the block test vectors.
+/// Makes sure that zebra-serialized transactions can be deserialized by librustzcash.
+#[test]
+fn fake_v5_librustzcash_round_trip() {
+    zebra_test::init();
+
+    fake_v5_librustzcash_round_trip_for_network(Network::Mainnet);
+    fake_v5_librustzcash_round_trip_for_network(Network::Testnet);
+}
+
+fn fake_v5_librustzcash_round_trip_for_network(network: Network) {
+    let block_iter = match network {
+        Network::Mainnet => zebra_test::vectors::MAINNET_BLOCKS.iter(),
+        Network::Testnet => zebra_test::vectors::TESTNET_BLOCKS.iter(),
+    };
+
+    let overwinter_activation_height = NetworkUpgrade::Overwinter
+        .activation_height(network)
+        .expect("a valid height")
+        .0;
+
+    // skip blocks that are before overwinter as they will not have a valid consensus branch id
+    let blocks_after_overwinter =
+        block_iter.skip_while(|(height, _)| **height < overwinter_activation_height);
+
+    for (height, original_bytes) in blocks_after_overwinter {
+        let original_block = original_bytes
+            .zcash_deserialize_into::<Block>()
+            .expect("block is structurally valid");
+
+        let mut fake_block = original_block.clone();
+        fake_block.transactions = fake_block
+            .transactions
+            .iter()
+            .map(AsRef::as_ref)
+            .map(|t| arbitrary::transaction_to_fake_v5(t, network, Height(*height)))
+            .map(Into::into)
+            .collect();
+
+        // test each transaction
+        for (original_tx, fake_tx) in original_block
+            .transactions
+            .iter()
+            .zip(fake_block.transactions.iter())
+        {
+            assert_ne!(
+                &original_tx, &fake_tx,
+                "v1-v4 transactions must change when converted to fake v5"
+            );
+
+            let fake_bytes = fake_tx
+                .zcash_serialize_to_vec()
+                .expect("vec serialization is infallible");
+
+            assert_ne!(
+                &original_bytes[..],
+                fake_bytes,
+                "v1-v4 transaction data must change when converted to fake v5"
+            );
+
+            let _alt_tx: zcash_primitives::transaction::Transaction = fake_tx
+                .as_ref()
+                .try_into()
+                .expect("librustzcash deserialization must work for zebra serialized transactions");
+        }
+    }
+}
+
+#[test]
+fn zip244_txid() -> Result<()> {
+    zebra_test::init();
+
+    for test in zip0244::TEST_VECTORS.iter() {
+        let transaction = test.tx.zcash_deserialize_into::<Transaction>()?;
+        let hasher = TxIdBuilder::new(&transaction);
+        let txid = hasher.txid()?;
+        assert_eq!(txid.0, test.txid);
+    }
+
+    Ok(())
+}

zebra-chain/src/transaction/txid.rs

@@ -0,0 +1,54 @@
+//! Transaction ID computation. Contains code for generating the Transaction ID
+//! from the transaction.
+use std::{convert::TryInto, io};
+
+use super::{Hash, Transaction};
+use crate::serialization::{sha256d, ZcashSerialize};
+
+/// A Transaction ID builder. It computes the transaction ID by hashing
+/// different parts of the transaction, depending on the transaction version.
+/// For V5 transactions, it follows [ZIP-244] and [ZIP-225].
+///
+/// [ZIP-244]: https://zips.z.cash/zip-0244
+/// [ZIP-225]: https://zips.z.cash/zip-0225
+pub(super) struct TxIdBuilder<'a> {
+    trans: &'a Transaction,
+}
+
+impl<'a> TxIdBuilder<'a> {
+    /// Return a new TxIdBuilder for the given transaction.
+    pub fn new(trans: &'a Transaction) -> Self {
+        TxIdBuilder { trans }
+    }
+
+    /// Compute the Transaction ID for the previously specified transaction.
+    pub(super) fn txid(self) -> Result<Hash, io::Error> {
+        match self.trans {
+            Transaction::V1 { .. }
+            | Transaction::V2 { .. }
+            | Transaction::V3 { .. }
+            | Transaction::V4 { .. } => self.txid_v1_to_v4(),
+            Transaction::V5 { .. } => self.txid_v5(),
+        }
+    }
+
+    /// Compute the Transaction ID for transactions V1 to V4.
+    /// In these cases it's simply the hash of the serialized transaction.
+    fn txid_v1_to_v4(self) -> Result<Hash, io::Error> {
+        let mut hash_writer = sha256d::Writer::default();
+        self.trans
+            .zcash_serialize(&mut hash_writer)
+            .expect("Transactions must serialize into the hash.");
+        Ok(Hash(hash_writer.finish()))
+    }
+
+    /// Compute the Transaction ID for a V5 transaction in the given network upgrade.
+    /// In this case it's the hash of a tree of hashes of specific parts of the
+    /// transaction, as specified in ZIP-244 and ZIP-225.
+    fn txid_v5(self) -> Result<Hash, io::Error> {
+        // The v5 txid (from ZIP-244) is computed using librustzcash. Convert the zebra
+        // transaction to a librustzcash transaction.
+        let alt_tx: zcash_primitives::transaction::Transaction = self.trans.try_into()?;
+        Ok(Hash(*alt_tx.txid().as_ref()))
+    }
+}

zebra-test/src/lib.rs

@@ -24,6 +24,7 @@ pub mod net;
 pub mod prelude;
 pub mod transcript;
 pub mod vectors;
+pub mod zip0244;
 
 /// A multi-threaded Tokio runtime that can be shared between tests.
 ///