From 081cda7990c8fd2970ec01aba052ed70996032c7 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 9 Mar 2022 11:22:00 +1000 Subject: [PATCH] 3. refactor(db): add disk serialization types for transactions (#3741) * refactor(db): simplify block height serialization * refactor(db): make height serialization length generic * refactor(db): create a TransactionIndex type This changes the names of some snapshot types, but doesn't change any data. * refactor(db): create transparent OutputIndex and OutputLocation types This keeps the same serialization, to avoid changing the database version. * doc(rfc/db): make transparent database type names consistent * doc(rfc/db): fix a bug in the Utxo.is_coinbase derivation * fix(db): use the correct serialized size for OutputLocation --- book/src/dev/rfcs/0005-state-updates.md | 27 +-- .../service/check/tests/utxo.txt | 10 ++ .../service/finalized_state/tests/prop.txt | 7 + .../proptest-regressions/service/tests.txt | 2 + .../src/service/finalized_state/arbitrary.rs | 23 +-- .../service/finalized_state/disk_format.rs | 32 +++- .../finalized_state/disk_format/block.rs | 87 +++++++--- .../disk_format/transparent.rs | 161 +++++++++++++++--- .../service/finalized_state/zebra_db/block.rs | 9 +- .../transaction_hashes@mainnet_0.snap | 4 +- .../transaction_hashes@mainnet_1.snap | 6 +- .../transaction_hashes@mainnet_2.snap | 8 +- .../transaction_hashes@testnet_0.snap | 4 +- .../transaction_hashes@testnet_1.snap | 6 +- .../transaction_hashes@testnet_2.snap | 8 +- .../snapshots/transactions@mainnet_0.snap | 4 +- .../snapshots/transactions@mainnet_1.snap | 6 +- .../snapshots/transactions@mainnet_2.snap | 8 +- .../snapshots/transactions@testnet_0.snap | 4 +- .../snapshots/transactions@testnet_1.snap | 6 +- .../snapshots/transactions@testnet_2.snap | 8 +- .../finalized_state/zebra_db/transparent.rs | 15 +- 22 files changed, 315 insertions(+), 130 deletions(-) create mode 100644 zebra-state/proptest-regressions/service/check/tests/utxo.txt create mode 100644 zebra-state/proptest-regressions/service/finalized_state/tests/prop.txt diff --git a/book/src/dev/rfcs/0005-state-updates.md b/book/src/dev/rfcs/0005-state-updates.md index f01d96740..1129a5c11 100644 --- a/book/src/dev/rfcs/0005-state-updates.md +++ b/book/src/dev/rfcs/0005-state-updates.md @@ -611,10 +611,10 @@ We use the following rocksdb column families: | `hash_by_tx_loc` | `TransactionLocation` | `transaction::Hash` | Never | | `tx_loc_by_hash` | `transaction::Hash` | `TransactionLocation` | Never | | *Transparent* | | | | -| `utxo_by_out_loc` | `OutLocation` | `transparent::Output` | Delete | -| `balance_by_transparent_addr` | `transparent::Address` | `Amount \|\| TransparentAddrLoc` | Update | -| `utxo_by_transparent_addr_loc` | `TransparentAddrLoc` | `AtLeastOne` | Up/Del | -| `tx_by_transparent_addr_loc` | `TransparentAddrLoc` | `AtLeastOne` | Append | +| `utxo_by_out_loc` | `OutputLocation` | `transparent::Output` | Delete | +| `balance_by_transparent_addr` | `transparent::Address` | `Amount \|\| AddressLocation` | Update | +| `utxo_by_transparent_addr_loc` | `AddressLocation` | `AtLeastOne` | Up/Del | +| `tx_by_transparent_addr_loc` | `AddressLocation` | `AtLeastOne` | Append | | *Sprout* | | | | | `sprout_nullifiers` | `sprout::Nullifier` | `()` | Never | | `sprout_anchors` | `sprout::tree::Root` | `sprout::tree::NoteCommitmentTree` | Never | @@ -640,12 +640,12 @@ Block and Transaction Data: - `TransactionCount`: same as `TransactionIndex` - `TransactionLocation`: `Height \|\| TransactionIndex` - `HeightTransactionCount`: `Height \|\| TransactionCount` -- `TransparentOutputIndex`: 24 bits, big-endian, unsigned (max ~223,000 transfers in the 2 MB block limit) +- `OutputIndex`: 24 bits, big-endian, unsigned (max ~223,000 transfers in the 2 MB block limit) - transparent and shielded input indexes, and shielded output indexes: 16 bits, big-endian, unsigned (max ~49,000 transfers in the 2 MB block limit) -- `OutLocation`: `TransactionLocation \|\| TransparentOutputIndex` -- `TransparentAddrLoc`: the first `OutLocation` used by a `transparent::Address`. +- `OutputLocation`: `TransactionLocation \|\| OutputIndex` +- `AddressLocation`: the first `OutputLocation` used by a `transparent::Address`. Always has the same value for each address, even if the first output is spent. -- `Utxo`: `Output`, derives extra fields from the `OutLocation` key +- `Utxo`: `Output`, derives extra fields from the `OutputLocation` key - `AtLeastOne`: `[T; AtLeastOne::len()]` (for known-size `T`) We use big-endian encoding for keys, to allow database index prefix searches. @@ -734,17 +734,18 @@ So they should not be used for consensus-critical checks. we store blocks by height, storing the height saves one level of indirection. Transaction hashes can be looked up using `hash_by_tx`. -- Similarly, UTXOs are stored in `utxo_by_outpoint` by `OutLocation`, +- Similarly, UTXOs are stored in `utxo_by_outpoint` by `OutputLocation`, rather than `OutPoint`. `OutPoint`s can be looked up using `tx_by_hash`, and reconstructed using `hash_by_tx`. - The `Utxo` type can be constructed from the `Output` data, `height: TransactionLocation.height`, and - `is_coinbase: OutLocation.output_index == 1`. + `is_coinbase: TransactionLocation.index == 0` + (coinbase transactions are always the first transaction in a block). - `balance_by_transparent_addr` is the sum of all `utxo_by_transparent_addr_loc`s that are still in `utxo_by_outpoint`. It is cached to improve performance for - addresses with large UTXO sets. It also stores the `TransparentAddrLoc` for each + addresses with large UTXO sets. It also stores the `AddressLocation` for each address, which allows for efficient lookups. - `utxo_by_transparent_addr_loc` stores unspent transparent output locations by address. @@ -752,12 +753,12 @@ So they should not be used for consensus-critical checks. has been spent in `utxo_by_outpoint`, that UTXO location can be deleted from `utxo_by_transparent_addr_loc`. (We don't do these deletions every time a block is committed, because that requires an expensive full index search.) - This list includes the `TransparentAddrLoc`, if it has not been spent. + This list includes the `AddressLocation`, if it has not been spent. (This duplicate data is small, and helps simplify the code.) - `tx_by_transparent_addr_loc` stores transaction locations by address. This list includes transactions containing spent UTXOs. - It also includes the `TransactionLocation` from the `TransparentAddrLoc`. + It also includes the `TransactionLocation` from the `AddressLocation`. (This duplicate data is small, and helps simplify the code.) - Each `*_note_commitment_tree` stores the note commitment tree state diff --git a/zebra-state/proptest-regressions/service/check/tests/utxo.txt b/zebra-state/proptest-regressions/service/check/tests/utxo.txt new file mode 100644 index 000000000..c49186362 --- /dev/null +++ b/zebra-state/proptest-regressions/service/check/tests/utxo.txt @@ -0,0 +1,10 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc e269485ce65fc50f093f8d979c5afb233709e0c18e56ab419afb065c2e0bf854 # shrinks to output = zebra_chain::transparent::Output, mut prevout_input = zebra_chain::transparent::Input, use_finalized_state = false +cc 2639971d2f0cad4354fa6a4b00f8d588e04638c33d884f8d31ca6b09e43a31d9 # shrinks to output = zebra_chain::transparent::Output, mut prevout_input = zebra_chain::transparent::Input, use_finalized_state_output = false, mut use_finalized_state_spend = false +cc 59045504569e389f48e0f8d1b7938e5fdfed84e1ba83af25c18df8300086788c # shrinks to unused_output = zebra_chain::transparent::Output, prevout_input = zebra_chain::transparent::Input +cc 65bbd1a767ce94e046fbab250fc8b9c8f3acc52bf9d032c9f198347052b62775 # shrinks to output = zebra_chain::transparent::Output, mut prevout_input = zebra_chain::transparent::Input diff --git a/zebra-state/proptest-regressions/service/finalized_state/tests/prop.txt b/zebra-state/proptest-regressions/service/finalized_state/tests/prop.txt new file mode 100644 index 000000000..5339d318d --- /dev/null +++ b/zebra-state/proptest-regressions/service/finalized_state/tests/prop.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 933c998cd42e62c9b80ceae375981200f1039e493262f7d931d973900c75812e # shrinks to (chain, count, network, _history_tree) = (alloc::vec::Vec, len=104, 2, Mainnet, HistoryTree(None)) diff --git a/zebra-state/proptest-regressions/service/tests.txt b/zebra-state/proptest-regressions/service/tests.txt index 2aa3a91ef..569e8416a 100644 --- a/zebra-state/proptest-regressions/service/tests.txt +++ b/zebra-state/proptest-regressions/service/tests.txt @@ -5,3 +5,5 @@ # It is recommended to check this file in to source control so that # everyone who runs the test benefits from these saved cases. cc 37aea4b0880d7d9029ea4fad0136bd8553f81eea0435122737ec513f4f6fb73c # shrinks to (network, nu_activation_height, chain) = (Mainnet, Height(1046400), alloc::vec::Vec>>, len=101) +cc 1a833b934966164ec7170c4bbdd7c48723ac0c873203af5f7880539ff1c095bf # shrinks to (network, finalized_blocks, non_finalized_blocks) = (Mainnet, alloc::vec::Vec, len=2, alloc::vec::Vec, len=9) +cc 5fe3b32843194422a1ed411c7187c013d0cfd5c5f4a238643df1d5a7decd12c0 # shrinks to (network, finalized_blocks, non_finalized_blocks) = (Mainnet, alloc::vec::Vec, len=2, alloc::vec::Vec, len=9) diff --git a/zebra-state/src/service/finalized_state/arbitrary.rs b/zebra-state/src/service/finalized_state/arbitrary.rs index 8f9af08b4..cb76bd10e 100644 --- a/zebra-state/src/service/finalized_state/arbitrary.rs +++ b/zebra-state/src/service/finalized_state/arbitrary.rs @@ -4,33 +4,14 @@ use std::sync::Arc; -use proptest::prelude::*; - -use zebra_chain::{ - amount::NonNegative, - block::{self, Block}, - sprout, - value_balance::ValueBalance, -}; +use zebra_chain::{amount::NonNegative, block::Block, sprout, value_balance::ValueBalance}; use crate::service::finalized_state::{ disk_db::{DiskWriteBatch, WriteDisk}, - disk_format::{FromDisk, IntoDisk, TransactionLocation}, + disk_format::{FromDisk, IntoDisk}, FinalizedState, }; -impl Arbitrary for TransactionLocation { - type Parameters = (); - - fn arbitrary_with(_args: Self::Parameters) -> Self::Strategy { - (any::(), any::()) - .prop_map(|(height, index)| Self { height, index }) - .boxed() - } - - type Strategy = BoxedStrategy; -} - pub fn round_trip(input: T) -> T where T: IntoDisk + FromDisk, diff --git a/zebra-state/src/service/finalized_state/disk_format.rs b/zebra-state/src/service/finalized_state/disk_format.rs index 3d05d55fd..00152526c 100644 --- a/zebra-state/src/service/finalized_state/disk_format.rs +++ b/zebra-state/src/service/finalized_state/disk_format.rs @@ -17,8 +17,8 @@ mod tests; pub use block::TransactionLocation; -/// Helper trait for defining the exact format used to interact with disk per -/// type. +/// Helper trait for defining the exact format used to store to disk, +/// for each type. pub trait IntoDisk { /// The type used to compare a value as a key to other keys stored in a /// database. @@ -29,6 +29,14 @@ pub trait IntoDisk { fn as_bytes(&self) -> Self::Bytes; } +/// Helper trait for types with fixed-length disk storage. +/// +/// This trait must not be implemented for types with variable-length disk storage. +pub trait IntoDiskFixedLen: IntoDisk { + /// Returns the fixed serialized length of `Bytes`. + fn fixed_byte_len() -> usize; +} + /// Helper type for retrieving types from the disk with the correct format. /// /// The ivec should be correctly encoded by IntoDisk. @@ -41,7 +49,7 @@ pub trait FromDisk: Sized { fn from_bytes(bytes: impl AsRef<[u8]>) -> Self; } -// Generic trait impls +// Generic serialization impls impl<'a, T> IntoDisk for &'a T where @@ -74,6 +82,8 @@ where } } +// Commonly used serialization impls + impl IntoDisk for () { type Bytes = [u8; 0]; @@ -81,3 +91,19 @@ impl IntoDisk for () { [] } } + +// Generic serialization length impls + +impl IntoDiskFixedLen for T +where + T: IntoDisk, + T::Bytes: Default + IntoIterator + Copy, +{ + /// Returns the fixed size of `Bytes`. + /// + /// Assumes that `Copy` types are fixed-sized byte arrays. + fn fixed_byte_len() -> usize { + // Bytes is probably a [u8; N] + Self::Bytes::default().into_iter().count() + } +} diff --git a/zebra-state/src/service/finalized_state/disk_format/block.rs b/zebra-state/src/service/finalized_state/disk_format/block.rs index f9dbdc765..beb3af40e 100644 --- a/zebra-state/src/service/finalized_state/disk_format/block.rs +++ b/zebra-state/src/service/finalized_state/disk_format/block.rs @@ -15,29 +15,58 @@ use zebra_chain::{ transaction, }; -use crate::service::finalized_state::disk_format::{FromDisk, IntoDisk}; +use crate::service::finalized_state::disk_format::{FromDisk, IntoDisk, IntoDiskFixedLen}; + +#[cfg(any(test, feature = "proptest-impl"))] +use proptest_derive::Arbitrary; + +// Transaction types + +/// A transaction's index in its block. +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))] +pub struct TransactionIndex(u32); + +impl TransactionIndex { + /// Create a transaction index from the native index integer type. + #[allow(dead_code)] + pub fn from_usize(transaction_index: usize) -> TransactionIndex { + TransactionIndex( + transaction_index + .try_into() + .expect("the maximum valid index fits in the inner type"), + ) + } + + /// Return this index as the native index integer type. + #[allow(dead_code)] + pub fn as_usize(&self) -> usize { + self.0 + .try_into() + .expect("the maximum valid index fits in usize") + } +} /// A transaction's location in the chain, by block height and transaction index. /// /// This provides a chain-order list of transactions. #[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))] pub struct TransactionLocation { /// The block height of the transaction. pub height: Height, /// The index of the transaction in its block. - pub index: u32, + pub index: TransactionIndex, } impl TransactionLocation { /// Create a transaction location from a block height and index (as the native index integer type). #[allow(dead_code)] - pub fn from_usize(height: Height, index: usize) -> TransactionLocation { + pub fn from_usize(height: Height, transaction_index: usize) -> TransactionLocation { TransactionLocation { height, - index: index - .try_into() - .expect("all valid indexes are much lower than u32::MAX"), + index: TransactionIndex::from_usize(transaction_index), } } } @@ -92,37 +121,39 @@ impl FromDisk for block::Hash { // Transaction trait impls +impl IntoDisk for TransactionIndex { + type Bytes = [u8; 4]; + + fn as_bytes(&self) -> Self::Bytes { + self.0.to_be_bytes() + } +} + +impl FromDisk for TransactionIndex { + fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self { + TransactionIndex(u32::from_be_bytes(disk_bytes.as_ref().try_into().unwrap())) + } +} + impl IntoDisk for TransactionLocation { type Bytes = [u8; 8]; fn as_bytes(&self) -> Self::Bytes { let height_bytes = self.height.as_bytes(); - let index_bytes = self.index.to_be_bytes(); + let index_bytes = self.index.as_bytes(); - let mut bytes = [0; 8]; - - bytes[0..4].copy_from_slice(&height_bytes); - bytes[4..8].copy_from_slice(&index_bytes); - - bytes + [height_bytes, index_bytes].concat().try_into().unwrap() } } impl FromDisk for TransactionLocation { fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self { - let disk_bytes = disk_bytes.as_ref(); - let height = { - let mut bytes = [0; 4]; - bytes.copy_from_slice(&disk_bytes[0..4]); - let height = u32::from_be_bytes(bytes); - Height(height) - }; + let height_len = Height::fixed_byte_len(); - let index = { - let mut bytes = [0; 4]; - bytes.copy_from_slice(&disk_bytes[4..8]); - u32::from_be_bytes(bytes) - }; + let (height_bytes, index_bytes) = disk_bytes.as_ref().split_at(height_len); + + let height = Height::from_bytes(height_bytes); + let index = TransactionIndex::from_bytes(index_bytes); TransactionLocation { height, index } } @@ -135,3 +166,9 @@ impl IntoDisk for transaction::Hash { self.0 } } + +impl FromDisk for transaction::Hash { + fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self { + transaction::Hash(disk_bytes.as_ref().try_into().unwrap()) + } +} diff --git a/zebra-state/src/service/finalized_state/disk_format/transparent.rs b/zebra-state/src/service/finalized_state/disk_format/transparent.rs index 4de232a89..5766c81c4 100644 --- a/zebra-state/src/service/finalized_state/disk_format/transparent.rs +++ b/zebra-state/src/service/finalized_state/disk_format/transparent.rs @@ -5,36 +5,164 @@ //! The [`crate::constants::DATABASE_FORMAT_VERSION`] constant must //! be incremented each time the database format (column, serialization, etc) changes. +use std::fmt::Debug; + +use serde::{Deserialize, Serialize}; + use zebra_chain::{ block::Height, serialization::{ZcashDeserializeInto, ZcashSerialize}, - transparent, + transaction, transparent, }; -use crate::service::finalized_state::disk_format::{FromDisk, IntoDisk}; +use crate::service::finalized_state::disk_format::{FromDisk, IntoDisk, IntoDiskFixedLen}; +#[cfg(any(test, feature = "proptest-impl"))] +use proptest_derive::Arbitrary; + +// Transparent types + +/// A transaction's index in its block. +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))] +pub struct OutputIndex(u32); + +impl OutputIndex { + /// Create a transparent output index from the native index integer type. + #[allow(dead_code)] + pub fn from_usize(output_index: usize) -> OutputIndex { + OutputIndex( + output_index + .try_into() + .expect("the maximum valid index fits in the inner type"), + ) + } + + /// Return this index as the native index integer type. + #[allow(dead_code)] + pub fn as_usize(&self) -> usize { + self.0 + .try_into() + .expect("the maximum valid index fits in usize") + } + + /// Create a transparent output index from the Zcash consensus integer type. + pub fn from_zcash(output_index: u32) -> OutputIndex { + OutputIndex(output_index) + } + + /// Return this index as the Zcash consensus integer type. + #[allow(dead_code)] + pub fn as_zcash(&self) -> u32 { + self.0 + } +} + +/// A transparent output's location in the chain, by block height and transaction index. +/// +/// TODO: provide a chain-order list of transactions (#3150) +/// derive Ord, PartialOrd (#3150) +#[derive(Copy, Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))] +pub struct OutputLocation { + /// The transaction hash. + pub hash: transaction::Hash, + + /// The index of the transparent output in its transaction. + pub index: OutputIndex, +} + +impl OutputLocation { + /// Create a transparent output location from a transaction hash and index + /// (as the native index integer type). + #[allow(dead_code)] + pub fn from_usize(hash: transaction::Hash, output_index: usize) -> OutputLocation { + OutputLocation { + hash, + index: OutputIndex::from_usize(output_index), + } + } + + /// Create a transparent output location from a [`transparent::OutPoint`]. + pub fn from_outpoint(outpoint: &transparent::OutPoint) -> OutputLocation { + OutputLocation { + hash: outpoint.hash, + index: OutputIndex::from_zcash(outpoint.index), + } + } +} + +// Transparent trait impls + +// TODO: serialize the index into a smaller number of bytes (#3152) +// serialize the index in big-endian order (#3150) +impl IntoDisk for OutputIndex { + type Bytes = [u8; 4]; + + fn as_bytes(&self) -> Self::Bytes { + self.0.to_le_bytes() + } +} + +impl FromDisk for OutputIndex { + fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self { + OutputIndex(u32::from_le_bytes(disk_bytes.as_ref().try_into().unwrap())) + } +} + +impl IntoDisk for OutputLocation { + type Bytes = [u8; 36]; + + fn as_bytes(&self) -> Self::Bytes { + let hash_bytes = self.hash.as_bytes().to_vec(); + let index_bytes = self.index.as_bytes().to_vec(); + + [hash_bytes, index_bytes].concat().try_into().unwrap() + } +} + +impl FromDisk for OutputLocation { + fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self { + let hash_len = transaction::Hash::fixed_byte_len(); + + let (hash_bytes, index_bytes) = disk_bytes.as_ref().split_at(hash_len); + + let hash = transaction::Hash::from_bytes(hash_bytes); + let index = OutputIndex::from_bytes(index_bytes); + + OutputLocation { hash, index } + } +} + +// TODO: just serialize the Output, and derive the Utxo data from OutputLocation (#3151) impl IntoDisk for transparent::Utxo { type Bytes = Vec; fn as_bytes(&self) -> Self::Bytes { - let mut bytes = vec![0; 5]; - bytes[0..4].copy_from_slice(&self.height.0.to_be_bytes()); - bytes[4] = self.from_coinbase as u8; - self.output - .zcash_serialize(&mut bytes) + let height_bytes = self.height.as_bytes().to_vec(); + let coinbase_flag_bytes = [self.from_coinbase as u8].to_vec(); + let output_bytes = self + .output + .zcash_serialize_to_vec() .expect("serialization to vec doesn't fail"); - bytes + + [height_bytes, coinbase_flag_bytes, output_bytes].concat() } } impl FromDisk for transparent::Utxo { fn from_bytes(bytes: impl AsRef<[u8]>) -> Self { - let (meta_bytes, output_bytes) = bytes.as_ref().split_at(5); - let height = Height(u32::from_be_bytes(meta_bytes[0..4].try_into().unwrap())); - let from_coinbase = meta_bytes[4] == 1u8; + let height_len = Height::fixed_byte_len(); + + let (height_bytes, rest_bytes) = bytes.as_ref().split_at(height_len); + let (coinbase_flag_bytes, output_bytes) = rest_bytes.split_at(1); + + let height = Height::from_bytes(height_bytes); + let from_coinbase = coinbase_flag_bytes[0] == 1u8; let output = output_bytes .zcash_deserialize_into() - .expect("db has serialized data"); + .expect("db has valid serialized data"); + Self { output, height, @@ -42,12 +170,3 @@ impl FromDisk for transparent::Utxo { } } } - -impl IntoDisk for transparent::OutPoint { - type Bytes = Vec; - - fn as_bytes(&self) -> Self::Bytes { - self.zcash_serialize_to_vec() - .expect("serialization to vec doesn't fail") - } -} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 34e039cd5..974fd1450 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -91,7 +91,7 @@ impl FinalizedState { .expect("block will exist if TransactionLocation does"); // TODO: store transactions in a separate database index (#3151) - block.transactions[index as usize].clone() + block.transactions[index.as_usize()].clone() }) } } @@ -224,12 +224,7 @@ impl DiskWriteBatch { .zip(transaction_hashes.iter()) .enumerate() { - let transaction_location = TransactionLocation { - height: *height, - index: transaction_index - .try_into() - .expect("no more than 4 billion transactions per block"), - }; + let transaction_location = TransactionLocation::from_usize(*height, transaction_index); self.zs_insert(tx_by_hash, transaction_hash, transaction_location); self.prepare_nullifier_batch(db, transaction)?; diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@mainnet_0.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@mainnet_0.snap index 4f81e89e8..efa98d76c 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@mainnet_0.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@mainnet_0.snap @@ -1,6 +1,6 @@ --- source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs -assertion_line: 245 +assertion_line: 274 expression: stored_transaction_hashes --- @@ -8,7 +8,7 @@ expression: stored_transaction_hashes TransactionHash( loc: TransactionLocation( height: Height(0), - index: 0, + index: TransactionIndex(0), ), hash: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", ), diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@mainnet_1.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@mainnet_1.snap index aa7a7743d..eabe89f3e 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@mainnet_1.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@mainnet_1.snap @@ -1,6 +1,6 @@ --- source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs -assertion_line: 245 +assertion_line: 274 expression: stored_transaction_hashes --- @@ -8,14 +8,14 @@ expression: stored_transaction_hashes TransactionHash( loc: TransactionLocation( height: Height(0), - index: 0, + index: TransactionIndex(0), ), hash: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", ), TransactionHash( loc: TransactionLocation( height: Height(1), - index: 0, + index: TransactionIndex(0), ), hash: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609", ), diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@mainnet_2.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@mainnet_2.snap index 6f5888228..a67b01f9b 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@mainnet_2.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@mainnet_2.snap @@ -1,6 +1,6 @@ --- source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs -assertion_line: 245 +assertion_line: 274 expression: stored_transaction_hashes --- @@ -8,21 +8,21 @@ expression: stored_transaction_hashes TransactionHash( loc: TransactionLocation( height: Height(0), - index: 0, + index: TransactionIndex(0), ), hash: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", ), TransactionHash( loc: TransactionLocation( height: Height(1), - index: 0, + index: TransactionIndex(0), ), hash: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609", ), TransactionHash( loc: TransactionLocation( height: Height(2), - index: 0, + index: TransactionIndex(0), ), hash: "8974d08d1c5f9c860d8b629d582a56659a4a1dcb2b5f98a25a5afcc2a784b0f4", ), diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@testnet_0.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@testnet_0.snap index 4f81e89e8..efa98d76c 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@testnet_0.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@testnet_0.snap @@ -1,6 +1,6 @@ --- source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs -assertion_line: 245 +assertion_line: 274 expression: stored_transaction_hashes --- @@ -8,7 +8,7 @@ expression: stored_transaction_hashes TransactionHash( loc: TransactionLocation( height: Height(0), - index: 0, + index: TransactionIndex(0), ), hash: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", ), diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@testnet_1.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@testnet_1.snap index e31e98e69..a4f68136c 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@testnet_1.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@testnet_1.snap @@ -1,6 +1,6 @@ --- source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs -assertion_line: 245 +assertion_line: 274 expression: stored_transaction_hashes --- @@ -8,14 +8,14 @@ expression: stored_transaction_hashes TransactionHash( loc: TransactionLocation( height: Height(0), - index: 0, + index: TransactionIndex(0), ), hash: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", ), TransactionHash( loc: TransactionLocation( height: Height(1), - index: 0, + index: TransactionIndex(0), ), hash: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75", ), diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@testnet_2.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@testnet_2.snap index 1e8206db3..bc1851f7a 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@testnet_2.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transaction_hashes@testnet_2.snap @@ -1,6 +1,6 @@ --- source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs -assertion_line: 245 +assertion_line: 274 expression: stored_transaction_hashes --- @@ -8,21 +8,21 @@ expression: stored_transaction_hashes TransactionHash( loc: TransactionLocation( height: Height(0), - index: 0, + index: TransactionIndex(0), ), hash: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", ), TransactionHash( loc: TransactionLocation( height: Height(1), - index: 0, + index: TransactionIndex(0), ), hash: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75", ), TransactionHash( loc: TransactionLocation( height: Height(2), - index: 0, + index: TransactionIndex(0), ), hash: "5822c0532da8a008259ac39933d3210e508c17e3ba21d2b2c428785efdccb3d5", ), diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@mainnet_0.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@mainnet_0.snap index f954efcd6..06f505d03 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@mainnet_0.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@mainnet_0.snap @@ -1,6 +1,6 @@ --- source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs -assertion_line: 246 +assertion_line: 275 expression: stored_transactions --- @@ -8,7 +8,7 @@ expression: stored_transactions TransactionData( loc: TransactionLocation( height: Height(0), - index: 0, + index: TransactionIndex(0), ), transaction: "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff071f0104455a6361736830623963346565663862376363343137656535303031653335303039383462366665613335363833613763616331343161303433633432303634383335643334ffffffff010000000000000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000", ), diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@mainnet_1.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@mainnet_1.snap index cc5fa362c..9bfa4b5da 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@mainnet_1.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@mainnet_1.snap @@ -1,6 +1,6 @@ --- source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs -assertion_line: 246 +assertion_line: 275 expression: stored_transactions --- @@ -8,14 +8,14 @@ expression: stored_transactions TransactionData( loc: TransactionLocation( height: Height(0), - index: 0, + index: TransactionIndex(0), ), transaction: "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff071f0104455a6361736830623963346565663862376363343137656535303031653335303039383462366665613335363833613763616331343161303433633432303634383335643334ffffffff010000000000000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000", ), TransactionData( loc: TransactionLocation( height: Height(1), - index: 0, + index: TransactionIndex(0), ), transaction: "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff025100ffffffff0250c30000000000002321027a46eb513588b01b37ea24303f4b628afd12cc20df789fede0921e43cad3e875acd43000000000000017a9147d46a730d31f97b1930d3368a967c309bd4d136a8700000000", ), diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@mainnet_2.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@mainnet_2.snap index 0a52c978f..c615d20c5 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@mainnet_2.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@mainnet_2.snap @@ -1,6 +1,6 @@ --- source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs -assertion_line: 246 +assertion_line: 275 expression: stored_transactions --- @@ -8,21 +8,21 @@ expression: stored_transactions TransactionData( loc: TransactionLocation( height: Height(0), - index: 0, + index: TransactionIndex(0), ), transaction: "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff071f0104455a6361736830623963346565663862376363343137656535303031653335303039383462366665613335363833613763616331343161303433633432303634383335643334ffffffff010000000000000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000", ), TransactionData( loc: TransactionLocation( height: Height(1), - index: 0, + index: TransactionIndex(0), ), transaction: "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff025100ffffffff0250c30000000000002321027a46eb513588b01b37ea24303f4b628afd12cc20df789fede0921e43cad3e875acd43000000000000017a9147d46a730d31f97b1930d3368a967c309bd4d136a8700000000", ), TransactionData( loc: TransactionLocation( height: Height(2), - index: 0, + index: TransactionIndex(0), ), transaction: "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff025200ffffffff02a0860100000000002321027a46eb513588b01b37ea24303f4b628afd12cc20df789fede0921e43cad3e875aca86100000000000017a9147d46a730d31f97b1930d3368a967c309bd4d136a8700000000", ), diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@testnet_0.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@testnet_0.snap index f954efcd6..06f505d03 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@testnet_0.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@testnet_0.snap @@ -1,6 +1,6 @@ --- source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs -assertion_line: 246 +assertion_line: 275 expression: stored_transactions --- @@ -8,7 +8,7 @@ expression: stored_transactions TransactionData( loc: TransactionLocation( height: Height(0), - index: 0, + index: TransactionIndex(0), ), transaction: "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff071f0104455a6361736830623963346565663862376363343137656535303031653335303039383462366665613335363833613763616331343161303433633432303634383335643334ffffffff010000000000000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000", ), diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@testnet_1.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@testnet_1.snap index ec66c71f2..e056c45fa 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@testnet_1.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@testnet_1.snap @@ -1,6 +1,6 @@ --- source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs -assertion_line: 246 +assertion_line: 275 expression: stored_transactions --- @@ -8,14 +8,14 @@ expression: stored_transactions TransactionData( loc: TransactionLocation( height: Height(0), - index: 0, + index: TransactionIndex(0), ), transaction: "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff071f0104455a6361736830623963346565663862376363343137656535303031653335303039383462366665613335363833613763616331343161303433633432303634383335643334ffffffff010000000000000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000", ), TransactionData( loc: TransactionLocation( height: Height(1), - index: 0, + index: TransactionIndex(0), ), transaction: "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff03510101ffffffff0250c30000000000002321025229e1240a21004cf8338db05679fa34753706e84f6aebba086ba04317fd8f99acd43000000000000017a914ef775f1f997f122a062fff1a2d7443abd1f9c6428700000000", ), diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@testnet_2.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@testnet_2.snap index b9b8c548a..95cfe0795 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@testnet_2.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/transactions@testnet_2.snap @@ -1,6 +1,6 @@ --- source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs -assertion_line: 227 +assertion_line: 275 expression: stored_transactions --- @@ -8,21 +8,21 @@ expression: stored_transactions TransactionData( loc: TransactionLocation( height: Height(0), - index: 0, + index: TransactionIndex(0), ), transaction: "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff071f0104455a6361736830623963346565663862376363343137656535303031653335303039383462366665613335363833613763616331343161303433633432303634383335643334ffffffff010000000000000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000", ), TransactionData( loc: TransactionLocation( height: Height(1), - index: 0, + index: TransactionIndex(0), ), transaction: "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff03510101ffffffff0250c30000000000002321025229e1240a21004cf8338db05679fa34753706e84f6aebba086ba04317fd8f99acd43000000000000017a914ef775f1f997f122a062fff1a2d7443abd1f9c6428700000000", ), TransactionData( loc: TransactionLocation( height: Height(2), - index: 0, + index: TransactionIndex(0), ), transaction: "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff03520103ffffffff02a086010000000000232102acce9f6c16986c525fd34759d851ef5b4b85b5019a57bd59747be0ef1ba62523aca86100000000000017a914ef775f1f997f122a062fff1a2d7443abd1f9c6428700000000", ), diff --git a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs index 22073fe6c..9a1b5791f 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs @@ -18,6 +18,7 @@ use zebra_chain::transparent; use crate::{ service::finalized_state::{ disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk}, + disk_format::transparent::OutputLocation, FinalizedBlock, FinalizedState, }, BoxError, @@ -30,7 +31,10 @@ impl FinalizedState { /// `transparent::OutPoint` if it is present. pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option { let utxo_by_outpoint = self.db.cf_handle("utxo_by_outpoint").unwrap(); - self.db.zs_get(utxo_by_outpoint, outpoint) + + let output_location = OutputLocation::from_outpoint(outpoint); + + self.db.zs_get(utxo_by_outpoint, &output_location) } } @@ -54,20 +58,23 @@ impl DiskWriteBatch { // Index all new transparent outputs, before deleting any we've spent for (outpoint, utxo) in new_outputs.borrow().iter() { - self.zs_insert(utxo_by_outpoint, outpoint, utxo); + let output_location = OutputLocation::from_outpoint(outpoint); + + self.zs_insert(utxo_by_outpoint, output_location, utxo); } // Mark all transparent inputs as spent. // // Coinbase inputs represent new coins, // so there are no UTXOs to mark as spent. - for outpoint in block + for output_location in block .transactions .iter() .flat_map(|tx| tx.inputs()) .flat_map(|input| input.outpoint()) + .map(|outpoint| OutputLocation::from_outpoint(&outpoint)) { - self.zs_delete(utxo_by_outpoint, outpoint); + self.zs_delete(utxo_by_outpoint, output_location); } Ok(())