From 7e8194c63f68dc1a4b9c5a1b61da0125b0c88fb2 Mon Sep 17 00:00:00 2001 From: teor Date: Sat, 9 Apr 2022 08:42:05 +1000 Subject: [PATCH] 3. change(db): Store UTXOs by transaction location rather than transaction hash (#3978) * Change OutputLocation to contain a TransactionLocation * Change OutputLocation reads from the database * Update some doc comments * Update some TODOs * Change deleting spent UTXOs and updating spent balances * Change adding new UTXOs and adding their values to balances * Disable dead code warnings * Update snapshot test code * Update round-trip tests for OutputLocations * Update snapshot test data * Increment the database format version * Remove a redundant try_into() Co-authored-by: Janito Vaqueiro Ferreira Filho * Refactor redundant code Co-authored-by: Janito Vaqueiro Ferreira Filho * ci: attempt at fixing 'Regenerate stateful disks' Co-authored-by: Janito Vaqueiro Ferreira Filho Co-authored-by: Conrado Gouvea --- .github/workflows/test.yml | 2 +- zebra-chain/src/transparent/utxo.rs | 6 + zebra-state/src/constants.rs | 2 +- zebra-state/src/request.rs | 8 +- zebra-state/src/service.rs | 7 +- .../src/service/finalized_state/disk_db.rs | 8 +- .../service/finalized_state/disk_format.rs | 2 +- .../finalized_state/disk_format/block.rs | 57 ++++-- .../finalized_state/disk_format/tests/prop.rs | 21 +- ...y_transparent_addr_raw_data@mainnet_1.snap | 2 +- ...y_transparent_addr_raw_data@mainnet_2.snap | 2 +- ...y_transparent_addr_raw_data@testnet_1.snap | 2 +- ...y_transparent_addr_raw_data@testnet_2.snap | 2 +- .../utxo_by_outpoint_raw_data@mainnet_1.snap | 4 +- .../utxo_by_outpoint_raw_data@mainnet_2.snap | 8 +- .../utxo_by_outpoint_raw_data@testnet_1.snap | 4 +- .../utxo_by_outpoint_raw_data@testnet_2.snap | 8 +- .../disk_format/transparent.rs | 189 +++++++++++++----- .../disk_format/transparent/arbitrary.rs | 19 ++ .../service/finalized_state/zebra_db/block.rs | 121 ++++++++--- .../zebra_db/block/tests/snapshot.rs | 17 +- .../snapshots/address_balances@mainnet_1.snap | 7 +- .../snapshots/address_balances@mainnet_2.snap | 7 +- .../snapshots/address_balances@testnet_1.snap | 7 +- .../snapshots/address_balances@testnet_2.snap | 7 +- .../tests/snapshots/utxos@mainnet_0.snap | 9 +- .../tests/snapshots/utxos@mainnet_1.snap | 27 ++- .../tests/snapshots/utxos@mainnet_2.snap | 45 +++-- .../tests/snapshots/utxos@testnet_0.snap | 9 +- .../tests/snapshots/utxos@testnet_1.snap | 27 ++- .../tests/snapshots/utxos@testnet_2.snap | 45 +++-- .../service/finalized_state/zebra_db/chain.rs | 4 +- .../finalized_state/zebra_db/transparent.rs | 57 +++--- 33 files changed, 523 insertions(+), 219 deletions(-) create mode 100644 zebra-state/src/service/finalized_state/disk_format/transparent/arbitrary.rs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c234f358d..9a6b85054 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -324,7 +324,7 @@ jobs: EXIT_CODE=$(\ gcloud compute ssh \ - sync-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + regenerate-disk-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ --zone ${{ env.ZONE }} \ --quiet \ --ssh-flag="-o ServerAliveInterval=5" \ diff --git a/zebra-chain/src/transparent/utxo.rs b/zebra-chain/src/transparent/utxo.rs index 4f836159a..4e27fc3da 100644 --- a/zebra-chain/src/transparent/utxo.rs +++ b/zebra-chain/src/transparent/utxo.rs @@ -17,6 +17,10 @@ use crate::{ pub struct Utxo { /// The output itself. pub output: transparent::Output, + + // TODO: replace the height and from_coinbase fields with OutputLocation, + // and provide lookup/calculation methods for height and from_coinbase + // /// The height at which the output was created. pub height: block::Height, /// Whether the output originated in a coinbase transaction. @@ -35,6 +39,8 @@ pub struct Utxo { any(test, feature = "proptest-impl"), derive(proptest_derive::Arbitrary) )] +// +// TODO: after modifying UTXO to contain an OutputLocation, replace this type with UTXO pub struct OrderedUtxo { /// An unspent transaction output. pub utxo: Utxo, diff --git a/zebra-state/src/constants.rs b/zebra-state/src/constants.rs index 583ccb2eb..8193857f2 100644 --- a/zebra-state/src/constants.rs +++ b/zebra-state/src/constants.rs @@ -18,7 +18,7 @@ pub use zebra_chain::transparent::MIN_TRANSPARENT_COINBASE_MATURITY; pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRANSPARENT_COINBASE_MATURITY - 1; /// The database format version, incremented each time the database format changes. -pub const DATABASE_FORMAT_VERSION: u32 = 17; +pub const DATABASE_FORMAT_VERSION: u32 = 18; /// The maximum number of blocks to check for NU5 transactions, /// before we assume we are on a pre-NU5 legacy chain. diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs index 985f6932e..3894cd4ce 100644 --- a/zebra-state/src/request.rs +++ b/zebra-state/src/request.rs @@ -10,8 +10,8 @@ use zebra_chain::{ value_balance::{ValueBalance, ValueBalanceError}, }; -// Allow *only* this unused import, so that rustdoc link resolution -// will work with inline links. +/// Allow *only* this unused import, so that rustdoc link resolution +/// will work with inline links. #[allow(unused_imports)] use crate::Response; @@ -119,10 +119,6 @@ pub struct FinalizedBlock { /// New transparent outputs created in this block, indexed by /// [`Outpoint`](transparent::Outpoint). /// - /// Each output is tagged with its transaction index in the block. - /// (The outputs of earlier transactions in a block can be spent by later - /// transactions.) - /// /// Note: although these transparent outputs are newly created, they may not /// be unspent, since a later transaction in a block can spend outputs of an /// earlier transaction. diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 0a342d9ca..ca1f540ca 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -497,7 +497,12 @@ impl StateService { .or_else(|| self.disk.db().height(hash)) } - /// Return the [`Utxo`] pointed to by `outpoint` if it exists in any chain. + /// Return the [`Utxo`] pointed to by `outpoint`, if it exists in any chain, + /// or in any pending block. + /// + /// Some of the returned UTXOs may be invalid, because: + /// - they are not in the best chain, or + /// - their block fails contextual validation. pub fn any_utxo(&self, outpoint: &transparent::OutPoint) -> Option { self.mem .any_utxo(outpoint) diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs index 9a4485808..d636505d0 100644 --- a/zebra-state/src/service/finalized_state/disk_db.rs +++ b/zebra-state/src/service/finalized_state/disk_db.rs @@ -361,15 +361,15 @@ impl DiskDb { // Transactions rocksdb::ColumnFamilyDescriptor::new("tx_by_loc", db_options.clone()), rocksdb::ColumnFamilyDescriptor::new("hash_by_tx_loc", db_options.clone()), - // TODO: rename to tx_loc_by_hash (#3151) + // TODO: rename to tx_loc_by_hash (#3950) rocksdb::ColumnFamilyDescriptor::new("tx_by_hash", db_options.clone()), // Transparent rocksdb::ColumnFamilyDescriptor::new("balance_by_transparent_addr", db_options.clone()), - // TODO: #3954 + // TODO: #3951 //rocksdb::ColumnFamilyDescriptor::new("tx_by_transparent_addr_loc", db_options.clone()), - // TODO: rename to utxo_by_out_loc (#3953) + // TODO: rename to utxo_by_out_loc (#3952) rocksdb::ColumnFamilyDescriptor::new("utxo_by_outpoint", db_options.clone()), - // TODO: #3952 + // TODO: #3953 //rocksdb::ColumnFamilyDescriptor::new("utxo_by_transparent_addr_loc", db_options.clone()), // Sprout rocksdb::ColumnFamilyDescriptor::new("sprout_nullifiers", db_options.clone()), diff --git a/zebra-state/src/service/finalized_state/disk_format.rs b/zebra-state/src/service/finalized_state/disk_format.rs index 0790ceda7..e663d0054 100644 --- a/zebra-state/src/service/finalized_state/disk_format.rs +++ b/zebra-state/src/service/finalized_state/disk_format.rs @@ -15,7 +15,7 @@ pub mod transparent; #[cfg(test)] mod tests; -pub use block::TransactionLocation; +pub use block::{TransactionIndex, TransactionLocation}; /// Helper type for writing types to disk as raw bytes. /// Also used to convert key types to raw bytes for disk lookups. diff --git a/zebra-state/src/service/finalized_state/disk_format/block.rs b/zebra-state/src/service/finalized_state/disk_format/block.rs index b6a038897..eda3ca56c 100644 --- a/zebra-state/src/service/finalized_state/disk_format/block.rs +++ b/zebra-state/src/service/finalized_state/disk_format/block.rs @@ -5,8 +5,6 @@ //! The [`crate::constants::DATABASE_FORMAT_VERSION`] constant must //! be incremented each time the database format (column, serialization, etc) changes. -use std::fmt::Debug; - use serde::{Deserialize, Serialize}; use zebra_chain::{ @@ -24,7 +22,7 @@ use proptest_derive::Arbitrary; /// The maximum value of an on-disk serialized [`Height`]. /// -/// This allows us to store [`OutputIndex`]es in 8 bytes, +/// This allows us to store [`OutputLocation`]s in 8 bytes, /// which makes database searches more efficient. /// /// # Consensus @@ -46,24 +44,39 @@ pub const HEIGHT_DISK_BYTES: usize = 3; /// This reduces database size and increases lookup performance. pub const TX_INDEX_DISK_BYTES: usize = 2; -// Transaction types +/// [`TransactionLocation`]s are stored as a 3 byte height and a 2 byte transaction index. +/// +/// This reduces database size and increases lookup performance. +pub const TRANSACTION_LOCATION_DISK_BYTES: usize = HEIGHT_DISK_BYTES + TX_INDEX_DISK_BYTES; + +// Block and transaction types /// A transaction's index in its block. /// /// # Consensus /// -/// This maximum height supports on-disk storage of transactions in blocks up to ~5 MB. +/// A 2-byte index supports on-disk storage of transactions in blocks up to ~5 MB. /// (The current maximum block size is 2 MB.) /// /// Since Zebra only stores fully verified blocks on disk, /// blocks larger than this size are rejected before reaching the database. /// /// (The maximum transaction count is tested by the large generated block serialization tests.) -#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] #[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))] pub struct TransactionIndex(u16); impl TransactionIndex { + /// Creates a transaction index from the inner type. + pub fn from_index(transaction_index: u16) -> TransactionIndex { + TransactionIndex(transaction_index) + } + + /// Returns this index as the inner type. + pub fn index(&self) -> u16 { + self.0 + } + /// Creates a transaction index from a `usize`. pub fn from_usize(transaction_index: usize) -> TransactionIndex { TransactionIndex( @@ -102,7 +115,7 @@ impl TransactionIndex { /// A transaction's location in the chain, by block height and transaction index. /// /// This provides a chain-order list of transactions. -#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] #[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))] pub struct TransactionLocation { /// The block height of the transaction. @@ -113,7 +126,16 @@ pub struct TransactionLocation { } impl TransactionLocation { - /// Creates a transaction location from a block height and `usize` index. + /// Creates a transaction location from a block height and transaction index. + #[allow(dead_code)] + pub fn from_index(height: Height, transaction_index: u16) -> TransactionLocation { + TransactionLocation { + height, + index: TransactionIndex::from_index(transaction_index), + } + } + + /// Creates a transaction location from a block height and `usize` transaction index. pub fn from_usize(height: Height, transaction_index: usize) -> TransactionLocation { TransactionLocation { height, @@ -121,7 +143,7 @@ impl TransactionLocation { } } - /// Creates a transaction location from a block height and `u64` index. + /// Creates a transaction location from a block height and `u64` transaction index. pub fn from_u64(height: Height, transaction_index: u64) -> TransactionLocation { TransactionLocation { height, @@ -130,7 +152,7 @@ impl TransactionLocation { } } -// Block trait impls +// Block and transaction trait impls impl IntoDisk for block::Header { type Bytes = Vec; @@ -164,12 +186,13 @@ impl IntoDisk for Height { } impl FromDisk for Height { - fn from_bytes(bytes: impl AsRef<[u8]>) -> Self { + fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self { let mem_len = u32::BITS / 8; let mem_len = mem_len.try_into().unwrap(); - let mem_bytes = expand_zero_be_bytes(bytes.as_ref(), mem_len); - Height(u32::from_be_bytes(mem_bytes.try_into().unwrap())) + let mem_bytes = expand_zero_be_bytes(disk_bytes.as_ref(), mem_len); + let mem_bytes = mem_bytes.try_into().unwrap(); + Height(u32::from_be_bytes(mem_bytes)) } } @@ -213,18 +236,20 @@ impl IntoDisk for TransactionIndex { type Bytes = [u8; TX_INDEX_DISK_BYTES]; fn as_bytes(&self) -> Self::Bytes { - self.0.to_be_bytes() + self.index().to_be_bytes() } } impl FromDisk for TransactionIndex { fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self { - TransactionIndex(u16::from_be_bytes(disk_bytes.as_ref().try_into().unwrap())) + let disk_bytes = disk_bytes.as_ref().try_into().unwrap(); + + TransactionIndex::from_index(u16::from_be_bytes(disk_bytes)) } } impl IntoDisk for TransactionLocation { - type Bytes = [u8; HEIGHT_DISK_BYTES + TX_INDEX_DISK_BYTES]; + type Bytes = [u8; TRANSACTION_LOCATION_DISK_BYTES]; fn as_bytes(&self) -> Self::Bytes { let height_bytes = self.height.as_bytes().to_vec(); diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/prop.rs b/zebra-state/src/service/finalized_state/disk_format/tests/prop.rs index 7a839d8c2..2ce245764 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/prop.rs +++ b/zebra-state/src/service/finalized_state/disk_format/tests/prop.rs @@ -126,21 +126,36 @@ fn roundtrip_transparent_address() { fn roundtrip_output_location() { zebra_test::init(); - proptest!(|(val in any::())| assert_value_properties(val)); + proptest!( + |(mut val in any::())| { + *val.height_mut() = val.height().clamp(Height(0), MAX_ON_DISK_HEIGHT); + assert_value_properties(val) + } + ); } #[test] fn roundtrip_address_location() { zebra_test::init(); - proptest!(|(val in any::())| assert_value_properties(val)); + proptest!( + |(mut val in any::())| { + *val.height_mut() = val.height().clamp(Height(0), MAX_ON_DISK_HEIGHT); + assert_value_properties(val) + } + ); } #[test] fn roundtrip_address_balance_location() { zebra_test::init(); - proptest!(|(val in any::())| assert_value_properties(val)); + proptest!( + |(mut val in any::())| { + *val.height_mut() = val.location().height().clamp(Height(0), MAX_ON_DISK_HEIGHT); + assert_value_properties(val) + } + ); } #[test] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@mainnet_1.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@mainnet_1.snap index 1aa792a1b..cbb88f6fa 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@mainnet_1.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@mainnet_1.snap @@ -5,6 +5,6 @@ expression: cf_data [ KV( k: "017d46a730d31f97b1930d3368a967c309bd4d136a", - v: "d4300000000000000946edb9c083c9942d92305444527765fad789c438c717783276a9f7fbf61b8501000000", + v: "d4300000000000000000010000000001", ), ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@mainnet_2.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@mainnet_2.snap index 678849fbb..646afbe07 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@mainnet_2.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@mainnet_2.snap @@ -5,6 +5,6 @@ expression: cf_data [ KV( k: "017d46a730d31f97b1930d3368a967c309bd4d136a", - v: "7c920000000000000946edb9c083c9942d92305444527765fad789c438c717783276a9f7fbf61b8501000000", + v: "7c920000000000000000010000000001", ), ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@testnet_1.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@testnet_1.snap index 0712b625a..3e6546dde 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@testnet_1.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@testnet_1.snap @@ -5,6 +5,6 @@ expression: cf_data [ KV( k: "03ef775f1f997f122a062fff1a2d7443abd1f9c642", - v: "d430000000000000755f7c7d27a811596e9fae6dd30ca45be86e901d499909de35b6ff1f699f7ef301000000", + v: "d4300000000000000000010000000001", ), ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@testnet_2.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@testnet_2.snap index 124789a52..b7cfb00fe 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@testnet_2.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@testnet_2.snap @@ -5,6 +5,6 @@ expression: cf_data [ KV( k: "03ef775f1f997f122a062fff1a2d7443abd1f9c642", - v: "7c92000000000000755f7c7d27a811596e9fae6dd30ca45be86e901d499909de35b6ff1f699f7ef301000000", + v: "7c920000000000000000010000000001", ), ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@mainnet_1.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@mainnet_1.snap index 61687a76f..4630a1d56 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@mainnet_1.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@mainnet_1.snap @@ -4,11 +4,11 @@ expression: cf_data --- [ KV( - k: "0946edb9c083c9942d92305444527765fad789c438c717783276a9f7fbf61b8500000000", + k: "0000010000000000", v: "0000010150c30000000000002321027a46eb513588b01b37ea24303f4b628afd12cc20df789fede0921e43cad3e875ac", ), KV( - k: "0946edb9c083c9942d92305444527765fad789c438c717783276a9f7fbf61b8501000000", + k: "0000010000000001", v: "00000101d43000000000000017a9147d46a730d31f97b1930d3368a967c309bd4d136a87", ), ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@mainnet_2.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@mainnet_2.snap index 2f2c98294..c2bf99ce4 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@mainnet_2.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@mainnet_2.snap @@ -4,19 +4,19 @@ expression: cf_data --- [ KV( - k: "0946edb9c083c9942d92305444527765fad789c438c717783276a9f7fbf61b8500000000", + k: "0000010000000000", v: "0000010150c30000000000002321027a46eb513588b01b37ea24303f4b628afd12cc20df789fede0921e43cad3e875ac", ), KV( - k: "0946edb9c083c9942d92305444527765fad789c438c717783276a9f7fbf61b8501000000", + k: "0000010000000001", v: "00000101d43000000000000017a9147d46a730d31f97b1930d3368a967c309bd4d136a87", ), KV( - k: "f4b084a7c2fc5a5aa2985f2bcb1d4a9a65562a589d628b0d869c5f1c8dd0748900000000", + k: "0000020000000000", v: "00000201a0860100000000002321027a46eb513588b01b37ea24303f4b628afd12cc20df789fede0921e43cad3e875ac", ), KV( - k: "f4b084a7c2fc5a5aa2985f2bcb1d4a9a65562a589d628b0d869c5f1c8dd0748901000000", + k: "0000020000000001", v: "00000201a86100000000000017a9147d46a730d31f97b1930d3368a967c309bd4d136a87", ), ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@testnet_1.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@testnet_1.snap index b153b108e..dfd6f813f 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@testnet_1.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@testnet_1.snap @@ -4,11 +4,11 @@ expression: cf_data --- [ KV( - k: "755f7c7d27a811596e9fae6dd30ca45be86e901d499909de35b6ff1f699f7ef300000000", + k: "0000010000000000", v: "0000010150c30000000000002321025229e1240a21004cf8338db05679fa34753706e84f6aebba086ba04317fd8f99ac", ), KV( - k: "755f7c7d27a811596e9fae6dd30ca45be86e901d499909de35b6ff1f699f7ef301000000", + k: "0000010000000001", v: "00000101d43000000000000017a914ef775f1f997f122a062fff1a2d7443abd1f9c64287", ), ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@testnet_2.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@testnet_2.snap index 28dcd57e3..ea774a64e 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@testnet_2.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@testnet_2.snap @@ -4,19 +4,19 @@ expression: cf_data --- [ KV( - k: "755f7c7d27a811596e9fae6dd30ca45be86e901d499909de35b6ff1f699f7ef300000000", + k: "0000010000000000", v: "0000010150c30000000000002321025229e1240a21004cf8338db05679fa34753706e84f6aebba086ba04317fd8f99ac", ), KV( - k: "755f7c7d27a811596e9fae6dd30ca45be86e901d499909de35b6ff1f699f7ef301000000", + k: "0000010000000001", v: "00000101d43000000000000017a914ef775f1f997f122a062fff1a2d7443abd1f9c64287", ), KV( - k: "d5b3ccfd5e7828c4b2d221bae3178c500e21d33399c39a2508a0a82d53c0225800000000", + k: "0000020000000000", v: "00000201a086010000000000232102acce9f6c16986c525fd34759d851ef5b4b85b5019a57bd59747be0ef1ba62523ac", ), KV( - k: "d5b3ccfd5e7828c4b2d221bae3178c500e21d33399c39a2508a0a82d53c0225801000000", + k: "0000020000000001", v: "00000201a86100000000000017a914ef775f1f997f122a062fff1a2d7443abd1f9c64287", ), ] diff --git a/zebra-state/src/service/finalized_state/disk_format/transparent.rs b/zebra-state/src/service/finalized_state/disk_format/transparent.rs index bcc44e167..57e25df90 100644 --- a/zebra-state/src/service/finalized_state/disk_format/transparent.rs +++ b/zebra-state/src/service/finalized_state/disk_format/transparent.rs @@ -14,37 +14,57 @@ use zebra_chain::{ block::Height, parameters::Network::*, serialization::{ZcashDeserializeInto, ZcashSerialize}, - transaction, transparent::{self, Address::*}, }; -use crate::service::finalized_state::disk_format::{block::HEIGHT_DISK_BYTES, FromDisk, IntoDisk}; +use crate::service::finalized_state::disk_format::{ + block::{ + TransactionIndex, TransactionLocation, HEIGHT_DISK_BYTES, TRANSACTION_LOCATION_DISK_BYTES, + }, + expand_zero_be_bytes, truncate_zero_be_bytes, FromDisk, IntoDisk, +}; #[cfg(any(test, feature = "proptest-impl"))] use proptest_derive::Arbitrary; +#[cfg(any(test, feature = "proptest-impl"))] +mod arbitrary; + /// Transparent balances are stored as an 8 byte integer on disk. pub const BALANCE_DISK_BYTES: usize = 8; -/// Output transaction locations are stored as a 32 byte transaction hash on disk. +/// [`OutputIndex`]es are stored as 3 bytes on disk. /// -/// TODO: change to TransactionLocation to reduce database size and increases lookup performance (#3953) -pub const OUTPUT_TX_HASH_DISK_BYTES: usize = 32; +/// This reduces database size and increases lookup performance. +pub const OUTPUT_INDEX_DISK_BYTES: usize = 3; -/// [`OutputIndex`]es are stored as 4 bytes on disk. +/// [`OutputLocation`]s are stored as a 3 byte height, 2 byte transaction index, +/// and 3 byte output index on disk. /// -/// TODO: change to 3 bytes to reduce database size and increases lookup performance (#3953) -pub const OUTPUT_INDEX_DISK_BYTES: usize = 4; +/// This reduces database size and increases lookup performance. +pub const OUTPUT_LOCATION_DISK_BYTES: usize = + TRANSACTION_LOCATION_DISK_BYTES + OUTPUT_INDEX_DISK_BYTES; // Transparent types -/// A transaction's index in its block. -#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] -#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))] +/// A transparent output's index in its transaction. +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] pub struct OutputIndex(u32); impl OutputIndex { - /// Create a transparent output index from the native index integer type. + /// Create a transparent output index from the Zcash consensus integer type. + /// + /// `u32` is also the inner type. + pub fn from_index(output_index: u32) -> OutputIndex { + OutputIndex(output_index) + } + + /// Returns this index as the inner type. + pub fn index(&self) -> u32 { + self.0 + } + + /// Create a transparent output index from `usize`. #[allow(dead_code)] pub fn from_usize(output_index: usize) -> OutputIndex { OutputIndex( @@ -54,7 +74,7 @@ impl OutputIndex { ) } - /// Return this index as the native index integer type. + /// Return this index as `usize`. #[allow(dead_code)] pub fn as_usize(&self) -> usize { self.0 @@ -62,51 +82,104 @@ impl OutputIndex { .expect("the maximum valid index fits in usize") } - /// Create a transparent output index from the Zcash consensus integer type. - pub fn from_zcash(output_index: u32) -> OutputIndex { - OutputIndex(output_index) + /// Create a transparent output index from `u64`. + #[allow(dead_code)] + pub fn from_u64(output_index: u64) -> OutputIndex { + OutputIndex( + output_index + .try_into() + .expect("the maximum u64 index fits in the inner type"), + ) } - /// Return this index as the Zcash consensus integer type. + /// Return this index as `u64`. #[allow(dead_code)] - pub fn as_zcash(&self) -> u32 { - self.0 + pub fn as_u64(&self) -> u64 { + self.0.into() } } /// A transparent output's location in the chain, by block height and transaction index. /// -/// TODO: provide a chain-order list of transactions (#3150) -/// derive Ord, PartialOrd (#3150) -#[derive(Copy, Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +/// [`OutputLocation`]s are sorted in increasing chain order, by height, transaction index, +/// and output index. +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] #[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))] pub struct OutputLocation { - /// The transaction hash. - #[serde(with = "hex")] - pub hash: transaction::Hash, + /// The location of the transparent input's transaction. + transaction_location: TransactionLocation, /// The index of the transparent output in its transaction. - pub index: OutputIndex, + output_index: OutputIndex, } impl OutputLocation { - /// Create a transparent output location from a transaction hash and index - /// (as the native index integer type). + /// Creates an output location from a block height, and `usize` transaction and output indexes. #[allow(dead_code)] - pub fn from_usize(hash: transaction::Hash, output_index: usize) -> OutputLocation { + pub fn from_usize( + height: Height, + transaction_index: usize, + output_index: usize, + ) -> OutputLocation { OutputLocation { - hash, - index: OutputIndex::from_usize(output_index), + transaction_location: TransactionLocation::from_usize(height, transaction_index), + output_index: OutputIndex::from_usize(output_index), } } - /// Create a transparent output location from a [`transparent::OutPoint`]. - pub fn from_outpoint(outpoint: &transparent::OutPoint) -> OutputLocation { + /// Creates an output location from an [`Outpoint`], + /// and the [`TransactionLocation`] of its transaction. + /// + /// The [`TransactionLocation`] is provided separately, + /// because the lookup is a database operation. + pub fn from_outpoint( + transaction_location: TransactionLocation, + outpoint: &transparent::OutPoint, + ) -> OutputLocation { + OutputLocation::from_output_index(transaction_location, outpoint.index) + } + + /// Creates an output location from a [`TransactionLocation`] and a `u32` output index. + /// + /// Output indexes are serialized to `u32` in the Zcash consensus-critical transaction format. + pub fn from_output_index( + transaction_location: TransactionLocation, + output_index: u32, + ) -> OutputLocation { OutputLocation { - hash: outpoint.hash, - index: OutputIndex::from_zcash(outpoint.index), + transaction_location, + output_index: OutputIndex::from_index(output_index), } } + + /// Returns the height of this [`transparent::Output`]. + #[allow(dead_code)] + pub fn height(&self) -> Height { + self.transaction_location.height + } + + /// Returns the transaction index of this [`transparent::Output`]. + #[allow(dead_code)] + pub fn transaction_index(&self) -> TransactionIndex { + self.transaction_location.index + } + + /// Returns the output index of this [`transparent::Output`]. + pub fn output_index(&self) -> OutputIndex { + self.output_index + } + + /// Returns the location of the transaction for this [`transparent::Output`]. + pub fn transaction_location(&self) -> TransactionLocation { + self.transaction_location + } + + /// Allows tests to set the height of this output location. + #[cfg(any(test, feature = "proptest-impl"))] + #[allow(dead_code)] + pub fn height_mut(&mut self) -> &mut Height { + &mut self.transaction_location.height + } } /// The location of the first [`transparent::Output`] sent to an address. @@ -166,6 +239,13 @@ impl AddressBalanceLocation { pub fn location(&self) -> AddressLocation { self.location } + + /// Allows tests to set the height of the address location. + #[cfg(any(test, feature = "proptest-impl"))] + #[allow(dead_code)] + pub fn height_mut(&mut self) -> &mut Height { + &mut self.location.transaction_location.height + } } // Transparent trait impls @@ -237,40 +317,57 @@ impl IntoDisk for OutputIndex { type Bytes = [u8; OUTPUT_INDEX_DISK_BYTES]; fn as_bytes(&self) -> Self::Bytes { - self.0.to_le_bytes() + let mem_bytes = self.index().to_be_bytes(); + + let disk_bytes = truncate_zero_be_bytes(&mem_bytes, OUTPUT_INDEX_DISK_BYTES); + + disk_bytes.try_into().unwrap() } } impl FromDisk for OutputIndex { fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self { - OutputIndex(u32::from_le_bytes(disk_bytes.as_ref().try_into().unwrap())) + let mem_len = u32::BITS / 8; + let mem_len = mem_len.try_into().unwrap(); + + let mem_bytes = expand_zero_be_bytes(disk_bytes.as_ref(), mem_len); + let mem_bytes = mem_bytes.try_into().unwrap(); + OutputIndex::from_index(u32::from_be_bytes(mem_bytes)) } } impl IntoDisk for OutputLocation { - type Bytes = [u8; OUTPUT_TX_HASH_DISK_BYTES + OUTPUT_INDEX_DISK_BYTES]; + type Bytes = [u8; OUTPUT_LOCATION_DISK_BYTES]; fn as_bytes(&self) -> Self::Bytes { - let hash_bytes = self.hash.as_bytes().to_vec(); - let index_bytes = self.index.as_bytes().to_vec(); + let transaction_location_bytes = self.transaction_location().as_bytes().to_vec(); + let output_index_bytes = self.output_index().as_bytes().to_vec(); - [hash_bytes, index_bytes].concat().try_into().unwrap() + [transaction_location_bytes, output_index_bytes] + .concat() + .try_into() + .unwrap() } } impl FromDisk for OutputLocation { fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self { - let (hash_bytes, index_bytes) = disk_bytes.as_ref().split_at(OUTPUT_TX_HASH_DISK_BYTES); + let (transaction_location_bytes, output_index_bytes) = disk_bytes + .as_ref() + .split_at(TRANSACTION_LOCATION_DISK_BYTES); - let hash = transaction::Hash::from_bytes(hash_bytes); - let index = OutputIndex::from_bytes(index_bytes); + let transaction_location = TransactionLocation::from_bytes(transaction_location_bytes); + let output_index = OutputIndex::from_bytes(output_index_bytes); - OutputLocation { hash, index } + OutputLocation { + transaction_location, + output_index, + } } } impl IntoDisk for AddressBalanceLocation { - type Bytes = [u8; BALANCE_DISK_BYTES + OUTPUT_TX_HASH_DISK_BYTES + OUTPUT_INDEX_DISK_BYTES]; + type Bytes = [u8; BALANCE_DISK_BYTES + OUTPUT_LOCATION_DISK_BYTES]; fn as_bytes(&self) -> Self::Bytes { let balance_bytes = self.balance().as_bytes().to_vec(); diff --git a/zebra-state/src/service/finalized_state/disk_format/transparent/arbitrary.rs b/zebra-state/src/service/finalized_state/disk_format/transparent/arbitrary.rs new file mode 100644 index 000000000..c99385945 --- /dev/null +++ b/zebra-state/src/service/finalized_state/disk_format/transparent/arbitrary.rs @@ -0,0 +1,19 @@ +//! Randomised data generation for disk format property tests. + +use proptest::prelude::*; + +use zebra_chain::{serialization::TrustedPreallocate, transparent}; + +use super::OutputIndex; + +impl Arbitrary for OutputIndex { + type Parameters = (); + + fn arbitrary_with(_args: ()) -> Self::Strategy { + (0..=transparent::Output::max_allocation()) + .prop_map(OutputIndex::from_u64) + .boxed() + } + + type Strategy = BoxedStrategy; +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 75b0189cd..220c9c9c1 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -9,13 +9,16 @@ //! The [`crate::constants::DATABASE_FORMAT_VERSION`] constant must //! be incremented each time the database format (column, serialization, etc) changes. -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; use itertools::Itertools; use zebra_chain::{ amount::NonNegative, - block::{self, Block}, + block::{self, Block, Height}, history_tree::HistoryTree, parameters::{Network, GENESIS_PREVIOUS_BLOCK_HASH}, serialization::TrustedPreallocate, @@ -27,7 +30,11 @@ use zebra_chain::{ use crate::{ service::finalized_state::{ disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk}, - disk_format::{transparent::AddressBalanceLocation, FromDisk, TransactionLocation}, + disk_format::{ + block::TransactionLocation, + transparent::{AddressBalanceLocation, OutputLocation}, + FromDisk, + }, zebra_db::{metrics::block_precommit_metrics, shielded::NoteCommitmentTrees, ZebraDb}, FinalizedBlock, }, @@ -150,14 +157,11 @@ impl ZebraDb { self.db.zs_get(&hash_by_tx_loc, &location) } - /// Returns the [`Transaction`] with [`transaction::Hash`], and its [`block::Height`], + /// Returns the [`Transaction`] with [`transaction::Hash`], and its [`Height`], /// if a transaction with that hash exists in the finalized chain. // // TODO: move this method to the start of the section - pub fn transaction( - &self, - hash: transaction::Hash, - ) -> Option<(Arc, block::Height)> { + pub fn transaction(&self, hash: transaction::Hash) -> Option<(Arc, Height)> { let tx_by_loc = self.db.cf_handle("tx_by_loc").unwrap(); let transaction_location = self.transaction_location(hash)?; @@ -189,26 +193,64 @@ impl ZebraDb { ) -> Result { let finalized_hash = finalized.hash; - // Get a list of the spent UTXOs, before we delete any from the database - let all_utxos_spent_by_block: HashMap = finalized - .block - .transactions + let tx_hash_indexes: HashMap = finalized + .transaction_hashes .iter() - .flat_map(|tx| tx.inputs().iter()) - .flat_map(|input| input.outpoint()) - .map(|outpoint| { + .enumerate() + .map(|(index, hash)| (*hash, index)) + .collect(); + + // Get a list of the new UTXOs in the format we need for database updates. + // + // TODO: index new_outputs by TransactionLocation, + // simplify the spent_utxos location lookup code, + // and remove the extra new_outputs_by_out_loc argument + let new_outputs_by_out_loc: BTreeMap = finalized + .new_outputs + .iter() + .map(|(outpoint, utxo)| { ( - outpoint, - // Some utxos are spent in the same block, so they will be in `new_outputs` - self.utxo(&outpoint) - .or_else(|| finalized.new_outputs.get(&outpoint).cloned()) - .expect("already checked UTXO was in state or block"), + lookup_out_loc(finalized.height, outpoint, &tx_hash_indexes), + utxo.clone(), ) }) .collect(); + // Get a list of the spent UTXOs, before we delete any from the database + let spent_utxos: Vec<(transparent::OutPoint, OutputLocation, transparent::Utxo)> = + finalized + .block + .transactions + .iter() + .flat_map(|tx| tx.inputs().iter()) + .flat_map(|input| input.outpoint()) + .map(|outpoint| { + ( + outpoint, + // Some utxos are spent in the same block, so they will be in + // `tx_hash_indexes` and `new_outputs` + self.output_location(&outpoint).unwrap_or_else(|| { + lookup_out_loc(finalized.height, &outpoint, &tx_hash_indexes) + }), + self.utxo(&outpoint) + .or_else(|| finalized.new_outputs.get(&outpoint).cloned()) + .expect("already checked UTXO was in state or block"), + ) + }) + .collect(); + + let spent_utxos_by_outpoint: HashMap = + spent_utxos + .iter() + .map(|(outpoint, _output_loc, utxo)| (*outpoint, utxo.clone())) + .collect(); + let spent_utxos_by_out_loc: BTreeMap = spent_utxos + .into_iter() + .map(|(_outpoint, out_loc, utxo)| (out_loc, utxo)) + .collect(); + // Get the current address balances, before the transactions in this block - let address_balances = all_utxos_spent_by_block + let address_balances = spent_utxos_by_out_loc .values() .chain(finalized.new_outputs.values()) .filter_map(|utxo| utxo.output.address(network)) @@ -222,7 +264,9 @@ impl ZebraDb { batch.prepare_block_batch( &self.db, finalized, - all_utxos_spent_by_block, + new_outputs_by_out_loc, + spent_utxos_by_outpoint, + spent_utxos_by_out_loc, address_balances, self.note_commitment_trees(), history_tree, @@ -237,6 +281,23 @@ impl ZebraDb { } } +/// Lookup the output location for an outpoint. +/// +/// `tx_hash_indexes` must contain `outpoint.hash` and that transaction's index in its block. +fn lookup_out_loc( + height: Height, + outpoint: &transparent::OutPoint, + tx_hash_indexes: &HashMap, +) -> OutputLocation { + let tx_index = tx_hash_indexes + .get(&outpoint.hash) + .expect("already checked UTXO was in state or block"); + + let tx_loc = TransactionLocation::from_usize(height, *tx_index); + + OutputLocation::from_outpoint(tx_loc, outpoint) +} + impl DiskWriteBatch { // Write block methods @@ -254,7 +315,9 @@ impl DiskWriteBatch { &mut self, db: &DiskDb, finalized: FinalizedBlock, - all_utxos_spent_by_block: HashMap, + new_outputs_by_out_loc: BTreeMap, + spent_utxos_by_outpoint: HashMap, + spent_utxos_by_out_loc: BTreeMap, address_balances: HashMap, mut note_commitment_trees: NoteCommitmentTrees, history_tree: HistoryTree, @@ -288,7 +351,8 @@ impl DiskWriteBatch { self.prepare_transaction_index_batch( db, &finalized, - &all_utxos_spent_by_block, + new_outputs_by_out_loc, + spent_utxos_by_out_loc, address_balances, &mut note_commitment_trees, )?; @@ -296,7 +360,7 @@ impl DiskWriteBatch { self.prepare_note_commitment_batch(db, &finalized, note_commitment_trees, history_tree)?; // Commit UTXOs and value pools - self.prepare_chain_value_pools_batch(db, &finalized, all_utxos_spent_by_block, value_pool)?; + self.prepare_chain_value_pools_batch(db, &finalized, spent_utxos_by_outpoint, value_pool)?; // The block has passed contextual validation, so update the metrics block_precommit_metrics(block, *hash, *height); @@ -396,7 +460,8 @@ impl DiskWriteBatch { &mut self, db: &DiskDb, finalized: &FinalizedBlock, - all_utxos_spent_by_block: &HashMap, + new_outputs_by_out_loc: BTreeMap, + utxos_spent_by_block: BTreeMap, address_balances: HashMap, note_commitment_trees: &mut NoteCommitmentTrees, ) -> Result<(), BoxError> { @@ -411,8 +476,8 @@ impl DiskWriteBatch { self.prepare_transparent_outputs_batch( db, - finalized, - all_utxos_spent_by_block, + new_outputs_by_out_loc, + utxos_spent_by_block, address_balances, ) } diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs index 2840deadb..92813c7f9 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs @@ -357,15 +357,23 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) { transparent::OutPoint::from_usize(transaction_hash, output_index); let output_location = - OutputLocation::from_usize(transaction_hash, output_index); + OutputLocation::from_usize(query_height, tx_index, output_index); - let stored_utxo = state.utxo(&outpoint); + let stored_output_location = state + .output_location(&outpoint) + .expect("all outpoints are indexed"); + + let stored_utxo_by_outpoint = state.utxo(&outpoint); + let stored_utxo_by_out_loc = state.utxo_by_location(output_location); + + assert_eq!(stored_output_location, output_location); + assert_eq!(stored_utxo_by_out_loc, stored_utxo_by_outpoint); // # Consensus // // The genesis transaction's UTXO is not indexed. // This check also ignores spent UTXOs. - if let Some(stored_utxo) = &stored_utxo { + if let Some(stored_utxo) = &stored_utxo_by_out_loc { assert_eq!(&stored_utxo.output, output); assert_eq!(stored_utxo.height, query_height); @@ -381,8 +389,7 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) { ); } - // TODO: use output_location in #3151 - stored_utxos.push((outpoint, stored_utxo)); + stored_utxos.push((output_location, stored_utxo_by_out_loc)); } } } diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_1.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_1.snap index 4fa05a947..616b2c42b 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_1.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_1.snap @@ -6,8 +6,11 @@ expression: stored_address_balances ("t3Vz22vK5z2LcKEdg16Yv4FFneEL1zg9ojd", AddressBalanceLocation( balance: Amount(12500), location: OutputLocation( - hash: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609", - index: OutputIndex(1), + transaction_location: TransactionLocation( + height: Height(1), + index: TransactionIndex(0), + ), + output_index: OutputIndex(1), ), )), ] diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_2.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_2.snap index fbd8f2bac..14fe6eace 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_2.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_2.snap @@ -6,8 +6,11 @@ expression: stored_address_balances ("t3Vz22vK5z2LcKEdg16Yv4FFneEL1zg9ojd", AddressBalanceLocation( balance: Amount(37500), location: OutputLocation( - hash: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609", - index: OutputIndex(1), + transaction_location: TransactionLocation( + height: Height(1), + index: TransactionIndex(0), + ), + output_index: OutputIndex(1), ), )), ] diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_1.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_1.snap index 843eb4fd2..4ea25efcd 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_1.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_1.snap @@ -6,8 +6,11 @@ expression: stored_address_balances ("t2UNzUUx8mWBCRYPRezvA363EYXyEpHokyi", AddressBalanceLocation( balance: Amount(12500), location: OutputLocation( - hash: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75", - index: OutputIndex(1), + transaction_location: TransactionLocation( + height: Height(1), + index: TransactionIndex(0), + ), + output_index: OutputIndex(1), ), )), ] diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_2.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_2.snap index 3f29fd701..6c792ba79 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_2.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_2.snap @@ -6,8 +6,11 @@ expression: stored_address_balances ("t2UNzUUx8mWBCRYPRezvA363EYXyEpHokyi", AddressBalanceLocation( balance: Amount(37500), location: OutputLocation( - hash: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75", - index: OutputIndex(1), + transaction_location: TransactionLocation( + height: Height(1), + index: TransactionIndex(0), + ), + output_index: OutputIndex(1), ), )), ] diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@mainnet_0.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@mainnet_0.snap index 1b421f4a2..c99f6d9af 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@mainnet_0.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@mainnet_0.snap @@ -3,8 +3,11 @@ source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs expression: stored_utxos --- [ - (OutPoint( - hash: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", - index: 0, + (OutputLocation( + transaction_location: TransactionLocation( + height: Height(0), + index: TransactionIndex(0), + ), + output_index: OutputIndex(0), ), None), ] diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@mainnet_1.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@mainnet_1.snap index 431016c01..0a3252110 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@mainnet_1.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@mainnet_1.snap @@ -3,13 +3,19 @@ source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs expression: stored_utxos --- [ - (OutPoint( - hash: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", - index: 0, + (OutputLocation( + transaction_location: TransactionLocation( + height: Height(0), + index: TransactionIndex(0), + ), + output_index: OutputIndex(0), ), None), - (OutPoint( - hash: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609", - index: 0, + (OutputLocation( + transaction_location: TransactionLocation( + height: Height(1), + index: TransactionIndex(0), + ), + output_index: OutputIndex(0), ), Some(Utxo( output: Output( value: Amount(50000), @@ -18,9 +24,12 @@ expression: stored_utxos height: Height(1), from_coinbase: true, ))), - (OutPoint( - hash: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609", - index: 1, + (OutputLocation( + transaction_location: TransactionLocation( + height: Height(1), + index: TransactionIndex(0), + ), + output_index: OutputIndex(1), ), Some(Utxo( output: Output( value: Amount(12500), diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@mainnet_2.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@mainnet_2.snap index 4c1e8e414..a64b3d04f 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@mainnet_2.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@mainnet_2.snap @@ -3,13 +3,19 @@ source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs expression: stored_utxos --- [ - (OutPoint( - hash: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", - index: 0, + (OutputLocation( + transaction_location: TransactionLocation( + height: Height(0), + index: TransactionIndex(0), + ), + output_index: OutputIndex(0), ), None), - (OutPoint( - hash: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609", - index: 0, + (OutputLocation( + transaction_location: TransactionLocation( + height: Height(1), + index: TransactionIndex(0), + ), + output_index: OutputIndex(0), ), Some(Utxo( output: Output( value: Amount(50000), @@ -18,9 +24,12 @@ expression: stored_utxos height: Height(1), from_coinbase: true, ))), - (OutPoint( - hash: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609", - index: 1, + (OutputLocation( + transaction_location: TransactionLocation( + height: Height(1), + index: TransactionIndex(0), + ), + output_index: OutputIndex(1), ), Some(Utxo( output: Output( value: Amount(12500), @@ -29,9 +38,12 @@ expression: stored_utxos height: Height(1), from_coinbase: true, ))), - (OutPoint( - hash: "8974d08d1c5f9c860d8b629d582a56659a4a1dcb2b5f98a25a5afcc2a784b0f4", - index: 0, + (OutputLocation( + transaction_location: TransactionLocation( + height: Height(2), + index: TransactionIndex(0), + ), + output_index: OutputIndex(0), ), Some(Utxo( output: Output( value: Amount(100000), @@ -40,9 +52,12 @@ expression: stored_utxos height: Height(2), from_coinbase: true, ))), - (OutPoint( - hash: "8974d08d1c5f9c860d8b629d582a56659a4a1dcb2b5f98a25a5afcc2a784b0f4", - index: 1, + (OutputLocation( + transaction_location: TransactionLocation( + height: Height(2), + index: TransactionIndex(0), + ), + output_index: OutputIndex(1), ), Some(Utxo( output: Output( value: Amount(25000), diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@testnet_0.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@testnet_0.snap index 1b421f4a2..c99f6d9af 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@testnet_0.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@testnet_0.snap @@ -3,8 +3,11 @@ source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs expression: stored_utxos --- [ - (OutPoint( - hash: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", - index: 0, + (OutputLocation( + transaction_location: TransactionLocation( + height: Height(0), + index: TransactionIndex(0), + ), + output_index: OutputIndex(0), ), None), ] diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@testnet_1.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@testnet_1.snap index 50a4bd7ce..45e2af9a2 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@testnet_1.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@testnet_1.snap @@ -3,13 +3,19 @@ source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs expression: stored_utxos --- [ - (OutPoint( - hash: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", - index: 0, + (OutputLocation( + transaction_location: TransactionLocation( + height: Height(0), + index: TransactionIndex(0), + ), + output_index: OutputIndex(0), ), None), - (OutPoint( - hash: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75", - index: 0, + (OutputLocation( + transaction_location: TransactionLocation( + height: Height(1), + index: TransactionIndex(0), + ), + output_index: OutputIndex(0), ), Some(Utxo( output: Output( value: Amount(50000), @@ -18,9 +24,12 @@ expression: stored_utxos height: Height(1), from_coinbase: true, ))), - (OutPoint( - hash: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75", - index: 1, + (OutputLocation( + transaction_location: TransactionLocation( + height: Height(1), + index: TransactionIndex(0), + ), + output_index: OutputIndex(1), ), Some(Utxo( output: Output( value: Amount(12500), diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@testnet_2.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@testnet_2.snap index 15aedd057..2738dda53 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@testnet_2.snap +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@testnet_2.snap @@ -3,13 +3,19 @@ source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs expression: stored_utxos --- [ - (OutPoint( - hash: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", - index: 0, + (OutputLocation( + transaction_location: TransactionLocation( + height: Height(0), + index: TransactionIndex(0), + ), + output_index: OutputIndex(0), ), None), - (OutPoint( - hash: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75", - index: 0, + (OutputLocation( + transaction_location: TransactionLocation( + height: Height(1), + index: TransactionIndex(0), + ), + output_index: OutputIndex(0), ), Some(Utxo( output: Output( value: Amount(50000), @@ -18,9 +24,12 @@ expression: stored_utxos height: Height(1), from_coinbase: true, ))), - (OutPoint( - hash: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75", - index: 1, + (OutputLocation( + transaction_location: TransactionLocation( + height: Height(1), + index: TransactionIndex(0), + ), + output_index: OutputIndex(1), ), Some(Utxo( output: Output( value: Amount(12500), @@ -29,9 +38,12 @@ expression: stored_utxos height: Height(1), from_coinbase: true, ))), - (OutPoint( - hash: "5822c0532da8a008259ac39933d3210e508c17e3ba21d2b2c428785efdccb3d5", - index: 0, + (OutputLocation( + transaction_location: TransactionLocation( + height: Height(2), + index: TransactionIndex(0), + ), + output_index: OutputIndex(0), ), Some(Utxo( output: Output( value: Amount(100000), @@ -40,9 +52,12 @@ expression: stored_utxos height: Height(2), from_coinbase: true, ))), - (OutPoint( - hash: "5822c0532da8a008259ac39933d3210e508c17e3ba21d2b2c428785efdccb3d5", - index: 1, + (OutputLocation( + transaction_location: TransactionLocation( + height: Height(2), + index: TransactionIndex(0), + ), + output_index: OutputIndex(1), ), Some(Utxo( output: Output( value: Amount(25000), diff --git a/zebra-state/src/service/finalized_state/zebra_db/chain.rs b/zebra-state/src/service/finalized_state/zebra_db/chain.rs index 231859ed4..76e5aceac 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/chain.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/chain.rs @@ -112,14 +112,14 @@ impl DiskWriteBatch { &mut self, db: &DiskDb, finalized: &FinalizedBlock, - all_utxos_spent_by_block: HashMap, + utxos_spent_by_block: HashMap, value_pool: ValueBalance, ) -> Result<(), BoxError> { let tip_chain_value_pool = db.cf_handle("tip_chain_value_pool").unwrap(); let FinalizedBlock { block, .. } = finalized; - let new_pool = value_pool.add_block(block.borrow(), &all_utxos_spent_by_block)?; + let new_pool = value_pool.add_block(block.borrow(), &utxos_spent_by_block)?; self.zs_insert(&tip_chain_value_pool, (), new_pool); Ok(()) diff --git a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs index 8bbe73b71..a24f06739 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs @@ -1,6 +1,6 @@ //! Provides high-level access to database: -//! - unspent [`transparent::Outputs`]s -//! - transparent address indexes +//! - unspent [`transparent::Outputs`]s (UTXOs), and +//! - transparent address indexes. //! //! This module makes sure that: //! - all disk writes happen inside a RocksDB transaction, and @@ -11,7 +11,7 @@ //! The [`crate::constants::DATABASE_FORMAT_VERSION`] constant must //! be incremented each time the database format (column, serialization, etc) changes. -use std::{borrow::Borrow, collections::HashMap}; +use std::collections::{BTreeMap, HashMap}; use zebra_chain::{ amount::{Amount, NonNegative}, @@ -23,7 +23,6 @@ use crate::{ disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk}, disk_format::transparent::{AddressBalanceLocation, AddressLocation, OutputLocation}, zebra_db::ZebraDb, - FinalizedBlock, }, BoxError, }; @@ -60,23 +59,39 @@ impl ZebraDb { .map(|abl| abl.location()) } + /// Returns the [`OutputLocation`] for a [`transparent::OutPoint`]. + /// + /// This method returns the locations of spent and unspent outpoints. + /// Returns `None` if the output was never in the finalized state. + pub fn output_location(&self, outpoint: &transparent::OutPoint) -> Option { + self.transaction_location(outpoint.hash) + .map(|transaction_location| { + OutputLocation::from_outpoint(transaction_location, outpoint) + }) + } + /// Returns the transparent output for a [`transparent::OutPoint`], - /// if it is still unspent in the finalized state. + /// if it is unspent in the finalized state. pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option { - let utxo_by_outpoint = self.db.cf_handle("utxo_by_outpoint").unwrap(); + let output_location = self.output_location(outpoint)?; - let output_location = OutputLocation::from_outpoint(outpoint); + self.utxo_by_location(output_location) + } - self.db.zs_get(&utxo_by_outpoint, &output_location) + /// Returns the transparent output for an [`OutputLocation`], + /// if it is unspent in the finalized state. + pub fn utxo_by_location(&self, output_location: OutputLocation) -> Option { + let utxo_by_out_loc = self.db.cf_handle("utxo_by_outpoint").unwrap(); + self.db.zs_get(&utxo_by_out_loc, &output_location) } } impl DiskWriteBatch { /// Prepare a database batch containing `finalized.block`'s: /// - transparent address balance changes, + /// - UTXO changes, and /// TODO: - /// - transparent address index changes (add in #3951, #3953), and - /// - UTXO changes (modify in #3952) + /// - transparent address index changes (add in #3951, #3953), /// and return it (without actually writing anything). /// /// # Errors @@ -85,21 +100,16 @@ impl DiskWriteBatch { pub fn prepare_transparent_outputs_batch( &mut self, db: &DiskDb, - finalized: &FinalizedBlock, - all_utxos_spent_by_block: &HashMap, + new_outputs_by_out_loc: BTreeMap, + utxos_spent_by_block: BTreeMap, mut address_balances: HashMap, ) -> Result<(), BoxError> { let utxo_by_outpoint = db.cf_handle("utxo_by_outpoint").unwrap(); let balance_by_transparent_addr = db.cf_handle("balance_by_transparent_addr").unwrap(); - let FinalizedBlock { - block, new_outputs, .. - } = finalized; - // Index all new transparent outputs, before deleting any we've spent - for (outpoint, utxo) in new_outputs.borrow().iter() { + for (output_location, utxo) in new_outputs_by_out_loc { let receiving_address = utxo.output.address(self.network()); - let output_location = OutputLocation::from_outpoint(outpoint); // Update the address balance by adding this UTXO's value if let Some(receiving_address) = receiving_address { @@ -120,15 +130,8 @@ impl DiskWriteBatch { // Mark all transparent inputs as spent. // // Coinbase inputs represent new coins, so there are no UTXOs to mark as spent. - for outpoint in block - .transactions - .iter() - .flat_map(|tx| tx.inputs()) - .flat_map(|input| input.outpoint()) - { - let output_location = OutputLocation::from_outpoint(&outpoint); - - let spent_output = &all_utxos_spent_by_block.get(&outpoint).unwrap().output; + for (output_location, utxo) in utxos_spent_by_block { + let spent_output = utxo.output; let sending_address = spent_output.address(self.network()); // Update the address balance by subtracting this UTXO's value