zecfoundation
/
zebra
mirror of https://github.com/ZcashFoundation/zebra.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

3. change(db): Store UTXOs by transaction location rather than transaction hash (#3978) 

* Change OutputLocation to contain a TransactionLocation

* Change OutputLocation reads from the database

* Update some doc comments

* Update some TODOs

* Change deleting spent UTXOs and updating spent balances

* Change adding new UTXOs and adding their values to balances

* Disable dead code warnings

* Update snapshot test code

* Update round-trip tests for OutputLocations

* Update snapshot test data

* Increment the database format version

* Remove a redundant try_into()

Co-authored-by: Janito Vaqueiro Ferreira Filho <janito.vff@gmail.com>

* Refactor redundant code

Co-authored-by: Janito Vaqueiro Ferreira Filho <janito.vff@gmail.com>

* ci: attempt at fixing 'Regenerate stateful disks'

Co-authored-by: Janito Vaqueiro Ferreira Filho <janito.vff@gmail.com>
Co-authored-by: Conrado Gouvea <conrado@zfnd.org>
This commit is contained in:
teor 2022-04-09 08:42:05 +10:00 committed by GitHub
parent 4e4ecb5a4d
commit 7e8194c63f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 523 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/test.yml vendored
View File

@ -324,7 +324,7 @@ jobs:
EXIT_CODE=$(\ EXIT_CODE=$(\
gcloud compute ssh \ gcloud compute ssh \
sync-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ regenerate-disk-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \ --zone ${{ env.ZONE }} \
--quiet \ --quiet \
--ssh-flag="-o ServerAliveInterval=5" \ --ssh-flag="-o ServerAliveInterval=5" \

6
zebra-chain/src/transparent/utxo.rs
View File

@ -17,6 +17,10 @@ use crate::{
pub struct Utxo { pub struct Utxo {
/// The output itself. /// The output itself.
pub output: transparent::Output, pub output: transparent::Output,
// TODO: replace the height and from_coinbase fields with OutputLocation,
// and provide lookup/calculation methods for height and from_coinbase
//
/// The height at which the output was created. /// The height at which the output was created.
pub height: block::Height, pub height: block::Height,
/// Whether the output originated in a coinbase transaction. /// Whether the output originated in a coinbase transaction.
@ -35,6 +39,8 @@ pub struct Utxo {
any(test, feature = "proptest-impl"), any(test, feature = "proptest-impl"),
derive(proptest_derive::Arbitrary) derive(proptest_derive::Arbitrary)
)] )]
//
// TODO: after modifying UTXO to contain an OutputLocation, replace this type with UTXO
pub struct OrderedUtxo { pub struct OrderedUtxo {
/// An unspent transaction output. /// An unspent transaction output.
pub utxo: Utxo, pub utxo: Utxo,

2
zebra-state/src/constants.rs
View File

@ -18,7 +18,7 @@ pub use zebra_chain::transparent::MIN_TRANSPARENT_COINBASE_MATURITY;
pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRANSPARENT_COINBASE_MATURITY - 1; pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRANSPARENT_COINBASE_MATURITY - 1;
/// The database format version, incremented each time the database format changes. /// The database format version, incremented each time the database format changes.
pub const DATABASE_FORMAT_VERSION: u32 = 17; pub const DATABASE_FORMAT_VERSION: u32 = 18;
/// The maximum number of blocks to check for NU5 transactions, /// The maximum number of blocks to check for NU5 transactions,
/// before we assume we are on a pre-NU5 legacy chain. /// before we assume we are on a pre-NU5 legacy chain.

8
zebra-state/src/request.rs
View File

@ -10,8 +10,8 @@ use zebra_chain::{
value_balance::{ValueBalance, ValueBalanceError}, value_balance::{ValueBalance, ValueBalanceError},
}; };
// Allow *only* this unused import, so that rustdoc link resolution /// Allow *only* this unused import, so that rustdoc link resolution
// will work with inline links. /// will work with inline links.
#[allow(unused_imports)] #[allow(unused_imports)]
use crate::Response; use crate::Response;
@ -119,10 +119,6 @@ pub struct FinalizedBlock {
/// New transparent outputs created in this block, indexed by /// New transparent outputs created in this block, indexed by
/// [`Outpoint`](transparent::Outpoint). /// [`Outpoint`](transparent::Outpoint).
/// ///
/// Each output is tagged with its transaction index in the block.
/// (The outputs of earlier transactions in a block can be spent by later
/// transactions.)
///
/// Note: although these transparent outputs are newly created, they may not /// Note: although these transparent outputs are newly created, they may not
/// be unspent, since a later transaction in a block can spend outputs of an /// be unspent, since a later transaction in a block can spend outputs of an
/// earlier transaction. /// earlier transaction.

7
zebra-state/src/service.rs
View File

@ -497,7 +497,12 @@ impl StateService {
.or_else(|| self.disk.db().height(hash)) .or_else(|| self.disk.db().height(hash))
} }
/// Return the [`Utxo`] pointed to by `outpoint` if it exists in any chain. /// Return the [`Utxo`] pointed to by `outpoint`, if it exists in any chain,
/// or in any pending block.
///
/// Some of the returned UTXOs may be invalid, because:
/// - they are not in the best chain, or
/// - their block fails contextual validation.
pub fn any_utxo(&self, outpoint: &transparent::OutPoint) -> Option<transparent::Utxo> { pub fn any_utxo(&self, outpoint: &transparent::OutPoint) -> Option<transparent::Utxo> {
self.mem self.mem
.any_utxo(outpoint) .any_utxo(outpoint)

8
zebra-state/src/service/finalized_state/disk_db.rs
View File

@ -361,15 +361,15 @@ impl DiskDb {
// Transactions // Transactions
rocksdb::ColumnFamilyDescriptor::new("tx_by_loc", db_options.clone()), rocksdb::ColumnFamilyDescriptor::new("tx_by_loc", db_options.clone()),
rocksdb::ColumnFamilyDescriptor::new("hash_by_tx_loc", db_options.clone()), rocksdb::ColumnFamilyDescriptor::new("hash_by_tx_loc", db_options.clone()),
// TODO: rename to tx_loc_by_hash (#3151) // TODO: rename to tx_loc_by_hash (#3950)
rocksdb::ColumnFamilyDescriptor::new("tx_by_hash", db_options.clone()), rocksdb::ColumnFamilyDescriptor::new("tx_by_hash", db_options.clone()),
// Transparent // Transparent
rocksdb::ColumnFamilyDescriptor::new("balance_by_transparent_addr", db_options.clone()), rocksdb::ColumnFamilyDescriptor::new("balance_by_transparent_addr", db_options.clone()),
// TODO: #3954 // TODO: #3951
//rocksdb::ColumnFamilyDescriptor::new("tx_by_transparent_addr_loc", db_options.clone()), //rocksdb::ColumnFamilyDescriptor::new("tx_by_transparent_addr_loc", db_options.clone()),
// TODO: rename to utxo_by_out_loc (#3953) // TODO: rename to utxo_by_out_loc (#3952)
rocksdb::ColumnFamilyDescriptor::new("utxo_by_outpoint", db_options.clone()), rocksdb::ColumnFamilyDescriptor::new("utxo_by_outpoint", db_options.clone()),
// TODO: #3952 // TODO: #3953
//rocksdb::ColumnFamilyDescriptor::new("utxo_by_transparent_addr_loc", db_options.clone()), //rocksdb::ColumnFamilyDescriptor::new("utxo_by_transparent_addr_loc", db_options.clone()),
// Sprout // Sprout
rocksdb::ColumnFamilyDescriptor::new("sprout_nullifiers", db_options.clone()), rocksdb::ColumnFamilyDescriptor::new("sprout_nullifiers", db_options.clone()),

2
zebra-state/src/service/finalized_state/disk_format.rs
View File

@ -15,7 +15,7 @@ pub mod transparent;
#[cfg(test)] #[cfg(test)]
mod tests; mod tests;
pub use block::TransactionLocation; pub use block::{TransactionIndex, TransactionLocation};
/// Helper type for writing types to disk as raw bytes. /// Helper type for writing types to disk as raw bytes.
/// Also used to convert key types to raw bytes for disk lookups. /// Also used to convert key types to raw bytes for disk lookups.

57
zebra-state/src/service/finalized_state/disk_format/block.rs
View File

@ -5,8 +5,6 @@
//! The [`crate::constants::DATABASE_FORMAT_VERSION`] constant must //! The [`crate::constants::DATABASE_FORMAT_VERSION`] constant must
//! be incremented each time the database format (column, serialization, etc) changes. //! be incremented each time the database format (column, serialization, etc) changes.
use std::fmt::Debug;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use zebra_chain::{ use zebra_chain::{
@ -24,7 +22,7 @@ use proptest_derive::Arbitrary;
/// The maximum value of an on-disk serialized [`Height`]. /// The maximum value of an on-disk serialized [`Height`].
/// ///
/// This allows us to store [`OutputIndex`]es in 8 bytes, /// This allows us to store [`OutputLocation`]s in 8 bytes,
/// which makes database searches more efficient. /// which makes database searches more efficient.
/// ///
/// # Consensus /// # Consensus
@ -46,24 +44,39 @@ pub const HEIGHT_DISK_BYTES: usize = 3;
/// This reduces database size and increases lookup performance. /// This reduces database size and increases lookup performance.
pub const TX_INDEX_DISK_BYTES: usize = 2; pub const TX_INDEX_DISK_BYTES: usize = 2;
// Transaction types /// [`TransactionLocation`]s are stored as a 3 byte height and a 2 byte transaction index.
///
/// This reduces database size and increases lookup performance.
pub const TRANSACTION_LOCATION_DISK_BYTES: usize = HEIGHT_DISK_BYTES + TX_INDEX_DISK_BYTES;
// Block and transaction types
/// A transaction's index in its block. /// A transaction's index in its block.
/// ///
/// # Consensus /// # Consensus
/// ///
/// This maximum height supports on-disk storage of transactions in blocks up to ~5 MB. /// A 2-byte index supports on-disk storage of transactions in blocks up to ~5 MB.
/// (The current maximum block size is 2 MB.) /// (The current maximum block size is 2 MB.)
/// ///
/// Since Zebra only stores fully verified blocks on disk, /// Since Zebra only stores fully verified blocks on disk,
/// blocks larger than this size are rejected before reaching the database. /// blocks larger than this size are rejected before reaching the database.
/// ///
/// (The maximum transaction count is tested by the large generated block serialization tests.) /// (The maximum transaction count is tested by the large generated block serialization tests.)
#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] #[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))] #[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))]
pub struct TransactionIndex(u16); pub struct TransactionIndex(u16);
impl TransactionIndex { impl TransactionIndex {
/// Creates a transaction index from the inner type.
pub fn from_index(transaction_index: u16) -> TransactionIndex {
TransactionIndex(transaction_index)
}
/// Returns this index as the inner type.
pub fn index(&self) -> u16 {
self.0
}
/// Creates a transaction index from a `usize`. /// Creates a transaction index from a `usize`.
pub fn from_usize(transaction_index: usize) -> TransactionIndex { pub fn from_usize(transaction_index: usize) -> TransactionIndex {
TransactionIndex( TransactionIndex(
@ -102,7 +115,7 @@ impl TransactionIndex {
/// A transaction's location in the chain, by block height and transaction index. /// A transaction's location in the chain, by block height and transaction index.
/// ///
/// This provides a chain-order list of transactions. /// This provides a chain-order list of transactions.
#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] #[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))] #[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))]
pub struct TransactionLocation { pub struct TransactionLocation {
/// The block height of the transaction. /// The block height of the transaction.
@ -113,7 +126,16 @@ pub struct TransactionLocation {
} }
impl TransactionLocation { impl TransactionLocation {
/// Creates a transaction location from a block height and `usize` index. /// Creates a transaction location from a block height and transaction index.
#[allow(dead_code)]
pub fn from_index(height: Height, transaction_index: u16) -> TransactionLocation {
TransactionLocation {
height,
index: TransactionIndex::from_index(transaction_index),
}
}
/// Creates a transaction location from a block height and `usize` transaction index.
pub fn from_usize(height: Height, transaction_index: usize) -> TransactionLocation { pub fn from_usize(height: Height, transaction_index: usize) -> TransactionLocation {
TransactionLocation { TransactionLocation {
height, height,
@ -121,7 +143,7 @@ impl TransactionLocation {
} }
} }
/// Creates a transaction location from a block height and `u64` index. /// Creates a transaction location from a block height and `u64` transaction index.
pub fn from_u64(height: Height, transaction_index: u64) -> TransactionLocation { pub fn from_u64(height: Height, transaction_index: u64) -> TransactionLocation {
TransactionLocation { TransactionLocation {
height, height,
@ -130,7 +152,7 @@ impl TransactionLocation {
} }
} }
// Block trait impls // Block and transaction trait impls
impl IntoDisk for block::Header { impl IntoDisk for block::Header {
type Bytes = Vec<u8>; type Bytes = Vec<u8>;
@ -164,12 +186,13 @@ impl IntoDisk for Height {
} }
impl FromDisk for Height { impl FromDisk for Height {
fn from_bytes(bytes: impl AsRef<[u8]>) -> Self { fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self {
let mem_len = u32::BITS / 8; let mem_len = u32::BITS / 8;
let mem_len = mem_len.try_into().unwrap(); let mem_len = mem_len.try_into().unwrap();
let mem_bytes = expand_zero_be_bytes(bytes.as_ref(), mem_len); let mem_bytes = expand_zero_be_bytes(disk_bytes.as_ref(), mem_len);
Height(u32::from_be_bytes(mem_bytes.try_into().unwrap())) let mem_bytes = mem_bytes.try_into().unwrap();
Height(u32::from_be_bytes(mem_bytes))
} }
} }
@ -213,18 +236,20 @@ impl IntoDisk for TransactionIndex {
type Bytes = [u8; TX_INDEX_DISK_BYTES]; type Bytes = [u8; TX_INDEX_DISK_BYTES];
fn as_bytes(&self) -> Self::Bytes { fn as_bytes(&self) -> Self::Bytes {
self.0.to_be_bytes() self.index().to_be_bytes()
} }
} }
impl FromDisk for TransactionIndex { impl FromDisk for TransactionIndex {
fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self { fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self {
TransactionIndex(u16::from_be_bytes(disk_bytes.as_ref().try_into().unwrap())) let disk_bytes = disk_bytes.as_ref().try_into().unwrap();
TransactionIndex::from_index(u16::from_be_bytes(disk_bytes))
} }
} }
impl IntoDisk for TransactionLocation { impl IntoDisk for TransactionLocation {
type Bytes = [u8; HEIGHT_DISK_BYTES + TX_INDEX_DISK_BYTES]; type Bytes = [u8; TRANSACTION_LOCATION_DISK_BYTES];
fn as_bytes(&self) -> Self::Bytes { fn as_bytes(&self) -> Self::Bytes {
let height_bytes = self.height.as_bytes().to_vec(); let height_bytes = self.height.as_bytes().to_vec();

21
zebra-state/src/service/finalized_state/disk_format/tests/prop.rs
View File

@ -126,21 +126,36 @@ fn roundtrip_transparent_address() {
fn roundtrip_output_location() { fn roundtrip_output_location() {
zebra_test::init(); zebra_test::init();
proptest!(|(val in any::<OutputLocation>())| assert_value_properties(val)); proptest!(
|(mut val in any::<OutputLocation>())| {
*val.height_mut() = val.height().clamp(Height(0), MAX_ON_DISK_HEIGHT);
assert_value_properties(val)
}
);
} }
#[test] #[test]
fn roundtrip_address_location() { fn roundtrip_address_location() {
zebra_test::init(); zebra_test::init();
proptest!(|(val in any::<AddressLocation>())| assert_value_properties(val)); proptest!(
|(mut val in any::<AddressLocation>())| {
*val.height_mut() = val.height().clamp(Height(0), MAX_ON_DISK_HEIGHT);
assert_value_properties(val)
}
);
} }
#[test] #[test]
fn roundtrip_address_balance_location() { fn roundtrip_address_balance_location() {
zebra_test::init(); zebra_test::init();
proptest!(|(val in any::<AddressBalanceLocation>())| assert_value_properties(val)); proptest!(
|(mut val in any::<AddressBalanceLocation>())| {
*val.height_mut() = val.location().height().clamp(Height(0), MAX_ON_DISK_HEIGHT);
assert_value_properties(val)
}
);
} }
#[test] #[test]

2
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@mainnet_1.snap
View File

@ -5,6 +5,6 @@ expression: cf_data
[ [
KV( KV(
k: "017d46a730d31f97b1930d3368a967c309bd4d136a", k: "017d46a730d31f97b1930d3368a967c309bd4d136a",
v: "d4300000000000000946edb9c083c9942d92305444527765fad789c438c717783276a9f7fbf61b8501000000", v: "d4300000000000000000010000000001",
), ),
] ]

2
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@mainnet_2.snap
View File

@ -5,6 +5,6 @@ expression: cf_data
[ [
KV( KV(
k: "017d46a730d31f97b1930d3368a967c309bd4d136a", k: "017d46a730d31f97b1930d3368a967c309bd4d136a",
v: "7c920000000000000946edb9c083c9942d92305444527765fad789c438c717783276a9f7fbf61b8501000000", v: "7c920000000000000000010000000001",
), ),
] ]

2
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@testnet_1.snap
View File

@ -5,6 +5,6 @@ expression: cf_data
[ [
KV( KV(
k: "03ef775f1f997f122a062fff1a2d7443abd1f9c642", k: "03ef775f1f997f122a062fff1a2d7443abd1f9c642",
v: "d430000000000000755f7c7d27a811596e9fae6dd30ca45be86e901d499909de35b6ff1f699f7ef301000000", v: "d4300000000000000000010000000001",
), ),
] ]

2
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@testnet_2.snap
View File

@ -5,6 +5,6 @@ expression: cf_data
[ [
KV( KV(
k: "03ef775f1f997f122a062fff1a2d7443abd1f9c642", k: "03ef775f1f997f122a062fff1a2d7443abd1f9c642",
v: "7c92000000000000755f7c7d27a811596e9fae6dd30ca45be86e901d499909de35b6ff1f699f7ef301000000", v: "7c920000000000000000010000000001",
), ),
] ]

4
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@mainnet_1.snap
View File

@ -4,11 +4,11 @@ expression: cf_data
--- ---
[ [
KV( KV(
k: "0946edb9c083c9942d92305444527765fad789c438c717783276a9f7fbf61b8500000000", k: "0000010000000000",
v: "0000010150c30000000000002321027a46eb513588b01b37ea24303f4b628afd12cc20df789fede0921e43cad3e875ac", v: "0000010150c30000000000002321027a46eb513588b01b37ea24303f4b628afd12cc20df789fede0921e43cad3e875ac",
), ),
KV( KV(
k: "0946edb9c083c9942d92305444527765fad789c438c717783276a9f7fbf61b8501000000", k: "0000010000000001",
v: "00000101d43000000000000017a9147d46a730d31f97b1930d3368a967c309bd4d136a87", v: "00000101d43000000000000017a9147d46a730d31f97b1930d3368a967c309bd4d136a87",
), ),
] ]

8
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@mainnet_2.snap
View File

@ -4,19 +4,19 @@ expression: cf_data
--- ---
[ [
KV( KV(
k: "0946edb9c083c9942d92305444527765fad789c438c717783276a9f7fbf61b8500000000", k: "0000010000000000",
v: "0000010150c30000000000002321027a46eb513588b01b37ea24303f4b628afd12cc20df789fede0921e43cad3e875ac", v: "0000010150c30000000000002321027a46eb513588b01b37ea24303f4b628afd12cc20df789fede0921e43cad3e875ac",
), ),
KV( KV(
k: "0946edb9c083c9942d92305444527765fad789c438c717783276a9f7fbf61b8501000000", k: "0000010000000001",
v: "00000101d43000000000000017a9147d46a730d31f97b1930d3368a967c309bd4d136a87", v: "00000101d43000000000000017a9147d46a730d31f97b1930d3368a967c309bd4d136a87",
), ),
KV( KV(
k: "f4b084a7c2fc5a5aa2985f2bcb1d4a9a65562a589d628b0d869c5f1c8dd0748900000000", k: "0000020000000000",
v: "00000201a0860100000000002321027a46eb513588b01b37ea24303f4b628afd12cc20df789fede0921e43cad3e875ac", v: "00000201a0860100000000002321027a46eb513588b01b37ea24303f4b628afd12cc20df789fede0921e43cad3e875ac",
), ),
KV( KV(
k: "f4b084a7c2fc5a5aa2985f2bcb1d4a9a65562a589d628b0d869c5f1c8dd0748901000000", k: "0000020000000001",
v: "00000201a86100000000000017a9147d46a730d31f97b1930d3368a967c309bd4d136a87", v: "00000201a86100000000000017a9147d46a730d31f97b1930d3368a967c309bd4d136a87",
), ),
] ]

4
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@testnet_1.snap
View File

@ -4,11 +4,11 @@ expression: cf_data
--- ---
[ [
KV( KV(
k: "755f7c7d27a811596e9fae6dd30ca45be86e901d499909de35b6ff1f699f7ef300000000", k: "0000010000000000",
v: "0000010150c30000000000002321025229e1240a21004cf8338db05679fa34753706e84f6aebba086ba04317fd8f99ac", v: "0000010150c30000000000002321025229e1240a21004cf8338db05679fa34753706e84f6aebba086ba04317fd8f99ac",
), ),
KV( KV(
k: "755f7c7d27a811596e9fae6dd30ca45be86e901d499909de35b6ff1f699f7ef301000000", k: "0000010000000001",
v: "00000101d43000000000000017a914ef775f1f997f122a062fff1a2d7443abd1f9c64287", v: "00000101d43000000000000017a914ef775f1f997f122a062fff1a2d7443abd1f9c64287",
), ),
] ]

8
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/utxo_by_outpoint_raw_data@testnet_2.snap
View File

@ -4,19 +4,19 @@ expression: cf_data
--- ---
[ [
KV( KV(
k: "755f7c7d27a811596e9fae6dd30ca45be86e901d499909de35b6ff1f699f7ef300000000", k: "0000010000000000",
v: "0000010150c30000000000002321025229e1240a21004cf8338db05679fa34753706e84f6aebba086ba04317fd8f99ac", v: "0000010150c30000000000002321025229e1240a21004cf8338db05679fa34753706e84f6aebba086ba04317fd8f99ac",
), ),
KV( KV(
k: "755f7c7d27a811596e9fae6dd30ca45be86e901d499909de35b6ff1f699f7ef301000000", k: "0000010000000001",
v: "00000101d43000000000000017a914ef775f1f997f122a062fff1a2d7443abd1f9c64287", v: "00000101d43000000000000017a914ef775f1f997f122a062fff1a2d7443abd1f9c64287",
), ),
KV( KV(
k: "d5b3ccfd5e7828c4b2d221bae3178c500e21d33399c39a2508a0a82d53c0225800000000", k: "0000020000000000",
v: "00000201a086010000000000232102acce9f6c16986c525fd34759d851ef5b4b85b5019a57bd59747be0ef1ba62523ac", v: "00000201a086010000000000232102acce9f6c16986c525fd34759d851ef5b4b85b5019a57bd59747be0ef1ba62523ac",
), ),
KV( KV(
k: "d5b3ccfd5e7828c4b2d221bae3178c500e21d33399c39a2508a0a82d53c0225801000000", k: "0000020000000001",
v: "00000201a86100000000000017a914ef775f1f997f122a062fff1a2d7443abd1f9c64287", v: "00000201a86100000000000017a914ef775f1f997f122a062fff1a2d7443abd1f9c64287",
), ),
] ]

189
zebra-state/src/service/finalized_state/disk_format/transparent.rs
View File

@ -14,37 +14,57 @@ use zebra_chain::{
block::Height, block::Height,
parameters::Network::*, parameters::Network::*,
serialization::{ZcashDeserializeInto, ZcashSerialize}, serialization::{ZcashDeserializeInto, ZcashSerialize},
transaction,
transparent::{self, Address::*}, transparent::{self, Address::*},
}; };
use crate::service::finalized_state::disk_format::{block::HEIGHT_DISK_BYTES, FromDisk, IntoDisk}; use crate::service::finalized_state::disk_format::{
block::{
TransactionIndex, TransactionLocation, HEIGHT_DISK_BYTES, TRANSACTION_LOCATION_DISK_BYTES,
},
expand_zero_be_bytes, truncate_zero_be_bytes, FromDisk, IntoDisk,
};
#[cfg(any(test, feature = "proptest-impl"))] #[cfg(any(test, feature = "proptest-impl"))]
use proptest_derive::Arbitrary; use proptest_derive::Arbitrary;
#[cfg(any(test, feature = "proptest-impl"))]
mod arbitrary;
/// Transparent balances are stored as an 8 byte integer on disk. /// Transparent balances are stored as an 8 byte integer on disk.
pub const BALANCE_DISK_BYTES: usize = 8; pub const BALANCE_DISK_BYTES: usize = 8;
/// Output transaction locations are stored as a 32 byte transaction hash on disk. /// [`OutputIndex`]es are stored as 3 bytes on disk.
/// ///
/// TODO: change to TransactionLocation to reduce database size and increases lookup performance (#3953) /// This reduces database size and increases lookup performance.
pub const OUTPUT_TX_HASH_DISK_BYTES: usize = 32; pub const OUTPUT_INDEX_DISK_BYTES: usize = 3;
/// [`OutputIndex`]es are stored as 4 bytes on disk. /// [`OutputLocation`]s are stored as a 3 byte height, 2 byte transaction index,
/// and 3 byte output index on disk.
/// ///
/// TODO: change to 3 bytes to reduce database size and increases lookup performance (#3953) /// This reduces database size and increases lookup performance.
pub const OUTPUT_INDEX_DISK_BYTES: usize = 4; pub const OUTPUT_LOCATION_DISK_BYTES: usize =
TRANSACTION_LOCATION_DISK_BYTES + OUTPUT_INDEX_DISK_BYTES;
// Transparent types // Transparent types
/// A transaction's index in its block. /// A transparent output's index in its transaction.
#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] #[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))]
pub struct OutputIndex(u32); pub struct OutputIndex(u32);
impl OutputIndex { impl OutputIndex {
/// Create a transparent output index from the native index integer type. /// Create a transparent output index from the Zcash consensus integer type.
///
/// `u32` is also the inner type.
pub fn from_index(output_index: u32) -> OutputIndex {
OutputIndex(output_index)
}
/// Returns this index as the inner type.
pub fn index(&self) -> u32 {
self.0
}
/// Create a transparent output index from `usize`.
#[allow(dead_code)] #[allow(dead_code)]
pub fn from_usize(output_index: usize) -> OutputIndex { pub fn from_usize(output_index: usize) -> OutputIndex {
OutputIndex( OutputIndex(
@ -54,7 +74,7 @@ impl OutputIndex {
) )
} }
/// Return this index as the native index integer type. /// Return this index as `usize`.
#[allow(dead_code)] #[allow(dead_code)]
pub fn as_usize(&self) -> usize { pub fn as_usize(&self) -> usize {
self.0 self.0
@ -62,51 +82,104 @@ impl OutputIndex {
.expect("the maximum valid index fits in usize") .expect("the maximum valid index fits in usize")
} }
/// Create a transparent output index from the Zcash consensus integer type. /// Create a transparent output index from `u64`.
pub fn from_zcash(output_index: u32) -> OutputIndex { #[allow(dead_code)]
OutputIndex(output_index) pub fn from_u64(output_index: u64) -> OutputIndex {
OutputIndex(
output_index
.try_into()
.expect("the maximum u64 index fits in the inner type"),
)
} }
/// Return this index as the Zcash consensus integer type. /// Return this index as `u64`.
#[allow(dead_code)] #[allow(dead_code)]
pub fn as_zcash(&self) -> u32 { pub fn as_u64(&self) -> u64 {
self.0 self.0.into()
} }
} }
/// A transparent output's location in the chain, by block height and transaction index. /// A transparent output's location in the chain, by block height and transaction index.
/// ///
/// TODO: provide a chain-order list of transactions (#3150) /// [`OutputLocation`]s are sorted in increasing chain order, by height, transaction index,
/// derive Ord, PartialOrd (#3150) /// and output index.
#[derive(Copy, Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] #[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))] #[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))]
pub struct OutputLocation { pub struct OutputLocation {
/// The transaction hash. /// The location of the transparent input's transaction.
#[serde(with = "hex")] transaction_location: TransactionLocation,
pub hash: transaction::Hash,
/// The index of the transparent output in its transaction. /// The index of the transparent output in its transaction.
pub index: OutputIndex, output_index: OutputIndex,
} }
impl OutputLocation { impl OutputLocation {
/// Create a transparent output location from a transaction hash and index /// Creates an output location from a block height, and `usize` transaction and output indexes.
/// (as the native index integer type).
#[allow(dead_code)] #[allow(dead_code)]
pub fn from_usize(hash: transaction::Hash, output_index: usize) -> OutputLocation { pub fn from_usize(
height: Height,
transaction_index: usize,
output_index: usize,
) -> OutputLocation {
OutputLocation { OutputLocation {
hash, transaction_location: TransactionLocation::from_usize(height, transaction_index),
index: OutputIndex::from_usize(output_index), output_index: OutputIndex::from_usize(output_index),
} }
} }
/// Create a transparent output location from a [`transparent::OutPoint`]. /// Creates an output location from an [`Outpoint`],
pub fn from_outpoint(outpoint: &transparent::OutPoint) -> OutputLocation { /// and the [`TransactionLocation`] of its transaction.
///
/// The [`TransactionLocation`] is provided separately,
/// because the lookup is a database operation.
pub fn from_outpoint(
transaction_location: TransactionLocation,
outpoint: &transparent::OutPoint,
) -> OutputLocation {
OutputLocation::from_output_index(transaction_location, outpoint.index)
}
/// Creates an output location from a [`TransactionLocation`] and a `u32` output index.
///
/// Output indexes are serialized to `u32` in the Zcash consensus-critical transaction format.
pub fn from_output_index(
transaction_location: TransactionLocation,
output_index: u32,
) -> OutputLocation {
OutputLocation { OutputLocation {
hash: outpoint.hash, transaction_location,
index: OutputIndex::from_zcash(outpoint.index), output_index: OutputIndex::from_index(output_index),
} }
} }
/// Returns the height of this [`transparent::Output`].
#[allow(dead_code)]
pub fn height(&self) -> Height {
self.transaction_location.height
}
/// Returns the transaction index of this [`transparent::Output`].
#[allow(dead_code)]
pub fn transaction_index(&self) -> TransactionIndex {
self.transaction_location.index
}
/// Returns the output index of this [`transparent::Output`].
pub fn output_index(&self) -> OutputIndex {
self.output_index
}
/// Returns the location of the transaction for this [`transparent::Output`].
pub fn transaction_location(&self) -> TransactionLocation {
self.transaction_location
}
/// Allows tests to set the height of this output location.
#[cfg(any(test, feature = "proptest-impl"))]
#[allow(dead_code)]
pub fn height_mut(&mut self) -> &mut Height {
&mut self.transaction_location.height
}
} }
/// The location of the first [`transparent::Output`] sent to an address. /// The location of the first [`transparent::Output`] sent to an address.
@ -166,6 +239,13 @@ impl AddressBalanceLocation {
pub fn location(&self) -> AddressLocation { pub fn location(&self) -> AddressLocation {
self.location self.location
} }
/// Allows tests to set the height of the address location.
#[cfg(any(test, feature = "proptest-impl"))]
#[allow(dead_code)]
pub fn height_mut(&mut self) -> &mut Height {
&mut self.location.transaction_location.height
}
} }
// Transparent trait impls // Transparent trait impls
@ -237,40 +317,57 @@ impl IntoDisk for OutputIndex {
type Bytes = [u8; OUTPUT_INDEX_DISK_BYTES]; type Bytes = [u8; OUTPUT_INDEX_DISK_BYTES];
fn as_bytes(&self) -> Self::Bytes { fn as_bytes(&self) -> Self::Bytes {
self.0.to_le_bytes() let mem_bytes = self.index().to_be_bytes();
let disk_bytes = truncate_zero_be_bytes(&mem_bytes, OUTPUT_INDEX_DISK_BYTES);
disk_bytes.try_into().unwrap()
} }
} }
impl FromDisk for OutputIndex { impl FromDisk for OutputIndex {
fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self { fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self {
OutputIndex(u32::from_le_bytes(disk_bytes.as_ref().try_into().unwrap())) let mem_len = u32::BITS / 8;
let mem_len = mem_len.try_into().unwrap();
let mem_bytes = expand_zero_be_bytes(disk_bytes.as_ref(), mem_len);
let mem_bytes = mem_bytes.try_into().unwrap();
OutputIndex::from_index(u32::from_be_bytes(mem_bytes))
} }
} }
impl IntoDisk for OutputLocation { impl IntoDisk for OutputLocation {
type Bytes = [u8; OUTPUT_TX_HASH_DISK_BYTES + OUTPUT_INDEX_DISK_BYTES]; type Bytes = [u8; OUTPUT_LOCATION_DISK_BYTES];
fn as_bytes(&self) -> Self::Bytes { fn as_bytes(&self) -> Self::Bytes {
let hash_bytes = self.hash.as_bytes().to_vec(); let transaction_location_bytes = self.transaction_location().as_bytes().to_vec();
let index_bytes = self.index.as_bytes().to_vec(); let output_index_bytes = self.output_index().as_bytes().to_vec();
[hash_bytes, index_bytes].concat().try_into().unwrap() [transaction_location_bytes, output_index_bytes]
.concat()
.try_into()
.unwrap()
} }
} }
impl FromDisk for OutputLocation { impl FromDisk for OutputLocation {
fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self { fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self {
let (hash_bytes, index_bytes) = disk_bytes.as_ref().split_at(OUTPUT_TX_HASH_DISK_BYTES); let (transaction_location_bytes, output_index_bytes) = disk_bytes
.as_ref()
.split_at(TRANSACTION_LOCATION_DISK_BYTES);
let hash = transaction::Hash::from_bytes(hash_bytes); let transaction_location = TransactionLocation::from_bytes(transaction_location_bytes);
let index = OutputIndex::from_bytes(index_bytes); let output_index = OutputIndex::from_bytes(output_index_bytes);
OutputLocation { hash, index } OutputLocation {
transaction_location,
output_index,
}
} }
} }
impl IntoDisk for AddressBalanceLocation { impl IntoDisk for AddressBalanceLocation {
type Bytes = [u8; BALANCE_DISK_BYTES + OUTPUT_TX_HASH_DISK_BYTES + OUTPUT_INDEX_DISK_BYTES]; type Bytes = [u8; BALANCE_DISK_BYTES + OUTPUT_LOCATION_DISK_BYTES];
fn as_bytes(&self) -> Self::Bytes { fn as_bytes(&self) -> Self::Bytes {
let balance_bytes = self.balance().as_bytes().to_vec(); let balance_bytes = self.balance().as_bytes().to_vec();

19
zebra-state/src/service/finalized_state/disk_format/transparent/arbitrary.rs Normal file
View File

@ -0,0 +1,19 @@
//! Randomised data generation for disk format property tests.
use proptest::prelude::*;
use zebra_chain::{serialization::TrustedPreallocate, transparent};
use super::OutputIndex;
impl Arbitrary for OutputIndex {
type Parameters = ();
fn arbitrary_with(_args: ()) -> Self::Strategy {
(0..=transparent::Output::max_allocation())
.prop_map(OutputIndex::from_u64)
.boxed()
}
type Strategy = BoxedStrategy<Self>;
}

121
zebra-state/src/service/finalized_state/zebra_db/block.rs
View File

@ -9,13 +9,16 @@
//! The [`crate::constants::DATABASE_FORMAT_VERSION`] constant must //! The [`crate::constants::DATABASE_FORMAT_VERSION`] constant must
//! be incremented each time the database format (column, serialization, etc) changes. //! be incremented each time the database format (column, serialization, etc) changes.
use std::{collections::HashMap, sync::Arc}; use std::{
collections::{BTreeMap, HashMap},
sync::Arc,
};
use itertools::Itertools; use itertools::Itertools;
use zebra_chain::{ use zebra_chain::{
amount::NonNegative, amount::NonNegative,
block::{self, Block}, block::{self, Block, Height},
history_tree::HistoryTree, history_tree::HistoryTree,
parameters::{Network, GENESIS_PREVIOUS_BLOCK_HASH}, parameters::{Network, GENESIS_PREVIOUS_BLOCK_HASH},
serialization::TrustedPreallocate, serialization::TrustedPreallocate,
@ -27,7 +30,11 @@ use zebra_chain::{
use crate::{ use crate::{
service::finalized_state::{ service::finalized_state::{
disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk}, disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk},
disk_format::{transparent::AddressBalanceLocation, FromDisk, TransactionLocation}, disk_format::{
block::TransactionLocation,
transparent::{AddressBalanceLocation, OutputLocation},
FromDisk,
},
zebra_db::{metrics::block_precommit_metrics, shielded::NoteCommitmentTrees, ZebraDb}, zebra_db::{metrics::block_precommit_metrics, shielded::NoteCommitmentTrees, ZebraDb},
FinalizedBlock, FinalizedBlock,
}, },
@ -150,14 +157,11 @@ impl ZebraDb {
self.db.zs_get(&hash_by_tx_loc, &location) self.db.zs_get(&hash_by_tx_loc, &location)
} }
/// Returns the [`Transaction`] with [`transaction::Hash`], and its [`block::Height`], /// Returns the [`Transaction`] with [`transaction::Hash`], and its [`Height`],
/// if a transaction with that hash exists in the finalized chain. /// if a transaction with that hash exists in the finalized chain.
// //
// TODO: move this method to the start of the section // TODO: move this method to the start of the section
pub fn transaction( pub fn transaction(&self, hash: transaction::Hash) -> Option<(Arc<Transaction>, Height)> {
&self,
hash: transaction::Hash,
) -> Option<(Arc<Transaction>, block::Height)> {
let tx_by_loc = self.db.cf_handle("tx_by_loc").unwrap(); let tx_by_loc = self.db.cf_handle("tx_by_loc").unwrap();
let transaction_location = self.transaction_location(hash)?; let transaction_location = self.transaction_location(hash)?;
@ -189,26 +193,64 @@ impl ZebraDb {
) -> Result<block::Hash, BoxError> { ) -> Result<block::Hash, BoxError> {
let finalized_hash = finalized.hash; let finalized_hash = finalized.hash;
// Get a list of the spent UTXOs, before we delete any from the database let tx_hash_indexes: HashMap<transaction::Hash, usize> = finalized
let all_utxos_spent_by_block: HashMap<transparent::OutPoint, transparent::Utxo> = finalized .transaction_hashes
.block
.transactions
.iter() .iter()
.flat_map(|tx| tx.inputs().iter()) .enumerate()
.flat_map(|input| input.outpoint()) .map(|(index, hash)| (*hash, index))
.map(|outpoint| { .collect();
// Get a list of the new UTXOs in the format we need for database updates.
//
// TODO: index new_outputs by TransactionLocation,
// simplify the spent_utxos location lookup code,
// and remove the extra new_outputs_by_out_loc argument
let new_outputs_by_out_loc: BTreeMap<OutputLocation, transparent::Utxo> = finalized
.new_outputs
.iter()
.map(|(outpoint, utxo)| {
( (
outpoint, lookup_out_loc(finalized.height, outpoint, &tx_hash_indexes),
// Some utxos are spent in the same block, so they will be in `new_outputs` utxo.clone(),
self.utxo(&outpoint)
.or_else(|| finalized.new_outputs.get(&outpoint).cloned())
.expect("already checked UTXO was in state or block"),
) )
}) })
.collect(); .collect();
// Get a list of the spent UTXOs, before we delete any from the database
let spent_utxos: Vec<(transparent::OutPoint, OutputLocation, transparent::Utxo)> =
finalized
.block
.transactions
.iter()
.flat_map(|tx| tx.inputs().iter())
.flat_map(|input| input.outpoint())
.map(|outpoint| {
(
outpoint,
// Some utxos are spent in the same block, so they will be in
// `tx_hash_indexes` and `new_outputs`
self.output_location(&outpoint).unwrap_or_else(|| {
lookup_out_loc(finalized.height, &outpoint, &tx_hash_indexes)
}),
self.utxo(&outpoint)
.or_else(|| finalized.new_outputs.get(&outpoint).cloned())
.expect("already checked UTXO was in state or block"),
)
})
.collect();
let spent_utxos_by_outpoint: HashMap<transparent::OutPoint, transparent::Utxo> =
spent_utxos
.iter()
.map(|(outpoint, _output_loc, utxo)| (*outpoint, utxo.clone()))
.collect();
let spent_utxos_by_out_loc: BTreeMap<OutputLocation, transparent::Utxo> = spent_utxos
.into_iter()
.map(|(_outpoint, out_loc, utxo)| (out_loc, utxo))
.collect();
// Get the current address balances, before the transactions in this block // Get the current address balances, before the transactions in this block
let address_balances = all_utxos_spent_by_block let address_balances = spent_utxos_by_out_loc
.values() .values()
.chain(finalized.new_outputs.values()) .chain(finalized.new_outputs.values())
.filter_map(|utxo| utxo.output.address(network)) .filter_map(|utxo| utxo.output.address(network))
@ -222,7 +264,9 @@ impl ZebraDb {
batch.prepare_block_batch( batch.prepare_block_batch(
&self.db, &self.db,
finalized, finalized,
all_utxos_spent_by_block, new_outputs_by_out_loc,
spent_utxos_by_outpoint,
spent_utxos_by_out_loc,
address_balances, address_balances,
self.note_commitment_trees(), self.note_commitment_trees(),
history_tree, history_tree,
@ -237,6 +281,23 @@ impl ZebraDb {
} }
} }
/// Lookup the output location for an outpoint.
///
/// `tx_hash_indexes` must contain `outpoint.hash` and that transaction's index in its block.
fn lookup_out_loc(
height: Height,
outpoint: &transparent::OutPoint,
tx_hash_indexes: &HashMap<transaction::Hash, usize>,
) -> OutputLocation {
let tx_index = tx_hash_indexes
.get(&outpoint.hash)
.expect("already checked UTXO was in state or block");
let tx_loc = TransactionLocation::from_usize(height, *tx_index);
OutputLocation::from_outpoint(tx_loc, outpoint)
}
impl DiskWriteBatch { impl DiskWriteBatch {
// Write block methods // Write block methods
@ -254,7 +315,9 @@ impl DiskWriteBatch {
&mut self, &mut self,
db: &DiskDb, db: &DiskDb,
finalized: FinalizedBlock, finalized: FinalizedBlock,
all_utxos_spent_by_block: HashMap<transparent::OutPoint, transparent::Utxo>, new_outputs_by_out_loc: BTreeMap<OutputLocation, transparent::Utxo>,
spent_utxos_by_outpoint: HashMap<transparent::OutPoint, transparent::Utxo>,
spent_utxos_by_out_loc: BTreeMap<OutputLocation, transparent::Utxo>,
address_balances: HashMap<transparent::Address, AddressBalanceLocation>, address_balances: HashMap<transparent::Address, AddressBalanceLocation>,
mut note_commitment_trees: NoteCommitmentTrees, mut note_commitment_trees: NoteCommitmentTrees,
history_tree: HistoryTree, history_tree: HistoryTree,
@ -288,7 +351,8 @@ impl DiskWriteBatch {
self.prepare_transaction_index_batch( self.prepare_transaction_index_batch(
db, db,
&finalized, &finalized,
&all_utxos_spent_by_block, new_outputs_by_out_loc,
spent_utxos_by_out_loc,
address_balances, address_balances,
&mut note_commitment_trees, &mut note_commitment_trees,
)?; )?;
@ -296,7 +360,7 @@ impl DiskWriteBatch {
self.prepare_note_commitment_batch(db, &finalized, note_commitment_trees, history_tree)?; self.prepare_note_commitment_batch(db, &finalized, note_commitment_trees, history_tree)?;
// Commit UTXOs and value pools // Commit UTXOs and value pools
self.prepare_chain_value_pools_batch(db, &finalized, all_utxos_spent_by_block, value_pool)?; self.prepare_chain_value_pools_batch(db, &finalized, spent_utxos_by_outpoint, value_pool)?;
// The block has passed contextual validation, so update the metrics // The block has passed contextual validation, so update the metrics
block_precommit_metrics(block, *hash, *height); block_precommit_metrics(block, *hash, *height);
@ -396,7 +460,8 @@ impl DiskWriteBatch {
&mut self, &mut self,
db: &DiskDb, db: &DiskDb,
finalized: &FinalizedBlock, finalized: &FinalizedBlock,
all_utxos_spent_by_block: &HashMap<transparent::OutPoint, transparent::Utxo>, new_outputs_by_out_loc: BTreeMap<OutputLocation, transparent::Utxo>,
utxos_spent_by_block: BTreeMap<OutputLocation, transparent::Utxo>,
address_balances: HashMap<transparent::Address, AddressBalanceLocation>, address_balances: HashMap<transparent::Address, AddressBalanceLocation>,
note_commitment_trees: &mut NoteCommitmentTrees, note_commitment_trees: &mut NoteCommitmentTrees,
) -> Result<(), BoxError> { ) -> Result<(), BoxError> {
@ -411,8 +476,8 @@ impl DiskWriteBatch {
self.prepare_transparent_outputs_batch( self.prepare_transparent_outputs_batch(
db, db,
finalized, new_outputs_by_out_loc,
all_utxos_spent_by_block, utxos_spent_by_block,
address_balances, address_balances,
) )
} }

17
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
View File

@ -357,15 +357,23 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) {
transparent::OutPoint::from_usize(transaction_hash, output_index); transparent::OutPoint::from_usize(transaction_hash, output_index);
let output_location = let output_location =
OutputLocation::from_usize(transaction_hash, output_index); OutputLocation::from_usize(query_height, tx_index, output_index);
let stored_utxo = state.utxo(&outpoint); let stored_output_location = state
.output_location(&outpoint)
.expect("all outpoints are indexed");
let stored_utxo_by_outpoint = state.utxo(&outpoint);
let stored_utxo_by_out_loc = state.utxo_by_location(output_location);
assert_eq!(stored_output_location, output_location);
assert_eq!(stored_utxo_by_out_loc, stored_utxo_by_outpoint);
// # Consensus // # Consensus
// //
// The genesis transaction's UTXO is not indexed. // The genesis transaction's UTXO is not indexed.
// This check also ignores spent UTXOs. // This check also ignores spent UTXOs.
if let Some(stored_utxo) = &stored_utxo { if let Some(stored_utxo) = &stored_utxo_by_out_loc {
assert_eq!(&stored_utxo.output, output); assert_eq!(&stored_utxo.output, output);
assert_eq!(stored_utxo.height, query_height); assert_eq!(stored_utxo.height, query_height);
@ -381,8 +389,7 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) {
); );
} }
// TODO: use output_location in #3151 stored_utxos.push((output_location, stored_utxo_by_out_loc));
stored_utxos.push((outpoint, stored_utxo));
} }
} }
} }

7
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_1.snap
View File

@ -6,8 +6,11 @@ expression: stored_address_balances
("t3Vz22vK5z2LcKEdg16Yv4FFneEL1zg9ojd", AddressBalanceLocation( ("t3Vz22vK5z2LcKEdg16Yv4FFneEL1zg9ojd", AddressBalanceLocation(
balance: Amount(12500), balance: Amount(12500),
location: OutputLocation( location: OutputLocation(
hash: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609", transaction_location: TransactionLocation(
index: OutputIndex(1), height: Height(1),
index: TransactionIndex(0),
),
output_index: OutputIndex(1),
), ),
)), )),
] ]

7
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_2.snap
View File

@ -6,8 +6,11 @@ expression: stored_address_balances
("t3Vz22vK5z2LcKEdg16Yv4FFneEL1zg9ojd", AddressBalanceLocation( ("t3Vz22vK5z2LcKEdg16Yv4FFneEL1zg9ojd", AddressBalanceLocation(
balance: Amount(37500), balance: Amount(37500),
location: OutputLocation( location: OutputLocation(
hash: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609", transaction_location: TransactionLocation(
index: OutputIndex(1), height: Height(1),
index: TransactionIndex(0),
),
output_index: OutputIndex(1),
), ),
)), )),
] ]

7
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_1.snap
View File

@ -6,8 +6,11 @@ expression: stored_address_balances
("t2UNzUUx8mWBCRYPRezvA363EYXyEpHokyi", AddressBalanceLocation( ("t2UNzUUx8mWBCRYPRezvA363EYXyEpHokyi", AddressBalanceLocation(
balance: Amount(12500), balance: Amount(12500),
location: OutputLocation( location: OutputLocation(
hash: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75", transaction_location: TransactionLocation(
index: OutputIndex(1), height: Height(1),
index: TransactionIndex(0),
),
output_index: OutputIndex(1),
), ),
)), )),
] ]

7
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_2.snap
View File

@ -6,8 +6,11 @@ expression: stored_address_balances
("t2UNzUUx8mWBCRYPRezvA363EYXyEpHokyi", AddressBalanceLocation( ("t2UNzUUx8mWBCRYPRezvA363EYXyEpHokyi", AddressBalanceLocation(
balance: Amount(37500), balance: Amount(37500),
location: OutputLocation( location: OutputLocation(
hash: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75", transaction_location: TransactionLocation(
index: OutputIndex(1), height: Height(1),
index: TransactionIndex(0),
),
output_index: OutputIndex(1),
), ),
)), )),
] ]

9
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@mainnet_0.snap
View File

@ -3,8 +3,11 @@ source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
expression: stored_utxos expression: stored_utxos
--- ---
[ [
(OutPoint( (OutputLocation(
hash: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", transaction_location: TransactionLocation(
index: 0, height: Height(0),
index: TransactionIndex(0),
),
output_index: OutputIndex(0),
), None), ), None),
] ]

27
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@mainnet_1.snap
View File

@ -3,13 +3,19 @@ source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
expression: stored_utxos expression: stored_utxos
--- ---
[ [
(OutPoint( (OutputLocation(
hash: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", transaction_location: TransactionLocation(
index: 0, height: Height(0),
index: TransactionIndex(0),
),
output_index: OutputIndex(0),
), None), ), None),
(OutPoint( (OutputLocation(
hash: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609", transaction_location: TransactionLocation(
index: 0, height: Height(1),
index: TransactionIndex(0),
),
output_index: OutputIndex(0),
), Some(Utxo( ), Some(Utxo(
output: Output( output: Output(
value: Amount(50000), value: Amount(50000),
@ -18,9 +24,12 @@ expression: stored_utxos
height: Height(1), height: Height(1),
from_coinbase: true, from_coinbase: true,
))), ))),
(OutPoint( (OutputLocation(
hash: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609", transaction_location: TransactionLocation(
index: 1, height: Height(1),
index: TransactionIndex(0),
),
output_index: OutputIndex(1),
), Some(Utxo( ), Some(Utxo(
output: Output( output: Output(
value: Amount(12500), value: Amount(12500),

45
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@mainnet_2.snap
View File

@ -3,13 +3,19 @@ source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
expression: stored_utxos expression: stored_utxos
--- ---
[ [
(OutPoint( (OutputLocation(
hash: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", transaction_location: TransactionLocation(
index: 0, height: Height(0),
index: TransactionIndex(0),
),
output_index: OutputIndex(0),
), None), ), None),
(OutPoint( (OutputLocation(
hash: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609", transaction_location: TransactionLocation(
index: 0, height: Height(1),
index: TransactionIndex(0),
),
output_index: OutputIndex(0),
), Some(Utxo( ), Some(Utxo(
output: Output( output: Output(
value: Amount(50000), value: Amount(50000),
@ -18,9 +24,12 @@ expression: stored_utxos
height: Height(1), height: Height(1),
from_coinbase: true, from_coinbase: true,
))), ))),
(OutPoint( (OutputLocation(
hash: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609", transaction_location: TransactionLocation(
index: 1, height: Height(1),
index: TransactionIndex(0),
),
output_index: OutputIndex(1),
), Some(Utxo( ), Some(Utxo(
output: Output( output: Output(
value: Amount(12500), value: Amount(12500),
@ -29,9 +38,12 @@ expression: stored_utxos
height: Height(1), height: Height(1),
from_coinbase: true, from_coinbase: true,
))), ))),
(OutPoint( (OutputLocation(
hash: "8974d08d1c5f9c860d8b629d582a56659a4a1dcb2b5f98a25a5afcc2a784b0f4", transaction_location: TransactionLocation(
index: 0, height: Height(2),
index: TransactionIndex(0),
),
output_index: OutputIndex(0),
), Some(Utxo( ), Some(Utxo(
output: Output( output: Output(
value: Amount(100000), value: Amount(100000),
@ -40,9 +52,12 @@ expression: stored_utxos
height: Height(2), height: Height(2),
from_coinbase: true, from_coinbase: true,
))), ))),
(OutPoint( (OutputLocation(
hash: "8974d08d1c5f9c860d8b629d582a56659a4a1dcb2b5f98a25a5afcc2a784b0f4", transaction_location: TransactionLocation(
index: 1, height: Height(2),
index: TransactionIndex(0),
),
output_index: OutputIndex(1),
), Some(Utxo( ), Some(Utxo(
output: Output( output: Output(
value: Amount(25000), value: Amount(25000),

9
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@testnet_0.snap
View File

@ -3,8 +3,11 @@ source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
expression: stored_utxos expression: stored_utxos
--- ---
[ [
(OutPoint( (OutputLocation(
hash: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", transaction_location: TransactionLocation(
index: 0, height: Height(0),
index: TransactionIndex(0),
),
output_index: OutputIndex(0),
), None), ), None),
] ]

27
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@testnet_1.snap
View File

@ -3,13 +3,19 @@ source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
expression: stored_utxos expression: stored_utxos
--- ---
[ [
(OutPoint( (OutputLocation(
hash: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", transaction_location: TransactionLocation(
index: 0, height: Height(0),
index: TransactionIndex(0),
),
output_index: OutputIndex(0),
), None), ), None),
(OutPoint( (OutputLocation(
hash: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75", transaction_location: TransactionLocation(
index: 0, height: Height(1),
index: TransactionIndex(0),
),
output_index: OutputIndex(0),
), Some(Utxo( ), Some(Utxo(
output: Output( output: Output(
value: Amount(50000), value: Amount(50000),
@ -18,9 +24,12 @@ expression: stored_utxos
height: Height(1), height: Height(1),
from_coinbase: true, from_coinbase: true,
))), ))),
(OutPoint( (OutputLocation(
hash: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75", transaction_location: TransactionLocation(
index: 1, height: Height(1),
index: TransactionIndex(0),
),
output_index: OutputIndex(1),
), Some(Utxo( ), Some(Utxo(
output: Output( output: Output(
value: Amount(12500), value: Amount(12500),

45
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/utxos@testnet_2.snap
View File

@ -3,13 +3,19 @@ source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
expression: stored_utxos expression: stored_utxos
--- ---
[ [
(OutPoint( (OutputLocation(
hash: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", transaction_location: TransactionLocation(
index: 0, height: Height(0),
index: TransactionIndex(0),
),
output_index: OutputIndex(0),
), None), ), None),
(OutPoint( (OutputLocation(
hash: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75", transaction_location: TransactionLocation(
index: 0, height: Height(1),
index: TransactionIndex(0),
),
output_index: OutputIndex(0),
), Some(Utxo( ), Some(Utxo(
output: Output( output: Output(
value: Amount(50000), value: Amount(50000),
@ -18,9 +24,12 @@ expression: stored_utxos
height: Height(1), height: Height(1),
from_coinbase: true, from_coinbase: true,
))), ))),
(OutPoint( (OutputLocation(
hash: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75", transaction_location: TransactionLocation(
index: 1, height: Height(1),
index: TransactionIndex(0),
),
output_index: OutputIndex(1),
), Some(Utxo( ), Some(Utxo(
output: Output( output: Output(
value: Amount(12500), value: Amount(12500),
@ -29,9 +38,12 @@ expression: stored_utxos
height: Height(1), height: Height(1),
from_coinbase: true, from_coinbase: true,
))), ))),
(OutPoint( (OutputLocation(
hash: "5822c0532da8a008259ac39933d3210e508c17e3ba21d2b2c428785efdccb3d5", transaction_location: TransactionLocation(
index: 0, height: Height(2),
index: TransactionIndex(0),
),
output_index: OutputIndex(0),
), Some(Utxo( ), Some(Utxo(
output: Output( output: Output(
value: Amount(100000), value: Amount(100000),
@ -40,9 +52,12 @@ expression: stored_utxos
height: Height(2), height: Height(2),
from_coinbase: true, from_coinbase: true,
))), ))),
(OutPoint( (OutputLocation(
hash: "5822c0532da8a008259ac39933d3210e508c17e3ba21d2b2c428785efdccb3d5", transaction_location: TransactionLocation(
index: 1, height: Height(2),
index: TransactionIndex(0),
),
output_index: OutputIndex(1),
), Some(Utxo( ), Some(Utxo(
output: Output( output: Output(
value: Amount(25000), value: Amount(25000),

4
zebra-state/src/service/finalized_state/zebra_db/chain.rs
View File

@ -112,14 +112,14 @@ impl DiskWriteBatch {
&mut self, &mut self,
db: &DiskDb, db: &DiskDb,
finalized: &FinalizedBlock, finalized: &FinalizedBlock,
all_utxos_spent_by_block: HashMap<transparent::OutPoint, transparent::Utxo>, utxos_spent_by_block: HashMap<transparent::OutPoint, transparent::Utxo>,
value_pool: ValueBalance<NonNegative>, value_pool: ValueBalance<NonNegative>,
) -> Result<(), BoxError> { ) -> Result<(), BoxError> {
let tip_chain_value_pool = db.cf_handle("tip_chain_value_pool").unwrap(); let tip_chain_value_pool = db.cf_handle("tip_chain_value_pool").unwrap();
let FinalizedBlock { block, .. } = finalized; let FinalizedBlock { block, .. } = finalized;
let new_pool = value_pool.add_block(block.borrow(), &all_utxos_spent_by_block)?; let new_pool = value_pool.add_block(block.borrow(), &utxos_spent_by_block)?;
self.zs_insert(&tip_chain_value_pool, (), new_pool); self.zs_insert(&tip_chain_value_pool, (), new_pool);
Ok(()) Ok(())

57
zebra-state/src/service/finalized_state/zebra_db/transparent.rs
View File

@ -1,6 +1,6 @@
//! Provides high-level access to database: //! Provides high-level access to database:
//! - unspent [`transparent::Outputs`]s //! - unspent [`transparent::Outputs`]s (UTXOs), and
//! - transparent address indexes //! - transparent address indexes.
//! //!
//! This module makes sure that: //! This module makes sure that:
//! - all disk writes happen inside a RocksDB transaction, and //! - all disk writes happen inside a RocksDB transaction, and
@ -11,7 +11,7 @@
//! The [`crate::constants::DATABASE_FORMAT_VERSION`] constant must //! The [`crate::constants::DATABASE_FORMAT_VERSION`] constant must
//! be incremented each time the database format (column, serialization, etc) changes. //! be incremented each time the database format (column, serialization, etc) changes.
use std::{borrow::Borrow, collections::HashMap}; use std::collections::{BTreeMap, HashMap};
use zebra_chain::{ use zebra_chain::{
amount::{Amount, NonNegative}, amount::{Amount, NonNegative},
@ -23,7 +23,6 @@ use crate::{
disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk}, disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk},
disk_format::transparent::{AddressBalanceLocation, AddressLocation, OutputLocation}, disk_format::transparent::{AddressBalanceLocation, AddressLocation, OutputLocation},
zebra_db::ZebraDb, zebra_db::ZebraDb,
FinalizedBlock,
}, },
BoxError, BoxError,
}; };
@ -60,23 +59,39 @@ impl ZebraDb {
.map(|abl| abl.location()) .map(|abl| abl.location())
} }
/// Returns the [`OutputLocation`] for a [`transparent::OutPoint`].
///
/// This method returns the locations of spent and unspent outpoints.
/// Returns `None` if the output was never in the finalized state.
pub fn output_location(&self, outpoint: &transparent::OutPoint) -> Option<OutputLocation> {
self.transaction_location(outpoint.hash)
.map(|transaction_location| {
OutputLocation::from_outpoint(transaction_location, outpoint)
})
}
/// Returns the transparent output for a [`transparent::OutPoint`], /// Returns the transparent output for a [`transparent::OutPoint`],
/// if it is still unspent in the finalized state. /// if it is unspent in the finalized state.
pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<transparent::Utxo> { pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<transparent::Utxo> {
let utxo_by_outpoint = self.db.cf_handle("utxo_by_outpoint").unwrap(); let output_location = self.output_location(outpoint)?;
let output_location = OutputLocation::from_outpoint(outpoint); self.utxo_by_location(output_location)
}
self.db.zs_get(&utxo_by_outpoint, &output_location) /// Returns the transparent output for an [`OutputLocation`],
/// if it is unspent in the finalized state.
pub fn utxo_by_location(&self, output_location: OutputLocation) -> Option<transparent::Utxo> {
let utxo_by_out_loc = self.db.cf_handle("utxo_by_outpoint").unwrap();
self.db.zs_get(&utxo_by_out_loc, &output_location)
} }
} }
impl DiskWriteBatch { impl DiskWriteBatch {
/// Prepare a database batch containing `finalized.block`'s: /// Prepare a database batch containing `finalized.block`'s:
/// - transparent address balance changes, /// - transparent address balance changes,
/// - UTXO changes, and
/// TODO: /// TODO:
/// - transparent address index changes (add in #3951, #3953), and /// - transparent address index changes (add in #3951, #3953),
/// - UTXO changes (modify in #3952)
/// and return it (without actually writing anything). /// and return it (without actually writing anything).
/// ///
/// # Errors /// # Errors
@ -85,21 +100,16 @@ impl DiskWriteBatch {
pub fn prepare_transparent_outputs_batch( pub fn prepare_transparent_outputs_batch(
&mut self, &mut self,
db: &DiskDb, db: &DiskDb,
finalized: &FinalizedBlock, new_outputs_by_out_loc: BTreeMap<OutputLocation, transparent::Utxo>,
all_utxos_spent_by_block: &HashMap<transparent::OutPoint, transparent::Utxo>, utxos_spent_by_block: BTreeMap<OutputLocation, transparent::Utxo>,
mut address_balances: HashMap<transparent::Address, AddressBalanceLocation>, mut address_balances: HashMap<transparent::Address, AddressBalanceLocation>,
) -> Result<(), BoxError> { ) -> Result<(), BoxError> {
let utxo_by_outpoint = db.cf_handle("utxo_by_outpoint").unwrap(); let utxo_by_outpoint = db.cf_handle("utxo_by_outpoint").unwrap();
let balance_by_transparent_addr = db.cf_handle("balance_by_transparent_addr").unwrap(); let balance_by_transparent_addr = db.cf_handle("balance_by_transparent_addr").unwrap();
let FinalizedBlock {
block, new_outputs, ..
} = finalized;
// Index all new transparent outputs, before deleting any we've spent // Index all new transparent outputs, before deleting any we've spent
for (outpoint, utxo) in new_outputs.borrow().iter() { for (output_location, utxo) in new_outputs_by_out_loc {
let receiving_address = utxo.output.address(self.network()); let receiving_address = utxo.output.address(self.network());
let output_location = OutputLocation::from_outpoint(outpoint);
// Update the address balance by adding this UTXO's value // Update the address balance by adding this UTXO's value
if let Some(receiving_address) = receiving_address { if let Some(receiving_address) = receiving_address {
@ -120,15 +130,8 @@ impl DiskWriteBatch {
// Mark all transparent inputs as spent. // Mark all transparent inputs as spent.
// //
// Coinbase inputs represent new coins, so there are no UTXOs to mark as spent. // Coinbase inputs represent new coins, so there are no UTXOs to mark as spent.
for outpoint in block for (output_location, utxo) in utxos_spent_by_block {
.transactions let spent_output = utxo.output;
.iter()
.flat_map(|tx| tx.inputs())
.flat_map(|input| input.outpoint())
{
let output_location = OutputLocation::from_outpoint(&outpoint);
let spent_output = &all_utxos_spent_by_block.get(&outpoint).unwrap().output;
let sending_address = spent_output.address(self.network()); let sending_address = spent_output.address(self.network());
// Update the address balance by subtracting this UTXO's value // Update the address balance by subtracting this UTXO's value