zecfoundation
/
zebra
mirror of https://github.com/ZcashFoundation/zebra.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

2. feat(db): Add address balance indexes to the finalized state (#3963) 

* Add an empty balance_by_transparent_addr column family

* Add an AddressBalanceLocation type for balance_by_transparent_addr

* Add serialization for balance_by_transparent_addr types

* Add round-trip tests for the new serialized types

* Add missing round-trip and serialized equality tests

* Add a network field to DiskWriteBatch

* Refactor confusing all_utxos_spent_by_block argument

It was actually just the UTXOs from the state spent by the block,
excluding the UTXOs created and spent within the block.

But now we need it to contain all the spent outputs,
including the ones created by the block.

* Read and update address balances in the finalized state

* Update raw data snapshots for transparent address balances

* Add test-only deserialization for transparent addresses

* Add high-level snapshot test code for address balances

* Add high-level snapshots for address balances

* Increment the state version after NU5 testnet 2 rollback
This commit is contained in:
teor 2022-04-08 09:15:17 +10:00 committed by GitHub
parent 6aba60d657
commit 7faa6a26c5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 837 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
zebra-chain/src/transparent/address.rs
View File

@ -6,15 +6,14 @@ use ripemd160::{Digest, Ripemd160};
use secp256k1::PublicKey;
use sha2::Sha256;
#[cfg(test)]
use proptest::{arbitrary::Arbitrary, collection::vec, prelude::*};
use crate::{
parameters::Network,
serialization::{SerializationError, ZcashDeserialize, ZcashSerialize},
transparent::Script,
};
use super::Script;
#[cfg(test)]
use proptest::prelude::*;
/// Magic numbers used to identify what networks Transparent Addresses
/// are associated with.
@ -41,7 +40,11 @@ mod magics {
/// to a Bitcoin address just by removing the "t".)
///
/// https://zips.z.cash/protocol/protocol.pdf#transparentaddrencoding
#[derive(Copy, Clone, Eq, PartialEq)]
#[derive(Copy, Clone, Eq, PartialEq, Hash)]
#[cfg_attr(
any(test, feature = "proptest-impl"),
derive(proptest_derive::Arbitrary)
)]
pub enum Address {
/// P2SH (Pay to Script Hash) addresses
PayToScriptHash {
@ -208,6 +211,26 @@ impl Address {
}
}
/// Returns the network for this address.
pub fn network(&self) -> Network {
match *self {
Address::PayToScriptHash { network, .. } => network,
Address::PayToPublicKeyHash { network, .. } => network,
}
}
/// Returns the hash bytes for this address, regardless of the address type.
///
/// # Correctness
///
/// Use [`ZcashSerialize`] and [`ZcashDeserialize`] for consensus-critical serialization.
pub fn hash_bytes(&self) -> [u8; 20] {
match *self {
Address::PayToScriptHash { script_hash, .. } => script_hash,
Address::PayToPublicKeyHash { pub_key_hash, .. } => pub_key_hash,
}
}
/// A hash of a transparent address payload, as used in
/// transparent pay-to-script-hash and pay-to-publickey-hash
/// addresses.
@ -224,46 +247,6 @@ impl Address {
}
}
#[cfg(test)]
impl Address {
fn p2pkh_strategy() -> impl Strategy<Value = Self> {
(any::<Network>(), vec(any::<u8>(), 20))
.prop_map(|(network, payload_bytes)| {
let mut bytes = [0; 20];
bytes.copy_from_slice(payload_bytes.as_slice());
Self::PayToPublicKeyHash {
network,
pub_key_hash: bytes,
}
})
.boxed()
}
fn p2sh_strategy() -> impl Strategy<Value = Self> {
(any::<Network>(), vec(any::<u8>(), 20))
.prop_map(|(network, payload_bytes)| {
let mut bytes = [0; 20];
bytes.copy_from_slice(payload_bytes.as_slice());
Self::PayToScriptHash {
network,
script_hash: bytes,
}
})
.boxed()
}
}
#[cfg(test)]
impl Arbitrary for Address {
type Parameters = ();
fn arbitrary_with(_args: Self::Parameters) -> Self::Strategy {
prop_oneof![Self::p2pkh_strategy(), Self::p2sh_strategy(),].boxed()
}
type Strategy = BoxedStrategy<Self>;
}
#[cfg(test)]
mod tests {

2
zebra-state/Cargo.toml
View File

@ -15,6 +15,7 @@ dirs = "4.0.0"
displaydoc = "0.2.3"
futures = "0.3.21"
hex = "0.4.3"
itertools = "0.10.3"
lazy_static = "1.4.0"
metrics = "0.17.1"
mset = "0.1.0"
@ -35,7 +36,6 @@ zebra-test = { path = "../zebra-test/", optional = true }
[dev-dependencies]
color-eyre = "0.6.0"
itertools = "0.10.3"
once_cell = "1.10.0"
spandoc = "0.2.1"

2
zebra-state/src/constants.rs
View File

@ -18,7 +18,7 @@ pub use zebra_chain::transparent::MIN_TRANSPARENT_COINBASE_MATURITY;
pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRANSPARENT_COINBASE_MATURITY - 1;
/// The database format version, incremented each time the database format changes.
pub const DATABASE_FORMAT_VERSION: u32 = 16;
pub const DATABASE_FORMAT_VERSION: u32 = 17;
/// The maximum number of blocks to check for NU5 transactions,
/// before we assume we are on a pre-NU5 legacy chain.

28
zebra-state/src/service/finalized_state/disk_db.rs
View File

@ -68,10 +68,17 @@ pub struct DiskDb {
///
/// [`rocksdb::WriteBatch`] is a batched set of database updates,
/// which must be written to the database using `DiskDb::write(batch)`.
//
// TODO: move DiskDb, FinalizedBlock, and the source String into this struct,
// (DiskDb can be cloned),
// and make them accessible via read-only methods
#[must_use = "batches must be written to the database"]
pub struct DiskWriteBatch {
/// The inner RocksDB write batch.
batch: rocksdb::WriteBatch,
/// The configured network.
network: Network,
}
/// Helper trait for inserting (Key, Value) pairs into rocksdb with a consistently
@ -298,11 +305,24 @@ impl ReadDisk for DiskDb {
}
impl DiskWriteBatch {
pub fn new() -> Self {
/// Creates and returns a new transactional batch write.
///
/// # Correctness
///
/// Each block must be written to the state inside a batch, so that:
/// - concurrent `ReadStateService` queries don't see half-written blocks, and
/// - if Zebra calls `exit`, panics, or crashes, half-written blocks are rolled back.
pub fn new(network: Network) -> Self {
DiskWriteBatch {
batch: rocksdb::WriteBatch::default(),
network,
}
}
/// Returns the configured network for this write batch.
pub fn network(&self) -> Network {
self.network
}
}
impl DiskDb {
@ -344,7 +364,13 @@ impl DiskDb {
// TODO: rename to tx_loc_by_hash (#3151)
rocksdb::ColumnFamilyDescriptor::new("tx_by_hash", db_options.clone()),
// Transparent
rocksdb::ColumnFamilyDescriptor::new("balance_by_transparent_addr", db_options.clone()),
// TODO: #3954
//rocksdb::ColumnFamilyDescriptor::new("tx_by_transparent_addr_loc", db_options.clone()),
// TODO: rename to utxo_by_out_loc (#3953)
rocksdb::ColumnFamilyDescriptor::new("utxo_by_outpoint", db_options.clone()),
// TODO: #3952
//rocksdb::ColumnFamilyDescriptor::new("utxo_by_transparent_addr_loc", db_options.clone()),
// Sprout
rocksdb::ColumnFamilyDescriptor::new("sprout_nullifiers", db_options.clone()),
rocksdb::ColumnFamilyDescriptor::new("sprout_anchors", db_options.clone()),

319
zebra-state/src/service/finalized_state/disk_format/tests/prop.rs
View File

@ -3,17 +3,26 @@
use proptest::{arbitrary::any, prelude::*};
use zebra_chain::{
amount::NonNegative,
amount::{Amount, NonNegative},
block::{self, Height},
orchard, sapling, sprout,
transaction::{self, Transaction},
transparent,
value_balance::ValueBalance,
};
use crate::service::finalized_state::{
arbitrary::assert_value_properties,
disk_format::{block::MAX_ON_DISK_HEIGHT, transparent::OutputLocation, TransactionLocation},
disk_format::{
block::MAX_ON_DISK_HEIGHT,
transparent::{AddressBalanceLocation, AddressLocation, OutputLocation},
IntoDisk, TransactionLocation,
},
};
// Block
// TODO: split these tests into the disk_format sub-modules
#[test]
fn roundtrip_block_height() {
zebra_test::init();
@ -29,6 +38,22 @@ fn roundtrip_block_height() {
);
}
#[test]
fn roundtrip_block_hash() {
zebra_test::init();
proptest!(|(val in any::<block::Hash>())| assert_value_properties(val));
}
#[test]
fn roundtrip_block_header() {
zebra_test::init();
proptest!(|(val in any::<block::Header>())| assert_value_properties(val));
}
// Transaction
#[test]
fn roundtrip_transaction_location() {
zebra_test::init();
@ -41,27 +66,85 @@ fn roundtrip_transaction_location() {
);
}
#[test]
fn roundtrip_transaction_hash() {
zebra_test::init();
proptest!(|(val in any::<transaction::Hash>())| assert_value_properties(val));
}
#[test]
fn roundtrip_transaction() {
zebra_test::init();
proptest!(|(val in any::<Transaction>())| assert_value_properties(val));
}
// Transparent
// TODO: turn this into a generic function like assert_value_properties()
#[test]
fn serialized_transparent_address_equal() {
zebra_test::init();
proptest!(|(val1 in any::<transparent::Address>(), val2 in any::<transparent::Address>())| {
if val1 == val2 {
prop_assert_eq!(
val1.as_bytes(),
val2.as_bytes(),
"struct values were equal, but serialized bytes were not.\n\
Values:\n\
{:?}\n\
{:?}",
val1,
val2,
);
} else {
prop_assert_ne!(
val1.as_bytes(),
val2.as_bytes(),
"struct values were not equal, but serialized bytes were equal.\n\
Values:\n\
{:?}\n\
{:?}",
val1,
val2,
);
}
}
);
}
#[test]
fn roundtrip_transparent_address() {
zebra_test::init();
proptest!(|(val in any::<transparent::Address>())| assert_value_properties(val));
}
#[test]
fn roundtrip_output_location() {
zebra_test::init();
proptest!(|(val in any::<OutputLocation>())| assert_value_properties(val));
}
#[test]
fn roundtrip_block_hash() {
fn roundtrip_address_location() {
zebra_test::init();
proptest!(|(val in any::<block::Hash>())| assert_value_properties(val));
proptest!(|(val in any::<AddressLocation>())| assert_value_properties(val));
}
#[test]
fn roundtrip_block_header() {
fn roundtrip_address_balance_location() {
zebra_test::init();
proptest!(|(val in any::<block::Header>())| assert_value_properties(val));
proptest!(|(val in any::<AddressBalanceLocation>())| assert_value_properties(val));
}
#[test]
fn roundtrip_transparent_output() {
fn roundtrip_unspent_transparent_output() {
zebra_test::init();
proptest!(
@ -72,6 +155,228 @@ fn roundtrip_transparent_output() {
);
}
#[test]
fn roundtrip_transparent_output() {
zebra_test::init();
proptest!(|(val in any::<transparent::Output>())| assert_value_properties(val));
}
#[test]
fn roundtrip_amount() {
zebra_test::init();
proptest!(|(val in any::<Amount::<NonNegative>>())| assert_value_properties(val));
}
// Sprout
#[test]
fn serialized_sprout_nullifier_equal() {
zebra_test::init();
proptest!(|(val1 in any::<sprout::Nullifier>(), val2 in any::<sprout::Nullifier>())| {
if val1 == val2 {
prop_assert_eq!(
val1.as_bytes(),
val2.as_bytes(),
"struct values were equal, but serialized bytes were not.\n\
Values:\n\
{:?}\n\
{:?}",
val1,
val2,
);
} else {
prop_assert_ne!(
val1.as_bytes(),
val2.as_bytes(),
"struct values were not equal, but serialized bytes were equal.\n\
Values:\n\
{:?}\n\
{:?}",
val1,
val2,
);
}
}
);
}
#[test]
fn serialized_sprout_tree_root_equal() {
zebra_test::init();
proptest!(|(val1 in any::<sprout::tree::Root>(), val2 in any::<sprout::tree::Root>())| {
if val1 == val2 {
prop_assert_eq!(
val1.as_bytes(),
val2.as_bytes(),
"struct values were equal, but serialized bytes were not.\n\
Values:\n\
{:?}\n\
{:?}",
val1,
val2,
);
} else {
prop_assert_ne!(
val1.as_bytes(),
val2.as_bytes(),
"struct values were not equal, but serialized bytes were equal.\n\
Values:\n\
{:?}\n\
{:?}",
val1,
val2,
);
}
}
);
}
// TODO: test note commitment tree round-trip, after implementing proptest::Arbitrary
// Sapling
#[test]
fn serialized_sapling_nullifier_equal() {
zebra_test::init();
proptest!(|(val1 in any::<sapling::Nullifier>(), val2 in any::<sapling::Nullifier>())| {
if val1 == val2 {
prop_assert_eq!(
val1.as_bytes(),
val2.as_bytes(),
"struct values were equal, but serialized bytes were not.\n\
Values:\n\
{:?}\n\
{:?}",
val1,
val2,
);
} else {
prop_assert_ne!(
val1.as_bytes(),
val2.as_bytes(),
"struct values were not equal, but serialized bytes were equal.\n\
Values:\n\
{:?}\n\
{:?}",
val1,
val2,
);
}
}
);
}
#[test]
fn serialized_sapling_tree_root_equal() {
zebra_test::init();
proptest!(|(val1 in any::<sapling::tree::Root>(), val2 in any::<sapling::tree::Root>())| {
if val1 == val2 {
prop_assert_eq!(
val1.as_bytes(),
val2.as_bytes(),
"struct values were equal, but serialized bytes were not.\n\
Values:\n\
{:?}\n\
{:?}",
val1,
val2,
);
} else {
prop_assert_ne!(
val1.as_bytes(),
val2.as_bytes(),
"struct values were not equal, but serialized bytes were equal.\n\
Values:\n\
{:?}\n\
{:?}",
val1,
val2,
);
}
}
);
}
// TODO: test note commitment tree round-trip, after implementing proptest::Arbitrary
// Orchard
#[test]
fn serialized_orchard_nullifier_equal() {
zebra_test::init();
proptest!(|(val1 in any::<orchard::Nullifier>(), val2 in any::<orchard::Nullifier>())| {
if val1 == val2 {
prop_assert_eq!(
val1.as_bytes(),
val2.as_bytes(),
"struct values were equal, but serialized bytes were not.\n\
Values:\n\
{:?}\n\
{:?}",
val1,
val2,
);
} else {
prop_assert_ne!(
val1.as_bytes(),
val2.as_bytes(),
"struct values were not equal, but serialized bytes were equal.\n\
Values:\n\
{:?}\n\
{:?}",
val1,
val2,
);
}
}
);
}
#[test]
fn serialized_orchard_tree_root_equal() {
zebra_test::init();
proptest!(|(val1 in any::<orchard::tree::Root>(), val2 in any::<orchard::tree::Root>())| {
if val1 == val2 {
prop_assert_eq!(
val1.as_bytes(),
val2.as_bytes(),
"struct values were equal, but serialized bytes were not.\n\
Values:\n\
{:?}\n\
{:?}",
val1,
val2,
);
} else {
prop_assert_ne!(
val1.as_bytes(),
val2.as_bytes(),
"struct values were not equal, but serialized bytes were equal.\n\
Values:\n\
{:?}\n\
{:?}",
val1,
val2,
);
}
}
);
}
// TODO: test note commitment tree round-trip, after implementing proptest::Arbitrary
// Chain
// TODO: test NonEmptyHistoryTree round-trip, after implementing proptest::Arbitrary
#[test]
fn roundtrip_value_balance() {
zebra_test::init();

10
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@mainnet_1.snap Normal file
View File

@ -0,0 +1,10 @@
---
source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
expression: cf_data
---
[
KV(
k: "017d46a730d31f97b1930d3368a967c309bd4d136a",
v: "d4300000000000000946edb9c083c9942d92305444527765fad789c438c717783276a9f7fbf61b8501000000",
),
]

10
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@mainnet_2.snap Normal file
View File

@ -0,0 +1,10 @@
---
source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
expression: cf_data
---
[
KV(
k: "017d46a730d31f97b1930d3368a967c309bd4d136a",
v: "7c920000000000000946edb9c083c9942d92305444527765fad789c438c717783276a9f7fbf61b8501000000",
),
]

10
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@testnet_1.snap Normal file
View File

@ -0,0 +1,10 @@
---
source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
expression: cf_data
---
[
KV(
k: "03ef775f1f997f122a062fff1a2d7443abd1f9c642",
v: "d430000000000000755f7c7d27a811596e9fae6dd30ca45be86e901d499909de35b6ff1f699f7ef301000000",
),
]

10
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@testnet_2.snap Normal file
View File

@ -0,0 +1,10 @@
---
source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
expression: cf_data
---
[
KV(
k: "03ef775f1f997f122a062fff1a2d7443abd1f9c642",
v: "7c92000000000000755f7c7d27a811596e9fae6dd30ca45be86e901d499909de35b6ff1f699f7ef301000000",
),
]

1
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/column_family_names.snap
View File

@ -3,6 +3,7 @@ source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
expression: cf_names
---
[
"balance_by_transparent_addr",
"block_by_height",
"default",
"hash_by_height",

1
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_0.snap
View File

@ -3,6 +3,7 @@ source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
expression: empty_column_families
---
[
"balance_by_transparent_addr: no entries",
"history_tree: no entries",
"orchard_anchors: no entries",
"orchard_nullifiers: no entries",

1
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@no_blocks.snap
View File

@ -3,6 +3,7 @@ source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
expression: empty_column_families
---
[
"balance_by_transparent_addr: no entries",
"block_by_height: no entries",
"hash_by_height: no entries",
"hash_by_tx_loc: no entries",

1
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_0.snap
View File

@ -3,6 +3,7 @@ source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
expression: empty_column_families
---
[
"balance_by_transparent_addr: no entries",
"history_tree: no entries",
"orchard_anchors: no entries",
"orchard_nullifiers: no entries",

178
zebra-state/src/service/finalized_state/disk_format/transparent.rs
View File

@ -10,9 +10,12 @@ use std::fmt::Debug;
use serde::{Deserialize, Serialize};
use zebra_chain::{
amount::{Amount, NonNegative},
block::Height,
parameters::Network::*,
serialization::{ZcashDeserializeInto, ZcashSerialize},
transaction, transparent,
transaction,
transparent::{self, Address::*},
};
use crate::service::finalized_state::disk_format::{block::HEIGHT_DISK_BYTES, FromDisk, IntoDisk};
@ -20,14 +23,17 @@ use crate::service::finalized_state::disk_format::{block::HEIGHT_DISK_BYTES, Fro
#[cfg(any(test, feature = "proptest-impl"))]
use proptest_derive::Arbitrary;
/// Transparent balances are stored as an 8 byte integer on disk.
pub const BALANCE_DISK_BYTES: usize = 8;
/// Output transaction locations are stored as a 32 byte transaction hash on disk.
///
/// TODO: change to TransactionLocation to reduce database size and increases lookup performance (#3151)
/// TODO: change to TransactionLocation to reduce database size and increases lookup performance (#3953)
pub const OUTPUT_TX_HASH_DISK_BYTES: usize = 32;
/// [`OutputIndex`]es are stored as 4 bytes on disk.
///
/// TODO: change to 3 bytes to reduce database size and increases lookup performance (#3151)
/// TODO: change to 3 bytes to reduce database size and increases lookup performance (#3953)
pub const OUTPUT_INDEX_DISK_BYTES: usize = 4;
// Transparent types
@ -76,6 +82,7 @@ impl OutputIndex {
#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))]
pub struct OutputLocation {
/// The transaction hash.
#[serde(with = "hex")]
pub hash: transaction::Hash,
/// The index of the transparent output in its transaction.
@ -102,10 +109,130 @@ impl OutputLocation {
}
}
/// The location of the first [`transparent::Output`] sent to an address.
///
/// The address location stays the same, even if the corresponding output
/// has been spent.
///
/// The first output location is used to represent the address in the database,
/// because output locations are significantly smaller than addresses.
///
/// TODO: make this a different type to OutputLocation?
/// derive IntoDisk and FromDisk?
pub type AddressLocation = OutputLocation;
/// Data which Zebra indexes for each [`transparent::Address`].
///
/// Currently, Zebra tracks this data 1:1 for each address:
/// - the balance [`Amount`] for a transparent address, and
/// - the [`OutputLocation`] for the first [`transparent::Output`] sent to that address
/// (regardless of whether that output is spent or unspent).
///
/// All other address data is tracked multiple times for each address
/// (UTXOs and transactions).
#[derive(Copy, Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))]
pub struct AddressBalanceLocation {
/// The total balance of all UTXOs sent to an address.
balance: Amount<NonNegative>,
/// The location of the first [`transparent::Output`] sent to an address.
location: AddressLocation,
}
impl AddressBalanceLocation {
/// Creates a new [`AddressBalanceLocation`] from the location of
/// the first [`transparent::Output`] sent to an address.
///
/// The returned value has a zero initial balance.
pub fn new(first_output: OutputLocation) -> AddressBalanceLocation {
AddressBalanceLocation {
balance: Amount::zero(),
location: first_output,
}
}
/// Returns the current balance for the address.
pub fn balance(&self) -> Amount<NonNegative> {
self.balance
}
/// Returns a mutable reference to the current balance for the address.
pub fn balance_mut(&mut self) -> &mut Amount<NonNegative> {
&mut self.balance
}
/// Returns the location of the first [`transparent::Output`] sent to an address.
pub fn location(&self) -> AddressLocation {
self.location
}
}
// Transparent trait impls
// TODO: serialize the index into a smaller number of bytes (#3152)
// serialize the index in big-endian order (#3150)
/// Returns a byte representing the [`transparent::Address`] variant.
fn address_variant(address: &transparent::Address) -> u8 {
// Return smaller values for more common variants.
//
// (This probably doesn't matter, but it might help slightly with data compression.)
match (address.network(), address) {
(Mainnet, PayToPublicKeyHash { .. }) => 0,
(Mainnet, PayToScriptHash { .. }) => 1,
(Testnet, PayToPublicKeyHash { .. }) => 2,
(Testnet, PayToScriptHash { .. }) => 3,
}
}
impl IntoDisk for transparent::Address {
type Bytes = [u8; 21];
fn as_bytes(&self) -> Self::Bytes {
let variant_bytes = vec![address_variant(self)];
let hash_bytes = self.hash_bytes().to_vec();
[variant_bytes, hash_bytes].concat().try_into().unwrap()
}
}
#[cfg(any(test, feature = "proptest-impl"))]
impl FromDisk for transparent::Address {
fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self {
let (address_variant, hash_bytes) = disk_bytes.as_ref().split_at(1);
let address_variant = address_variant[0];
let hash_bytes = hash_bytes.try_into().unwrap();
let network = if address_variant < 2 {
Mainnet
} else {
Testnet
};
if address_variant % 2 == 0 {
transparent::Address::from_pub_key_hash(network, hash_bytes)
} else {
transparent::Address::from_script_hash(network, hash_bytes)
}
}
}
impl IntoDisk for Amount<NonNegative> {
type Bytes = [u8; BALANCE_DISK_BYTES];
fn as_bytes(&self) -> Self::Bytes {
self.to_bytes()
}
}
impl FromDisk for Amount<NonNegative> {
fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
let array = bytes.as_ref().try_into().unwrap();
Amount::from_bytes(array).unwrap()
}
}
// TODO: serialize the index into a smaller number of bytes (#3953)
// serialize the index in big-endian order (#3953)
impl IntoDisk for OutputIndex {
type Bytes = [u8; OUTPUT_INDEX_DISK_BYTES];
@ -142,7 +269,46 @@ impl FromDisk for OutputLocation {
}
}
// TODO: just serialize the Output, and derive the Utxo data from OutputLocation (#3151)
impl IntoDisk for AddressBalanceLocation {
type Bytes = [u8; BALANCE_DISK_BYTES + OUTPUT_TX_HASH_DISK_BYTES + OUTPUT_INDEX_DISK_BYTES];
fn as_bytes(&self) -> Self::Bytes {
let balance_bytes = self.balance().as_bytes().to_vec();
let location_bytes = self.location().as_bytes().to_vec();
[balance_bytes, location_bytes].concat().try_into().unwrap()
}
}
impl FromDisk for AddressBalanceLocation {
fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self {
let (balance_bytes, location_bytes) = disk_bytes.as_ref().split_at(BALANCE_DISK_BYTES);
let balance = Amount::from_bytes(balance_bytes.try_into().unwrap()).unwrap();
let location = AddressLocation::from_bytes(location_bytes);
let mut balance_location = AddressBalanceLocation::new(location);
*balance_location.balance_mut() = balance;
balance_location
}
}
impl IntoDisk for transparent::Output {
type Bytes = Vec<u8>;
fn as_bytes(&self) -> Self::Bytes {
self.zcash_serialize_to_vec().unwrap()
}
}
impl FromDisk for transparent::Output {
fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
bytes.as_ref().zcash_deserialize_into().unwrap()
}
}
// TODO: delete UTXO serialization (#3953)
impl IntoDisk for transparent::Utxo {
type Bytes = Vec<u8>;

8
zebra-state/src/service/finalized_state/zebra_db/arbitrary.rs
View File

@ -4,7 +4,9 @@
use std::ops::Deref;
use zebra_chain::{amount::NonNegative, block::Block, sprout, value_balance::ValueBalance};
use zebra_chain::{
amount::NonNegative, block::Block, parameters::Network::*, sprout, value_balance::ValueBalance,
};
use crate::service::finalized_state::{
disk_db::{DiskDb, DiskWriteBatch, WriteDisk},
@ -31,7 +33,7 @@ impl ZebraDb {
/// Allow to set up a fake value pool in the database for testing purposes.
pub fn set_finalized_value_pool(&self, fake_value_pool: ValueBalance<NonNegative>) {
let mut batch = DiskWriteBatch::new();
let mut batch = DiskWriteBatch::new(Mainnet);
let value_pool_cf = self.db().cf_handle("tip_chain_value_pool").unwrap();
batch.zs_insert(&value_pool_cf, (), fake_value_pool);
@ -41,7 +43,7 @@ impl ZebraDb {
/// Artificially prime the note commitment tree anchor sets with anchors
/// referenced in a block, for testing purposes _only_.
pub fn populate_with_anchors(&self, block: &Block) {
let mut batch = DiskWriteBatch::new();
let mut batch = DiskWriteBatch::new(Mainnet);
let sprout_anchors = self.db().cf_handle("sprout_anchors").unwrap();
let sapling_anchors = self.db().cf_handle("sapling_anchors").unwrap();

53
zebra-state/src/service/finalized_state/zebra_db/block.rs
View File

@ -11,6 +11,8 @@
use std::{collections::HashMap, sync::Arc};
use itertools::Itertools;
use zebra_chain::{
amount::NonNegative,
block::{self, Block},
@ -25,7 +27,7 @@ use zebra_chain::{
use crate::{
service::finalized_state::{
disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk},
disk_format::{FromDisk, TransactionLocation},
disk_format::{transparent::AddressBalanceLocation, FromDisk, TransactionLocation},
zebra_db::{metrics::block_precommit_metrics, shielded::NoteCommitmentTrees, ZebraDb},
FinalizedBlock,
},
@ -188,23 +190,40 @@ impl ZebraDb {
let finalized_hash = finalized.hash;
// Get a list of the spent UTXOs, before we delete any from the database
let all_utxos_spent_by_block = finalized
let all_utxos_spent_by_block: HashMap<transparent::OutPoint, transparent::Utxo> = finalized
.block
.transactions
.iter()
.flat_map(|tx| tx.inputs().iter())
.flat_map(|input| input.outpoint())
.flat_map(|outpoint| self.utxo(&outpoint).map(|utxo| (outpoint, utxo)))
.map(|outpoint| {
(
outpoint,
// Some utxos are spent in the same block, so they will be in `new_outputs`
self.utxo(&outpoint)
.or_else(|| finalized.new_outputs.get(&outpoint).cloned())
.expect("already checked UTXO was in state or block"),
)
})
.collect();
let mut batch = DiskWriteBatch::new();
// Get the current address balances, before the transactions in this block
let address_balances = all_utxos_spent_by_block
.values()
.chain(finalized.new_outputs.values())
.filter_map(|utxo| utxo.output.address(network))
.unique()
.filter_map(|address| Some((address, self.address_balance_location(&address)?)))
.collect();
let mut batch = DiskWriteBatch::new(network);
// In case of errors, propagate and do not write the batch.
batch.prepare_block_batch(
&self.db,
finalized,
network,
all_utxos_spent_by_block,
address_balances,
self.note_commitment_trees(),
history_tree,
self.finalized_value_pool(),
@ -235,9 +254,8 @@ impl DiskWriteBatch {
&mut self,
db: &DiskDb,
finalized: FinalizedBlock,
network: Network,
all_utxos_spent_by_block: HashMap<transparent::OutPoint, transparent::Utxo>,
// TODO: make an argument struct for all the current note commitment trees & history
address_balances: HashMap<transparent::Address, AddressBalanceLocation>,
mut note_commitment_trees: NoteCommitmentTrees,
history_tree: HistoryTree,
value_pool: ValueBalance<NonNegative>,
@ -267,16 +285,16 @@ impl DiskWriteBatch {
}
// Commit transaction indexes
self.prepare_transaction_index_batch(db, &finalized, &mut note_commitment_trees)?;
self.prepare_note_commitment_batch(
self.prepare_transaction_index_batch(
db,
&finalized,
network,
note_commitment_trees,
history_tree,
&all_utxos_spent_by_block,
address_balances,
&mut note_commitment_trees,
)?;
self.prepare_note_commitment_batch(db, &finalized, note_commitment_trees, history_tree)?;
// Commit UTXOs and value pools
self.prepare_chain_value_pools_batch(db, &finalized, all_utxos_spent_by_block, value_pool)?;
@ -378,6 +396,8 @@ impl DiskWriteBatch {
&mut self,
db: &DiskDb,
finalized: &FinalizedBlock,
all_utxos_spent_by_block: &HashMap<transparent::OutPoint, transparent::Utxo>,
address_balances: HashMap<transparent::Address, AddressBalanceLocation>,
note_commitment_trees: &mut NoteCommitmentTrees,
) -> Result<(), BoxError> {
let FinalizedBlock { block, .. } = finalized;
@ -389,6 +409,11 @@ impl DiskWriteBatch {
DiskWriteBatch::update_note_commitment_trees(transaction, note_commitment_trees)?;
}
self.prepare_transparent_outputs_batch(db, finalized)
self.prepare_transparent_outputs_batch(
db,
finalized,
all_utxos_spent_by_block,
address_balances,
)
}
}

58
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
View File

@ -46,7 +46,9 @@ use zebra_chain::{
use crate::{
service::finalized_state::{
disk_format::{block::TransactionIndex, transparent::OutputLocation, TransactionLocation},
disk_format::{
block::TransactionIndex, transparent::OutputLocation, FromDisk, TransactionLocation,
},
FinalizedState,
},
Config,
@ -194,6 +196,7 @@ fn test_block_and_transaction_data_with_network(network: Network) {
settings.set_snapshot_suffix(format!("{}_{}", net_suffix, height));
settings.bind(|| snapshot_block_and_transaction_data(&state));
settings.bind(|| snapshot_transparent_address_data(&state));
}
}
@ -216,17 +219,23 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) {
assert_eq!(sapling_tree, sapling::tree::NoteCommitmentTree::default());
assert_eq!(orchard_tree, orchard::tree::NoteCommitmentTree::default());
// Blocks
let mut stored_block_hashes = Vec::new();
let mut stored_blocks = Vec::new();
let mut stored_sapling_trees = Vec::new();
let mut stored_orchard_trees = Vec::new();
// Transactions
let mut stored_transaction_hashes = Vec::new();
let mut stored_transactions = Vec::new();
// Transparent
let mut stored_utxos = Vec::new();
// Shielded
let mut stored_sapling_trees = Vec::new();
let mut stored_orchard_trees = Vec::new();
let sapling_tree_at_tip = state.sapling_note_commitment_tree();
let orchard_tree_at_tip = state.orchard_note_commitment_tree();
@ -412,6 +421,47 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) {
}
}
/// Snapshot transparent address data, using `cargo insta` and RON serialization.
fn snapshot_transparent_address_data(state: &FinalizedState) {
let balance_by_transparent_addr = state.cf_handle("balance_by_transparent_addr").unwrap();
let mut stored_address_balances = Vec::new();
// TODO: UTXOs for each address (#3953)
// transactions for each address (#3951)
// Correctness: Multi-key iteration causes hangs in concurrent code, but seems ok in tests.
let addresses =
state.full_iterator_cf(&balance_by_transparent_addr, rocksdb::IteratorMode::Start);
// The default raw data serialization is very verbose, so we hex-encode the bytes.
let addresses: Vec<transparent::Address> = addresses
.map(|(key, _value)| transparent::Address::from_bytes(key))
.collect();
for address in addresses {
let stored_address_balance = state
.address_balance_location(&address)
.expect("address indexes are consistent");
stored_address_balances.push((address.to_string(), stored_address_balance));
}
// TODO: check that the UTXO and transaction lists are in chain order.
/*
assert!(
is_sorted(&stored_address_utxos),
"unsorted: {:?}",
stored_address_utxos,
);
*/
// We want to snapshot the order in the database,
// because sometimes it is significant for performance or correctness.
// So we don't sort the vectors before snapshotting.
insta::assert_ron_snapshot!("address_balances", stored_address_balances);
}
/// Return true if `list` is sorted in ascending order.
///
/// TODO: replace with Vec::is_sorted when it stabilises

5
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_0.snap Normal file
View File

@ -0,0 +1,5 @@
---
source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
expression: stored_address_balances
---
[]

13
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_1.snap Normal file
View File

@ -0,0 +1,13 @@
---
source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
expression: stored_address_balances
---
[
("t3Vz22vK5z2LcKEdg16Yv4FFneEL1zg9ojd", AddressBalanceLocation(
balance: Amount(12500),
location: OutputLocation(
hash: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609",
index: OutputIndex(1),
),
)),
]

13
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_2.snap Normal file
View File

@ -0,0 +1,13 @@
---
source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
expression: stored_address_balances
---
[
("t3Vz22vK5z2LcKEdg16Yv4FFneEL1zg9ojd", AddressBalanceLocation(
balance: Amount(37500),
location: OutputLocation(
hash: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609",
index: OutputIndex(1),
),
)),
]

5
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_0.snap Normal file
View File

@ -0,0 +1,5 @@
---
source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
expression: stored_address_balances
---
[]

13
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_1.snap Normal file
View File

@ -0,0 +1,13 @@
---
source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
expression: stored_address_balances
---
[
("t2UNzUUx8mWBCRYPRezvA363EYXyEpHokyi", AddressBalanceLocation(
balance: Amount(12500),
location: OutputLocation(
hash: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75",
index: OutputIndex(1),
),
)),
]

13
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_2.snap Normal file
View File

@ -0,0 +1,13 @@
---
source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
expression: stored_address_balances
---
[
("t2UNzUUx8mWBCRYPRezvA363EYXyEpHokyi", AddressBalanceLocation(
balance: Amount(37500),
location: OutputLocation(
hash: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75",
index: OutputIndex(1),
),
)),
]

2
zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs
View File

@ -115,7 +115,7 @@ fn test_block_db_round_trip_with(
};
// Skip validation by writing the block directly to the database
let mut batch = DiskWriteBatch::new();
let mut batch = DiskWriteBatch::new(Mainnet);
batch
.prepare_block_header_transactions_batch(&state.db, &finalized)
.expect("block is valid for batch");

16
zebra-state/src/service/finalized_state/zebra_db/chain.rs
View File

@ -16,9 +16,7 @@ use std::{borrow::Borrow, collections::HashMap};
use zebra_chain::{
amount::NonNegative,
history_tree::{HistoryTree, NonEmptyHistoryTree},
orchard,
parameters::Network,
sapling, transparent,
orchard, sapling, transparent,
value_balance::ValueBalance,
};
@ -73,7 +71,6 @@ impl DiskWriteBatch {
&mut self,
db: &DiskDb,
finalized: &FinalizedBlock,
network: Network,
sapling_root: sapling::tree::Root,
orchard_root: orchard::tree::Root,
mut history_tree: HistoryTree,
@ -82,7 +79,7 @@ impl DiskWriteBatch {
let FinalizedBlock { block, height, .. } = finalized;
history_tree.push(network, block.clone(), sapling_root, orchard_root)?;
history_tree.push(self.network(), block.clone(), sapling_root, orchard_root)?;
// Update the tree in state
let current_tip_height = *height - 1;
@ -115,17 +112,12 @@ impl DiskWriteBatch {
&mut self,
db: &DiskDb,
finalized: &FinalizedBlock,
mut all_utxos_spent_by_block: HashMap<transparent::OutPoint, transparent::Utxo>,
all_utxos_spent_by_block: HashMap<transparent::OutPoint, transparent::Utxo>,
value_pool: ValueBalance<NonNegative>,
) -> Result<(), BoxError> {
let tip_chain_value_pool = db.cf_handle("tip_chain_value_pool").unwrap();
let FinalizedBlock {
block, new_outputs, ..
} = finalized;
// Some utxos are spent in the same block, so they will be in `new_outputs`.
all_utxos_spent_by_block.extend(new_outputs.clone());
let FinalizedBlock { block, .. } = finalized;
let new_pool = value_pool.add_block(block.borrow(), &all_utxos_spent_by_block)?;
self.zs_insert(&tip_chain_value_pool, (), new_pool);

14
zebra-state/src/service/finalized_state/zebra_db/shielded.rs
View File

@ -13,8 +13,7 @@
//! be incremented each time the database format (column, serialization, etc) changes.
use zebra_chain::{
block::Height, history_tree::HistoryTree, orchard, parameters::Network, sapling, sprout,
transaction::Transaction,
block::Height, history_tree::HistoryTree, orchard, sapling, sprout, transaction::Transaction,
};
use crate::{
@ -241,8 +240,6 @@ impl DiskWriteBatch {
&mut self,
db: &DiskDb,
finalized: &FinalizedBlock,
network: Network,
// TODO: make an argument struct for all the note commitment trees & history
note_commitment_trees: NoteCommitmentTrees,
history_tree: HistoryTree,
) -> Result<(), BoxError> {
@ -294,14 +291,7 @@ impl DiskWriteBatch {
note_commitment_trees.orchard,
);
self.prepare_history_batch(
db,
finalized,
network,
sapling_root,
orchard_root,
history_tree,
)
self.prepare_history_batch(db, finalized, sapling_root, orchard_root, history_tree)
}
/// Prepare a database batch containing the initial note commitment trees,

95
zebra-state/src/service/finalized_state/zebra_db/transparent.rs
View File

@ -11,14 +11,17 @@
//! The [`crate::constants::DATABASE_FORMAT_VERSION`] constant must
//! be incremented each time the database format (column, serialization, etc) changes.
use std::borrow::Borrow;
use std::{borrow::Borrow, collections::HashMap};
use zebra_chain::transparent;
use zebra_chain::{
amount::{Amount, NonNegative},
transparent,
};
use crate::{
service::finalized_state::{
disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk},
disk_format::transparent::OutputLocation,
disk_format::transparent::{AddressBalanceLocation, AddressLocation, OutputLocation},
zebra_db::ZebraDb,
FinalizedBlock,
},
@ -28,8 +31,37 @@ use crate::{
impl ZebraDb {
// Read transparent methods
/// Returns the `transparent::Output` pointed to by the given
/// `transparent::OutPoint` if it is present.
/// Returns the [`AddressBalanceLocation`] for a [`transparent::Address`],
/// if it is in the finalized state.
pub fn address_balance_location(
&self,
address: &transparent::Address,
) -> Option<AddressBalanceLocation> {
let balance_by_transparent_addr = self.db.cf_handle("balance_by_transparent_addr").unwrap();
self.db.zs_get(&balance_by_transparent_addr, address)
}
/// Returns the balance for a [`transparent::Address`],
/// if it is in the finalized state.
#[allow(dead_code)]
pub fn address_balance(&self, address: &transparent::Address) -> Option<Amount<NonNegative>> {
self.address_balance_location(address)
.map(|abl| abl.balance())
}
/// Returns the first output that sent funds to a [`transparent::Address`],
/// if it is in the finalized state.
///
/// This location is used as an efficient index key for addresses.
#[allow(dead_code)]
pub fn address_location(&self, address: &transparent::Address) -> Option<AddressLocation> {
self.address_balance_location(address)
.map(|abl| abl.location())
}
/// Returns the transparent output for a [`transparent::OutPoint`],
/// if it is still unspent in the finalized state.
pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<transparent::Utxo> {
let utxo_by_outpoint = self.db.cf_handle("utxo_by_outpoint").unwrap();
@ -40,7 +72,11 @@ impl ZebraDb {
}
impl DiskWriteBatch {
/// Prepare a database batch containing `finalized.block`'s UTXO changes,
/// Prepare a database batch containing `finalized.block`'s:
/// - transparent address balance changes,
/// TODO:
/// - transparent address index changes (add in #3951, #3953), and
/// - UTXO changes (modify in #3952)
/// and return it (without actually writing anything).
///
/// # Errors
@ -50,8 +86,11 @@ impl DiskWriteBatch {
&mut self,
db: &DiskDb,
finalized: &FinalizedBlock,
all_utxos_spent_by_block: &HashMap<transparent::OutPoint, transparent::Utxo>,
mut address_balances: HashMap<transparent::Address, AddressBalanceLocation>,
) -> Result<(), BoxError> {
let utxo_by_outpoint = db.cf_handle("utxo_by_outpoint").unwrap();
let balance_by_transparent_addr = db.cf_handle("balance_by_transparent_addr").unwrap();
let FinalizedBlock {
block, new_outputs, ..
@ -59,25 +98,61 @@ impl DiskWriteBatch {
// Index all new transparent outputs, before deleting any we've spent
for (outpoint, utxo) in new_outputs.borrow().iter() {
let receiving_address = utxo.output.address(self.network());
let output_location = OutputLocation::from_outpoint(outpoint);
// Update the address balance by adding this UTXO's value
if let Some(receiving_address) = receiving_address {
let address_balance = address_balances
.entry(receiving_address)
.or_insert_with(|| AddressBalanceLocation::new(output_location))
.balance_mut();
let new_address_balance = (*address_balance + utxo.output.value())
.expect("balance overflow already checked");
*address_balance = new_address_balance;
}
self.zs_insert(&utxo_by_outpoint, output_location, utxo);
}
// Mark all transparent inputs as spent.
//
// Coinbase inputs represent new coins,
// so there are no UTXOs to mark as spent.
for output_location in block
// Coinbase inputs represent new coins, so there are no UTXOs to mark as spent.
for outpoint in block
.transactions
.iter()
.flat_map(|tx| tx.inputs())
.flat_map(|input| input.outpoint())
.map(|outpoint| OutputLocation::from_outpoint(&outpoint))
{
let output_location = OutputLocation::from_outpoint(&outpoint);
let spent_output = &all_utxos_spent_by_block.get(&outpoint).unwrap().output;
let sending_address = spent_output.address(self.network());
// Update the address balance by subtracting this UTXO's value
if let Some(sending_address) = sending_address {
let address_balance = address_balances
.entry(sending_address)
.or_insert_with(|| panic!("spent outputs must already have an address balance"))
.balance_mut();
let new_address_balance = (*address_balance - spent_output.value())
.expect("balance underflow already checked");
*address_balance = new_address_balance;
}
self.zs_delete(&utxo_by_outpoint, output_location);
}
// Write the new address balances to the database
for (address, address_balance) in address_balances.into_iter() {
// Some of these balances are new, and some are updates
self.zs_insert(&balance_by_transparent_addr, address, address_balance);
}
Ok(())
}
}