test(scan): Add raw database format snapshots to the scanner (#8075)
* Make some scanner storage methods more flexible * Move tests to a submodule and expose test functions and constants * Make scanner functions clearer and easier to use * Simplify state snapshot test code * Add raw data snapshot tests for the scanner * Add snapshots * Fix import path * Fix import conditional compilation * fix imports * fix imports 2 * Put read and write db exports together * Remove confusing IntoDisk/FromDisk impl * Fix an incorrect unused method that could panic * Delete a test that is no longer valid --------- Co-authored-by: Alfredo Garcia <oxarbitrage@gmail.com>
This commit is contained in:
teor
GitHub
parent
5bdad1bcaa
commit
3318eaaa22
|
|
@ -5885,8 +5885,11 @@ dependencies = [
|
|||
"ff",
|
||||
"group",
|
||||
"indexmap 2.1.0",
|
||||
"insta",
|
||||
"itertools 0.12.0",
|
||||
"jubjub",
|
||||
"proptest",
|
||||
"proptest-derive",
|
||||
"rand 0.8.5",
|
||||
"semver 1.0.20",
|
||||
"serde",
|
||||
|
|
|
|||
|
|
@ -18,6 +18,21 @@ categories = ["cryptography::cryptocurrencies"]
|
|||
|
||||
# Production features that activate extra dependencies, or extra features in dependencies
|
||||
|
||||
# Test features
|
||||
|
||||
proptest-impl = [
|
||||
"proptest",
|
||||
"proptest-derive",
|
||||
"zebra-state/proptest-impl",
|
||||
"zebra-chain/proptest-impl",
|
||||
"bls12_381",
|
||||
"ff",
|
||||
"group",
|
||||
"jubjub",
|
||||
"rand",
|
||||
"zcash_note_encryption",
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
|
||||
color-eyre = "0.6.2"
|
||||
|
|
@ -37,15 +52,29 @@ zebra-state = { path = "../zebra-state", version = "1.0.0-beta.31", features = [
|
|||
|
||||
chrono = { version = "0.4.31", default-features = false, features = ["clock", "std", "serde"] }
|
||||
|
||||
# test feature proptest-impl
|
||||
proptest = { version = "1.4.0", optional = true }
|
||||
proptest-derive = { version = "0.4.0", optional = true }
|
||||
|
||||
bls12_381 = { version = "0.8.0", optional = true }
|
||||
ff = { version = "0.13.0", optional = true }
|
||||
group = { version = "0.13.0", optional = true }
|
||||
jubjub = { version = "0.10.0", optional = true }
|
||||
rand = { version = "0.8.5", optional = true }
|
||||
zcash_note_encryption = { version = "0.4.0", optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
insta = { version = "1.33.0", features = ["ron", "redactions"] }
|
||||
tokio = { version = "1.34.0", features = ["test-util"] }
|
||||
|
||||
proptest = "1.4.0"
|
||||
proptest-derive = "0.4.0"
|
||||
bls12_381 = "0.8.0"
|
||||
ff = "0.13.0"
|
||||
group = "0.13.0"
|
||||
jubjub = "0.10.0"
|
||||
rand = "0.8.5"
|
||||
tokio = { version = "1.34.0", features = ["test-util"] }
|
||||
|
||||
zcash_note_encryption = "0.4.0"
|
||||
|
||||
zebra-state = { path = "../zebra-state", version = "1.0.0-beta.31", features = ["proptest-impl"] }
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ pub mod init;
|
|||
pub mod scan;
|
||||
pub mod storage;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
#[cfg(any(test, feature = "proptest-impl"))]
|
||||
pub mod tests;
|
||||
|
||||
pub use config::Config;
|
||||
pub use init::{init, spawn_init};
|
||||
|
|
|
|||
|
|
@ -211,8 +211,8 @@ pub async fn scan_height_and_store_results(
|
|||
let dfvk_res = scanned_block_to_db_result(dfvk_res);
|
||||
let ivk_res = scanned_block_to_db_result(ivk_res);
|
||||
|
||||
storage.add_sapling_results(sapling_key.clone(), height, dfvk_res);
|
||||
storage.add_sapling_results(sapling_key, height, ivk_res);
|
||||
storage.add_sapling_results(&sapling_key, height, dfvk_res);
|
||||
storage.add_sapling_results(&sapling_key, height, ivk_res);
|
||||
|
||||
Ok::<_, Report>(())
|
||||
})
|
||||
|
|
@ -398,7 +398,7 @@ fn scanned_block_to_db_result<Nf>(
|
|||
.map(|tx| {
|
||||
(
|
||||
TransactionIndex::from_usize(tx.index),
|
||||
SaplingScannedResult::from(tx.txid.as_ref()),
|
||||
SaplingScannedResult::from_bytes_in_display_order(*tx.txid.as_ref()),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
|
|
|
|||
|
|
@ -71,7 +71,13 @@ impl Storage {
|
|||
///
|
||||
/// This method can block while writing database files, so it must be inside spawn_blocking()
|
||||
/// in async code.
|
||||
pub fn add_sapling_key(&mut self, sapling_key: &SaplingScanningKey, birthday: Option<Height>) {
|
||||
pub fn add_sapling_key(
|
||||
&mut self,
|
||||
sapling_key: &SaplingScanningKey,
|
||||
birthday: impl Into<Option<Height>>,
|
||||
) {
|
||||
let birthday = birthday.into();
|
||||
|
||||
// It's ok to write some keys and not others during shutdown, so each key can get its own
|
||||
// batch. (They will be re-written on startup anyway.)
|
||||
let mut batch = ScannerWriteBatch::default();
|
||||
|
|
@ -93,7 +99,8 @@ impl Storage {
|
|||
self.sapling_keys_and_birthday_heights()
|
||||
}
|
||||
|
||||
/// Add the sapling results for `height` to the storage.
|
||||
/// Add the sapling results for `height` to the storage. The results can be any map of
|
||||
/// [`TransactionIndex`] to [`SaplingScannedResult`].
|
||||
///
|
||||
/// # Performance / Hangs
|
||||
///
|
||||
|
|
@ -101,7 +108,7 @@ impl Storage {
|
|||
/// in async code.
|
||||
pub fn add_sapling_results(
|
||||
&mut self,
|
||||
sapling_key: SaplingScanningKey,
|
||||
sapling_key: &SaplingScanningKey,
|
||||
height: Height,
|
||||
sapling_results: BTreeMap<TransactionIndex, SaplingScannedResult>,
|
||||
) {
|
||||
|
|
|
|||
|
|
@ -19,6 +19,9 @@ pub use zebra_state::{
|
|||
|
||||
pub mod sapling;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
/// The directory name used to distinguish the scanner database from Zebra's other databases or
|
||||
/// flat files.
|
||||
///
|
||||
|
|
|
|||
|
|
@ -47,13 +47,15 @@ impl Storage {
|
|||
// Reading Sapling database entries
|
||||
|
||||
/// Returns the result for a specific database index (key, block height, transaction index).
|
||||
/// Returns `None` if the result is missing or an empty marker for a birthday or progress
|
||||
/// height.
|
||||
//
|
||||
// TODO: add tests for this method
|
||||
pub fn sapling_result_for_index(
|
||||
&self,
|
||||
index: &SaplingScannedDatabaseIndex,
|
||||
) -> Option<SaplingScannedResult> {
|
||||
self.db.zs_get(&self.sapling_tx_ids_cf(), &index)
|
||||
self.db.zs_get(&self.sapling_tx_ids_cf(), &index).flatten()
|
||||
}
|
||||
|
||||
/// Returns the results for a specific key and block height.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
//! General scanner database tests.
|
||||
|
||||
mod snapshot;
|
||||
|
|
@ -0,0 +1,164 @@
|
|||
//! Raw data snapshot tests for the scanner database format.
|
||||
//!
|
||||
//! These tests check:
|
||||
//! - the name of each column family
|
||||
//! - the number of key-value entries
|
||||
//! - the bytes in each key and value
|
||||
//!
|
||||
//! These tests currently use fixed test vectors.
|
||||
//!
|
||||
//! # Fixing Test Failures
|
||||
//!
|
||||
//! If this test fails, run:
|
||||
//! ```sh
|
||||
//! cd zebra-scan
|
||||
//! cargo insta test --review --features shielded-scan
|
||||
//! ```
|
||||
//! to update the test snapshots, then commit the `test_*.snap` files using git.
|
||||
//!
|
||||
//! # Snapshot Format
|
||||
//!
|
||||
//! These snapshots use [RON (Rusty Object Notation)](https://github.com/ron-rs/ron#readme),
|
||||
//! a text format similar to Rust syntax. Raw byte data is encoded in hexadecimal.
|
||||
//!
|
||||
//! Due to `serde` limitations, some object types can't be represented exactly,
|
||||
//! so RON uses the closest equivalent structure.
|
||||
|
||||
use std::{collections::BTreeMap, sync::Arc};
|
||||
|
||||
use zebra_chain::{
|
||||
block::{Block, Height},
|
||||
parameters::Network::{self, *},
|
||||
serialization::ZcashDeserializeInto,
|
||||
};
|
||||
use zebra_state::{RawBytes, ReadDisk, TransactionIndex, KV};
|
||||
|
||||
use crate::{
|
||||
storage::{db::ScannerDb, Storage},
|
||||
tests::{FAKE_SAPLING_VIEWING_KEY, ZECPAGES_SAPLING_VIEWING_KEY},
|
||||
Config,
|
||||
};
|
||||
|
||||
/// Snapshot test for RocksDB column families, and their key-value data.
|
||||
///
|
||||
/// These snapshots contain the `default` column family, but it is not used by Zebra.
|
||||
#[test]
|
||||
fn test_raw_rocksdb_column_families() {
|
||||
let _init_guard = zebra_test::init();
|
||||
|
||||
test_raw_rocksdb_column_families_with_network(Mainnet);
|
||||
test_raw_rocksdb_column_families_with_network(Testnet);
|
||||
}
|
||||
|
||||
/// Snapshot raw column families for `network`.
|
||||
///
|
||||
/// See [`test_raw_rocksdb_column_families`].
|
||||
fn test_raw_rocksdb_column_families_with_network(network: Network) {
|
||||
let mut net_suffix = network.to_string();
|
||||
net_suffix.make_ascii_lowercase();
|
||||
|
||||
let mut storage = Storage::new(&Config::ephemeral(), network);
|
||||
|
||||
// Snapshot the column family names
|
||||
let mut cf_names = storage.db.list_cf().expect("empty database is valid");
|
||||
|
||||
// The order that RocksDB returns column families is irrelevant,
|
||||
// because we always access them by name.
|
||||
cf_names.sort();
|
||||
|
||||
// Assert that column family names are the same, regardless of the network.
|
||||
// Later, we check they are also the same regardless of the block height.
|
||||
insta::assert_ron_snapshot!("column_family_names", cf_names);
|
||||
|
||||
// Assert that empty databases are the same, regardless of the network.
|
||||
let mut settings = insta::Settings::clone_current();
|
||||
|
||||
settings.set_snapshot_suffix("empty");
|
||||
settings.bind(|| snapshot_raw_rocksdb_column_family_data(&storage.db, &cf_names));
|
||||
|
||||
// Snapshot a birthday that is automatically set to activation height
|
||||
storage.add_sapling_key(&ZECPAGES_SAPLING_VIEWING_KEY.to_string(), None);
|
||||
// Snapshot a birthday above activation height
|
||||
storage.add_sapling_key(&FAKE_SAPLING_VIEWING_KEY.to_string(), Height(1_000_000));
|
||||
|
||||
settings.set_snapshot_suffix(format!("{net_suffix}_keys"));
|
||||
settings.bind(|| snapshot_raw_rocksdb_column_family_data(&storage.db, &cf_names));
|
||||
|
||||
// Snapshot raw database data for:
|
||||
// - mainnet and testnet
|
||||
// - genesis, block 1, and block 2
|
||||
let blocks = match network {
|
||||
Mainnet => &*zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS,
|
||||
Testnet => &*zebra_test::vectors::CONTINUOUS_TESTNET_BLOCKS,
|
||||
};
|
||||
|
||||
// We limit the number of blocks, because the serialized data is a few kilobytes per block.
|
||||
for height in 0..=2 {
|
||||
let block: Arc<Block> = blocks
|
||||
.get(&height)
|
||||
.expect("block height has test data")
|
||||
.zcash_deserialize_into()
|
||||
.expect("test data deserializes");
|
||||
|
||||
// Fake results from the first few blocks
|
||||
storage.add_sapling_results(
|
||||
&ZECPAGES_SAPLING_VIEWING_KEY.to_string(),
|
||||
Height(height),
|
||||
block
|
||||
.transactions
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, tx)| (TransactionIndex::from_usize(index), tx.hash().into()))
|
||||
.collect(),
|
||||
);
|
||||
|
||||
let mut settings = insta::Settings::clone_current();
|
||||
settings.set_snapshot_suffix(format!("{net_suffix}_{height}"));
|
||||
|
||||
settings.bind(|| snapshot_raw_rocksdb_column_family_data(&storage.db, &cf_names));
|
||||
}
|
||||
}
|
||||
|
||||
/// Snapshot the data in each column family, using `cargo insta` and RON serialization.
|
||||
fn snapshot_raw_rocksdb_column_family_data(db: &ScannerDb, original_cf_names: &[String]) {
|
||||
let mut new_cf_names = db.list_cf().expect("empty database is valid");
|
||||
new_cf_names.sort();
|
||||
|
||||
// Assert that column family names are the same, regardless of the network or block height.
|
||||
assert_eq!(
|
||||
original_cf_names, new_cf_names,
|
||||
"unexpected extra column families",
|
||||
);
|
||||
|
||||
let mut empty_column_families = Vec::new();
|
||||
|
||||
// Now run the data snapshots
|
||||
for cf_name in original_cf_names {
|
||||
let cf_handle = db
|
||||
.cf_handle(cf_name)
|
||||
.expect("RocksDB API provides correct names");
|
||||
|
||||
// Correctness: Multi-key iteration causes hangs in concurrent code, but seems ok in tests.
|
||||
let cf_items: BTreeMap<RawBytes, RawBytes> = db.zs_items_in_range_ordered(&cf_handle, ..);
|
||||
|
||||
// The default raw data serialization is very verbose, so we hex-encode the bytes.
|
||||
let cf_data: Vec<KV> = cf_items
|
||||
.iter()
|
||||
.map(|(key, value)| KV::new(key.raw_bytes(), value.raw_bytes()))
|
||||
.collect();
|
||||
|
||||
if cf_name == "default" {
|
||||
assert_eq!(cf_data.len(), 0, "default column family is never used");
|
||||
} else if cf_data.is_empty() {
|
||||
// distinguish column family names from empty column families
|
||||
empty_column_families.push(format!("{cf_name}: no entries"));
|
||||
} else {
|
||||
// The note commitment tree snapshots will change if the trees do not have cached roots.
|
||||
// But we expect them to always have cached roots,
|
||||
// because those roots are used to populate the anchor column families.
|
||||
insta::assert_ron_snapshot!(format!("{cf_name}_raw_data"), cf_data);
|
||||
}
|
||||
}
|
||||
|
||||
insta::assert_ron_snapshot!("empty_column_families", empty_column_families);
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
---
|
||||
source: zebra-scan/src/storage/db/tests/snapshot.rs
|
||||
expression: cf_names
|
||||
---
|
||||
[
|
||||
"default",
|
||||
"sapling_tx_ids",
|
||||
]
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
---
|
||||
source: zebra-scan/src/storage/db/tests/snapshot.rs
|
||||
expression: empty_column_families
|
||||
---
|
||||
[
|
||||
"sapling_tx_ids: no entries",
|
||||
]
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
source: zebra-scan/src/storage/db/tests/snapshot.rs
|
||||
expression: empty_column_families
|
||||
---
|
||||
[]
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
source: zebra-scan/src/storage/db/tests/snapshot.rs
|
||||
expression: empty_column_families
|
||||
---
|
||||
[]
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
source: zebra-scan/src/storage/db/tests/snapshot.rs
|
||||
expression: empty_column_families
|
||||
---
|
||||
[]
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
source: zebra-scan/src/storage/db/tests/snapshot.rs
|
||||
expression: empty_column_families
|
||||
---
|
||||
[]
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
source: zebra-scan/src/storage/db/tests/snapshot.rs
|
||||
expression: empty_column_families
|
||||
---
|
||||
[]
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
source: zebra-scan/src/storage/db/tests/snapshot.rs
|
||||
expression: empty_column_families
|
||||
---
|
||||
[]
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
source: zebra-scan/src/storage/db/tests/snapshot.rs
|
||||
expression: empty_column_families
|
||||
---
|
||||
[]
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
source: zebra-scan/src/storage/db/tests/snapshot.rs
|
||||
expression: empty_column_families
|
||||
---
|
||||
[]
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
source: zebra-scan/src/storage/db/tests/snapshot.rs
|
||||
expression: cf_data
|
||||
---
|
||||
[
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000000000",
|
||||
v: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000",
|
||||
v: "",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777366616b650f423f0000",
|
||||
v: "",
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
---
|
||||
source: zebra-scan/src/storage/db/tests/snapshot.rs
|
||||
expression: cf_data
|
||||
---
|
||||
[
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000000000",
|
||||
v: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000010000",
|
||||
v: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000",
|
||||
v: "",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777366616b650f423f0000",
|
||||
v: "",
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
---
|
||||
source: zebra-scan/src/storage/db/tests/snapshot.rs
|
||||
expression: cf_data
|
||||
---
|
||||
[
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000000000",
|
||||
v: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000010000",
|
||||
v: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000020000",
|
||||
v: "8974d08d1c5f9c860d8b629d582a56659a4a1dcb2b5f98a25a5afcc2a784b0f4",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000",
|
||||
v: "",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777366616b650f423f0000",
|
||||
v: "",
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
---
|
||||
source: zebra-scan/src/storage/db/tests/snapshot.rs
|
||||
expression: cf_data
|
||||
---
|
||||
[
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000",
|
||||
v: "",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777366616b650f423f0000",
|
||||
v: "",
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
source: zebra-scan/src/storage/db/tests/snapshot.rs
|
||||
expression: cf_data
|
||||
---
|
||||
[
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000000000",
|
||||
v: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000",
|
||||
v: "",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777366616b650f423f0000",
|
||||
v: "",
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
---
|
||||
source: zebra-scan/src/storage/db/tests/snapshot.rs
|
||||
expression: cf_data
|
||||
---
|
||||
[
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000000000",
|
||||
v: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000010000",
|
||||
v: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000",
|
||||
v: "",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777366616b650f423f0000",
|
||||
v: "",
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
---
|
||||
source: zebra-scan/src/storage/db/tests/snapshot.rs
|
||||
expression: cf_data
|
||||
---
|
||||
[
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000000000",
|
||||
v: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000010000",
|
||||
v: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000020000",
|
||||
v: "5822c0532da8a008259ac39933d3210e508c17e3ba21d2b2c428785efdccb3d5",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000",
|
||||
v: "",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777366616b650f423f0000",
|
||||
v: "",
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
---
|
||||
source: zebra-scan/src/storage/db/tests/snapshot.rs
|
||||
expression: cf_data
|
||||
---
|
||||
[
|
||||
KV(
|
||||
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000",
|
||||
v: "",
|
||||
),
|
||||
KV(
|
||||
k: "7a78766965777366616b650f423f0000",
|
||||
v: "",
|
||||
),
|
||||
]
|
||||
|
|
@ -12,17 +12,14 @@ use ff::{Field, PrimeField};
|
|||
use group::GroupEncoding;
|
||||
use rand::{rngs::OsRng, thread_rng, RngCore};
|
||||
|
||||
use zcash_client_backend::{
|
||||
encoding::decode_extended_full_viewing_key,
|
||||
proto::compact_formats::{
|
||||
ChainMetadata, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx,
|
||||
},
|
||||
use zcash_client_backend::proto::compact_formats::{
|
||||
ChainMetadata, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx,
|
||||
};
|
||||
use zcash_note_encryption::Domain;
|
||||
use zcash_primitives::{
|
||||
block::BlockHash,
|
||||
consensus::BlockHeight,
|
||||
constants::{mainnet::HRP_SAPLING_EXTENDED_FULL_VIEWING_KEY, SPENDING_KEY_GENERATOR},
|
||||
constants::SPENDING_KEY_GENERATOR,
|
||||
memo::MemoBytes,
|
||||
sapling::{
|
||||
note_encryption::{sapling_note_encryption, SaplingDomain},
|
||||
|
|
@ -30,200 +27,29 @@ use zcash_primitives::{
|
|||
value::NoteValue,
|
||||
Note, Nullifier,
|
||||
},
|
||||
zip32::{DiversifiableFullViewingKey, ExtendedSpendingKey},
|
||||
zip32::DiversifiableFullViewingKey,
|
||||
};
|
||||
|
||||
use zebra_chain::{
|
||||
amount::{Amount, NegativeAllowed},
|
||||
block::{self, merkle, Block, Header, Height},
|
||||
chain_tip::ChainTip,
|
||||
fmt::HexDebug,
|
||||
parameters::Network,
|
||||
primitives::{redjubjub, Groth16Proof},
|
||||
sapling::{self, PerSpendAnchor, Spend, TransferData},
|
||||
serialization::{AtLeastOne, ZcashDeserializeInto},
|
||||
serialization::AtLeastOne,
|
||||
transaction::{LockTime, Transaction},
|
||||
transparent::{CoinbaseData, Input},
|
||||
work::{difficulty::CompactDifficulty, equihash::Solution},
|
||||
};
|
||||
use zebra_state::{SaplingScannedResult, TransactionIndex};
|
||||
|
||||
use crate::{
|
||||
config::Config,
|
||||
scan::{block_to_compact, scan_block},
|
||||
};
|
||||
#[cfg(test)]
|
||||
mod vectors;
|
||||
|
||||
/// This test:
|
||||
/// - Creates a viewing key and a fake block containing a Sapling output decryptable by the key.
|
||||
/// - Scans the block.
|
||||
/// - Checks that the result contains the txid of the tx containing the Sapling output.
|
||||
#[tokio::test]
|
||||
async fn scanning_from_fake_generated_blocks() -> Result<()> {
|
||||
let extsk = ExtendedSpendingKey::master(&[]);
|
||||
let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key();
|
||||
let nf = Nullifier([7; 32]);
|
||||
/// The extended Sapling viewing key of [ZECpages](https://zecpages.com/boardinfo)
|
||||
pub const ZECPAGES_SAPLING_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz";
|
||||
|
||||
let (block, sapling_tree_size) = fake_block(1u32.into(), nf, &dfvk, 1, true, Some(0));
|
||||
|
||||
assert_eq!(block.transactions.len(), 4);
|
||||
|
||||
let res = scan_block(Network::Mainnet, &block, sapling_tree_size, &[&dfvk]).unwrap();
|
||||
|
||||
// The response should have one transaction relevant to the key we provided.
|
||||
assert_eq!(res.transactions().len(), 1);
|
||||
|
||||
// Check that the original block contains the txid in the scanning result.
|
||||
assert!(block
|
||||
.transactions
|
||||
.iter()
|
||||
.map(|tx| tx.hash().bytes_in_display_order())
|
||||
.any(|txid| &txid == res.transactions()[0].txid.as_ref()));
|
||||
|
||||
// Check that the txid in the scanning result matches the third tx in the original block.
|
||||
assert_eq!(
|
||||
res.transactions()[0].txid.as_ref(),
|
||||
&block.transactions[2].hash().bytes_in_display_order()
|
||||
);
|
||||
|
||||
// The block hash of the response should be the same as the one provided.
|
||||
assert_eq!(res.block_hash().0, block.hash().0);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Scan a populated state for the ZECpages viewing key.
|
||||
/// This test is very similar to `scanning_from_populated_zebra_state` but with the ZECpages key.
|
||||
/// There are no zechub transactions in the test data so we should get empty related transactions.
|
||||
#[tokio::test]
|
||||
async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> {
|
||||
/// The extended Sapling viewing key of [ZECpages](https://zecpages.com/boardinfo)
|
||||
const ZECPAGES_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz";
|
||||
|
||||
// Parse the key from ZECpages
|
||||
let efvk = decode_extended_full_viewing_key(
|
||||
HRP_SAPLING_EXTENDED_FULL_VIEWING_KEY,
|
||||
ZECPAGES_VIEWING_KEY,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Build a vector of viewing keys `vks` to scan for.
|
||||
let fvk = efvk.fvk;
|
||||
let ivk = fvk.vk.ivk();
|
||||
let ivks = vec![ivk];
|
||||
|
||||
let network = Network::Mainnet;
|
||||
|
||||
// Create a continuous chain of mainnet blocks from genesis
|
||||
let blocks: Vec<Arc<Block>> = zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS
|
||||
.iter()
|
||||
.map(|(_height, block_bytes)| block_bytes.zcash_deserialize_into().unwrap())
|
||||
.collect();
|
||||
|
||||
// Create a populated state service.
|
||||
let (_state_service, read_only_state_service, latest_chain_tip, _chain_tip_change) =
|
||||
zebra_state::populated_state(blocks.clone(), network).await;
|
||||
|
||||
let db = read_only_state_service.db();
|
||||
|
||||
// use the tip as starting height
|
||||
let mut height = latest_chain_tip.best_tip_height().unwrap();
|
||||
|
||||
let mut transactions_found = 0;
|
||||
let mut transactions_scanned = 0;
|
||||
let mut blocks_scanned = 0;
|
||||
while let Some(block) = db.block(height.into()) {
|
||||
// We use a dummy size of the Sapling note commitment tree. We can't set the size to zero
|
||||
// because the underlying scanning function would return
|
||||
// `zcash_client_backeng::scanning::ScanError::TreeSizeUnknown`.
|
||||
let sapling_commitment_tree_size = 1;
|
||||
|
||||
let orchard_commitment_tree_size = 0;
|
||||
|
||||
let chain_metadata = ChainMetadata {
|
||||
sapling_commitment_tree_size,
|
||||
orchard_commitment_tree_size,
|
||||
};
|
||||
|
||||
let compact_block = block_to_compact(&block, chain_metadata);
|
||||
|
||||
let res = scan_block(network, &block, sapling_commitment_tree_size, &ivks)
|
||||
.expect("scanning block for the ZECpages viewing key should work");
|
||||
|
||||
transactions_found += res.transactions().len();
|
||||
transactions_scanned += compact_block.vtx.len();
|
||||
blocks_scanned += 1;
|
||||
|
||||
// scan backwards
|
||||
if height.is_min() {
|
||||
break;
|
||||
}
|
||||
height = height.previous()?;
|
||||
}
|
||||
|
||||
// make sure all blocks and transactions were scanned
|
||||
assert_eq!(blocks_scanned, 11);
|
||||
assert_eq!(transactions_scanned, 11);
|
||||
|
||||
// no relevant transactions should be found
|
||||
assert_eq!(transactions_found, 0);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Creates a viewing key and a fake block containing a Sapling output decryptable by the key, scans
|
||||
/// the block using the key, and adds the results to the database.
|
||||
///
|
||||
/// The purpose of this test is to check if our database and our scanning code are compatible.
|
||||
#[test]
|
||||
#[allow(deprecated)]
|
||||
fn scanning_fake_blocks_store_key_and_results() -> Result<()> {
|
||||
// Generate a key
|
||||
let extsk = ExtendedSpendingKey::master(&[]);
|
||||
// TODO: find out how to do it with `to_diversifiable_full_viewing_key` as `to_extended_full_viewing_key` is deprecated.
|
||||
let extfvk = extsk.to_extended_full_viewing_key();
|
||||
let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key();
|
||||
let key_to_be_stored =
|
||||
zcash_client_backend::encoding::encode_extended_full_viewing_key("zxviews", &extfvk);
|
||||
|
||||
// Create a database
|
||||
let mut s = crate::storage::Storage::new(&Config::ephemeral(), Network::Mainnet);
|
||||
|
||||
// Insert the generated key to the database
|
||||
s.add_sapling_key(&key_to_be_stored, None);
|
||||
|
||||
// Check key was added
|
||||
assert_eq!(s.sapling_keys().len(), 1);
|
||||
assert_eq!(
|
||||
s.sapling_keys().get(&key_to_be_stored),
|
||||
Some(&s.min_sapling_birthday_height())
|
||||
);
|
||||
|
||||
let nf = Nullifier([7; 32]);
|
||||
|
||||
let (block, sapling_tree_size) = fake_block(1u32.into(), nf, &dfvk, 1, true, Some(0));
|
||||
|
||||
let result = scan_block(Network::Mainnet, &block, sapling_tree_size, &[&dfvk]).unwrap();
|
||||
|
||||
// The response should have one transaction relevant to the key we provided.
|
||||
assert_eq!(result.transactions().len(), 1);
|
||||
|
||||
let result = SaplingScannedResult::from(result.transactions()[0].txid.as_ref());
|
||||
|
||||
// Add result to database
|
||||
s.add_sapling_results(
|
||||
key_to_be_stored.clone(),
|
||||
Height(1),
|
||||
[(TransactionIndex::from_usize(0), result)].into(),
|
||||
);
|
||||
|
||||
// Check the result was added
|
||||
assert_eq!(
|
||||
s.sapling_results(&key_to_be_stored).get(&Height(1)),
|
||||
Some(&vec![result])
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
/// A fake viewing key in an incorrect format.
|
||||
pub const FAKE_SAPLING_VIEWING_KEY: &str = "zxviewsfake";
|
||||
|
||||
/// Generates a fake block containing a Sapling output decryptable by `dfvk`.
|
||||
///
|
||||
|
|
@ -232,7 +58,7 @@ fn scanning_fake_blocks_store_key_and_results() -> Result<()> {
|
|||
/// 2. a V4 tx containing a random Sapling output,
|
||||
/// 3. a V4 tx containing a Sapling output decryptable by `dfvk`,
|
||||
/// 4. depending on the value of `tx_after`, another V4 tx containing a random Sapling output.
|
||||
fn fake_block(
|
||||
pub fn fake_block(
|
||||
height: BlockHeight,
|
||||
nf: Nullifier,
|
||||
dfvk: &DiversifiableFullViewingKey,
|
||||
|
|
@ -305,7 +131,7 @@ fn fake_block(
|
|||
// be a number for easier conversion:
|
||||
// https://github.com/zcash/librustzcash/blob/zcash_primitives-0.13.0/zcash_client_backend/src/scanning.rs#L635
|
||||
// We need to copy because this is a test private function upstream.
|
||||
fn fake_compact_block(
|
||||
pub fn fake_compact_block(
|
||||
height: BlockHeight,
|
||||
prev_hash: BlockHash,
|
||||
nf: Nullifier,
|
||||
|
|
@ -394,7 +220,7 @@ fn fake_compact_block(
|
|||
// This is an exact copy of `zcash_client_backend::scanning::random_compact_tx`:
|
||||
// https://github.com/zcash/librustzcash/blob/zcash_primitives-0.13.0/zcash_client_backend/src/scanning.rs#L597
|
||||
// We need to copy because this is a test private function upstream.
|
||||
fn random_compact_tx(mut rng: impl RngCore) -> CompactTx {
|
||||
pub fn random_compact_tx(mut rng: impl RngCore) -> CompactTx {
|
||||
let fake_nf = {
|
||||
let mut nf = vec![0; 32];
|
||||
rng.fill_bytes(&mut nf);
|
||||
|
|
@ -427,7 +253,7 @@ fn random_compact_tx(mut rng: impl RngCore) -> CompactTx {
|
|||
}
|
||||
|
||||
/// Converts [`CompactTx`] to [`Transaction::V4`].
|
||||
fn compact_to_v4(tx: &CompactTx) -> Result<Transaction> {
|
||||
pub fn compact_to_v4(tx: &CompactTx) -> Result<Transaction> {
|
||||
let sk = redjubjub::SigningKey::<redjubjub::SpendAuth>::new(thread_rng());
|
||||
let vk = redjubjub::VerificationKey::from(&sk);
|
||||
let dummy_rk = sapling::keys::ValidatingKey::try_from(vk)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,198 @@
|
|||
//! Fixed integration test vectors for the scanner.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use color_eyre::Result;
|
||||
|
||||
use zcash_client_backend::{
|
||||
encoding::decode_extended_full_viewing_key, proto::compact_formats::ChainMetadata,
|
||||
};
|
||||
use zcash_primitives::{
|
||||
constants::mainnet::HRP_SAPLING_EXTENDED_FULL_VIEWING_KEY,
|
||||
sapling::Nullifier,
|
||||
zip32::{DiversifiableFullViewingKey, ExtendedSpendingKey},
|
||||
};
|
||||
|
||||
use zebra_chain::{
|
||||
block::{Block, Height},
|
||||
chain_tip::ChainTip,
|
||||
parameters::Network,
|
||||
serialization::ZcashDeserializeInto,
|
||||
};
|
||||
use zebra_state::{SaplingScannedResult, TransactionIndex};
|
||||
|
||||
use crate::{
|
||||
config::Config,
|
||||
scan::{block_to_compact, scan_block},
|
||||
tests::{fake_block, ZECPAGES_SAPLING_VIEWING_KEY},
|
||||
};
|
||||
|
||||
/// This test:
|
||||
/// - Creates a viewing key and a fake block containing a Sapling output decryptable by the key.
|
||||
/// - Scans the block.
|
||||
/// - Checks that the result contains the txid of the tx containing the Sapling output.
|
||||
#[tokio::test]
|
||||
async fn scanning_from_fake_generated_blocks() -> Result<()> {
|
||||
let extsk = ExtendedSpendingKey::master(&[]);
|
||||
let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key();
|
||||
let nf = Nullifier([7; 32]);
|
||||
|
||||
let (block, sapling_tree_size) = fake_block(1u32.into(), nf, &dfvk, 1, true, Some(0));
|
||||
|
||||
assert_eq!(block.transactions.len(), 4);
|
||||
|
||||
let res = scan_block(Network::Mainnet, &block, sapling_tree_size, &[&dfvk]).unwrap();
|
||||
|
||||
// The response should have one transaction relevant to the key we provided.
|
||||
assert_eq!(res.transactions().len(), 1);
|
||||
|
||||
// Check that the original block contains the txid in the scanning result.
|
||||
assert!(block
|
||||
.transactions
|
||||
.iter()
|
||||
.map(|tx| tx.hash().bytes_in_display_order())
|
||||
.any(|txid| &txid == res.transactions()[0].txid.as_ref()));
|
||||
|
||||
// Check that the txid in the scanning result matches the third tx in the original block.
|
||||
assert_eq!(
|
||||
res.transactions()[0].txid.as_ref(),
|
||||
&block.transactions[2].hash().bytes_in_display_order()
|
||||
);
|
||||
|
||||
// The block hash of the response should be the same as the one provided.
|
||||
assert_eq!(res.block_hash().0, block.hash().0);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Scan a populated state for the ZECpages viewing key.
|
||||
/// This test is very similar to `scanning_from_populated_zebra_state` but with the ZECpages key.
|
||||
/// There are no zechub transactions in the test data so we should get empty related transactions.
|
||||
#[tokio::test]
|
||||
async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> {
|
||||
// Parse the key from ZECpages
|
||||
let efvk = decode_extended_full_viewing_key(
|
||||
HRP_SAPLING_EXTENDED_FULL_VIEWING_KEY,
|
||||
ZECPAGES_SAPLING_VIEWING_KEY,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Build a vector of viewing keys `vks` to scan for.
|
||||
let fvk = efvk.fvk;
|
||||
let ivk = fvk.vk.ivk();
|
||||
let ivks = vec![ivk];
|
||||
|
||||
let network = Network::Mainnet;
|
||||
|
||||
// Create a continuous chain of mainnet blocks from genesis
|
||||
let blocks: Vec<Arc<Block>> = zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS
|
||||
.iter()
|
||||
.map(|(_height, block_bytes)| block_bytes.zcash_deserialize_into().unwrap())
|
||||
.collect();
|
||||
|
||||
// Create a populated state service.
|
||||
let (_state_service, read_only_state_service, latest_chain_tip, _chain_tip_change) =
|
||||
zebra_state::populated_state(blocks.clone(), network).await;
|
||||
|
||||
let db = read_only_state_service.db();
|
||||
|
||||
// use the tip as starting height
|
||||
let mut height = latest_chain_tip.best_tip_height().unwrap();
|
||||
|
||||
let mut transactions_found = 0;
|
||||
let mut transactions_scanned = 0;
|
||||
let mut blocks_scanned = 0;
|
||||
while let Some(block) = db.block(height.into()) {
|
||||
// We use a dummy size of the Sapling note commitment tree. We can't set the size to zero
|
||||
// because the underlying scanning function would return
|
||||
// `zcash_client_backeng::scanning::ScanError::TreeSizeUnknown`.
|
||||
let sapling_commitment_tree_size = 1;
|
||||
|
||||
let orchard_commitment_tree_size = 0;
|
||||
|
||||
let chain_metadata = ChainMetadata {
|
||||
sapling_commitment_tree_size,
|
||||
orchard_commitment_tree_size,
|
||||
};
|
||||
|
||||
let compact_block = block_to_compact(&block, chain_metadata);
|
||||
|
||||
let res = scan_block(network, &block, sapling_commitment_tree_size, &ivks)
|
||||
.expect("scanning block for the ZECpages viewing key should work");
|
||||
|
||||
transactions_found += res.transactions().len();
|
||||
transactions_scanned += compact_block.vtx.len();
|
||||
blocks_scanned += 1;
|
||||
|
||||
// scan backwards
|
||||
if height.is_min() {
|
||||
break;
|
||||
}
|
||||
height = height.previous()?;
|
||||
}
|
||||
|
||||
// make sure all blocks and transactions were scanned
|
||||
assert_eq!(blocks_scanned, 11);
|
||||
assert_eq!(transactions_scanned, 11);
|
||||
|
||||
// no relevant transactions should be found
|
||||
assert_eq!(transactions_found, 0);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Creates a viewing key and a fake block containing a Sapling output decryptable by the key, scans
|
||||
/// the block using the key, and adds the results to the database.
|
||||
///
|
||||
/// The purpose of this test is to check if our database and our scanning code are compatible.
|
||||
#[test]
|
||||
#[allow(deprecated)]
|
||||
fn scanning_fake_blocks_store_key_and_results() -> Result<()> {
|
||||
// Generate a key
|
||||
let extsk = ExtendedSpendingKey::master(&[]);
|
||||
// TODO: find out how to do it with `to_diversifiable_full_viewing_key` as `to_extended_full_viewing_key` is deprecated.
|
||||
let extfvk = extsk.to_extended_full_viewing_key();
|
||||
let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key();
|
||||
let key_to_be_stored =
|
||||
zcash_client_backend::encoding::encode_extended_full_viewing_key("zxviews", &extfvk);
|
||||
|
||||
// Create a database
|
||||
let mut s = crate::storage::Storage::new(&Config::ephemeral(), Network::Mainnet);
|
||||
|
||||
// Insert the generated key to the database
|
||||
s.add_sapling_key(&key_to_be_stored, None);
|
||||
|
||||
// Check key was added
|
||||
assert_eq!(s.sapling_keys().len(), 1);
|
||||
assert_eq!(
|
||||
s.sapling_keys().get(&key_to_be_stored),
|
||||
Some(&s.min_sapling_birthday_height())
|
||||
);
|
||||
|
||||
let nf = Nullifier([7; 32]);
|
||||
|
||||
let (block, sapling_tree_size) = fake_block(1u32.into(), nf, &dfvk, 1, true, Some(0));
|
||||
|
||||
let result = scan_block(Network::Mainnet, &block, sapling_tree_size, &[&dfvk]).unwrap();
|
||||
|
||||
// The response should have one transaction relevant to the key we provided.
|
||||
assert_eq!(result.transactions().len(), 1);
|
||||
|
||||
let result =
|
||||
SaplingScannedResult::from_bytes_in_display_order(*result.transactions()[0].txid.as_ref());
|
||||
|
||||
// Add result to database
|
||||
s.add_sapling_results(
|
||||
&key_to_be_stored,
|
||||
Height(1),
|
||||
[(TransactionIndex::from_usize(0), result)].into(),
|
||||
);
|
||||
|
||||
// Check the result was added
|
||||
assert_eq!(
|
||||
s.sapling_results(&key_to_be_stored).get(&Height(1)),
|
||||
Some(&vec![result])
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -63,12 +63,12 @@ pub use service::{
|
|||
pub use rocksdb::AsColumnFamilyRef;
|
||||
#[cfg(feature = "shielded-scan")]
|
||||
pub use service::finalized_state::{
|
||||
FromDisk, IntoDisk, ReadDisk, SaplingScannedDatabaseEntry, SaplingScannedDatabaseIndex,
|
||||
FromDisk, IntoDisk, SaplingScannedDatabaseEntry, SaplingScannedDatabaseIndex,
|
||||
SaplingScannedResult, SaplingScanningKey, ZebraDb,
|
||||
};
|
||||
|
||||
#[cfg(any(test, feature = "proptest-impl", feature = "shielded-scan"))]
|
||||
pub use service::finalized_state::{DiskWriteBatch, WriteDisk};
|
||||
pub use service::finalized_state::{DiskWriteBatch, ReadDisk, WriteDisk};
|
||||
|
||||
#[cfg(feature = "getblocktemplate-rpcs")]
|
||||
pub use response::GetBlockTemplateChainInfo;
|
||||
|
|
@ -77,7 +77,7 @@ pub use response::GetBlockTemplateChainInfo;
|
|||
pub use service::{
|
||||
arbitrary::{populated_state, CHAIN_TIP_UPDATE_WAIT_LIMIT},
|
||||
chain_tip::{ChainTipBlock, ChainTipSender},
|
||||
finalized_state::MAX_ON_DISK_HEIGHT,
|
||||
finalized_state::{RawBytes, KV, MAX_ON_DISK_HEIGHT},
|
||||
init_test, init_test_services, ReadStateService,
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -42,8 +42,8 @@ mod tests;
|
|||
pub use disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk};
|
||||
#[allow(unused_imports)]
|
||||
pub use disk_format::{
|
||||
FromDisk, IntoDisk, OutputIndex, OutputLocation, TransactionIndex, TransactionLocation,
|
||||
MAX_ON_DISK_HEIGHT,
|
||||
FromDisk, IntoDisk, OutputIndex, OutputLocation, RawBytes, TransactionIndex,
|
||||
TransactionLocation, MAX_ON_DISK_HEIGHT,
|
||||
};
|
||||
pub use zebra_db::ZebraDb;
|
||||
|
||||
|
|
@ -53,6 +53,9 @@ pub use disk_format::{
|
|||
SaplingScanningKey,
|
||||
};
|
||||
|
||||
#[cfg(any(test, feature = "proptest-impl"))]
|
||||
pub use disk_format::KV;
|
||||
|
||||
/// The column families supported by the running `zebra-state` database code.
|
||||
///
|
||||
/// Existing column families that aren't listed here are preserved when the database is opened.
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ pub mod upgrade;
|
|||
#[cfg(feature = "shielded-scan")]
|
||||
pub mod scan;
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(any(test, feature = "proptest-impl"))]
|
||||
mod tests;
|
||||
|
||||
pub use block::{TransactionIndex, TransactionLocation, MAX_ON_DISK_HEIGHT};
|
||||
|
|
@ -28,6 +28,9 @@ pub use scan::{
|
|||
SaplingScanningKey,
|
||||
};
|
||||
|
||||
#[cfg(any(test, feature = "proptest-impl"))]
|
||||
pub use tests::KV;
|
||||
|
||||
/// Helper type for writing types to disk as raw bytes.
|
||||
/// Also used to convert key types to raw bytes for disk lookups.
|
||||
pub trait IntoDisk {
|
||||
|
|
|
|||
|
|
@ -37,9 +37,21 @@ impl From<SaplingScannedResult> for transaction::Hash {
|
|||
}
|
||||
}
|
||||
|
||||
impl From<&[u8; 32]> for SaplingScannedResult {
|
||||
fn from(bytes: &[u8; 32]) -> Self {
|
||||
Self(*bytes)
|
||||
impl From<transaction::Hash> for SaplingScannedResult {
|
||||
fn from(hash: transaction::Hash) -> Self {
|
||||
SaplingScannedResult(hash.bytes_in_display_order())
|
||||
}
|
||||
}
|
||||
|
||||
impl SaplingScannedResult {
|
||||
/// Creates a `SaplingScannedResult` from bytes in display order.
|
||||
pub fn from_bytes_in_display_order(bytes: [u8; 32]) -> Self {
|
||||
Self(bytes)
|
||||
}
|
||||
|
||||
/// Returns the inner bytes in display order.
|
||||
pub fn bytes_in_display_order(&self) -> [u8; 32] {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -162,19 +174,8 @@ impl FromDisk for SaplingScannedDatabaseIndex {
|
|||
}
|
||||
}
|
||||
|
||||
impl IntoDisk for SaplingScannedResult {
|
||||
type Bytes = [u8; 32];
|
||||
|
||||
fn as_bytes(&self) -> Self::Bytes {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl FromDisk for SaplingScannedResult {
|
||||
fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
|
||||
SaplingScannedResult(bytes.as_ref().try_into().unwrap())
|
||||
}
|
||||
}
|
||||
// We can't implement IntoDisk or FromDisk for SaplingScannedResult,
|
||||
// because the format is actually Option<SaplingScannedResult>.
|
||||
|
||||
impl IntoDisk for Option<SaplingScannedResult> {
|
||||
type Bytes = Vec<u8>;
|
||||
|
|
@ -183,7 +184,7 @@ impl IntoDisk for Option<SaplingScannedResult> {
|
|||
let mut bytes = Vec::new();
|
||||
|
||||
if let Some(result) = self.as_ref() {
|
||||
bytes.extend(result.as_bytes());
|
||||
bytes.extend(result.bytes_in_display_order());
|
||||
}
|
||||
|
||||
bytes
|
||||
|
|
@ -191,13 +192,18 @@ impl IntoDisk for Option<SaplingScannedResult> {
|
|||
}
|
||||
|
||||
impl FromDisk for Option<SaplingScannedResult> {
|
||||
#[allow(clippy::unwrap_in_result)]
|
||||
fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
|
||||
let bytes = bytes.as_ref();
|
||||
|
||||
if bytes.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(SaplingScannedResult::from_bytes(bytes))
|
||||
Some(SaplingScannedResult::from_bytes_in_display_order(
|
||||
bytes
|
||||
.try_into()
|
||||
.expect("unexpected incorrect SaplingScannedResult data length"),
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,13 +29,6 @@ fn roundtrip_sapling_db_index() {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrip_sapling_result() {
|
||||
let _init_guard = zebra_test::init();
|
||||
|
||||
proptest!(|(val in any::<SaplingScannedResult>())| assert_value_properties(val));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrip_option_sapling_result() {
|
||||
let _init_guard = zebra_test::init();
|
||||
|
|
|
|||
|
|
@ -2,12 +2,14 @@
|
|||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[cfg(test)]
|
||||
mod prop;
|
||||
#[cfg(test)]
|
||||
mod snapshot;
|
||||
|
||||
/// A formatting struct for raw key-value data
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
|
||||
struct KV {
|
||||
pub struct KV {
|
||||
/// The raw key bytes, as a hexadecimal-encoded string.
|
||||
k: String,
|
||||
|
||||
|
|
@ -17,7 +19,7 @@ struct KV {
|
|||
|
||||
impl KV {
|
||||
/// Create a new `KV` from raw key-value data.
|
||||
fn new<K, V>(key: K, value: V) -> KV
|
||||
pub fn new<K, V>(key: K, value: V) -> KV
|
||||
where
|
||||
K: AsRef<[u8]>,
|
||||
V: AsRef<[u8]>,
|
||||
|
|
@ -26,7 +28,7 @@ impl KV {
|
|||
}
|
||||
|
||||
/// Create a new `KV` from hex-encoded key-value data.
|
||||
fn new_hex(key: String, value: String) -> KV {
|
||||
pub fn new_hex(key: String, value: String) -> KV {
|
||||
KV { k: key, v: value }
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@
|
|||
//!
|
||||
//! Test shielded data, and data activated in Overwinter and later network upgrades.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::{collections::BTreeMap, sync::Arc};
|
||||
|
||||
use zebra_chain::{
|
||||
block::Block,
|
||||
|
|
@ -37,11 +37,11 @@ use zebra_chain::{
|
|||
|
||||
use crate::{
|
||||
service::finalized_state::{
|
||||
disk_db::{DiskDb, DB},
|
||||
disk_format::tests::KV,
|
||||
disk_db::DiskDb,
|
||||
disk_format::{tests::KV, RawBytes},
|
||||
FinalizedState,
|
||||
},
|
||||
Config,
|
||||
Config, ReadDisk,
|
||||
};
|
||||
|
||||
/// Snapshot test for RocksDB column families, and their key-value data.
|
||||
|
|
@ -133,13 +133,12 @@ fn snapshot_raw_rocksdb_column_family_data(db: &DiskDb, original_cf_names: &[Str
|
|||
.expect("RocksDB API provides correct names");
|
||||
|
||||
// Correctness: Multi-key iteration causes hangs in concurrent code, but seems ok in tests.
|
||||
let mut cf_iter = db.full_iterator_cf(&cf_handle, rocksdb::IteratorMode::Start);
|
||||
let cf_items: BTreeMap<RawBytes, RawBytes> = db.zs_items_in_range_ordered(&cf_handle, ..);
|
||||
|
||||
// The default raw data serialization is very verbose, so we hex-encode the bytes.
|
||||
let cf_data: Vec<KV> = cf_iter
|
||||
.by_ref()
|
||||
.map(|result| result.expect("unexpected database error"))
|
||||
.map(|(key, value)| KV::new(key, value))
|
||||
let cf_data: Vec<KV> = cf_items
|
||||
.iter()
|
||||
.map(|(key, value)| KV::new(key.raw_bytes(), value.raw_bytes()))
|
||||
.collect();
|
||||
|
||||
if cf_name == "default" {
|
||||
|
|
@ -153,14 +152,6 @@ fn snapshot_raw_rocksdb_column_family_data(db: &DiskDb, original_cf_names: &[Str
|
|||
// because those roots are used to populate the anchor column families.
|
||||
insta::assert_ron_snapshot!(format!("{cf_name}_raw_data"), cf_data);
|
||||
}
|
||||
|
||||
let raw_cf_iter: rocksdb::DBRawIteratorWithThreadMode<DB> = cf_iter.into();
|
||||
|
||||
assert_eq!(
|
||||
raw_cf_iter.status(),
|
||||
Ok(()),
|
||||
"unexpected column family iterator error",
|
||||
);
|
||||
}
|
||||
|
||||
insta::assert_ron_snapshot!("empty_column_families", empty_column_families);
|
||||
|
|
|
|||
|
|
@ -118,6 +118,7 @@ proptest-impl = [
|
|||
"zebra-state/proptest-impl",
|
||||
"zebra-network/proptest-impl",
|
||||
"zebra-chain/proptest-impl",
|
||||
"zebra-scan?/proptest-impl",
|
||||
]
|
||||
|
||||
# Build the zebra-checkpoints utility for checkpoint generation tests
|
||||
|
|
@ -271,6 +272,7 @@ zebra-chain = { path = "../zebra-chain", features = ["proptest-impl"] }
|
|||
zebra-consensus = { path = "../zebra-consensus", features = ["proptest-impl"] }
|
||||
zebra-network = { path = "../zebra-network", features = ["proptest-impl"] }
|
||||
zebra-state = { path = "../zebra-state", features = ["proptest-impl"] }
|
||||
zebra-scan = { path = "../zebra-scan", features = ["proptest-impl"] }
|
||||
|
||||
zebra-node-services = { path = "../zebra-node-services", features = ["rpc-client"] }
|
||||
|
||||
|
|
|
|||
|
|
@ -2821,13 +2821,13 @@ async fn fully_synced_rpc_z_getsubtreesbyindex_snapshot_test() -> Result<()> {
|
|||
fn scan_task_starts() -> Result<()> {
|
||||
use indexmap::IndexMap;
|
||||
|
||||
const ZECPAGES_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz";
|
||||
use zebra_scan::tests::ZECPAGES_SAPLING_VIEWING_KEY;
|
||||
|
||||
let _init_guard = zebra_test::init();
|
||||
|
||||
let mut config = default_test_config(Mainnet)?;
|
||||
let mut keys = IndexMap::new();
|
||||
keys.insert(ZECPAGES_VIEWING_KEY.to_string(), 1);
|
||||
keys.insert(ZECPAGES_SAPLING_VIEWING_KEY.to_string(), 1);
|
||||
config.shielded_scan.sapling_keys_to_scan = keys;
|
||||
|
||||
let testdir = testdir()?.with_config(&mut config)?;
|
||||
|
|
|
|||
Loading…
Reference in New Issue