zecfoundation
/
zebra
mirror of https://github.com/ZcashFoundation/zebra.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

test: Add progress markers to the snapshot tests (#8106) 

* Add progress tracking markers to snapshot tests

* Simplify sapling_keys_and_last_scanned_heights()
This commit is contained in:
teor 2023-12-15 08:32:09 +11:00 committed by GitHub
parent 8734fd98f3
commit a14cb40c1f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 103 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
zebra-scan/src/storage.rs
View File

@ -104,6 +104,8 @@ impl Storage {
/// Add the sapling results for `height` to the storage. The results can be any map of
/// [`TransactionIndex`] to [`SaplingScannedResult`].
///
/// Also adds empty progress tracking entries every `INSERT_CONTROL_INTERVAL` blocks if needed.
///
/// # Performance / Hangs
///
/// This method can block while writing database files, so it must be inside spawn_blocking()
@ -120,7 +122,15 @@ impl Storage {
// Every `INSERT_CONTROL_INTERVAL` we add a new entry to the scanner database for each key
// so we can track progress made in the last interval even if no transaction was yet found.
let is_control_time = height.0 % INSERT_CONTROL_INTERVAL == 0 && sapling_results.is_empty();
let needs_control_entry =
height.0 % INSERT_CONTROL_INTERVAL == 0 && sapling_results.is_empty();
// Add scanner progress tracking entry for key.
// Defensive programming: add the tracking entry first, so that we don't accidentally
// overwrite real results with it. (This is currently prevented by the empty check.)
if needs_control_entry {
batch.insert_sapling_height(self, sapling_key, height);
}
for (index, sapling_result) in sapling_results {
let index = SaplingScannedDatabaseIndex {
@ -136,11 +146,6 @@ impl Storage {
batch.insert_sapling_result(self, entry);
}
// Add tracking entry for key.
if is_control_time {
batch.insert_sapling_height(self, sapling_key, height);
}
self.write_batch(batch);
}

18
zebra-scan/src/storage/db/sapling.rs
View File

@ -107,27 +107,25 @@ impl Storage {
Option<SaplingScannedResult>,
)> = self.db.zs_last_key_value(&sapling_tx_ids);
loop {
let Some((mut last_stored_record_index, _result)) = last_stored_record else {
return keys;
};
while let Some((last_stored_record_index, _result)) = last_stored_record {
let sapling_key = last_stored_record_index.sapling_key.clone();
let height = last_stored_record_index.tx_loc.height;
let prev_height = keys.insert(sapling_key.clone(), height);
assert_eq!(
prev_height, None,
"unexpected duplicate key: keys must only be inserted once\
"unexpected duplicate key: keys must only be inserted once \
last_stored_record_index: {last_stored_record_index:?}",
);
// Skip all the results until the next key.
last_stored_record_index = SaplingScannedDatabaseIndex::min_for_key(&sapling_key);
last_stored_record = self
.db
.zs_prev_key_value_strictly_before(&sapling_tx_ids, &last_stored_record_index);
last_stored_record = self.db.zs_prev_key_value_strictly_before(
&sapling_tx_ids,
&SaplingScannedDatabaseIndex::min_for_key(&sapling_key),
);
}
keys
}
/// Returns the Sapling indexes and results in the supplied range.

46
zebra-scan/src/storage/db/tests.rs
View File

@ -1,6 +1,6 @@
//! General scanner database tests.
use std::sync::Arc;
use std::{collections::BTreeMap, sync::Arc};
use zebra_chain::{
block::{Block, Height},
@ -10,7 +10,7 @@ use zebra_chain::{
use zebra_state::TransactionIndex;
use crate::{
storage::Storage,
storage::{Storage, INSERT_CONTROL_INTERVAL},
tests::{FAKE_SAPLING_VIEWING_KEY, ZECPAGES_SAPLING_VIEWING_KEY},
Config,
};
@ -32,7 +32,15 @@ pub fn add_fake_keys(storage: &mut Storage) {
}
/// Add fake results to `storage` for testing purposes.
pub fn add_fake_results(storage: &mut Storage, network: Network, height: Height) {
///
/// If `add_progress_marker` is `true`, adds a progress marker.
/// If it is `false`, adds the transaction hashes from `height`.
pub fn add_fake_results(
storage: &mut Storage,
network: Network,
height: Height,
add_progress_marker: bool,
) {
let blocks = match network {
Mainnet => &*zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS,
Testnet => &*zebra_test::vectors::CONTINUOUS_TESTNET_BLOCKS,
@ -44,15 +52,25 @@ pub fn add_fake_results(storage: &mut Storage, network: Network, height: Height)
.zcash_deserialize_into()
.expect("test data deserializes");
// Fake results from the first few blocks
storage.add_sapling_results(
&ZECPAGES_SAPLING_VIEWING_KEY.to_string(),
height,
block
.transactions
.iter()
.enumerate()
.map(|(index, tx)| (TransactionIndex::from_usize(index), tx.hash().into()))
.collect(),
);
if add_progress_marker {
// Fake a progress marker.
let next_progress_height = height.0.next_multiple_of(INSERT_CONTROL_INTERVAL);
storage.add_sapling_results(
&FAKE_SAPLING_VIEWING_KEY.to_string(),
Height(next_progress_height),
BTreeMap::new(),
);
} else {
// Fake results from the block at `height`.
storage.add_sapling_results(
&ZECPAGES_SAPLING_VIEWING_KEY.to_string(),
height,
block
.transactions
.iter()
.enumerate()
.map(|(index, tx)| (TransactionIndex::from_usize(index), tx.hash().into()))
.collect(),
);
}
}

5
zebra-scan/src/storage/db/tests/snapshot.rs
View File

@ -89,7 +89,8 @@ fn test_database_format_with_network(network: Network) {
//
// We limit the number of blocks, because we create 2 snapshots per block, one for each network.
for height in 0..=2 {
super::add_fake_results(&mut storage, network, Height(height));
super::add_fake_results(&mut storage, network, Height(height), true);
super::add_fake_results(&mut storage, network, Height(height), false);
let mut settings = insta::Settings::clone_current();
settings.set_snapshot_suffix(format!("{net_suffix}_{height}"));
@ -98,8 +99,6 @@ fn test_database_format_with_network(network: Network) {
settings.bind(|| snapshot_raw_rocksdb_column_family_data(&storage.db, &cf_names));
settings.bind(|| snapshot_typed_result_data(&storage));
}
// TODO: add an empty marker result after PR #8080 merges
}
/// Snapshot the data in each column family, using `cargo insta` and RON serialization.

1
zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@mainnet_0.snap
View File

@ -3,5 +3,6 @@ source: zebra-scan/src/storage/db/tests/snapshot.rs
expression: sapling_results
---
{
Height(0): [],
Height(999999): [],
}

2
zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@mainnet_1.snap
View File

@ -3,5 +3,7 @@ source: zebra-scan/src/storage/db/tests/snapshot.rs
expression: sapling_results
---
{
Height(0): [],
Height(1000): [],
Height(999999): [],
}

2
zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@mainnet_2.snap
View File

@ -3,5 +3,7 @@ source: zebra-scan/src/storage/db/tests/snapshot.rs
expression: sapling_results
---
{
Height(0): [],
Height(1000): [],
Height(999999): [],
}

1
zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@testnet_0.snap
View File

@ -3,5 +3,6 @@ source: zebra-scan/src/storage/db/tests/snapshot.rs
expression: sapling_results
---
{
Height(0): [],
Height(999999): [],
}

2
zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@testnet_1.snap
View File

@ -3,5 +3,7 @@ source: zebra-scan/src/storage/db/tests/snapshot.rs
expression: sapling_results
---
{
Height(0): [],
Height(1000): [],
Height(999999): [],
}

2
zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@testnet_2.snap
View File

@ -3,5 +3,7 @@ source: zebra-scan/src/storage/db/tests/snapshot.rs
expression: sapling_results
---
{
Height(0): [],
Height(1000): [],
Height(999999): [],
}

4
zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_0.snap
View File

@ -11,6 +11,10 @@ expression: cf_data
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000",
v: "",
),
KV(
k: "7a78766965777366616b650000000000",
v: "",
),
KV(
k: "7a78766965777366616b650f423f0000",
v: "",

8
zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_1.snap
View File

@ -15,6 +15,14 @@ expression: cf_data
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000",
v: "",
),
KV(
k: "7a78766965777366616b650000000000",
v: "",
),
KV(
k: "7a78766965777366616b650003e80000",
v: "",
),
KV(
k: "7a78766965777366616b650f423f0000",
v: "",

8
zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_2.snap
View File

@ -19,6 +19,14 @@ expression: cf_data
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000",
v: "",
),
KV(
k: "7a78766965777366616b650000000000",
v: "",
),
KV(
k: "7a78766965777366616b650003e80000",
v: "",
),
KV(
k: "7a78766965777366616b650f423f0000",
v: "",

4
zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_0.snap
View File

@ -11,6 +11,10 @@ expression: cf_data
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000",
v: "",
),
KV(
k: "7a78766965777366616b650000000000",
v: "",
),
KV(
k: "7a78766965777366616b650f423f0000",
v: "",

8
zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_1.snap
View File

@ -15,6 +15,14 @@ expression: cf_data
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000",
v: "",
),
KV(
k: "7a78766965777366616b650000000000",
v: "",
),
KV(
k: "7a78766965777366616b650003e80000",
v: "",
),
KV(
k: "7a78766965777366616b650f423f0000",
v: "",

8
zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_2.snap
View File

@ -19,6 +19,14 @@ expression: cf_data
k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000",
v: "",
),
KV(
k: "7a78766965777366616b650000000000",
v: "",
),
KV(
k: "7a78766965777366616b650003e80000",
v: "",
),
KV(
k: "7a78766965777366616b650f423f0000",
v: "",