From a14cb40c1f653b21584b5c816f8e132f32c9b576 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 15 Dec 2023 08:32:09 +1100 Subject: [PATCH] test: Add progress markers to the snapshot tests (#8106) * Add progress tracking markers to snapshot tests * Simplify sapling_keys_and_last_scanned_heights() --- zebra-scan/src/storage.rs | 17 ++++--- zebra-scan/src/storage/db/sapling.rs | 18 ++++---- zebra-scan/src/storage/db/tests.rs | 46 +++++++++++++------ zebra-scan/src/storage/db/tests/snapshot.rs | 5 +- .../sapling_key_1_results@mainnet_0.snap | 1 + .../sapling_key_1_results@mainnet_1.snap | 2 + .../sapling_key_1_results@mainnet_2.snap | 2 + .../sapling_key_1_results@testnet_0.snap | 1 + .../sapling_key_1_results@testnet_1.snap | 2 + .../sapling_key_1_results@testnet_2.snap | 2 + .../sapling_tx_ids_raw_data@mainnet_0.snap | 4 ++ .../sapling_tx_ids_raw_data@mainnet_1.snap | 8 ++++ .../sapling_tx_ids_raw_data@mainnet_2.snap | 8 ++++ .../sapling_tx_ids_raw_data@testnet_0.snap | 4 ++ .../sapling_tx_ids_raw_data@testnet_1.snap | 8 ++++ .../sapling_tx_ids_raw_data@testnet_2.snap | 8 ++++ 16 files changed, 103 insertions(+), 33 deletions(-) diff --git a/zebra-scan/src/storage.rs b/zebra-scan/src/storage.rs index 5830ace47..745d09830 100644 --- a/zebra-scan/src/storage.rs +++ b/zebra-scan/src/storage.rs @@ -104,6 +104,8 @@ impl Storage { /// Add the sapling results for `height` to the storage. The results can be any map of /// [`TransactionIndex`] to [`SaplingScannedResult`]. /// + /// Also adds empty progress tracking entries every `INSERT_CONTROL_INTERVAL` blocks if needed. + /// /// # Performance / Hangs /// /// This method can block while writing database files, so it must be inside spawn_blocking() @@ -120,7 +122,15 @@ impl Storage { // Every `INSERT_CONTROL_INTERVAL` we add a new entry to the scanner database for each key // so we can track progress made in the last interval even if no transaction was yet found. - let is_control_time = height.0 % INSERT_CONTROL_INTERVAL == 0 && sapling_results.is_empty(); + let needs_control_entry = + height.0 % INSERT_CONTROL_INTERVAL == 0 && sapling_results.is_empty(); + + // Add scanner progress tracking entry for key. + // Defensive programming: add the tracking entry first, so that we don't accidentally + // overwrite real results with it. (This is currently prevented by the empty check.) + if needs_control_entry { + batch.insert_sapling_height(self, sapling_key, height); + } for (index, sapling_result) in sapling_results { let index = SaplingScannedDatabaseIndex { @@ -136,11 +146,6 @@ impl Storage { batch.insert_sapling_result(self, entry); } - // Add tracking entry for key. - if is_control_time { - batch.insert_sapling_height(self, sapling_key, height); - } - self.write_batch(batch); } diff --git a/zebra-scan/src/storage/db/sapling.rs b/zebra-scan/src/storage/db/sapling.rs index c36d9ad00..eb561a11c 100644 --- a/zebra-scan/src/storage/db/sapling.rs +++ b/zebra-scan/src/storage/db/sapling.rs @@ -107,27 +107,25 @@ impl Storage { Option, )> = self.db.zs_last_key_value(&sapling_tx_ids); - loop { - let Some((mut last_stored_record_index, _result)) = last_stored_record else { - return keys; - }; - + while let Some((last_stored_record_index, _result)) = last_stored_record { let sapling_key = last_stored_record_index.sapling_key.clone(); let height = last_stored_record_index.tx_loc.height; let prev_height = keys.insert(sapling_key.clone(), height); assert_eq!( prev_height, None, - "unexpected duplicate key: keys must only be inserted once\ + "unexpected duplicate key: keys must only be inserted once \ last_stored_record_index: {last_stored_record_index:?}", ); // Skip all the results until the next key. - last_stored_record_index = SaplingScannedDatabaseIndex::min_for_key(&sapling_key); - last_stored_record = self - .db - .zs_prev_key_value_strictly_before(&sapling_tx_ids, &last_stored_record_index); + last_stored_record = self.db.zs_prev_key_value_strictly_before( + &sapling_tx_ids, + &SaplingScannedDatabaseIndex::min_for_key(&sapling_key), + ); } + + keys } /// Returns the Sapling indexes and results in the supplied range. diff --git a/zebra-scan/src/storage/db/tests.rs b/zebra-scan/src/storage/db/tests.rs index b813a8b91..765ec77e6 100644 --- a/zebra-scan/src/storage/db/tests.rs +++ b/zebra-scan/src/storage/db/tests.rs @@ -1,6 +1,6 @@ //! General scanner database tests. -use std::sync::Arc; +use std::{collections::BTreeMap, sync::Arc}; use zebra_chain::{ block::{Block, Height}, @@ -10,7 +10,7 @@ use zebra_chain::{ use zebra_state::TransactionIndex; use crate::{ - storage::Storage, + storage::{Storage, INSERT_CONTROL_INTERVAL}, tests::{FAKE_SAPLING_VIEWING_KEY, ZECPAGES_SAPLING_VIEWING_KEY}, Config, }; @@ -32,7 +32,15 @@ pub fn add_fake_keys(storage: &mut Storage) { } /// Add fake results to `storage` for testing purposes. -pub fn add_fake_results(storage: &mut Storage, network: Network, height: Height) { +/// +/// If `add_progress_marker` is `true`, adds a progress marker. +/// If it is `false`, adds the transaction hashes from `height`. +pub fn add_fake_results( + storage: &mut Storage, + network: Network, + height: Height, + add_progress_marker: bool, +) { let blocks = match network { Mainnet => &*zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS, Testnet => &*zebra_test::vectors::CONTINUOUS_TESTNET_BLOCKS, @@ -44,15 +52,25 @@ pub fn add_fake_results(storage: &mut Storage, network: Network, height: Height) .zcash_deserialize_into() .expect("test data deserializes"); - // Fake results from the first few blocks - storage.add_sapling_results( - &ZECPAGES_SAPLING_VIEWING_KEY.to_string(), - height, - block - .transactions - .iter() - .enumerate() - .map(|(index, tx)| (TransactionIndex::from_usize(index), tx.hash().into())) - .collect(), - ); + if add_progress_marker { + // Fake a progress marker. + let next_progress_height = height.0.next_multiple_of(INSERT_CONTROL_INTERVAL); + storage.add_sapling_results( + &FAKE_SAPLING_VIEWING_KEY.to_string(), + Height(next_progress_height), + BTreeMap::new(), + ); + } else { + // Fake results from the block at `height`. + storage.add_sapling_results( + &ZECPAGES_SAPLING_VIEWING_KEY.to_string(), + height, + block + .transactions + .iter() + .enumerate() + .map(|(index, tx)| (TransactionIndex::from_usize(index), tx.hash().into())) + .collect(), + ); + } } diff --git a/zebra-scan/src/storage/db/tests/snapshot.rs b/zebra-scan/src/storage/db/tests/snapshot.rs index 6abf955e8..f3f18e2ef 100644 --- a/zebra-scan/src/storage/db/tests/snapshot.rs +++ b/zebra-scan/src/storage/db/tests/snapshot.rs @@ -89,7 +89,8 @@ fn test_database_format_with_network(network: Network) { // // We limit the number of blocks, because we create 2 snapshots per block, one for each network. for height in 0..=2 { - super::add_fake_results(&mut storage, network, Height(height)); + super::add_fake_results(&mut storage, network, Height(height), true); + super::add_fake_results(&mut storage, network, Height(height), false); let mut settings = insta::Settings::clone_current(); settings.set_snapshot_suffix(format!("{net_suffix}_{height}")); @@ -98,8 +99,6 @@ fn test_database_format_with_network(network: Network) { settings.bind(|| snapshot_raw_rocksdb_column_family_data(&storage.db, &cf_names)); settings.bind(|| snapshot_typed_result_data(&storage)); } - - // TODO: add an empty marker result after PR #8080 merges } /// Snapshot the data in each column family, using `cargo insta` and RON serialization. diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@mainnet_0.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@mainnet_0.snap index 2b1f3dc5d..160153b31 100644 --- a/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@mainnet_0.snap +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@mainnet_0.snap @@ -3,5 +3,6 @@ source: zebra-scan/src/storage/db/tests/snapshot.rs expression: sapling_results --- { + Height(0): [], Height(999999): [], } diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@mainnet_1.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@mainnet_1.snap index 2b1f3dc5d..c48c115fa 100644 --- a/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@mainnet_1.snap +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@mainnet_1.snap @@ -3,5 +3,7 @@ source: zebra-scan/src/storage/db/tests/snapshot.rs expression: sapling_results --- { + Height(0): [], + Height(1000): [], Height(999999): [], } diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@mainnet_2.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@mainnet_2.snap index 2b1f3dc5d..c48c115fa 100644 --- a/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@mainnet_2.snap +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@mainnet_2.snap @@ -3,5 +3,7 @@ source: zebra-scan/src/storage/db/tests/snapshot.rs expression: sapling_results --- { + Height(0): [], + Height(1000): [], Height(999999): [], } diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@testnet_0.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@testnet_0.snap index 2b1f3dc5d..160153b31 100644 --- a/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@testnet_0.snap +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@testnet_0.snap @@ -3,5 +3,6 @@ source: zebra-scan/src/storage/db/tests/snapshot.rs expression: sapling_results --- { + Height(0): [], Height(999999): [], } diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@testnet_1.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@testnet_1.snap index 2b1f3dc5d..c48c115fa 100644 --- a/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@testnet_1.snap +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@testnet_1.snap @@ -3,5 +3,7 @@ source: zebra-scan/src/storage/db/tests/snapshot.rs expression: sapling_results --- { + Height(0): [], + Height(1000): [], Height(999999): [], } diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@testnet_2.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@testnet_2.snap index 2b1f3dc5d..c48c115fa 100644 --- a/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@testnet_2.snap +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@testnet_2.snap @@ -3,5 +3,7 @@ source: zebra-scan/src/storage/db/tests/snapshot.rs expression: sapling_results --- { + Height(0): [], + Height(1000): [], Height(999999): [], } diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_0.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_0.snap index 7e4978dba..7cb2ced61 100644 --- a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_0.snap +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_0.snap @@ -11,6 +11,10 @@ expression: cf_data k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000", v: "", ), + KV( + k: "7a78766965777366616b650000000000", + v: "", + ), KV( k: "7a78766965777366616b650f423f0000", v: "", diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_1.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_1.snap index 710d65d2d..e3560142e 100644 --- a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_1.snap +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_1.snap @@ -15,6 +15,14 @@ expression: cf_data k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000", v: "", ), + KV( + k: "7a78766965777366616b650000000000", + v: "", + ), + KV( + k: "7a78766965777366616b650003e80000", + v: "", + ), KV( k: "7a78766965777366616b650f423f0000", v: "", diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_2.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_2.snap index fae6f22c0..a88179cbc 100644 --- a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_2.snap +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_2.snap @@ -19,6 +19,14 @@ expression: cf_data k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000", v: "", ), + KV( + k: "7a78766965777366616b650000000000", + v: "", + ), + KV( + k: "7a78766965777366616b650003e80000", + v: "", + ), KV( k: "7a78766965777366616b650f423f0000", v: "", diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_0.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_0.snap index 53089e848..d87f517e8 100644 --- a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_0.snap +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_0.snap @@ -11,6 +11,10 @@ expression: cf_data k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000", v: "", ), + KV( + k: "7a78766965777366616b650000000000", + v: "", + ), KV( k: "7a78766965777366616b650f423f0000", v: "", diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_1.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_1.snap index 4384d29f9..3f9edcab6 100644 --- a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_1.snap +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_1.snap @@ -15,6 +15,14 @@ expression: cf_data k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000", v: "", ), + KV( + k: "7a78766965777366616b650000000000", + v: "", + ), + KV( + k: "7a78766965777366616b650003e80000", + v: "", + ), KV( k: "7a78766965777366616b650f423f0000", v: "", diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_2.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_2.snap index a2e7dfcea..779346791 100644 --- a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_2.snap +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_2.snap @@ -19,6 +19,14 @@ expression: cf_data k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000", v: "", ), + KV( + k: "7a78766965777366616b650000000000", + v: "", + ), + KV( + k: "7a78766965777366616b650003e80000", + v: "", + ), KV( k: "7a78766965777366616b650f423f0000", v: "",