From 22852bc81d830c8377c691e09f419293c6edf219 Mon Sep 17 00:00:00 2001 From: Marek Date: Wed, 13 Dec 2023 22:54:00 +0100 Subject: [PATCH] change(state): Allow opening the database in a read-only mode (#8079) * Allow opening the database read-only mode * Update zebra-scan/src/storage/db.rs Co-authored-by: teor * Refactor skipping database upgrades * Fix errors caused by the merge * Add new argument in new_test_storage() * Simplify test storage code * Fix importing the `new_test_storate` fn --------- Co-authored-by: teor --- zebra-scan/src/init.rs | 2 +- zebra-scan/src/storage.rs | 4 ++-- zebra-scan/src/storage/db.rs | 8 +++++--- zebra-scan/src/storage/db/tests.rs | 2 +- zebra-scan/src/tests/vectors.rs | 4 ++-- zebra-state/src/service/finalized_state.rs | 3 +++ zebra-state/src/service/finalized_state/disk_db.rs | 7 ++++++- .../src/service/finalized_state/zebra_db.rs | 14 ++++++++++---- .../zebra_db/block/tests/vectors.rs | 1 + zebra-state/src/service/read/tests/vectors.rs | 1 + zebra-state/src/tests/setup.rs | 1 + 11 files changed, 33 insertions(+), 14 deletions(-) diff --git a/zebra-scan/src/init.rs b/zebra-scan/src/init.rs index a94676ead..03cea9a90 100644 --- a/zebra-scan/src/init.rs +++ b/zebra-scan/src/init.rs @@ -33,7 +33,7 @@ pub async fn init( state: scan::State, chain_tip_change: ChainTipChange, ) -> Result<(), Report> { - let storage = tokio::task::spawn_blocking(move || Storage::new(&config, network)) + let storage = tokio::task::spawn_blocking(move || Storage::new(&config, network, false)) .wait_for_panics() .await; diff --git a/zebra-scan/src/storage.rs b/zebra-scan/src/storage.rs index 664ba5f37..5830ace47 100644 --- a/zebra-scan/src/storage.rs +++ b/zebra-scan/src/storage.rs @@ -59,8 +59,8 @@ impl Storage { /// /// This method can block while creating or reading database files, so it must be inside /// spawn_blocking() in async code. - pub fn new(config: &Config, network: Network) -> Self { - let mut storage = Self::new_db(config, network); + pub fn new(config: &Config, network: Network, read_only: bool) -> Self { + let mut storage = Self::new_db(config, network, read_only); for (sapling_key, birthday) in config.sapling_keys_to_scan.iter() { storage.add_sapling_key(sapling_key, Some(zebra_chain::block::Height(*birthday))); diff --git a/zebra-scan/src/storage/db.rs b/zebra-scan/src/storage/db.rs index 5d40c54f1..a99278bcd 100644 --- a/zebra-scan/src/storage/db.rs +++ b/zebra-scan/src/storage/db.rs @@ -49,11 +49,11 @@ impl Storage { /// If there is no existing database, creates a new database on disk. /// /// New keys in `config` are not inserted into the database. - pub(crate) fn new_db(config: &Config, network: Network) -> Self { + pub(crate) fn new_db(config: &Config, network: Network, read_only: bool) -> Self { Self::new_with_debug( config, network, - // TODO: make format upgrades work with any database, then change this to `false` - true, + // TODO: make format upgrades work with any database, then change debug_skip_format_upgrades to `false` + true, read_only, ) } @@ -67,6 +67,7 @@ impl Storage { config: &Config, network: Network, debug_skip_format_upgrades: bool, + read_only: bool, ) -> Self { let db = ScannerDb::new( config.db_config(), @@ -77,6 +78,7 @@ impl Storage { SCANNER_COLUMN_FAMILIES_IN_CODE .iter() .map(ToString::to_string), + read_only, ); let new_storage = Self { db }; diff --git a/zebra-scan/src/storage/db/tests.rs b/zebra-scan/src/storage/db/tests.rs index 6b6365be9..b813a8b91 100644 --- a/zebra-scan/src/storage/db/tests.rs +++ b/zebra-scan/src/storage/db/tests.rs @@ -20,7 +20,7 @@ mod snapshot; /// Returns an empty `Storage` suitable for testing. pub fn new_test_storage(network: Network) -> Storage { - Storage::new(&Config::ephemeral(), network) + Storage::new(&Config::ephemeral(), network, false) } /// Add fake keys to `storage` for testing purposes. diff --git a/zebra-scan/src/tests/vectors.rs b/zebra-scan/src/tests/vectors.rs index 1352a6bfd..1b0dfd645 100644 --- a/zebra-scan/src/tests/vectors.rs +++ b/zebra-scan/src/tests/vectors.rs @@ -22,8 +22,8 @@ use zebra_chain::{ use zebra_state::{SaplingScannedResult, TransactionIndex}; use crate::{ - config::Config, scan::{block_to_compact, scan_block}, + storage::db::tests::new_test_storage, tests::{fake_block, ZECPAGES_SAPLING_VIEWING_KEY}, }; @@ -157,7 +157,7 @@ fn scanning_fake_blocks_store_key_and_results() -> Result<()> { zcash_client_backend::encoding::encode_extended_full_viewing_key("zxviews", &extfvk); // Create a database - let mut s = crate::storage::Storage::new(&Config::ephemeral(), Network::Mainnet); + let mut s = new_test_storage(Network::Mainnet); // Insert the generated key to the database s.add_sapling_key(&key_to_be_stored, None); diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index 151ed0125..895a043be 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -146,6 +146,7 @@ impl FinalizedState { false, #[cfg(feature = "elasticsearch")] elastic_db, + false, ) } @@ -158,6 +159,7 @@ impl FinalizedState { network: Network, debug_skip_format_upgrades: bool, #[cfg(feature = "elasticsearch")] elastic_db: Option, + read_only: bool, ) -> Self { let db = ZebraDb::new( config, @@ -168,6 +170,7 @@ impl FinalizedState { STATE_COLUMN_FAMILIES_IN_CODE .iter() .map(ToString::to_string), + read_only, ); #[cfg(feature = "elasticsearch")] diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs index a9bad0cb4..c835971bf 100644 --- a/zebra-state/src/service/finalized_state/disk_db.rs +++ b/zebra-state/src/service/finalized_state/disk_db.rs @@ -658,6 +658,7 @@ impl DiskDb { format_version_in_code: &Version, network: Network, column_families_in_code: impl IntoIterator, + read_only: bool, ) -> DiskDb { let db_kind = db_kind.as_ref(); let path = config.db_path(db_kind, format_version_in_code.major, network); @@ -680,7 +681,11 @@ impl DiskDb { .unique() .map(|cf_name| rocksdb::ColumnFamilyDescriptor::new(cf_name, db_options.clone())); - let db_result = DB::open_cf_descriptors(&db_options, &path, column_families); + let db_result = if read_only { + DB::open_cf_descriptors_read_only(&db_options, &path, column_families, false) + } else { + DB::open_cf_descriptors(&db_options, &path, column_families) + }; match db_result { Ok(db) => { diff --git a/zebra-state/src/service/finalized_state/zebra_db.rs b/zebra-state/src/service/finalized_state/zebra_db.rs index 6b7c6823d..810feff87 100644 --- a/zebra-state/src/service/finalized_state/zebra_db.rs +++ b/zebra-state/src/service/finalized_state/zebra_db.rs @@ -96,6 +96,7 @@ impl ZebraDb { network: Network, debug_skip_format_upgrades: bool, column_families_in_code: impl IntoIterator, + read_only: bool, ) -> ZebraDb { let disk_version = database_format_version_on_disk( config, @@ -108,16 +109,20 @@ impl ZebraDb { // Log any format changes before opening the database, in case opening fails. let format_change = DbFormatChange::open_database(format_version_in_code, disk_version); - // Always do format upgrades in production, but allow them to be skipped by the scanner - // (because it doesn't support them yet). + // Format upgrades try to write to the database, so we always skip them if `read_only` is + // `true`. + // + // We allow skipping the upgrades by the scanner because it doesn't support them yet and we + // also allow skipping them when we are running tests. // // TODO: Make scanner support format upgrades, then remove `shielded-scan` here. - let can_skip_format_upgrades = cfg!(test) || cfg!(feature = "shielded-scan"); + let debug_skip_format_upgrades = read_only + || ((cfg!(test) || cfg!(feature = "shielded-scan")) && debug_skip_format_upgrades); // Open the database and do initial checks. let mut db = ZebraDb { config: Arc::new(config.clone()), - debug_skip_format_upgrades: can_skip_format_upgrades && debug_skip_format_upgrades, + debug_skip_format_upgrades, format_change_handle: None, // After the database directory is created, a newly created database temporarily // changes to the default database version. Then we set the correct version in the @@ -129,6 +134,7 @@ impl ZebraDb { format_version_in_code, network, column_families_in_code, + read_only, ), }; diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs index 4a3385ebe..f0ce0d0f4 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs @@ -89,6 +89,7 @@ fn test_block_db_round_trip_with( STATE_COLUMN_FAMILIES_IN_CODE .iter() .map(ToString::to_string), + false, ); // Check that each block round-trips to the database diff --git a/zebra-state/src/service/read/tests/vectors.rs b/zebra-state/src/service/read/tests/vectors.rs index 38d0887aa..f1f39b6cc 100644 --- a/zebra-state/src/service/read/tests/vectors.rs +++ b/zebra-state/src/service/read/tests/vectors.rs @@ -377,5 +377,6 @@ fn new_ephemeral_db() -> ZebraDb { STATE_COLUMN_FAMILIES_IN_CODE .iter() .map(ToString::to_string), + false, ) } diff --git a/zebra-state/src/tests/setup.rs b/zebra-state/src/tests/setup.rs index 1432e72f3..c5a900c05 100644 --- a/zebra-state/src/tests/setup.rs +++ b/zebra-state/src/tests/setup.rs @@ -99,6 +99,7 @@ pub(crate) fn new_state_with_mainnet_genesis( true, #[cfg(feature = "elasticsearch")] None, + false, ); let non_finalized_state = NonFinalizedState::new(network);