From e6a8654c44517af54f2b16fdcfa554edd9fcef5d Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 24 Oct 2024 21:04:18 -0400 Subject: [PATCH] - adds build metadata when writing db version file, if any. - adds the build metadata to the db version file before adding indexes. - deletes indexes when running without the `indexer` feature --- zebra-state/src/config.rs | 6 +- .../src/service/finalized_state/disk_db.rs | 15 +++++ .../finalized_state/disk_format/upgrade.rs | 41 +++++++++++++ .../upgrade/drop_tx_locs_by_spends.rs | 59 +++++++++++++++++++ .../upgrade/track_tx_locs_by_spends.rs | 2 +- 5 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 zebra-state/src/service/finalized_state/disk_format/upgrade/drop_tx_locs_by_spends.rs diff --git a/zebra-state/src/config.rs b/zebra-state/src/config.rs index 71188945f..7d175d4d6 100644 --- a/zebra-state/src/config.rs +++ b/zebra-state/src/config.rs @@ -500,7 +500,11 @@ pub(crate) mod hidden { ) -> Result<(), BoxError> { let version_path = config.version_file_path(db_kind, changed_version.major, network); - let version = format!("{}.{}", changed_version.minor, changed_version.patch); + let mut version = format!("{}.{}", changed_version.minor, changed_version.patch); + + if !changed_version.build.is_empty() { + version.push_str(&format!("+{}", changed_version.build)); + } // Write the version file atomically so the cache is not corrupted if Zebra shuts down or // crashes. diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs index 69be5a458..adf782967 100644 --- a/zebra-state/src/service/finalized_state/disk_db.rs +++ b/zebra-state/src/service/finalized_state/disk_db.rs @@ -32,6 +32,9 @@ use crate::{ Config, }; +#[cfg(not(feature = "indexer"))] +use crate::constants::STATE_DATABASE_KIND; + // Doc-only imports #[allow(unused_imports)] use super::{TypedColumnFamily, WriteTypedBatch}; @@ -863,6 +866,18 @@ impl DiskDb { DB::open_cf_descriptors(&db_options, &path, column_families) }; + #[cfg(not(feature = "indexer"))] + let db_result = match db_result { + Ok(mut db) if db_kind == STATE_DATABASE_KIND => { + if let Err(err) = db.drop_cf(super::zebra_db::transparent::TX_LOC_BY_SPENT_OUT_LOC) + { + warn!(?err, "failed to drop unused column family"); + } + Ok(db) + } + other => other, + }; + match db_result { Ok(db) => { info!("Opened Zebra state cache at {}", path.display()); diff --git a/zebra-state/src/service/finalized_state/disk_format/upgrade.rs b/zebra-state/src/service/finalized_state/disk_format/upgrade.rs index f81a477d6..93625a848 100644 --- a/zebra-state/src/service/finalized_state/disk_format/upgrade.rs +++ b/zebra-state/src/service/finalized_state/disk_format/upgrade.rs @@ -29,6 +29,9 @@ pub(crate) mod add_subtrees; pub(crate) mod cache_genesis_roots; pub(crate) mod fix_tree_key_type; +#[cfg(not(feature = "indexer"))] +pub(crate) mod drop_tx_locs_by_spends; + #[cfg(feature = "indexer")] pub(crate) mod track_tx_locs_by_spends; @@ -350,6 +353,16 @@ impl DbFormatChange { // Indexing transaction locations by their spent outpoints and revealed nullifiers. let timer = CodeTimer::start(); + // Add build metadata to on-disk version file just before starting to add indexes + let mut version = db + .format_version_on_disk() + .expect("unable to read database format version file") + .expect("should write database format version file above"); + version.build = db.format_version_in_code().build; + + db.update_format_version_on_disk(&version) + .expect("unable to write database format version file to disk"); + info!("started checking/adding indexes for spending tx ids"); track_tx_locs_by_spends::run(initial_tip_height, db, cancel_receiver)?; info!("finished checking/adding indexes for spending tx ids"); @@ -357,6 +370,34 @@ impl DbFormatChange { timer.finish(module_path!(), line!(), "indexing spending transaction ids"); }; + #[cfg(not(feature = "indexer"))] + if let ( + Upgrade { .. } | CheckOpenCurrent { .. } | Downgrade { .. }, + Some(initial_tip_height), + ) = (self, initial_tip_height) + { + let mut version = db + .format_version_on_disk() + .expect("unable to read database format version file") + .expect("should write database format version file above"); + + if version.build.contains("indexer") { + // Indexing transaction locations by their spent outpoints and revealed nullifiers. + let timer = CodeTimer::start(); + + info!("started removing indexes for spending tx ids"); + drop_tx_locs_by_spends::run(initial_tip_height, db, cancel_receiver)?; + info!("finished removing indexes for spending tx ids"); + + // Remove build metadata to on-disk version file after indexes have been dropped. + version.build = db.format_version_in_code().build; + db.update_format_version_on_disk(&version) + .expect("unable to write database format version file to disk"); + + timer.finish(module_path!(), line!(), "removing spending transaction ids"); + } + }; + // These checks should pass for all format changes: // - upgrades should produce a valid format (and they already do that check) // - an empty state should pass all the format checks diff --git a/zebra-state/src/service/finalized_state/disk_format/upgrade/drop_tx_locs_by_spends.rs b/zebra-state/src/service/finalized_state/disk_format/upgrade/drop_tx_locs_by_spends.rs new file mode 100644 index 000000000..db1297280 --- /dev/null +++ b/zebra-state/src/service/finalized_state/disk_format/upgrade/drop_tx_locs_by_spends.rs @@ -0,0 +1,59 @@ +//! Tracks transaction locations by their inputs and revealed nullifiers. + +use crossbeam_channel::{Receiver, TryRecvError}; +use rayon::iter::{IntoParallelIterator, ParallelIterator}; + +use zebra_chain::block::Height; + +use crate::service::finalized_state::ZebraDb; + +use super::{super::super::DiskWriteBatch, CancelFormatChange}; + +/// Runs disk format upgrade for tracking transaction locations by their inputs and revealed nullifiers. +/// +/// Returns `Ok` if the upgrade completed, and `Err` if it was cancelled. +#[allow(clippy::unwrap_in_result)] +#[instrument(skip(zebra_db, cancel_receiver))] +pub fn run( + initial_tip_height: Height, + zebra_db: &ZebraDb, + cancel_receiver: &Receiver, +) -> Result<(), CancelFormatChange> { + if !matches!(cancel_receiver.try_recv(), Err(TryRecvError::Empty)) { + return Err(CancelFormatChange); + } + + // The `TX_LOC_BY_SPENT_OUT_LOC` column family should be dropped + // when opening the database without the `indexer` feature. + + (0..=initial_tip_height.0) + .into_par_iter() + .try_for_each(|height| { + let height = Height(height); + let mut batch = DiskWriteBatch::new(); + + for (_tx_loc, tx) in zebra_db.transactions_by_height(height) { + if tx.is_coinbase() { + continue; + } + + batch + .prepare_nullifier_batch(zebra_db, &tx) + .expect("method should never return an error"); + } + + if !matches!(cancel_receiver.try_recv(), Err(TryRecvError::Empty)) { + return Err(CancelFormatChange); + } + + zebra_db + .write_batch(batch) + .expect("unexpected database write failure"); + + if !matches!(cancel_receiver.try_recv(), Err(TryRecvError::Empty)) { + return Err(CancelFormatChange); + } + + Ok(()) + }) +} diff --git a/zebra-state/src/service/finalized_state/disk_format/upgrade/track_tx_locs_by_spends.rs b/zebra-state/src/service/finalized_state/disk_format/upgrade/track_tx_locs_by_spends.rs index 7156442bb..175d670d6 100644 --- a/zebra-state/src/service/finalized_state/disk_format/upgrade/track_tx_locs_by_spends.rs +++ b/zebra-state/src/service/finalized_state/disk_format/upgrade/track_tx_locs_by_spends.rs @@ -84,7 +84,7 @@ pub fn run( batch .prepare_nullifier_batch(zebra_db, &tx, tx_loc) - .expect("should not return an error"); + .expect("method should never return an error"); } if !matches!(cancel_receiver.try_recv(), Err(TryRecvError::Empty)) {