change(state): Support in-place disk format upgrades for major database version bumps (#8748)

* Reuse existing db after a major upgrade

* Don't delete dbs that can be reused

* Apply suggestions from code review

Co-authored-by: Conrado Gouvea <conrado@zfnd.org>

* Fix formatting

* Create only the parent dir for the db

---------

Co-authored-by: Pili Guerra <mpguerra@users.noreply.github.com>
Co-authored-by: Conrado Gouvea <conrado@zfnd.org>

zebra-state/src/config.rs

@@ -15,7 +15,7 @@ use tracing::Span;
 use zebra_chain::parameters::Network;
 
 use crate::{
-    constants::{DATABASE_FORMAT_VERSION_FILE_NAME, STATE_DATABASE_KIND},
+    constants::{DATABASE_FORMAT_VERSION_FILE_NAME, RESTORABLE_DB_VERSIONS, STATE_DATABASE_KIND},
     state_database_format_version_in_code, BoxError,
 };
 
@@ -317,6 +317,15 @@ fn check_and_delete_database(
         return None;
     }
 
+    // Don't delete databases that can be reused.
+    if RESTORABLE_DB_VERSIONS
+        .iter()
+        .map(|v| v - 1)
+        .any(|v| v == dir_major_version)
+    {
+        return None;
+    }
+
     let outdated_path = entry.path();
 
     // # Correctness

zebra-state/src/constants.rs

@@ -122,6 +122,9 @@ const MAX_FIND_BLOCK_HEADERS_RESULTS_FOR_PROTOCOL: u32 = 160;
 pub const MAX_FIND_BLOCK_HEADERS_RESULTS_FOR_ZEBRA: u32 =
     MAX_FIND_BLOCK_HEADERS_RESULTS_FOR_PROTOCOL - 2;
 
+/// These database versions can be recreated from their directly preceding versions.
+pub const RESTORABLE_DB_VERSIONS: [u64; 1] = [26];
+
 lazy_static! {
     /// Regex that matches the RocksDB error when its lock file is already open.
     pub static ref LOCK_FILE_ERROR: Regex = Regex::new("(lock file).*(temporarily unavailable)|(in use)|(being used by another process)").expect("regex is valid");

zebra-state/src/service/finalized_state/disk_db.rs

@@ -12,8 +12,8 @@
 
 use std::{
     collections::{BTreeMap, HashMap},
-    fmt::Debug,
+    fmt::{Debug, Write},
-    fmt::Write,
+    fs,
     ops::RangeBounds,
     path::Path,
     sync::Arc,
@@ -27,6 +27,7 @@ use semver::Version;
 use zebra_chain::{parameters::Network, primitives::byte_array::increment_big_endian};
 
 use crate::{
+    constants::DATABASE_FORMAT_VERSION_FILE_NAME,
     service::finalized_state::disk_format::{FromDisk, IntoDisk},
     Config,
 };
@@ -522,7 +523,9 @@ impl DiskDb {
         let db_options = DiskDb::options();
         let column_families = DiskDb::construct_column_families(&db_options, db.path(), &[]);
         let mut column_families_log_string = String::from("");
+
         write!(column_families_log_string, "Column families and sizes: ").unwrap();
+
         for cf_descriptor in column_families.iter() {
             let cf_name = &cf_descriptor.name();
             let cf_handle = db
@@ -940,6 +943,123 @@ impl DiskDb {
 
     // Private methods
 
+    /// Tries to reuse an existing db after a major upgrade.
+    ///
+    /// If the current db version belongs to `restorable_db_versions`, the function moves a previous
+    /// db to a new path so it can be used again. It does so by merely trying to rename the path
+    /// corresponding to the db version directly preceding the current version to the path that is
+    /// used by the current db. If successful, it also deletes the db version file.
+    pub(crate) fn try_reusing_previous_db_after_major_upgrade(
+        restorable_db_versions: &[u64],
+        format_version_in_code: &Version,
+        config: &Config,
+        db_kind: impl AsRef<str>,
+        network: &Network,
+    ) {
+        if let Some(&major_db_ver) = restorable_db_versions
+            .iter()
+            .find(|v| **v == format_version_in_code.major)
+        {
+            let db_kind = db_kind.as_ref();
+
+            let old_path = config.db_path(db_kind, major_db_ver - 1, network);
+            let new_path = config.db_path(db_kind, major_db_ver, network);
+
+            let old_path = match fs::canonicalize(&old_path) {
+                Ok(canonicalized_old_path) => canonicalized_old_path,
+                Err(e) => {
+                    warn!("could not canonicalize {old_path:?}: {e}");
+                    return;
+                }
+            };
+
+            let cache_path = match fs::canonicalize(&config.cache_dir) {
+                Ok(canonicalized_cache_path) => canonicalized_cache_path,
+                Err(e) => {
+                    warn!("could not canonicalize {:?}: {e}", config.cache_dir);
+                    return;
+                }
+            };
+
+            // # Correctness
+            //
+            // Check that the path we're about to move is inside the cache directory.
+            //
+            // If the user has symlinked the state directory to a non-cache directory, we don't want
+            // to move it, because it might contain other files.
+            //
+            // We don't attempt to guard against malicious symlinks created by attackers
+            // (TOCTOU attacks). Zebra should not be run with elevated privileges.
+            if !old_path.starts_with(&cache_path) {
+                info!("skipped reusing previous state cache: state is outside cache directory");
+                return;
+            }
+
+            let opts = DiskDb::options();
+            let old_db_exists = DB::list_cf(&opts, &old_path).is_ok_and(|cf| !cf.is_empty());
+            let new_db_exists = DB::list_cf(&opts, &new_path).is_ok_and(|cf| !cf.is_empty());
+
+            if old_db_exists && !new_db_exists {
+                // Create the parent directory for the new db. This is because we can't directly
+                // rename e.g. `state/v25/mainnet/` to `state/v26/mainnet/` with `fs::rename()` if
+                // `state/v26/` does not exist.
+                match fs::create_dir_all(
+                    new_path
+                        .parent()
+                        .expect("new state cache must have a parent path"),
+                ) {
+                    Ok(()) => info!("created new directory for state cache at {new_path:?}"),
+                    Err(e) => {
+                        warn!(
+                            "could not create new directory for state cache at {new_path:?}: {e}"
+                        );
+                        return;
+                    }
+                };
+
+                match fs::rename(&old_path, &new_path) {
+                    Ok(()) => {
+                        info!("moved state cache from {old_path:?} to {new_path:?}");
+
+                        match fs::remove_file(new_path.join(DATABASE_FORMAT_VERSION_FILE_NAME)) {
+                            Ok(()) => info!("removed version file at {new_path:?}"),
+                            Err(e) => {
+                                warn!("could not remove version file at {new_path:?}: {e}")
+                            }
+                        }
+
+                        // Get the parent of the old path, e.g. `state/v25/` and delete it if it is
+                        // empty.
+                        let old_path = old_path
+                            .parent()
+                            .expect("old state cache must have parent path");
+
+                        if fs::read_dir(old_path)
+                            .expect("cached state dir needs to be readable")
+                            .next()
+                            .is_none()
+                        {
+                            match fs::remove_dir_all(old_path) {
+                                Ok(()) => {
+                                    info!("removed empty old state cache directory at {old_path:?}")
+                                }
+                                Err(e) => {
+                                    warn!(
+                                        "could not remove empty old state cache directory \
+                                           at {old_path:?}: {e}"
+                                    )
+                                }
+                            }
+                        }
+                    }
+                    Err(e) => {
+                        warn!("could not move state cache from {old_path:?} to {new_path:?}: {e}")
+                    }
+                }
+            }
+        }
+    }
+
     /// Returns the database options for the finalized state database.
     fn options() -> rocksdb::Options {
         let mut opts = rocksdb::Options::default();

zebra-state/src/service/finalized_state/zebra_db.rs

@@ -19,6 +19,7 @@ use zebra_chain::parameters::Network;
 
 use crate::{
     config::database_format_version_on_disk,
+    constants::RESTORABLE_DB_VERSIONS,
     service::finalized_state::{
         disk_db::DiskDb,
         disk_format::{
@@ -106,6 +107,14 @@ impl ZebraDb {
         )
         .expect("unable to read database format version file");
 
+        DiskDb::try_reusing_previous_db_after_major_upgrade(
+            &RESTORABLE_DB_VERSIONS,
+            format_version_in_code,
+            config,
+            &db_kind,
+            network,
+        );
+
         // Log any format changes before opening the database, in case opening fails.
         let format_change = DbFormatChange::open_database(format_version_in_code, disk_version);