From 49fca309cfe51b6a23a94e6404bd9036bd812588 Mon Sep 17 00:00:00 2001 From: Elijah Hampton Date: Thu, 28 Mar 2024 20:00:28 -0400 Subject: [PATCH] =?UTF-8?q?Adds=20logging=20of=20column=20family=20size=20?= =?UTF-8?q?and=20database=20size=20on=20startup=20and=20s=E2=80=A6=20(#833?= =?UTF-8?q?6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Adds logging of column family size and database size on startup and shutdown * Changes log level of column families size strings to debug. Adds TODO comment to use human_bytes crate for human-readable format of metrics. Adds print_db_metrics function to ZebraDb struct. * Calls enumerate() on column_families to access index var * Resolves cargo fmt checker results * Resolves clippy lint * Runs and fixes changes from fmt, clippy, check and test * Removes prop.txt * minor doc changes --------- Co-authored-by: Elijah Hampton Co-authored-by: Pili Guerra Co-authored-by: Alfredo Garcia --- zebra-state/src/service.rs | 14 ++++ .../src/service/finalized_state/disk_db.rs | 78 ++++++++++++++++++- .../src/service/finalized_state/zebra_db.rs | 8 ++ zebrad/src/commands/start.rs | 3 + 4 files changed, 102 insertions(+), 1 deletion(-) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 39f6f5e8d..bd0abe867 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -254,6 +254,10 @@ impl Drop for StateService { "dropping the state: dropped unused non-finalized state queue block", ); + // Log database metrics before shutting down + info!("dropping the state: logging database metrics"); + self.log_db_metrics(); + // Then drop self.read_service, which checks the block write task for panics, // and tries to shut down the database. } @@ -451,6 +455,11 @@ impl StateService { (state, read_service, latest_chain_tip, chain_tip_change) } + /// Call read only state service to log rocksdb database metrics. + pub fn log_db_metrics(&self) { + self.read_service.db.print_db_metrics(); + } + /// Queue a checkpoint verified block for verification and storage in the finalized state. /// /// Returns a channel receiver that provides the result of the block commit. @@ -853,6 +862,11 @@ impl ReadStateService { pub fn db(&self) -> &ZebraDb { &self.db } + + /// Logs rocksdb metrics using the read only state service. + pub fn log_db_metrics(&self) { + self.db.print_db_metrics(); + } } impl Service for StateService { diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs index 24ca542f9..cc3917752 100644 --- a/zebra-state/src/service/finalized_state/disk_db.rs +++ b/zebra-state/src/service/finalized_state/disk_db.rs @@ -13,6 +13,7 @@ use std::{ collections::{BTreeMap, HashMap}, fmt::Debug, + fmt::Write, ops::RangeBounds, path::Path, sync::Arc, @@ -21,7 +22,7 @@ use std::{ use itertools::Itertools; use rlimit::increase_nofile_limit; -use rocksdb::ReadOptions; +use rocksdb::{ColumnFamilyDescriptor, Options, ReadOptions}; use semver::Version; use zebra_chain::{parameters::Network, primitives::byte_array::increment_big_endian}; @@ -512,6 +513,55 @@ impl DiskWriteBatch { } impl DiskDb { + /// Prints rocksdb metrics for each column family along with total database disk size, live data disk size and database memory size. + pub fn print_db_metrics(&self) { + let mut total_size_on_disk = 0; + let mut total_live_size_on_disk = 0; + let mut total_size_in_mem = 0; + let db: &Arc = &self.db; + let db_options = DiskDb::options(); + let column_families = DiskDb::construct_column_families(&db_options, db.path(), &[]); + let mut column_families_log_string = String::from(""); + write!(column_families_log_string, "Column families and sizes: ").unwrap(); + for cf_descriptor in column_families.iter() { + let cf_name = &cf_descriptor.name(); + let cf_handle = db + .cf_handle(cf_name) + .expect("Column family handle must exist"); + let live_data_size = db + .property_int_value_cf(cf_handle, "rocksdb.estimate-live-data-size") + .unwrap_or(Some(0)); + let total_sst_files_size = db + .property_int_value_cf(cf_handle, "rocksdb.total-sst-files-size") + .unwrap_or(Some(0)); + let cf_disk_size = live_data_size.unwrap_or(0) + total_sst_files_size.unwrap_or(0); + total_size_on_disk += cf_disk_size; + total_live_size_on_disk += live_data_size.unwrap_or(0); + let mem_table_size = db + .property_int_value_cf(cf_handle, "rocksdb.size-all-mem-tables") + .unwrap_or(Some(0)); + total_size_in_mem += mem_table_size.unwrap_or(0); + + // TODO: Consider displaying the disk and memory sizes in a human-readable format - #8380. + write!( + column_families_log_string, + "{} (Disk: {} bytes, Memory: {} bytes)", + cf_name, + cf_disk_size, + mem_table_size.unwrap_or(0) + ) + .unwrap(); + } + + debug!("{}", column_families_log_string); + info!("Total Database Disk Size: {} bytes", total_size_on_disk); + info!( + "Total Live Data Disk Size: {} bytes", + total_live_size_on_disk + ); + info!("Total Database Memory Size: {} bytes", total_size_in_mem); + } + /// Returns a forward iterator over the items in `cf` in `range`. /// /// Holding this iterator open might delay block commit transactions. @@ -720,6 +770,32 @@ impl DiskDb { /// const MEMTABLE_RAM_CACHE_MEGABYTES: usize = 128; + /// Build a vector of current column families on the disk and optionally any new column families. + /// Returns an iterable collection of all column families. + fn construct_column_families( + db_options: &Options, + path: &Path, + column_families_in_code: &[String], + ) -> Vec { + // When opening the database in read/write mode, all column families must be opened. + // + // To make Zebra forward-compatible with databases updated by later versions, + // we read any existing column families off the disk, then add any new column families + // from the current implementation. + // + // + let column_families_on_disk = DB::list_cf(db_options, path).unwrap_or_default(); + let column_families = column_families_on_disk + .into_iter() + .chain(column_families_in_code.iter().cloned()) + .unique() + .collect::>(); + column_families + .into_iter() + .map(|cf_name| ColumnFamilyDescriptor::new(cf_name, db_options.clone())) + .collect() + } + /// Opens or creates the database at a path based on the kind, major version and network, /// with the supplied column families, preserving any existing column families, /// and returns a shared low-level database wrapper. diff --git a/zebra-state/src/service/finalized_state/zebra_db.rs b/zebra-state/src/service/finalized_state/zebra_db.rs index ee324a2cf..71c619e4f 100644 --- a/zebra-state/src/service/finalized_state/zebra_db.rs +++ b/zebra-state/src/service/finalized_state/zebra_db.rs @@ -322,6 +322,14 @@ impl ZebraDb { Ok(()) } + + /// Logs metrics related to the underlying RocksDB instance. + /// + /// This function prints various metrics and statistics about the RocksDB database, + /// such as disk usage, memory usage, and other performance-related metrics. + pub fn print_db_metrics(&self) { + self.db.print_db_metrics(); + } } impl Drop for ZebraDb { diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index 35c4a5b36..a6f2ec77f 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -131,6 +131,9 @@ impl StartCmd { ) .await?; + info!("logging database metrics on startup"); + read_only_state_service.log_db_metrics(); + let state = ServiceBuilder::new() .buffer(Self::state_buffer_bound()) .service(state_service);