Derive rocksdb_fifo_shred_storage_size based on limit-ledger-size (#27459)

### Problem
When FIFO compaction is used while --rocksdb_fifo_shred_storage_size
is unspecified, the FIFO shred storage size is set to a const default based
on the default `--limit-ledger-size`.

### Summary of the Change
When --rocksdb_fifo_shred_storage_size is unspecified, it is now
derived from `--limit-ledger-size` by reserving 1500 bytes for each
shred.

ledger-tool/src/main.rs

@@ -32,7 +32,7 @@ use {
         blockstore_db::{self, columns as cf, Column, ColumnName, Database},
         blockstore_options::{
             AccessType, BlockstoreOptions, BlockstoreRecoveryMode, BlockstoreRocksFifoOptions,
-            LedgerColumnOptions, ShredStorageType,
+            LedgerColumnOptions, ShredStorageType, MAX_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
         },
         blockstore_processor::{self, BlockstoreProcessorError, ProcessOptions},
         shred::Shred,
@@ -871,14 +871,11 @@ fn open_blockstore_with_temporary_primary_access(
 }
 
 fn get_shred_storage_type(ledger_path: &Path, warn_message: &str) -> ShredStorageType {
-    // TODO: the following shred_storage_type inference must be updated once the
-    // rocksdb options can be constructed via load_options_file() as the
-    // temporary use of DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES
-    // could affect the persisted rocksdb options file.
-    match ShredStorageType::from_ledger_path(
-        ledger_path,
-        DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
-    ) {
+    // TODO: the following shred_storage_type inference must be updated once
+    // the rocksdb options can be constructed via load_options_file() as the
+    // temporary use of MAX_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES could
+    // affect the persisted rocksdb options file.
+    match ShredStorageType::from_ledger_path(ledger_path, MAX_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES) {
         Some(s) => s,
         None => {
             warn!("{}", warn_message);
@@ -1278,10 +1275,6 @@ use jemallocator::Jemalloc;
 #[global_allocator]
 static GLOBAL: Jemalloc = Jemalloc;
 
-/// The default size for data and coding shred column families in FIFO compaction.
-/// u64::MAX as the default value means it won't delete any files by default.
-const DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES: u64 = std::u64::MAX;
-
 #[allow(clippy::cognitive_complexity)]
 fn main() {
     // Ignore SIGUSR1 to prevent long-running calls being killed by logrotate

ledger/src/blockstore_options.rs

@@ -217,6 +217,8 @@ pub struct BlockstoreRocksFifoOptions {
 // in bytes will equally allocated to both data shreds and coding shreds.
 pub const DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES: u64 = 250 * 1024 * 1024 * 1024;
 
+pub const MAX_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES: u64 = std::u64::MAX;
+
 impl Default for BlockstoreRocksFifoOptions {
     fn default() -> Self {
         BlockstoreRocksFifoOptions::new(DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES)

validator/src/main.rs

@@ -28,7 +28,7 @@ use {
     solana_gossip::{cluster_info::Node, contact_info::ContactInfo},
     solana_ledger::blockstore_options::{
         BlockstoreCompressionType, BlockstoreRecoveryMode, LedgerColumnOptions, ShredStorageType,
-        DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
+        MAX_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
     },
     solana_net_utils::VALIDATOR_PORT_RANGE,
     solana_perf::recycler::enable_recycler_warming,
@@ -392,6 +392,21 @@ fn hash_validator(hash: String) -> Result<(), String> {
         .map_err(|e| format!("{:?}", e))
 }
 
+/// Returns the default shred storage size (include both data and coding
+/// shreds) based on the validator config.
+fn default_fifo_shred_storage_size(vc: &ValidatorConfig) -> u64 {
+    // The max shred size is around 1228 bytes.
+    // Here we reserve a little bit more than that to give extra storage for FIFO
+    // to prevent it from purging data that have not yet being marked as obsoleted
+    // by LedgerCleanupService.
+    const RESERVED_BYTES_PER_SHRED: u64 = 1500;
+    match vc.max_ledger_shreds {
+        // x2 as we have data shred and coding shred.
+        Some(max_ledger_shreds) => max_ledger_shreds * RESERVED_BYTES_PER_SHRED * 2,
+        None => MAX_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
+    }
+}
+
 // This function is duplicated in ledger-tool/src/main.rs...
 fn hardforks_of(matches: &ArgMatches<'_>, name: &str) -> Option<Vec<Slot>> {
     if matches.is_present(name) {
@@ -489,8 +504,6 @@ pub fn main() {
     let default_accounts_shrink_optimize_total_space =
         &DEFAULT_ACCOUNTS_SHRINK_OPTIMIZE_TOTAL_SPACE.to_string();
     let default_accounts_shrink_ratio = &DEFAULT_ACCOUNTS_SHRINK_RATIO.to_string();
-    let default_rocksdb_fifo_shred_storage_size =
-        &DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES.to_string();
     let default_tpu_connection_pool_size = &DEFAULT_TPU_CONNECTION_POOL_SIZE.to_string();
     let default_rpc_max_request_body_size = &MAX_REQUEST_BODY_SIZE.to_string();
 
@@ -1043,7 +1056,7 @@ pub fn main() {
                 .takes_value(true)
                 .possible_values(&["level", "fifo"])
                 .default_value("level")
-                .help("EXPERIMENTAL: Controls how RocksDB compacts shreds. \
+                .help("Controls how RocksDB compacts shreds. \
                        *WARNING*: You will lose your ledger data when you switch between options. \
                        Possible values are: \
                        'level': stores shreds using RocksDB's default (level) compaction. \
@@ -1057,9 +1070,13 @@ pub fn main() {
                 .value_name("SHRED_STORAGE_SIZE_BYTES")
                 .takes_value(true)
                 .validator(is_parsable::<u64>)
-                .default_value(default_rocksdb_fifo_shred_storage_size)
                 .help("The shred storage size in bytes. \
-                       The suggested value is 50% of your ledger storage size in bytes."),
+                       The suggested value is at least 50% of your ledger storage size. \
+                       If this argument is unspecified, we will assign a proper \
+                       value based on --limit-ledger-size.  If --limit-ledger-size \
+                       is not presented, it means there is no limitation on the ledger \
+                       size and thus rocksdb_fifo_shred_storage_size will also be \
+                       unbounded."),
         )
         .arg(
             Arg::with_name("rocksdb_ledger_compression")
@@ -3034,11 +3051,16 @@ pub fn main() {
             None => ShredStorageType::default(),
             Some(shred_compaction_string) => match shred_compaction_string {
                 "level" => ShredStorageType::RocksLevel,
-                "fifo" => {
-                    let shred_storage_size =
-                        value_t_or_exit!(matches, "rocksdb_fifo_shred_storage_size", u64);
-                    ShredStorageType::rocks_fifo(shred_storage_size)
-                }
+                "fifo" => match matches.value_of("rocksdb_fifo_shred_storage_size") {
+                    None => ShredStorageType::rocks_fifo(default_fifo_shred_storage_size(
+                        &validator_config,
+                    )),
+                    Some(_) => ShredStorageType::rocks_fifo(value_t_or_exit!(
+                        matches,
+                        "rocksdb_fifo_shred_storage_size",
+                        u64
+                    )),
+                },
                 _ => panic!(
                     "Unrecognized rocksdb-shred-compaction: {}",
                     shred_compaction_string