Make ShredStorageType::RocksLevel public (#23272)

#### Summary of Changes
This PR adds two hidden arguments to the validator that allow users to use RocksDB's FIFO compaction for storing shreds.

        --shred-storage <SHRED_STORAGE>
            EXPERIMENTAL: Controls how RocksDB compacts shreds.  *WARNING*: You will lose your ledger data
            when you switch between options. Possible values are: 'level': stores shreds using RocksDB's default (level)
            compaction. 'fifo': stores shreds under RocksDB's FIFO compaction. This option is more efficient on
            disk-write-bytes of the ledger store. [default: level]  [possible values: level, fifo]

        --shred-storage-size <SHRED_STORAGE_SIZE_BYTES>
            The shred storage size in bytes. The suggested value is 50% of your ledger storage size in bytes. [default:
            268435456000]

core/src/validator.rs

@@ -36,7 +36,7 @@ use {
     solana_ledger::{
         bank_forks_utils,
         blockstore::{Blockstore, BlockstoreSignals, CompletedSlotsReceiver, PurgeType},
-        blockstore_db::{BlockstoreOptions, BlockstoreRecoveryMode},
+        blockstore_db::{BlockstoreOptions, BlockstoreRecoveryMode, ShredStorageType},
         blockstore_processor::{self, TransactionStatusSender},
         leader_schedule::FixedSchedule,
         leader_schedule_cache::LeaderScheduleCache,
@@ -165,6 +165,7 @@ pub struct ValidatorConfig {
     pub no_wait_for_vote_to_start_leader: bool,
     pub accounts_shrink_ratio: AccountShrinkThreshold,
     pub wait_to_vote_slot: Option<Slot>,
+    pub shred_storage_type: ShredStorageType,
 }
 
 impl Default for ValidatorConfig {
@@ -225,6 +226,7 @@ impl Default for ValidatorConfig {
             accounts_shrink_ratio: AccountShrinkThreshold::default(),
             accounts_db_config: None,
             wait_to_vote_slot: None,
+            shred_storage_type: ShredStorageType::RocksLevel,
         }
     }
 }
@@ -1256,6 +1258,7 @@ fn new_banks_from_ledger(
         BlockstoreOptions {
             recovery_mode: config.wal_recovery_mode.clone(),
             enforce_ulimit_nofile,
+            shred_storage_type: config.shred_storage_type.clone(),
             ..BlockstoreOptions::default()
         },
     )

ledger/src/blockstore_db.rs

@@ -35,6 +35,11 @@ use {
     thiserror::Error,
 };
 
+// The default storage size for storing shreds when `rocksdb-shred-compaction`
+// is set to `fifo` in the validator arguments.  This amount of storage size
+// in bytes will equally allocated to both data shreds and coding shreds.
+pub const DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES: u64 = 250 * 1024 * 1024 * 1024;
+
 const MAX_WRITE_BUFFER_SIZE: u64 = 256 * 1024 * 1024; // 256MB
 const FIFO_WRITE_BUFFER_SIZE: u64 = 2 * MAX_WRITE_BUFFER_SIZE;
 // Maximum size of cf::DataShred.  Used when `shred_storage_type`

local-cluster/src/validator_configs.rs

@@ -62,6 +62,7 @@ pub fn safe_clone_config(config: &ValidatorConfig) -> ValidatorConfig {
         accounts_shrink_ratio: config.accounts_shrink_ratio,
         accounts_db_config: config.accounts_db_config.clone(),
         wait_to_vote_slot: config.wait_to_vote_slot,
+        shred_storage_type: config.shred_storage_type.clone(),
     }
 }
 

validator/src/main.rs

@@ -32,7 +32,10 @@ use {
         cluster_info::{Node, VALIDATOR_PORT_RANGE},
         contact_info::ContactInfo,
     },
-    solana_ledger::blockstore_db::BlockstoreRecoveryMode,
+    solana_ledger::blockstore_db::{
+        BlockstoreRecoveryMode, BlockstoreRocksFifoOptions, ShredStorageType,
+        DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
+    },
     solana_perf::recycler::enable_recycler_warming,
     solana_poh::poh_service,
     solana_replica_lib::accountsdb_repl_server::AccountsDbReplServiceConfig,
@@ -451,6 +454,8 @@ pub fn main() {
     let default_accounts_shrink_optimize_total_space =
         &DEFAULT_ACCOUNTS_SHRINK_OPTIMIZE_TOTAL_SPACE.to_string();
     let default_accounts_shrink_ratio = &DEFAULT_ACCOUNTS_SHRINK_RATIO.to_string();
+    let default_rocksdb_fifo_shred_storage_size =
+        &DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES.to_string();
 
     let matches = App::new(crate_name!()).about(crate_description!())
         .version(solana_version::version!())
@@ -960,6 +965,32 @@ pub fn main() {
                 /* .default_value() intentionally not used here! */
                 .help("Keep this amount of shreds in root slots."),
         )
+        .arg(
+            Arg::with_name("rocksdb_shred_compaction")
+                .hidden(true)
+                .long("rocksdb-shred-compaction")
+                .value_name("ROCKSDB_COMPACTION_STYLE")
+                .takes_value(true)
+                .possible_values(&["level", "fifo"])
+                .default_value("level")
+                .help("EXPERIMENTAL: Controls how RocksDB compacts shreds. \
+                       *WARNING*: You will lose your ledger data when you switch between options. \
+                       Possible values are: \
+                       'level': stores shreds using RocksDB's default (level) compaction. \
+                       'fifo': stores shreds under RocksDB's FIFO compaction. \
+                           This option is more efficient on disk-write-bytes of the ledger store."),
+        )
+        .arg(
+            Arg::with_name("rocksdb_fifo_shred_storage_size")
+                .hidden(true)
+                .long("rocksdb-fifo-shred-storage-size")
+                .value_name("SHRED_STORAGE_SIZE_BYTES")
+                .takes_value(true)
+                .validator(is_parsable::<u64>)
+                .default_value(default_rocksdb_fifo_shred_storage_size)
+                .help("The shred storage size in bytes. \
+                       The suggested value is 50% of your ledger storage size in bytes."),
+        )
         .arg(
             Arg::with_name("skip_poh_verify")
                 .long("skip-poh-verify")
@@ -2522,6 +2553,25 @@ pub fn main() {
         validator_config.max_ledger_shreds = Some(limit_ledger_size);
     }
 
+    validator_config.shred_storage_type = match matches.value_of("rocksdb_shred_compaction") {
+        None => ShredStorageType::default(),
+        Some(shred_compaction_string) => match shred_compaction_string {
+            "level" => ShredStorageType::RocksLevel,
+            "fifo" => {
+                let shred_storage_size =
+                    value_t_or_exit!(matches, "rocksdb_fifo_shred_storage_size", u64);
+                ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions {
+                    shred_data_cf_size: shred_storage_size / 2,
+                    shred_code_cf_size: shred_storage_size / 2,
+                })
+            }
+            _ => panic!(
+                "Unrecognized rocksdb-shred-compaction: {}",
+                shred_compaction_string
+            ),
+        },
+    };
+
     if matches.is_present("halt_on_known_validators_accounts_hash_mismatch") {
         validator_config.halt_on_known_validators_accounts_hash_mismatch = true;
     }