diff --git a/core/src/validator.rs b/core/src/validator.rs index 0227cda2d..8952207a7 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -36,7 +36,7 @@ use { solana_ledger::{ bank_forks_utils, blockstore::{Blockstore, BlockstoreSignals, CompletedSlotsReceiver, PurgeType}, - blockstore_db::{BlockstoreOptions, BlockstoreRecoveryMode}, + blockstore_db::{BlockstoreOptions, BlockstoreRecoveryMode, ShredStorageType}, blockstore_processor::{self, TransactionStatusSender}, leader_schedule::FixedSchedule, leader_schedule_cache::LeaderScheduleCache, @@ -165,6 +165,7 @@ pub struct ValidatorConfig { pub no_wait_for_vote_to_start_leader: bool, pub accounts_shrink_ratio: AccountShrinkThreshold, pub wait_to_vote_slot: Option, + pub shred_storage_type: ShredStorageType, } impl Default for ValidatorConfig { @@ -225,6 +226,7 @@ impl Default for ValidatorConfig { accounts_shrink_ratio: AccountShrinkThreshold::default(), accounts_db_config: None, wait_to_vote_slot: None, + shred_storage_type: ShredStorageType::RocksLevel, } } } @@ -1256,6 +1258,7 @@ fn new_banks_from_ledger( BlockstoreOptions { recovery_mode: config.wal_recovery_mode.clone(), enforce_ulimit_nofile, + shred_storage_type: config.shred_storage_type.clone(), ..BlockstoreOptions::default() }, ) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 4c111ea38..c59a2e434 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -35,6 +35,11 @@ use { thiserror::Error, }; +// The default storage size for storing shreds when `rocksdb-shred-compaction` +// is set to `fifo` in the validator arguments. This amount of storage size +// in bytes will equally allocated to both data shreds and coding shreds. +pub const DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES: u64 = 250 * 1024 * 1024 * 1024; + const MAX_WRITE_BUFFER_SIZE: u64 = 256 * 1024 * 1024; // 256MB const FIFO_WRITE_BUFFER_SIZE: u64 = 2 * MAX_WRITE_BUFFER_SIZE; // Maximum size of cf::DataShred. Used when `shred_storage_type` diff --git a/local-cluster/src/validator_configs.rs b/local-cluster/src/validator_configs.rs index 4cca70621..389317fa5 100644 --- a/local-cluster/src/validator_configs.rs +++ b/local-cluster/src/validator_configs.rs @@ -62,6 +62,7 @@ pub fn safe_clone_config(config: &ValidatorConfig) -> ValidatorConfig { accounts_shrink_ratio: config.accounts_shrink_ratio, accounts_db_config: config.accounts_db_config.clone(), wait_to_vote_slot: config.wait_to_vote_slot, + shred_storage_type: config.shred_storage_type.clone(), } } diff --git a/validator/src/main.rs b/validator/src/main.rs index c169b97d9..da4225c83 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -32,7 +32,10 @@ use { cluster_info::{Node, VALIDATOR_PORT_RANGE}, contact_info::ContactInfo, }, - solana_ledger::blockstore_db::BlockstoreRecoveryMode, + solana_ledger::blockstore_db::{ + BlockstoreRecoveryMode, BlockstoreRocksFifoOptions, ShredStorageType, + DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES, + }, solana_perf::recycler::enable_recycler_warming, solana_poh::poh_service, solana_replica_lib::accountsdb_repl_server::AccountsDbReplServiceConfig, @@ -451,6 +454,8 @@ pub fn main() { let default_accounts_shrink_optimize_total_space = &DEFAULT_ACCOUNTS_SHRINK_OPTIMIZE_TOTAL_SPACE.to_string(); let default_accounts_shrink_ratio = &DEFAULT_ACCOUNTS_SHRINK_RATIO.to_string(); + let default_rocksdb_fifo_shred_storage_size = + &DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES.to_string(); let matches = App::new(crate_name!()).about(crate_description!()) .version(solana_version::version!()) @@ -960,6 +965,32 @@ pub fn main() { /* .default_value() intentionally not used here! */ .help("Keep this amount of shreds in root slots."), ) + .arg( + Arg::with_name("rocksdb_shred_compaction") + .hidden(true) + .long("rocksdb-shred-compaction") + .value_name("ROCKSDB_COMPACTION_STYLE") + .takes_value(true) + .possible_values(&["level", "fifo"]) + .default_value("level") + .help("EXPERIMENTAL: Controls how RocksDB compacts shreds. \ + *WARNING*: You will lose your ledger data when you switch between options. \ + Possible values are: \ + 'level': stores shreds using RocksDB's default (level) compaction. \ + 'fifo': stores shreds under RocksDB's FIFO compaction. \ + This option is more efficient on disk-write-bytes of the ledger store."), + ) + .arg( + Arg::with_name("rocksdb_fifo_shred_storage_size") + .hidden(true) + .long("rocksdb-fifo-shred-storage-size") + .value_name("SHRED_STORAGE_SIZE_BYTES") + .takes_value(true) + .validator(is_parsable::) + .default_value(default_rocksdb_fifo_shred_storage_size) + .help("The shred storage size in bytes. \ + The suggested value is 50% of your ledger storage size in bytes."), + ) .arg( Arg::with_name("skip_poh_verify") .long("skip-poh-verify") @@ -2522,6 +2553,25 @@ pub fn main() { validator_config.max_ledger_shreds = Some(limit_ledger_size); } + validator_config.shred_storage_type = match matches.value_of("rocksdb_shred_compaction") { + None => ShredStorageType::default(), + Some(shred_compaction_string) => match shred_compaction_string { + "level" => ShredStorageType::RocksLevel, + "fifo" => { + let shred_storage_size = + value_t_or_exit!(matches, "rocksdb_fifo_shred_storage_size", u64); + ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions { + shred_data_cf_size: shred_storage_size / 2, + shred_code_cf_size: shred_storage_size / 2, + }) + } + _ => panic!( + "Unrecognized rocksdb-shred-compaction: {}", + shred_compaction_string + ), + }, + }; + if matches.is_present("halt_on_known_validators_accounts_hash_mismatch") { validator_config.halt_on_known_validators_accounts_hash_mismatch = true; }