diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index f7229f061..943078ea8 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -33,7 +33,7 @@ use { solana_measure::measure::Measure, solana_runtime::{ accounts_db::{AccountsDbConfig, FillerAccountsConfig}, - accounts_index::{AccountsIndexConfig, ScanConfig}, + accounts_index::{AccountsIndexConfig, IndexLimitMb, ScanConfig}, bank::{Bank, RewardCalculationEvent}, bank_forks::BankForks, cost_model::CostModel, @@ -2059,13 +2059,15 @@ fn main() { let system_monitor_service = SystemMonitorService::new(Arc::clone(&exit_signal), true, false); - if let Some(limit) = + accounts_index_config.index_limit_mb = if let Some(limit) = value_t!(arg_matches, "accounts_index_memory_limit_mb", usize).ok() { - accounts_index_config.index_limit_mb = Some(limit); + IndexLimitMb::Limit(limit) } else if arg_matches.is_present("disable_accounts_disk_index") { - accounts_index_config.index_limit_mb = None; - } + IndexLimitMb::InMemOnly + } else { + IndexLimitMb::Unspecified + }; { let mut accounts_index_paths: Vec = diff --git a/runtime/src/accounts_index.rs b/runtime/src/accounts_index.rs index 8241db2dc..5ee58cf49 100644 --- a/runtime/src/accounts_index.rs +++ b/runtime/src/accounts_index.rs @@ -50,7 +50,7 @@ pub const ACCOUNTS_INDEX_CONFIG_FOR_TESTING: AccountsIndexConfig = AccountsIndex bins: Some(BINS_FOR_TESTING), flush_threads: Some(FLUSH_THREADS_TESTING), drives: None, - index_limit_mb: None, + index_limit_mb: IndexLimitMb::Unspecified, ages_to_stay_in_cache: None, scan_results_limit_bytes: None, started_from_validator: false, @@ -59,7 +59,7 @@ pub const ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS: AccountsIndexConfig = AccountsIn bins: Some(BINS_FOR_BENCHMARKS), flush_threads: Some(FLUSH_THREADS_TESTING), drives: None, - index_limit_mb: None, + index_limit_mb: IndexLimitMb::Unspecified, ages_to_stay_in_cache: None, scan_results_limit_bytes: None, started_from_validator: false, @@ -157,12 +157,29 @@ pub struct AccountSecondaryIndexesIncludeExclude { pub keys: HashSet, } +/// specification of how much memory in-mem portion of account index can use +#[derive(Debug, Clone)] +pub enum IndexLimitMb { + /// nothing explicit specified, so default + Unspecified, + /// limit was specified, use disk index for rest + Limit(usize), + /// in-mem-only was specified, no disk index + InMemOnly, +} + +impl Default for IndexLimitMb { + fn default() -> Self { + Self::Unspecified + } +} + #[derive(Debug, Default, Clone)] pub struct AccountsIndexConfig { pub bins: Option, pub flush_threads: Option, pub drives: Option>, - pub index_limit_mb: Option, + pub index_limit_mb: IndexLimitMb, pub ages_to_stay_in_cache: Option, pub scan_results_limit_bytes: Option, /// true if the accounts index is being created as a result of being started as a validator (as opposed to test, etc.) diff --git a/runtime/src/bucket_map_holder.rs b/runtime/src/bucket_map_holder.rs index 87b2ee26f..23ab83f2d 100644 --- a/runtime/src/bucket_map_holder.rs +++ b/runtime/src/bucket_map_holder.rs @@ -1,6 +1,6 @@ use { crate::{ - accounts_index::{AccountsIndexConfig, IndexValue}, + accounts_index::{AccountsIndexConfig, IndexLimitMb, IndexValue}, bucket_map_holder_stats::BucketMapHolderStats, in_mem_accounts_index::{InMemAccountsIndex, SlotT}, waitable_condvar::WaitableCondvar, @@ -153,7 +153,13 @@ impl BucketMapHolder { let mut bucket_config = BucketMapConfig::new(bins); bucket_config.drives = config.as_ref().and_then(|config| config.drives.clone()); - let mut mem_budget_mb = config.as_ref().and_then(|config| config.index_limit_mb); + let mut mem_budget_mb = config.as_ref().and_then(|config| { + if let IndexLimitMb::Limit(mb) = config.index_limit_mb { + Some(mb) + } else { + None + } + }); let bucket_map_tests_allowed = mem_budget_mb.is_none() && !config .as_ref() @@ -391,7 +397,7 @@ pub mod tests { fn test_disk_index_enabled() { let bins = 1; let config = AccountsIndexConfig { - index_limit_mb: Some(0), + index_limit_mb: IndexLimitMb::Limit(0), ..AccountsIndexConfig::default() }; let test = BucketMapHolder::::new(bins, &Some(config), 1); diff --git a/runtime/src/in_mem_accounts_index.rs b/runtime/src/in_mem_accounts_index.rs index fbb3a4f61..32cde7d41 100644 --- a/runtime/src/in_mem_accounts_index.rs +++ b/runtime/src/in_mem_accounts_index.rs @@ -1263,7 +1263,7 @@ impl Drop for FlushGuard<'_> { mod tests { use { super::*, - crate::accounts_index::{AccountsIndexConfig, BINS_FOR_TESTING}, + crate::accounts_index::{AccountsIndexConfig, IndexLimitMb, BINS_FOR_TESTING}, itertools::Itertools, }; @@ -1281,7 +1281,7 @@ mod tests { let holder = Arc::new(BucketMapHolder::new( BINS_FOR_TESTING, &Some(AccountsIndexConfig { - index_limit_mb: Some(1), + index_limit_mb: IndexLimitMb::Limit(1), ..AccountsIndexConfig::default() }), 1, diff --git a/validator/src/main.rs b/validator/src/main.rs index 181a522f1..3f523792b 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -47,7 +47,7 @@ use { }, accounts_index::{ AccountIndex, AccountSecondaryIndexes, AccountSecondaryIndexesIncludeExclude, - AccountsIndexConfig, + AccountsIndexConfig, IndexLimitMb, }, hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, runtime_config::RuntimeConfig, @@ -2234,11 +2234,14 @@ pub fn main() { accounts_index_config.bins = Some(bins); } - if let Some(limit) = value_t!(matches, "accounts_index_memory_limit_mb", usize).ok() { - accounts_index_config.index_limit_mb = Some(limit); - } else if matches.is_present("disable_accounts_disk_index") { - accounts_index_config.index_limit_mb = None; - } + accounts_index_config.index_limit_mb = + if let Some(limit) = value_t!(matches, "accounts_index_memory_limit_mb", usize).ok() { + IndexLimitMb::Limit(limit) + } else if matches.is_present("disable_accounts_disk_index") { + IndexLimitMb::InMemOnly + } else { + IndexLimitMb::Unspecified + }; { let mut accounts_index_paths: Vec = if matches.is_present("accounts_index_path") {