IndexLimitMb option adds 'Unspecified' state (#24249)
This commit is contained in:
parent
605036c117
commit
1bc49d219d
|
@ -33,7 +33,7 @@ use {
|
||||||
solana_measure::measure::Measure,
|
solana_measure::measure::Measure,
|
||||||
solana_runtime::{
|
solana_runtime::{
|
||||||
accounts_db::{AccountsDbConfig, FillerAccountsConfig},
|
accounts_db::{AccountsDbConfig, FillerAccountsConfig},
|
||||||
accounts_index::{AccountsIndexConfig, ScanConfig},
|
accounts_index::{AccountsIndexConfig, IndexLimitMb, ScanConfig},
|
||||||
bank::{Bank, RewardCalculationEvent},
|
bank::{Bank, RewardCalculationEvent},
|
||||||
bank_forks::BankForks,
|
bank_forks::BankForks,
|
||||||
cost_model::CostModel,
|
cost_model::CostModel,
|
||||||
|
@ -2059,13 +2059,15 @@ fn main() {
|
||||||
let system_monitor_service =
|
let system_monitor_service =
|
||||||
SystemMonitorService::new(Arc::clone(&exit_signal), true, false);
|
SystemMonitorService::new(Arc::clone(&exit_signal), true, false);
|
||||||
|
|
||||||
if let Some(limit) =
|
accounts_index_config.index_limit_mb = if let Some(limit) =
|
||||||
value_t!(arg_matches, "accounts_index_memory_limit_mb", usize).ok()
|
value_t!(arg_matches, "accounts_index_memory_limit_mb", usize).ok()
|
||||||
{
|
{
|
||||||
accounts_index_config.index_limit_mb = Some(limit);
|
IndexLimitMb::Limit(limit)
|
||||||
} else if arg_matches.is_present("disable_accounts_disk_index") {
|
} else if arg_matches.is_present("disable_accounts_disk_index") {
|
||||||
accounts_index_config.index_limit_mb = None;
|
IndexLimitMb::InMemOnly
|
||||||
}
|
} else {
|
||||||
|
IndexLimitMb::Unspecified
|
||||||
|
};
|
||||||
|
|
||||||
{
|
{
|
||||||
let mut accounts_index_paths: Vec<PathBuf> =
|
let mut accounts_index_paths: Vec<PathBuf> =
|
||||||
|
|
|
@ -50,7 +50,7 @@ pub const ACCOUNTS_INDEX_CONFIG_FOR_TESTING: AccountsIndexConfig = AccountsIndex
|
||||||
bins: Some(BINS_FOR_TESTING),
|
bins: Some(BINS_FOR_TESTING),
|
||||||
flush_threads: Some(FLUSH_THREADS_TESTING),
|
flush_threads: Some(FLUSH_THREADS_TESTING),
|
||||||
drives: None,
|
drives: None,
|
||||||
index_limit_mb: None,
|
index_limit_mb: IndexLimitMb::Unspecified,
|
||||||
ages_to_stay_in_cache: None,
|
ages_to_stay_in_cache: None,
|
||||||
scan_results_limit_bytes: None,
|
scan_results_limit_bytes: None,
|
||||||
started_from_validator: false,
|
started_from_validator: false,
|
||||||
|
@ -59,7 +59,7 @@ pub const ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS: AccountsIndexConfig = AccountsIn
|
||||||
bins: Some(BINS_FOR_BENCHMARKS),
|
bins: Some(BINS_FOR_BENCHMARKS),
|
||||||
flush_threads: Some(FLUSH_THREADS_TESTING),
|
flush_threads: Some(FLUSH_THREADS_TESTING),
|
||||||
drives: None,
|
drives: None,
|
||||||
index_limit_mb: None,
|
index_limit_mb: IndexLimitMb::Unspecified,
|
||||||
ages_to_stay_in_cache: None,
|
ages_to_stay_in_cache: None,
|
||||||
scan_results_limit_bytes: None,
|
scan_results_limit_bytes: None,
|
||||||
started_from_validator: false,
|
started_from_validator: false,
|
||||||
|
@ -157,12 +157,29 @@ pub struct AccountSecondaryIndexesIncludeExclude {
|
||||||
pub keys: HashSet<Pubkey>,
|
pub keys: HashSet<Pubkey>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// specification of how much memory in-mem portion of account index can use
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub enum IndexLimitMb {
|
||||||
|
/// nothing explicit specified, so default
|
||||||
|
Unspecified,
|
||||||
|
/// limit was specified, use disk index for rest
|
||||||
|
Limit(usize),
|
||||||
|
/// in-mem-only was specified, no disk index
|
||||||
|
InMemOnly,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for IndexLimitMb {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::Unspecified
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Default, Clone)]
|
#[derive(Debug, Default, Clone)]
|
||||||
pub struct AccountsIndexConfig {
|
pub struct AccountsIndexConfig {
|
||||||
pub bins: Option<usize>,
|
pub bins: Option<usize>,
|
||||||
pub flush_threads: Option<usize>,
|
pub flush_threads: Option<usize>,
|
||||||
pub drives: Option<Vec<PathBuf>>,
|
pub drives: Option<Vec<PathBuf>>,
|
||||||
pub index_limit_mb: Option<usize>,
|
pub index_limit_mb: IndexLimitMb,
|
||||||
pub ages_to_stay_in_cache: Option<Age>,
|
pub ages_to_stay_in_cache: Option<Age>,
|
||||||
pub scan_results_limit_bytes: Option<usize>,
|
pub scan_results_limit_bytes: Option<usize>,
|
||||||
/// true if the accounts index is being created as a result of being started as a validator (as opposed to test, etc.)
|
/// true if the accounts index is being created as a result of being started as a validator (as opposed to test, etc.)
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
use {
|
use {
|
||||||
crate::{
|
crate::{
|
||||||
accounts_index::{AccountsIndexConfig, IndexValue},
|
accounts_index::{AccountsIndexConfig, IndexLimitMb, IndexValue},
|
||||||
bucket_map_holder_stats::BucketMapHolderStats,
|
bucket_map_holder_stats::BucketMapHolderStats,
|
||||||
in_mem_accounts_index::{InMemAccountsIndex, SlotT},
|
in_mem_accounts_index::{InMemAccountsIndex, SlotT},
|
||||||
waitable_condvar::WaitableCondvar,
|
waitable_condvar::WaitableCondvar,
|
||||||
|
@ -153,7 +153,13 @@ impl<T: IndexValue> BucketMapHolder<T> {
|
||||||
|
|
||||||
let mut bucket_config = BucketMapConfig::new(bins);
|
let mut bucket_config = BucketMapConfig::new(bins);
|
||||||
bucket_config.drives = config.as_ref().and_then(|config| config.drives.clone());
|
bucket_config.drives = config.as_ref().and_then(|config| config.drives.clone());
|
||||||
let mut mem_budget_mb = config.as_ref().and_then(|config| config.index_limit_mb);
|
let mut mem_budget_mb = config.as_ref().and_then(|config| {
|
||||||
|
if let IndexLimitMb::Limit(mb) = config.index_limit_mb {
|
||||||
|
Some(mb)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
});
|
||||||
let bucket_map_tests_allowed = mem_budget_mb.is_none()
|
let bucket_map_tests_allowed = mem_budget_mb.is_none()
|
||||||
&& !config
|
&& !config
|
||||||
.as_ref()
|
.as_ref()
|
||||||
|
@ -391,7 +397,7 @@ pub mod tests {
|
||||||
fn test_disk_index_enabled() {
|
fn test_disk_index_enabled() {
|
||||||
let bins = 1;
|
let bins = 1;
|
||||||
let config = AccountsIndexConfig {
|
let config = AccountsIndexConfig {
|
||||||
index_limit_mb: Some(0),
|
index_limit_mb: IndexLimitMb::Limit(0),
|
||||||
..AccountsIndexConfig::default()
|
..AccountsIndexConfig::default()
|
||||||
};
|
};
|
||||||
let test = BucketMapHolder::<u64>::new(bins, &Some(config), 1);
|
let test = BucketMapHolder::<u64>::new(bins, &Some(config), 1);
|
||||||
|
|
|
@ -1263,7 +1263,7 @@ impl Drop for FlushGuard<'_> {
|
||||||
mod tests {
|
mod tests {
|
||||||
use {
|
use {
|
||||||
super::*,
|
super::*,
|
||||||
crate::accounts_index::{AccountsIndexConfig, BINS_FOR_TESTING},
|
crate::accounts_index::{AccountsIndexConfig, IndexLimitMb, BINS_FOR_TESTING},
|
||||||
itertools::Itertools,
|
itertools::Itertools,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -1281,7 +1281,7 @@ mod tests {
|
||||||
let holder = Arc::new(BucketMapHolder::new(
|
let holder = Arc::new(BucketMapHolder::new(
|
||||||
BINS_FOR_TESTING,
|
BINS_FOR_TESTING,
|
||||||
&Some(AccountsIndexConfig {
|
&Some(AccountsIndexConfig {
|
||||||
index_limit_mb: Some(1),
|
index_limit_mb: IndexLimitMb::Limit(1),
|
||||||
..AccountsIndexConfig::default()
|
..AccountsIndexConfig::default()
|
||||||
}),
|
}),
|
||||||
1,
|
1,
|
||||||
|
|
|
@ -47,7 +47,7 @@ use {
|
||||||
},
|
},
|
||||||
accounts_index::{
|
accounts_index::{
|
||||||
AccountIndex, AccountSecondaryIndexes, AccountSecondaryIndexesIncludeExclude,
|
AccountIndex, AccountSecondaryIndexes, AccountSecondaryIndexesIncludeExclude,
|
||||||
AccountsIndexConfig,
|
AccountsIndexConfig, IndexLimitMb,
|
||||||
},
|
},
|
||||||
hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
|
hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
|
||||||
runtime_config::RuntimeConfig,
|
runtime_config::RuntimeConfig,
|
||||||
|
@ -2234,11 +2234,14 @@ pub fn main() {
|
||||||
accounts_index_config.bins = Some(bins);
|
accounts_index_config.bins = Some(bins);
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(limit) = value_t!(matches, "accounts_index_memory_limit_mb", usize).ok() {
|
accounts_index_config.index_limit_mb =
|
||||||
accounts_index_config.index_limit_mb = Some(limit);
|
if let Some(limit) = value_t!(matches, "accounts_index_memory_limit_mb", usize).ok() {
|
||||||
} else if matches.is_present("disable_accounts_disk_index") {
|
IndexLimitMb::Limit(limit)
|
||||||
accounts_index_config.index_limit_mb = None;
|
} else if matches.is_present("disable_accounts_disk_index") {
|
||||||
}
|
IndexLimitMb::InMemOnly
|
||||||
|
} else {
|
||||||
|
IndexLimitMb::Unspecified
|
||||||
|
};
|
||||||
|
|
||||||
{
|
{
|
||||||
let mut accounts_index_paths: Vec<PathBuf> = if matches.is_present("accounts_index_path") {
|
let mut accounts_index_paths: Vec<PathBuf> = if matches.is_present("accounts_index_path") {
|
||||||
|
|
Loading…
Reference in New Issue