remove cli arg to specify # accounts hash scan passes (#28976)
This commit is contained in:
parent
4d338ed882
commit
e42649a8ba
|
@ -337,7 +337,6 @@ pub const ACCOUNTS_DB_CONFIG_FOR_TESTING: AccountsDbConfig = AccountsDbConfig {
|
||||||
index: Some(ACCOUNTS_INDEX_CONFIG_FOR_TESTING),
|
index: Some(ACCOUNTS_INDEX_CONFIG_FOR_TESTING),
|
||||||
accounts_hash_cache_path: None,
|
accounts_hash_cache_path: None,
|
||||||
filler_accounts_config: FillerAccountsConfig::const_default(),
|
filler_accounts_config: FillerAccountsConfig::const_default(),
|
||||||
hash_calc_num_passes: None,
|
|
||||||
write_cache_limit_bytes: None,
|
write_cache_limit_bytes: None,
|
||||||
skip_rewrites: false,
|
skip_rewrites: false,
|
||||||
ancient_append_vec_offset: None,
|
ancient_append_vec_offset: None,
|
||||||
|
@ -348,7 +347,6 @@ pub const ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS: AccountsDbConfig = AccountsDbConfig
|
||||||
index: Some(ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS),
|
index: Some(ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS),
|
||||||
accounts_hash_cache_path: None,
|
accounts_hash_cache_path: None,
|
||||||
filler_accounts_config: FillerAccountsConfig::const_default(),
|
filler_accounts_config: FillerAccountsConfig::const_default(),
|
||||||
hash_calc_num_passes: None,
|
|
||||||
write_cache_limit_bytes: None,
|
write_cache_limit_bytes: None,
|
||||||
skip_rewrites: false,
|
skip_rewrites: false,
|
||||||
ancient_append_vec_offset: None,
|
ancient_append_vec_offset: None,
|
||||||
|
@ -406,7 +404,6 @@ pub struct AccountsDbConfig {
|
||||||
pub index: Option<AccountsIndexConfig>,
|
pub index: Option<AccountsIndexConfig>,
|
||||||
pub accounts_hash_cache_path: Option<PathBuf>,
|
pub accounts_hash_cache_path: Option<PathBuf>,
|
||||||
pub filler_accounts_config: FillerAccountsConfig,
|
pub filler_accounts_config: FillerAccountsConfig,
|
||||||
pub hash_calc_num_passes: Option<usize>,
|
|
||||||
pub write_cache_limit_bytes: Option<u64>,
|
pub write_cache_limit_bytes: Option<u64>,
|
||||||
pub skip_rewrites: bool,
|
pub skip_rewrites: bool,
|
||||||
/// if None, ancient append vecs are disabled
|
/// if None, ancient append vecs are disabled
|
||||||
|
@ -1390,12 +1387,6 @@ pub struct AccountsDb {
|
||||||
|
|
||||||
pub(crate) verify_accounts_hash_in_bg: VerifyAccountsHashInBackground,
|
pub(crate) verify_accounts_hash_in_bg: VerifyAccountsHashInBackground,
|
||||||
|
|
||||||
// # of passes should be a function of the total # of accounts that are active.
|
|
||||||
// higher passes = slower total time, lower dynamic memory usage
|
|
||||||
// lower passes = faster total time, higher dynamic memory usage
|
|
||||||
// passes=2 cuts dynamic memory usage in approximately half.
|
|
||||||
pub num_hash_scan_passes: Option<usize>,
|
|
||||||
|
|
||||||
/// Used to disable logging dead slots during removal.
|
/// Used to disable logging dead slots during removal.
|
||||||
/// allow disabling noisy log
|
/// allow disabling noisy log
|
||||||
pub(crate) log_dead_slots: AtomicBool,
|
pub(crate) log_dead_slots: AtomicBool,
|
||||||
|
@ -2277,12 +2268,12 @@ impl<'a> AppendVecScan for ScanState<'a> {
|
||||||
|
|
||||||
impl AccountsDb {
|
impl AccountsDb {
|
||||||
pub fn default_for_tests() -> Self {
|
pub fn default_for_tests() -> Self {
|
||||||
Self::default_with_accounts_index(AccountInfoAccountsIndex::default_for_tests(), None, None)
|
Self::default_with_accounts_index(AccountInfoAccountsIndex::default_for_tests(), None)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// return (num_hash_scan_passes, bins_per_pass)
|
/// return (num_hash_scan_passes, bins_per_pass)
|
||||||
fn bins_per_pass(num_hash_scan_passes: Option<usize>) -> (usize, usize) {
|
fn bins_per_pass() -> (usize, usize) {
|
||||||
let num_hash_scan_passes = num_hash_scan_passes.unwrap_or(NUM_SCAN_PASSES_DEFAULT);
|
let num_hash_scan_passes = NUM_SCAN_PASSES_DEFAULT;
|
||||||
let bins_per_pass = PUBKEY_BINS_FOR_CALCULATING_HASHES / num_hash_scan_passes;
|
let bins_per_pass = PUBKEY_BINS_FOR_CALCULATING_HASHES / num_hash_scan_passes;
|
||||||
assert!(
|
assert!(
|
||||||
num_hash_scan_passes <= PUBKEY_BINS_FOR_CALCULATING_HASHES,
|
num_hash_scan_passes <= PUBKEY_BINS_FOR_CALCULATING_HASHES,
|
||||||
|
@ -2300,7 +2291,6 @@ impl AccountsDb {
|
||||||
fn default_with_accounts_index(
|
fn default_with_accounts_index(
|
||||||
accounts_index: AccountInfoAccountsIndex,
|
accounts_index: AccountInfoAccountsIndex,
|
||||||
accounts_hash_cache_path: Option<PathBuf>,
|
accounts_hash_cache_path: Option<PathBuf>,
|
||||||
num_hash_scan_passes: Option<usize>,
|
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let num_threads = get_thread_count();
|
let num_threads = get_thread_count();
|
||||||
const MAX_READ_ONLY_CACHE_DATA_SIZE: usize = 400_000_000; // 400M bytes
|
const MAX_READ_ONLY_CACHE_DATA_SIZE: usize = 400_000_000; // 400M bytes
|
||||||
|
@ -2318,9 +2308,6 @@ impl AccountsDb {
|
||||||
let mut bank_hashes = HashMap::new();
|
let mut bank_hashes = HashMap::new();
|
||||||
bank_hashes.insert(0, BankHashInfo::default());
|
bank_hashes.insert(0, BankHashInfo::default());
|
||||||
|
|
||||||
// validate inside here
|
|
||||||
Self::bins_per_pass(num_hash_scan_passes);
|
|
||||||
|
|
||||||
// Increase the stack for accounts threads
|
// Increase the stack for accounts threads
|
||||||
// rayon needs a lot of stack
|
// rayon needs a lot of stack
|
||||||
const ACCOUNTS_STACK_SIZE: usize = 8 * 1024 * 1024;
|
const ACCOUNTS_STACK_SIZE: usize = 8 * 1024 * 1024;
|
||||||
|
@ -2381,7 +2368,6 @@ impl AccountsDb {
|
||||||
accounts_update_notifier: None,
|
accounts_update_notifier: None,
|
||||||
filler_accounts_config: FillerAccountsConfig::default(),
|
filler_accounts_config: FillerAccountsConfig::default(),
|
||||||
filler_account_suffix: None,
|
filler_account_suffix: None,
|
||||||
num_hash_scan_passes,
|
|
||||||
log_dead_slots: AtomicBool::new(true),
|
log_dead_slots: AtomicBool::new(true),
|
||||||
exhaustively_verify_refcounts: false,
|
exhaustively_verify_refcounts: false,
|
||||||
epoch_accounts_hash_manager: EpochAccountsHashManager::new_invalid(),
|
epoch_accounts_hash_manager: EpochAccountsHashManager::new_invalid(),
|
||||||
|
@ -2477,13 +2463,7 @@ impl AccountsDb {
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(|x| x.write_cache_limit_bytes),
|
.and_then(|x| x.write_cache_limit_bytes),
|
||||||
exhaustively_verify_refcounts,
|
exhaustively_verify_refcounts,
|
||||||
..Self::default_with_accounts_index(
|
..Self::default_with_accounts_index(accounts_index, accounts_hash_cache_path)
|
||||||
accounts_index,
|
|
||||||
accounts_hash_cache_path,
|
|
||||||
accounts_db_config
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|cfg| cfg.hash_calc_num_passes),
|
|
||||||
)
|
|
||||||
};
|
};
|
||||||
if paths_is_empty {
|
if paths_is_empty {
|
||||||
// Create a temporary set of accounts directories, used primarily
|
// Create a temporary set of accounts directories, used primarily
|
||||||
|
@ -7629,7 +7609,7 @@ impl AccountsDb {
|
||||||
|
|
||||||
self.mark_old_slots_as_dirty(storages, config.epoch_schedule.slots_per_epoch, &mut stats);
|
self.mark_old_slots_as_dirty(storages, config.epoch_schedule.slots_per_epoch, &mut stats);
|
||||||
|
|
||||||
let (num_hash_scan_passes, bins_per_pass) = Self::bins_per_pass(self.num_hash_scan_passes);
|
let (num_hash_scan_passes, bins_per_pass) = Self::bins_per_pass();
|
||||||
let use_bg_thread_pool = config.use_bg_thread_pool;
|
let use_bg_thread_pool = config.use_bg_thread_pool;
|
||||||
let mut scan_and_hash = || {
|
let mut scan_and_hash = || {
|
||||||
let mut previous_pass = PreviousPass::default();
|
let mut previous_pass = PreviousPass::default();
|
||||||
|
|
|
@ -1223,14 +1223,6 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> {
|
||||||
.takes_value(true)
|
.takes_value(true)
|
||||||
.help("Number of bins to divide the accounts index into"),
|
.help("Number of bins to divide the accounts index into"),
|
||||||
)
|
)
|
||||||
.arg(
|
|
||||||
Arg::with_name("accounts_hash_num_passes")
|
|
||||||
.long("accounts-hash-num-passes")
|
|
||||||
.value_name("PASSES")
|
|
||||||
.validator(is_pow2)
|
|
||||||
.takes_value(true)
|
|
||||||
.help("Number of passes to calculate the hash of all accounts"),
|
|
||||||
)
|
|
||||||
.arg(
|
.arg(
|
||||||
Arg::with_name("accounts_index_path")
|
Arg::with_name("accounts_index_path")
|
||||||
.long("accounts-index-path")
|
.long("accounts-index-path")
|
||||||
|
|
|
@ -958,7 +958,7 @@ pub fn main() {
|
||||||
size: value_t_or_exit!(matches, "accounts_filler_size", usize),
|
size: value_t_or_exit!(matches, "accounts_filler_size", usize),
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut accounts_db_config = AccountsDbConfig {
|
let accounts_db_config = AccountsDbConfig {
|
||||||
index: Some(accounts_index_config),
|
index: Some(accounts_index_config),
|
||||||
accounts_hash_cache_path: Some(ledger_path.clone()),
|
accounts_hash_cache_path: Some(ledger_path.clone()),
|
||||||
filler_accounts_config,
|
filler_accounts_config,
|
||||||
|
@ -971,9 +971,6 @@ pub fn main() {
|
||||||
..AccountsDbConfig::default()
|
..AccountsDbConfig::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Some(passes) = value_t!(matches, "accounts_hash_num_passes", usize).ok() {
|
|
||||||
accounts_db_config.hash_calc_num_passes = Some(passes);
|
|
||||||
}
|
|
||||||
let accounts_db_config = Some(accounts_db_config);
|
let accounts_db_config = Some(accounts_db_config);
|
||||||
|
|
||||||
let geyser_plugin_config_files = if matches.is_present("geyser_plugin_config") {
|
let geyser_plugin_config_files = if matches.is_present("geyser_plugin_config") {
|
||||||
|
|
Loading…
Reference in New Issue