diff --git a/clap-utils/src/input_validators.rs b/clap-utils/src/input_validators.rs index bb9fad1722..e9a8b9f3f9 100644 --- a/clap-utils/src/input_validators.rs +++ b/clap-utils/src/input_validators.rs @@ -237,6 +237,22 @@ where is_parsable_generic::(slot) } +pub fn is_bin(bins: T) -> Result<(), String> +where + T: AsRef + Display, +{ + bins.as_ref() + .parse::() + .map_err(|e| format!("Unable to parse bins, provided: {}, err: {}", bins, e)) + .and_then(|v| { + if !v.is_power_of_two() { + Err(format!("Bins must be a power of 2: {}", v)) + } else { + Ok(()) + } + }) +} + pub fn is_port(port: T) -> Result<(), String> where T: AsRef + Display, diff --git a/core/src/validator.rs b/core/src/validator.rs index 4ee13192e4..555c3d8edd 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -138,6 +138,7 @@ pub struct ValidatorConfig { pub poh_hashes_per_batch: u64, pub account_indexes: AccountSecondaryIndexes, pub accounts_db_caching_enabled: bool, + pub accounts_index_bins: Option, pub warp_slot: Option, pub accounts_db_test_hash_calculation: bool, pub accounts_db_skip_shrink: bool, @@ -203,6 +204,7 @@ impl Default for ValidatorConfig { validator_exit: Arc::new(RwLock::new(Exit::default())), no_wait_for_vote_to_start_leader: true, accounts_shrink_ratio: AccountShrinkThreshold::default(), + accounts_index_bins: None, } } } @@ -1131,6 +1133,7 @@ fn new_banks_from_ledger( debug_keys: config.debug_keys.clone(), account_indexes: config.account_indexes.clone(), accounts_db_caching_enabled: config.accounts_db_caching_enabled, + accounts_index_bins: config.accounts_index_bins, shrink_ratio: config.accounts_shrink_ratio, accounts_db_test_hash_calculation: config.accounts_db_test_hash_calculation, accounts_db_skip_shrink: config.accounts_db_skip_shrink, diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 1409aa9e39..841cc2408b 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -11,7 +11,7 @@ use serde_json::json; use solana_clap_utils::{ input_parsers::{cluster_type_of, pubkey_of, pubkeys_of}, input_validators::{ - is_parsable, is_pubkey, is_pubkey_or_keypair, is_slot, is_valid_percentage, + is_bin, is_parsable, is_pubkey, is_pubkey_or_keypair, is_slot, is_valid_percentage, }, }; use solana_core::cost_model::CostModel; @@ -849,6 +849,12 @@ fn main() { .long("no-accounts-db-caching") .takes_value(false) .help("Disables accounts-db caching"); + let accounts_index_bins = Arg::with_name("accounts_index_bins") + .long("accounts-index-bins") + .value_name("BINS") + .validator(is_bin) + .takes_value(true) + .help("Number of bins to divide the accounts index into"); let account_paths_arg = Arg::with_name("account_paths") .long("accounts") .value_name("PATHS") @@ -1143,6 +1149,7 @@ fn main() { .arg(&account_paths_arg) .arg(&halt_at_slot_arg) .arg(&limit_load_slot_count_from_snapshot_arg) + .arg(&accounts_index_bins) .arg(&verify_index_arg) .arg(&hard_forks_arg) .arg(&no_accounts_db_caching_arg) @@ -1868,6 +1875,7 @@ fn main() { usize ) .ok(), + accounts_index_bins: value_t!(arg_matches, "accounts_index_bins", usize).ok(), verify_index: arg_matches.is_present("verify_accounts_index"), allow_dead_slots: arg_matches.is_present("allow_dead_slots"), accounts_db_test_hash_calculation: arg_matches diff --git a/ledger/src/bank_forks_utils.rs b/ledger/src/bank_forks_utils.rs index 6dc5661358..04937bbdda 100644 --- a/ledger/src/bank_forks_utils.rs +++ b/ledger/src/bank_forks_utils.rs @@ -132,7 +132,7 @@ fn load_from_snapshot( process_options.accounts_db_test_hash_calculation, process_options.accounts_db_skip_shrink, process_options.verify_index, - None, + process_options.accounts_index_bins, ) .expect("Load from snapshot failed"); diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index 05d7d5ffc9..a8385c7427 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -385,6 +385,7 @@ pub struct ProcessOptions { pub allow_dead_slots: bool, pub accounts_db_test_hash_calculation: bool, pub accounts_db_skip_shrink: bool, + pub accounts_index_bins: Option, pub verify_index: bool, pub shrink_ratio: AccountShrinkThreshold, } @@ -416,7 +417,7 @@ pub fn process_blockstore( opts.accounts_db_caching_enabled, opts.shrink_ratio, false, - None, // later, this will be passed from ProcessOptions + opts.accounts_index_bins, ); let bank0 = Arc::new(bank0); info!("processing ledger for slot 0..."); diff --git a/local-cluster/src/validator_configs.rs b/local-cluster/src/validator_configs.rs index dc514f5810..c218947640 100644 --- a/local-cluster/src/validator_configs.rs +++ b/local-cluster/src/validator_configs.rs @@ -57,6 +57,7 @@ pub fn safe_clone_config(config: &ValidatorConfig) -> ValidatorConfig { poh_hashes_per_batch: config.poh_hashes_per_batch, no_wait_for_vote_to_start_leader: config.no_wait_for_vote_to_start_leader, accounts_shrink_ratio: config.accounts_shrink_ratio, + accounts_index_bins: config.accounts_index_bins, } } diff --git a/replica-node/src/replica_node.rs b/replica-node/src/replica_node.rs index d91b8a9053..a1c98ce7be 100644 --- a/replica-node/src/replica_node.rs +++ b/replica-node/src/replica_node.rs @@ -127,7 +127,7 @@ fn initialize_from_snapshot( process_options.accounts_db_test_hash_calculation, false, process_options.verify_index, - None, + process_options.accounts_index_bins, ) .unwrap(); diff --git a/validator/src/main.rs b/validator/src/main.rs index e493ebc816..632051b825 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -10,8 +10,8 @@ use { solana_clap_utils::{ input_parsers::{keypair_of, keypairs_of, pubkey_of, value_of}, input_validators::{ - is_keypair, is_keypair_or_ask_keyword, is_parsable, is_pubkey, is_pubkey_or_keypair, - is_slot, + is_bin, is_keypair, is_keypair_or_ask_keyword, is_parsable, is_pubkey, + is_pubkey_or_keypair, is_slot, }, keypair::SKIP_SEED_PHRASE_VALIDATION_ARG, }, @@ -1820,6 +1820,14 @@ pub fn main() { .help("Enables faster starting of validators by skipping shrink. \ This option is for use during testing."), ) + .arg( + Arg::with_name("accounts_index_bins") + .long("accounts-index-bins") + .value_name("BINS") + .validator(is_bin) + .takes_value(true) + .help("Number of bins to divide the accounts index into"), + ) .arg( Arg::with_name("accounts_db_test_hash_calculation") .long("accounts-db-test-hash-calculation") @@ -2389,6 +2397,7 @@ pub fn main() { account_indexes, accounts_db_caching_enabled: !matches.is_present("no_accounts_db_caching"), accounts_db_test_hash_calculation: matches.is_present("accounts_db_test_hash_calculation"), + accounts_index_bins: value_t!(matches, "accounts_index_bins", usize).ok(), accounts_db_skip_shrink: matches.is_present("accounts_db_skip_shrink"), accounts_db_use_index_hash_calculation: matches.is_present("accounts_db_index_hashing"), tpu_coalesce_ms,