cli for num account index bins (#19085)

clap-utils/src/input_validators.rs

@@ -237,6 +237,22 @@ where
     is_parsable_generic::<Slot, _>(slot)
 }
 
+pub fn is_bin<T>(bins: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    bins.as_ref()
+        .parse::<usize>()
+        .map_err(|e| format!("Unable to parse bins, provided: {}, err: {}", bins, e))
+        .and_then(|v| {
+            if !v.is_power_of_two() {
+                Err(format!("Bins must be a power of 2: {}", v))
+            } else {
+                Ok(())
+            }
+        })
+}
+
 pub fn is_port<T>(port: T) -> Result<(), String>
 where
     T: AsRef<str> + Display,

core/src/validator.rs

@@ -138,6 +138,7 @@ pub struct ValidatorConfig {
     pub poh_hashes_per_batch: u64,
     pub account_indexes: AccountSecondaryIndexes,
     pub accounts_db_caching_enabled: bool,
+    pub accounts_index_bins: Option<usize>,
     pub warp_slot: Option<Slot>,
     pub accounts_db_test_hash_calculation: bool,
     pub accounts_db_skip_shrink: bool,
@@ -203,6 +204,7 @@ impl Default for ValidatorConfig {
             validator_exit: Arc::new(RwLock::new(Exit::default())),
             no_wait_for_vote_to_start_leader: true,
             accounts_shrink_ratio: AccountShrinkThreshold::default(),
+            accounts_index_bins: None,
         }
     }
 }
@@ -1131,6 +1133,7 @@ fn new_banks_from_ledger(
         debug_keys: config.debug_keys.clone(),
         account_indexes: config.account_indexes.clone(),
         accounts_db_caching_enabled: config.accounts_db_caching_enabled,
+        accounts_index_bins: config.accounts_index_bins,
         shrink_ratio: config.accounts_shrink_ratio,
         accounts_db_test_hash_calculation: config.accounts_db_test_hash_calculation,
         accounts_db_skip_shrink: config.accounts_db_skip_shrink,

ledger-tool/src/main.rs

@@ -11,7 +11,7 @@ use serde_json::json;
 use solana_clap_utils::{
     input_parsers::{cluster_type_of, pubkey_of, pubkeys_of},
     input_validators::{
-        is_parsable, is_pubkey, is_pubkey_or_keypair, is_slot, is_valid_percentage,
+        is_bin, is_parsable, is_pubkey, is_pubkey_or_keypair, is_slot, is_valid_percentage,
     },
 };
 use solana_core::cost_model::CostModel;
@@ -849,6 +849,12 @@ fn main() {
         .long("no-accounts-db-caching")
         .takes_value(false)
         .help("Disables accounts-db caching");
+    let accounts_index_bins = Arg::with_name("accounts_index_bins")
+        .long("accounts-index-bins")
+        .value_name("BINS")
+        .validator(is_bin)
+        .takes_value(true)
+        .help("Number of bins to divide the accounts index into");
     let account_paths_arg = Arg::with_name("account_paths")
         .long("accounts")
         .value_name("PATHS")
@@ -1143,6 +1149,7 @@ fn main() {
             .arg(&account_paths_arg)
             .arg(&halt_at_slot_arg)
             .arg(&limit_load_slot_count_from_snapshot_arg)
+            .arg(&accounts_index_bins)
             .arg(&verify_index_arg)
             .arg(&hard_forks_arg)
             .arg(&no_accounts_db_caching_arg)
@@ -1868,6 +1875,7 @@ fn main() {
                     usize
                 )
                 .ok(),
+                accounts_index_bins: value_t!(arg_matches, "accounts_index_bins", usize).ok(),
                 verify_index: arg_matches.is_present("verify_accounts_index"),
                 allow_dead_slots: arg_matches.is_present("allow_dead_slots"),
                 accounts_db_test_hash_calculation: arg_matches

ledger/src/bank_forks_utils.rs

@@ -132,7 +132,7 @@ fn load_from_snapshot(
         process_options.accounts_db_test_hash_calculation,
         process_options.accounts_db_skip_shrink,
         process_options.verify_index,
-        None,
+        process_options.accounts_index_bins,
     )
     .expect("Load from snapshot failed");
 

ledger/src/blockstore_processor.rs

@@ -385,6 +385,7 @@ pub struct ProcessOptions {
     pub allow_dead_slots: bool,
     pub accounts_db_test_hash_calculation: bool,
     pub accounts_db_skip_shrink: bool,
+    pub accounts_index_bins: Option<usize>,
     pub verify_index: bool,
     pub shrink_ratio: AccountShrinkThreshold,
 }
@@ -416,7 +417,7 @@ pub fn process_blockstore(
         opts.accounts_db_caching_enabled,
         opts.shrink_ratio,
         false,
-        None, // later, this will be passed from ProcessOptions
+        opts.accounts_index_bins,
     );
     let bank0 = Arc::new(bank0);
     info!("processing ledger for slot 0...");

local-cluster/src/validator_configs.rs

@@ -57,6 +57,7 @@ pub fn safe_clone_config(config: &ValidatorConfig) -> ValidatorConfig {
         poh_hashes_per_batch: config.poh_hashes_per_batch,
         no_wait_for_vote_to_start_leader: config.no_wait_for_vote_to_start_leader,
         accounts_shrink_ratio: config.accounts_shrink_ratio,
+        accounts_index_bins: config.accounts_index_bins,
     }
 }
 

replica-node/src/replica_node.rs

@@ -127,7 +127,7 @@ fn initialize_from_snapshot(
         process_options.accounts_db_test_hash_calculation,
         false,
         process_options.verify_index,
-        None,
+        process_options.accounts_index_bins,
     )
     .unwrap();
 

validator/src/main.rs

@@ -10,8 +10,8 @@ use {
     solana_clap_utils::{
         input_parsers::{keypair_of, keypairs_of, pubkey_of, value_of},
         input_validators::{
-            is_keypair, is_keypair_or_ask_keyword, is_parsable, is_pubkey, is_pubkey_or_keypair,
-            is_slot,
+            is_bin, is_keypair, is_keypair_or_ask_keyword, is_parsable, is_pubkey,
+            is_pubkey_or_keypair, is_slot,
         },
         keypair::SKIP_SEED_PHRASE_VALIDATION_ARG,
     },
@@ -1820,6 +1820,14 @@ pub fn main() {
                 .help("Enables faster starting of validators by skipping shrink. \
                       This option is for use during testing."),
         )
+        .arg(
+            Arg::with_name("accounts_index_bins")
+                .long("accounts-index-bins")
+                .value_name("BINS")
+                .validator(is_bin)
+                .takes_value(true)
+                .help("Number of bins to divide the accounts index into"),
+        )
         .arg(
             Arg::with_name("accounts_db_test_hash_calculation")
                 .long("accounts-db-test-hash-calculation")
@@ -2389,6 +2397,7 @@ pub fn main() {
         account_indexes,
         accounts_db_caching_enabled: !matches.is_present("no_accounts_db_caching"),
         accounts_db_test_hash_calculation: matches.is_present("accounts_db_test_hash_calculation"),
+        accounts_index_bins: value_t!(matches, "accounts_index_bins", usize).ok(),
         accounts_db_skip_shrink: matches.is_present("accounts_db_skip_shrink"),
         accounts_db_use_index_hash_calculation: matches.is_present("accounts_db_index_hashing"),
         tpu_coalesce_ms,