Add accounts-filler-size command line option (#23896)

core/src/validator.rs

@@ -798,8 +798,7 @@ impl Validator {
                 let enable_gossip_push = config
                     .accounts_db_config
                     .as_ref()
-                    .and_then(|config| config.filler_account_count)
+                    .map(|config| config.filler_accounts_config.count == 0)
-                    .map(|count| count == 0)
                     .unwrap_or(true);
 
                 let snapshot_packager_service = SnapshotPackagerService::new(

ledger-tool/src/main.rs

@@ -32,7 +32,7 @@ use {
     },
     solana_measure::measure::Measure,
     solana_runtime::{
-        accounts_db::AccountsDbConfig,
+        accounts_db::{AccountsDbConfig, FillerAccountsConfig},
         accounts_index::{AccountsIndexConfig, ScanConfig},
         bank::{Bank, RewardCalculationEvent},
         bank_forks::BankForks,
@@ -928,7 +928,16 @@ fn main() {
         .value_name("COUNT")
         .validator(is_parsable::<usize>)
         .takes_value(true)
+        .default_value("0")
         .help("How many accounts to add to stress the system. Accounts are ignored in operations related to correctness.");
+    let accounts_filler_size = Arg::with_name("accounts_filler_size")
+        .long("accounts-filler-size")
+        .value_name("BYTES")
+        .validator(is_parsable::<usize>)
+        .takes_value(true)
+        .default_value("0")
+        .requires("accounts_filler_count")
+        .help("Size per filler account in bytes.");
     let account_paths_arg = Arg::with_name("account_paths")
         .long("accounts")
         .value_name("PATHS")
@@ -1283,6 +1292,7 @@ fn main() {
             .arg(&disable_disk_index)
             .arg(&accountsdb_skip_shrink)
             .arg(&accounts_filler_count)
+            .arg(&accounts_filler_size)
             .arg(&verify_index_arg)
             .arg(&hard_forks_arg)
             .arg(&no_accounts_db_caching_arg)
@@ -2066,13 +2076,15 @@ fn main() {
                     accounts_index_config.drives = Some(accounts_index_paths);
                 }
 
-                let filler_account_count =
+                let filler_accounts_config = FillerAccountsConfig {
-                    value_t!(arg_matches, "accounts_filler_count", usize).ok();
+                    count: value_t_or_exit!(arg_matches, "accounts_filler_count", usize),
+                    size: value_t_or_exit!(arg_matches, "accounts_filler_size", usize),
+                };
 
                 let accounts_db_config = Some(AccountsDbConfig {
                     index: Some(accounts_index_config),
                     accounts_hash_cache_path: Some(ledger_path.clone()),
-                    filler_account_count,
+                    filler_accounts_config,
                     ..AccountsDbConfig::default()
                 });
 

ledger/src/bank_forks_utils.rs

@@ -125,13 +125,12 @@ pub fn load_bank_forks(
             accounts_update_notifier,
         )
     } else {
-        if process_options
+        let maybe_filler_accounts = process_options
             .accounts_db_config
             .as_ref()
-            .and_then(|config| config.filler_account_count)
+            .map(|config| config.filler_accounts_config.count > 0);
-            .unwrap_or_default()
+
-            > 0
+        if let Some(true) = maybe_filler_accounts {
-        {
             panic!("filler accounts specified, but not loading from snapshot");
         }
 

runtime/src/accounts_db.rs

@@ -65,6 +65,7 @@ use {
         genesis_config::{ClusterType, GenesisConfig},
         hash::Hash,
         pubkey::Pubkey,
+        rent::Rent,
         timing::AtomicInterval,
     },
     std::{
@@ -128,14 +129,14 @@ const CACHE_VIRTUAL_STORED_SIZE: StoredSize = 0;
 pub const ACCOUNTS_DB_CONFIG_FOR_TESTING: AccountsDbConfig = AccountsDbConfig {
     index: Some(ACCOUNTS_INDEX_CONFIG_FOR_TESTING),
     accounts_hash_cache_path: None,
-    filler_account_count: None,
+    filler_accounts_config: FillerAccountsConfig::const_default(),
     hash_calc_num_passes: None,
     write_cache_limit_bytes: None,
 };
 pub const ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS: AccountsDbConfig = AccountsDbConfig {
     index: Some(ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS),
     accounts_hash_cache_path: None,
-    filler_account_count: None,
+    filler_accounts_config: FillerAccountsConfig::const_default(),
     hash_calc_num_passes: None,
     write_cache_limit_bytes: None,
 };
@@ -148,11 +149,31 @@ pub struct AccountsAddRootTiming {
     pub store_us: u64,
 }
 
+#[derive(Debug, Clone, Copy)]
+pub struct FillerAccountsConfig {
+    /// Number of filler accounts
+    pub count: usize,
+    /// Data size per account, in bytes
+    pub size: usize,
+}
+
+impl FillerAccountsConfig {
+    pub const fn const_default() -> Self {
+        Self { count: 0, size: 0 }
+    }
+}
+
+impl Default for FillerAccountsConfig {
+    fn default() -> Self {
+        Self::const_default()
+    }
+}
+
 #[derive(Debug, Default, Clone)]
 pub struct AccountsDbConfig {
     pub index: Option<AccountsIndexConfig>,
     pub accounts_hash_cache_path: Option<PathBuf>,
-    pub filler_account_count: Option<usize>,
+    pub filler_accounts_config: FillerAccountsConfig,
     pub hash_calc_num_passes: Option<usize>,
     pub write_cache_limit_bytes: Option<u64>,
 }
@@ -1090,7 +1111,7 @@ pub struct AccountsDb {
     /// GeyserPlugin accounts update notifier
     accounts_update_notifier: Option<AccountsUpdateNotifier>,
 
-    filler_account_count: usize,
+    filler_accounts_config: FillerAccountsConfig,
     pub filler_account_suffix: Option<Pubkey>,
 
     active_stats: ActiveStats,
@@ -1633,7 +1654,7 @@ impl AccountsDb {
             dirty_stores: DashMap::default(),
             zero_lamport_accounts_to_purge_after_full_snapshot: DashSet::default(),
             accounts_update_notifier: None,
-            filler_account_count: 0,
+            filler_accounts_config: FillerAccountsConfig::default(),
             filler_account_suffix: None,
             num_hash_scan_passes,
         }
@@ -1677,11 +1698,13 @@ impl AccountsDb {
         let accounts_hash_cache_path = accounts_db_config
             .as_ref()
             .and_then(|x| x.accounts_hash_cache_path.clone());
-        let filler_account_count = accounts_db_config
+
+        let filler_accounts_config = accounts_db_config
             .as_ref()
-            .and_then(|cfg| cfg.filler_account_count)
+            .map(|config| config.filler_accounts_config)
             .unwrap_or_default();
-        let filler_account_suffix = if filler_account_count > 0 {
+
+        let filler_account_suffix = if filler_accounts_config.count > 0 {
             Some(solana_sdk::pubkey::new_rand())
         } else {
             None
@@ -1694,7 +1717,7 @@ impl AccountsDb {
             caching_enabled,
             shrink_ratio,
             accounts_update_notifier,
-            filler_account_count,
+            filler_accounts_config,
             filler_account_suffix,
             write_cache_limit_bytes: accounts_db_config
                 .as_ref()
@@ -5741,7 +5764,7 @@ impl AccountsDb {
                 };
 
                 let hash = AccountsHash {
-                    filler_account_suffix: if self.filler_account_count > 0 {
+                    filler_account_suffix: if self.filler_accounts_config.count > 0 {
                         self.filler_account_suffix
                     } else {
                         None
@@ -6857,15 +6880,18 @@ impl AccountsDb {
     /// The filler accounts are added to each slot in the snapshot after index generation.
     /// The accounts added in a slot are setup to have pubkeys such that rent will be collected from them before (or when?) their slot becomes an epoch old.
     /// Thus, the filler accounts are rewritten by rent and the old slot can be thrown away successfully.
-    pub fn maybe_add_filler_accounts(&self, epoch_schedule: &EpochSchedule) {
+    pub fn maybe_add_filler_accounts(&self, epoch_schedule: &EpochSchedule, rent: &Rent) {
-        if self.filler_account_count == 0 {
+        if self.filler_accounts_config.count == 0 {
             return;
         }
 
         let max_root_inclusive = self.accounts_index.max_root_inclusive();
         let epoch = epoch_schedule.get_epoch(max_root_inclusive);
 
-        info!("adding {} filler accounts", self.filler_account_count);
+        info!(
+            "adding {} filler accounts with size {}",
+            self.filler_accounts_config.count, self.filler_accounts_config.size,
+        );
         // break this up to force the accounts out of memory after each pass
         let passes = 100;
         let mut roots = self.storage.all_slots();
@@ -6879,8 +6905,9 @@ impl AccountsDb {
         let string = "FiLLERACCoUNTooooooooooooooooooooooooooooooo";
         let hash = Hash::from_str(string).unwrap();
         let owner = Pubkey::from_str(string).unwrap();
-        let lamports = 100_000_000;
+        let space = self.filler_accounts_config.size;
-        let space = 0;
+        let rent_exempt_reserve = rent.minimum_balance(space);
+        let lamports = rent_exempt_reserve;
         let account = AccountSharedData::new(lamports, space, &owner);
         let added = AtomicUsize::default();
         for pass in 0..=passes {
@@ -6907,8 +6934,8 @@ impl AccountsDb {
                 let subrange = crate::bank::Bank::pubkey_range_from_partition(partition);
 
                 let idx = overall_index.fetch_add(1, Ordering::Relaxed);
-                let filler_entries = (idx + 1) * self.filler_account_count / root_count
+                let filler_entries = (idx + 1) * self.filler_accounts_config.count / root_count
-                    - idx * self.filler_account_count / root_count;
+                    - idx * self.filler_accounts_config.count / root_count;
                 let accounts = (0..filler_entries)
                     .map(|_| {
                         let my_id = added.fetch_add(1, Ordering::Relaxed);

runtime/src/serde_snapshot.rs

@@ -642,7 +642,7 @@ where
         genesis_config,
     );
 
-    accounts_db.maybe_add_filler_accounts(&genesis_config.epoch_schedule);
+    accounts_db.maybe_add_filler_accounts(&genesis_config.epoch_schedule, &genesis_config.rent);
 
     handle.join().unwrap();
     measure_notify.stop();

validator/src/main.rs

@@ -42,8 +42,8 @@ use {
     },
     solana_runtime::{
         accounts_db::{
-            AccountShrinkThreshold, AccountsDbConfig, DEFAULT_ACCOUNTS_SHRINK_OPTIMIZE_TOTAL_SPACE,
+            AccountShrinkThreshold, AccountsDbConfig, FillerAccountsConfig,
-            DEFAULT_ACCOUNTS_SHRINK_RATIO,
+            DEFAULT_ACCOUNTS_SHRINK_OPTIMIZE_TOTAL_SPACE, DEFAULT_ACCOUNTS_SHRINK_RATIO,
         },
         accounts_index::{
             AccountIndex, AccountSecondaryIndexes, AccountSecondaryIndexesIncludeExclude,
@@ -1600,7 +1600,16 @@ pub fn main() {
             .value_name("COUNT")
             .validator(is_parsable::<usize>)
             .takes_value(true)
+            .default_value("0")
             .help("How many accounts to add to stress the system. Accounts are ignored in operations related to correctness."))
+         .arg(Arg::with_name("accounts_filler_size")
+            .long("accounts-filler-size")
+            .value_name("BYTES")
+            .validator(is_parsable::<usize>)
+            .takes_value(true)
+            .default_value("0")
+            .requires("accounts_filler_count")
+            .help("Size per filler account in bytes."))
          .arg(
             Arg::with_name("accounts_db_test_hash_calculation")
                 .long("accounts-db-test-hash-calculation")
@@ -2251,11 +2260,15 @@ pub fn main() {
             .ok()
             .map(|mb| mb * MB);
 
-    let filler_account_count = value_t!(matches, "accounts_filler_count", usize).ok();
+    let filler_accounts_config = FillerAccountsConfig {
+        count: value_t_or_exit!(matches, "accounts_filler_count", usize),
+        size: value_t_or_exit!(matches, "accounts_filler_size", usize),
+    };
+
     let mut accounts_db_config = AccountsDbConfig {
         index: Some(accounts_index_config),
         accounts_hash_cache_path: Some(ledger_path.clone()),
-        filler_account_count,
+        filler_accounts_config,
         write_cache_limit_bytes: value_t!(matches, "accounts_db_cache_limit_mb", u64)
             .ok()
             .map(|mb| mb * MB as u64),