diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs index 7439c6bffa..01037d4ad6 100644 --- a/runtime/src/accounts_db.rs +++ b/runtime/src/accounts_db.rs @@ -51,6 +51,7 @@ use { pubkey_bins::PubkeyBinCalculator24, read_only_accounts_cache::ReadOnlyAccountsCache, rent_collector::RentCollector, + rent_paying_accounts_by_partition::RentPayingAccountsByPartition, sorted_storages::SortedStorages, storable_accounts::StorableAccounts, }, @@ -254,18 +255,20 @@ pub enum ScanStorageResult { Stored(B), } -#[derive(Debug, Default, Clone, Copy)] +#[derive(Debug, Default)] pub struct IndexGenerationInfo { pub accounts_data_len: u64, + pub rent_paying_accounts_by_partition: RentPayingAccountsByPartition, } -#[derive(Debug, Default, Clone, Copy)] +#[derive(Debug, Default)] struct SlotIndexGenerationInfo { insert_time_us: u64, num_accounts: u64, num_accounts_rent_paying: usize, accounts_data_len: u64, amount_to_top_off_rent: u64, + rent_paying_accounts_by_partition: Vec, } #[derive(Default, Debug)] @@ -2016,11 +2019,11 @@ impl AccountsDb { account_indexes: AccountSecondaryIndexes, caching_enabled: bool, shrink_ratio: AccountShrinkThreshold, - accounts_db_config: Option, + mut accounts_db_config: Option, accounts_update_notifier: Option, ) -> Self { let accounts_index = - AccountsIndex::new(accounts_db_config.as_ref().and_then(|x| x.index.clone())); + AccountsIndex::new(accounts_db_config.as_mut().and_then(|x| x.index.take())); let accounts_hash_cache_path = accounts_db_config .as_ref() .and_then(|x| x.accounts_hash_cache_path.clone()); @@ -7990,6 +7993,7 @@ impl AccountsDb { let secondary = !self.account_indexes.is_empty(); + let mut rent_paying_accounts_by_partition = Vec::default(); let mut accounts_data_len = 0; let mut num_accounts_rent_paying = 0; let num_accounts = accounts_map.len(); @@ -8019,6 +8023,8 @@ impl AccountsDb { { amount_to_top_off_rent += amount_to_top_off_rent_this_account; num_accounts_rent_paying += 1; + // remember this rent-paying account pubkey + rent_paying_accounts_by_partition.push(pubkey); } ( @@ -8048,6 +8054,7 @@ impl AccountsDb { num_accounts_rent_paying, accounts_data_len, amount_to_top_off_rent, + rent_paying_accounts_by_partition, } } @@ -8251,6 +8258,9 @@ impl AccountsDb { ); let accounts_data_len = AtomicU64::new(0); + let rent_paying_accounts_by_partition = + Mutex::new(RentPayingAccountsByPartition::new(&schedule)); + // pass == 0 always runs and generates the index // pass == 1 only runs if verify == true. // verify checks that all the expected items are in the accounts index and measures how long it takes to look them all up @@ -8311,6 +8321,8 @@ impl AccountsDb { num_accounts_rent_paying: rent_paying_this_slot, accounts_data_len: accounts_data_len_this_slot, amount_to_top_off_rent: amount_to_top_off_rent_this_slot, + rent_paying_accounts_by_partition: + rent_paying_accounts_by_partition_this_slot, } = self.generate_index_for_slot(accounts_map, slot, &rent_collector); rent_paying.fetch_add(rent_paying_this_slot, Ordering::Relaxed); amount_to_top_off_rent @@ -8318,6 +8330,14 @@ impl AccountsDb { total_duplicates.fetch_add(total_this_slot, Ordering::Relaxed); accounts_data_len .fetch_add(accounts_data_len_this_slot, Ordering::Relaxed); + let mut rent_paying_accounts_by_partition = + rent_paying_accounts_by_partition.lock().unwrap(); + rent_paying_accounts_by_partition_this_slot + .iter() + .for_each(|k| { + rent_paying_accounts_by_partition.add_account(k); + }); + insert_us } else { // verify index matches expected and measure the time to get all items @@ -8471,6 +8491,9 @@ impl AccountsDb { IndexGenerationInfo { accounts_data_len: accounts_data_len.load(Ordering::Relaxed), + rent_paying_accounts_by_partition: rent_paying_accounts_by_partition + .into_inner() + .unwrap(), } } diff --git a/runtime/src/accounts_index.rs b/runtime/src/accounts_index.rs index 722e2406c6..5a69d0b7fe 100644 --- a/runtime/src/accounts_index.rs +++ b/runtime/src/accounts_index.rs @@ -8,6 +8,7 @@ use { inline_spl_token::{self, GenericTokenAccount}, inline_spl_token_2022, pubkey_bins::PubkeyBinCalculator24, + rent_paying_accounts_by_partition::RentPayingAccountsByPartition, rolling_bit_field::RollingBitField, secondary_index::*, }, @@ -694,6 +695,9 @@ pub struct AccountsIndex { pub active_scans: AtomicUsize, /// # of slots between latest max and latest scan pub max_distance_to_min_scan_slot: AtomicU64, + + /// populated at generate_index time - accounts that could possibly be rent paying + pub rent_paying_accounts_by_partition: RwLock, } impl AccountsIndex { @@ -727,6 +731,7 @@ impl AccountsIndex { roots_removed: AtomicUsize::default(), active_scans: AtomicUsize::default(), max_distance_to_min_scan_slot: AtomicU64::default(), + rent_paying_accounts_by_partition: RwLock::default(), } } diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 4b04053336..dbc695cec2 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -268,7 +268,7 @@ pub type BankSlotDelta = SlotDelta>; // Each cycle is composed of number of tiny pubkey subranges // to scan, which is always multiple of the number of slots in epoch. pub(crate) type PartitionIndex = u64; -type PartitionsPerCycle = u64; +pub type PartitionsPerCycle = u64; type Partition = (PartitionIndex, PartitionIndex, PartitionsPerCycle); type RentCollectionCycleParams = ( Epoch, @@ -5217,6 +5217,28 @@ impl Bank { }); } + /// return all end partition indexes for the given partition + /// partition could be (0, 1, N). In this case we only return [1] + /// the single 'end_index' that covers this partition. + /// partition could be (0, 2, N). In this case, we return [1, 2], which are all + /// the 'end_index' values contained in that range. + /// (0, 0, N) returns [0] as a special case. + /// There is a relationship between + /// 1. 'pubkey_range_from_partition' + /// 2. 'partition_from_pubkey' + /// 3. this function + fn get_partition_end_indexes(partition: &Partition) -> Vec { + if partition.0 == partition.1 && partition.0 == 0 { + // special case for start=end=0. ie. (0, 0, N). This returns [0] + vec![0] + } else { + // normal case of (start, end, N) + // so, we want [start+1, start+2, ..=end] + // if start == end, then return [] + (partition.0..partition.1).map(|index| index + 1).collect() + } + } + fn collect_rent_eagerly(&self, just_rewrites: bool) { if self.lazy_rent_collection.load(Relaxed) { return; @@ -5231,7 +5253,7 @@ impl Bank { if parallel { let ranges = partitions .iter() - .map(|partition| Self::pubkey_range_from_partition(*partition)) + .map(|partition| (*partition, Self::pubkey_range_from_partition(*partition))) .collect::>(); // test every range to make sure ranges are not overlapping // some tests collect rent from overlapping ranges @@ -5243,8 +5265,8 @@ impl Bank { continue; } - let i = &ranges[i]; - let j = &ranges[j]; + let i = &ranges[i].1; + let j = &ranges[j].1; // make sure i doesn't contain j if i.contains(j.start()) || i.contains(j.end()) { parallel = false; @@ -5257,7 +5279,7 @@ impl Bank { let thread_pool = &self.rc.accounts.accounts_db.thread_pool; thread_pool.install(|| { ranges.into_par_iter().for_each(|range| { - self.collect_rent_in_range(range, just_rewrites, &rent_metrics) + self.collect_rent_in_range(range.0, range.1, just_rewrites, &rent_metrics) }); }); } @@ -5329,6 +5351,8 @@ impl Bank { &self, mut accounts: Vec<(Pubkey, AccountSharedData, Slot)>, just_rewrites: bool, + rent_paying_pubkeys: Option<&HashSet>, + partition_index: PartitionIndex, ) -> CollectRentFromAccountsInfo { let mut rent_debits = RentDebits::default(); let mut total_rent_collected_info = CollectedInfo::default(); @@ -5379,6 +5403,22 @@ impl Bank { rewrites_skipped.push((*pubkey, hash)); assert_eq!(rent_collected_info, CollectedInfo::default()); } else if !just_rewrites { + if rent_collected_info.rent_amount > 0 { + if let Some(rent_paying_pubkeys) = rent_paying_pubkeys { + if !rent_paying_pubkeys.contains(pubkey) { + // inc counter instead of assert while we verify this is correct + inc_new_counter_info!("unexpected-rent-paying-pubkey", 1); + warn!( + "Collecting rent from unexpected pubkey: {}, slot: {}, parent_slot: {:?}, partition_index: {}, partition_from_pubkey: {}", + pubkey, + self.slot(), + self.parent().map(|bank| bank.slot()), + partition_index, + Bank::partition_from_pubkey(pubkey, self.epoch_schedule.slots_per_epoch), + ); + } + } + } total_rent_collected_info += rent_collected_info; accounts_to_store.push((pubkey, account)); } @@ -5411,7 +5451,28 @@ impl Bank { metrics: &RentMetrics, ) { let subrange_full = Self::pubkey_range_from_partition(partition); - self.collect_rent_in_range(subrange_full, just_rewrites, metrics) + self.collect_rent_in_range(partition, subrange_full, just_rewrites, metrics) + } + + /// get all pubkeys that we expect to be rent-paying or None, if this was not initialized at load time (that should only exist in test cases) + fn get_rent_paying_pubkeys(&self, partition: &Partition) -> Option> { + let rent_paying_accounts = &self + .rc + .accounts + .accounts_db + .accounts_index + .rent_paying_accounts_by_partition + .read() + .unwrap(); + rent_paying_accounts.is_initialized().then(|| { + Self::get_partition_end_indexes(partition) + .into_iter() + .flat_map(|end_index| { + rent_paying_accounts.get_pubkeys_in_partition_index(end_index) + }) + .cloned() + .collect::>() + }) } /// load accounts with pubkeys in 'subrange_full' @@ -5422,6 +5483,7 @@ impl Bank { /// This flag is used when restoring from a snapshot to calculate and verify the initial bank's delta hash. fn collect_rent_in_range( &self, + partition: Partition, subrange_full: RangeInclusive, just_rewrites: bool, metrics: &RentMetrics, @@ -5435,6 +5497,9 @@ impl Bank { hold_range.stop(); metrics.hold_range_us.fetch_add(hold_range.as_us(), Relaxed); + let rent_paying_pubkeys_ = self.get_rent_paying_pubkeys(&partition); + let rent_paying_pubkeys = rent_paying_pubkeys_.as_ref(); + // divide the range into num_threads smaller ranges and process in parallel // Note that 'pubkey_range_from_partition' cannot easily be re-used here to break the range smaller. // It has special handling of 0..0 and partition_count changes affect all ranges unevenly. @@ -5469,7 +5534,12 @@ impl Bank { .load_to_collect_rent_eagerly(&self.ancestors, subrange) }); CollectRentInPartitionInfo::new( - self.collect_rent_from_accounts(accounts, just_rewrites), + self.collect_rent_from_accounts( + accounts, + just_rewrites, + rent_paying_pubkeys, + partition.1, + ), Duration::from_nanos(measure_load_accounts.as_ns()), ) }) @@ -5478,6 +5548,9 @@ impl Bank { CollectRentInPartitionInfo::reduce, ); + // We cannot assert here that we collected from all expected keys. + // Some accounts may have been topped off or may have had all funds removed and gone to 0 lamports. + self.rc .accounts .hold_range_in_memory(&subrange_full, false, thread_pool); @@ -7819,6 +7892,7 @@ pub(crate) mod tests { genesis_sysvar_and_builtin_program_lamports, GenesisConfigInfo, ValidatorVoteKeypairs, }, + rent_paying_accounts_by_partition::RentPayingAccountsByPartition, status_cache::MAX_CACHE_ENTRIES, }, crossbeam_channel::{bounded, unbounded}, @@ -9863,12 +9937,16 @@ pub(crate) mod tests { let result = later_bank.collect_rent_from_accounts( vec![(zero_lamport_pubkey, account.clone(), later_slot)], just_rewrites, + None, + PartitionIndex::default(), ); assert!(result.rewrites_skipped.is_empty()); // loaded from previous slot, so we skip rent collection on it let result = later_bank.collect_rent_from_accounts( vec![(zero_lamport_pubkey, account, later_slot - 1)], just_rewrites, + None, + PartitionIndex::default(), ); assert!(result.rewrites_skipped[0].0 == zero_lamport_pubkey); } @@ -19209,6 +19287,76 @@ pub(crate) mod tests { } } + #[test] + fn test_get_partition_end_indexes() { + for n in 5..7 { + assert_eq!(vec![0], Bank::get_partition_end_indexes(&(0, 0, n))); + assert!(Bank::get_partition_end_indexes(&(1, 1, n)).is_empty()); + assert_eq!(vec![1], Bank::get_partition_end_indexes(&(0, 1, n))); + assert_eq!(vec![1, 2], Bank::get_partition_end_indexes(&(0, 2, n))); + assert_eq!(vec![3, 4], Bank::get_partition_end_indexes(&(2, 4, n))); + } + } + + #[test] + fn test_get_rent_paying_pubkeys() { + let lamports = 1; + let bank = create_simple_test_bank(lamports); + + let n = 432_000; + assert!(bank.get_rent_paying_pubkeys(&(0, 1, n)).is_none()); + assert!(bank.get_rent_paying_pubkeys(&(0, 2, n)).is_none()); + assert!(bank.get_rent_paying_pubkeys(&(0, 0, n)).is_none()); + + let pk1 = Pubkey::new(&[2; 32]); + let pk2 = Pubkey::new(&[3; 32]); + let index1 = Bank::partition_from_pubkey(&pk1, n); + let index2 = Bank::partition_from_pubkey(&pk2, n); + assert!(index1 > 0, "{}", index1); + assert!(index2 > index1, "{}, {}", index2, index1); + + let epoch_schedule = EpochSchedule::custom(n, 0, false); + + let mut rent_paying_accounts_by_partition = + RentPayingAccountsByPartition::new(&epoch_schedule); + rent_paying_accounts_by_partition.add_account(&pk1); + rent_paying_accounts_by_partition.add_account(&pk2); + + *bank + .rc + .accounts + .accounts_db + .accounts_index + .rent_paying_accounts_by_partition + .write() + .unwrap() = rent_paying_accounts_by_partition; + + assert_eq!( + bank.get_rent_paying_pubkeys(&(0, 1, n)), + Some(HashSet::default()) + ); + assert_eq!( + bank.get_rent_paying_pubkeys(&(0, 2, n)), + Some(HashSet::default()) + ); + assert_eq!( + bank.get_rent_paying_pubkeys(&(index1.saturating_sub(1), index1, n)), + Some(HashSet::from([pk1])) + ); + assert_eq!( + bank.get_rent_paying_pubkeys(&(index2.saturating_sub(1), index2, n)), + Some(HashSet::from([pk2])) + ); + assert_eq!( + bank.get_rent_paying_pubkeys(&(index1.saturating_sub(1), index2, n)), + Some(HashSet::from([pk2, pk1])) + ); + assert_eq!( + bank.get_rent_paying_pubkeys(&(0, 0, n)), + Some(HashSet::default()) + ); + } + /// Ensure that accounts data size is updated correctly by rent collection #[test] fn test_accounts_data_size_and_rent_collection() { diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 5274da9edf..a52a6ed07c 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -50,6 +50,7 @@ mod nonce_keyed_account; mod pubkey_bins; mod read_only_accounts_cache; pub mod rent_collector; +mod rent_paying_accounts_by_partition; mod rolling_bit_field; pub mod runtime_config; pub mod secondary_index; diff --git a/runtime/src/rent_paying_accounts_by_partition.rs b/runtime/src/rent_paying_accounts_by_partition.rs new file mode 100644 index 0000000000..286e5eebd5 --- /dev/null +++ b/runtime/src/rent_paying_accounts_by_partition.rs @@ -0,0 +1,72 @@ +//! Provide fast iteration of all pubkeys which could possibly be rent paying, grouped by rent collection partition +use { + crate::bank::{Bank, PartitionIndex, PartitionsPerCycle}, + solana_sdk::{epoch_schedule::EpochSchedule, pubkey::Pubkey}, + std::collections::HashSet, +}; + +/// populated at startup with the accounts that were found that are rent paying. +/// These are the 'possible' rent paying accounts. +/// This set can never grow during runtime since it is not possible to create rent paying accounts now. +/// It can shrink during execution if a rent paying account is dropped to lamports=0 or is topped off. +/// The next time the validator restarts, it will remove the account from this list. +#[derive(Debug, Default)] +pub struct RentPayingAccountsByPartition { + /// 1st index is partition end index, 0..=432_000 + /// 2nd dimension is list of pubkeys which were identified at startup to be rent paying + /// At the moment, we use this data structure to verify all rent paying accounts are expected. + /// When we stop iterating the accounts index to FIND rent paying accounts, we will no longer need this to be a hashset. + /// It can just be a vec. + pub accounts: Vec>, + partition_count: PartitionsPerCycle, +} + +impl RentPayingAccountsByPartition { + /// create new struct. Need slots per epoch from 'epoch_schedule' + pub fn new(epoch_schedule: &EpochSchedule) -> Self { + let partition_count = epoch_schedule.slots_per_epoch; + Self { + partition_count, + accounts: (0..=partition_count) + .into_iter() + .map(|_| HashSet::::default()) + .collect(), + } + } + /// Remember that 'pubkey' can possibly be rent paying. + pub fn add_account(&mut self, pubkey: &Pubkey) { + let partition_end_index = Bank::partition_from_pubkey(pubkey, self.partition_count); + let list = &mut self.accounts[partition_end_index as usize]; + + list.insert(*pubkey); + } + /// return all pubkeys that can possibly be rent paying with this partition end_index + pub fn get_pubkeys_in_partition_index( + &self, + partition_end_index: PartitionIndex, + ) -> &HashSet { + &self.accounts[partition_end_index as usize] + } + pub fn is_initialized(&self) -> bool { + self.partition_count != 0 + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + #[test] + fn test_add() { + let mut test = RentPayingAccountsByPartition::new(&EpochSchedule::custom(32, 0, false)); + let pk = Pubkey::new(&[1; 32]); + test.add_account(&pk); + // make sure duplicate adds only result in a single item + test.add_account(&pk); + assert_eq!(test.get_pubkeys_in_partition_index(0).len(), 1); + assert!(test.get_pubkeys_in_partition_index(1).is_empty()); + assert!(test.is_initialized()); + let test = RentPayingAccountsByPartition::default(); + assert!(!test.is_initialized()); + } +} diff --git a/runtime/src/serde_snapshot.rs b/runtime/src/serde_snapshot.rs index 0be5abf492..c5976f2720 100644 --- a/runtime/src/serde_snapshot.rs +++ b/runtime/src/serde_snapshot.rs @@ -714,11 +714,19 @@ where }) .unwrap(); - let IndexGenerationInfo { accounts_data_len } = accounts_db.generate_index( + let IndexGenerationInfo { + accounts_data_len, + rent_paying_accounts_by_partition, + } = accounts_db.generate_index( limit_load_slot_count_from_snapshot, verify_index, genesis_config, ); + *accounts_db + .accounts_index + .rent_paying_accounts_by_partition + .write() + .unwrap() = rent_paying_accounts_by_partition; accounts_db.maybe_add_filler_accounts( &genesis_config.epoch_schedule,