blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

handle rent paying accounts differently (#26487) 

* handle rent paying accounts differently

* restore collection

* downgrade assert to metric and warn until we get more runtime
This commit is contained in:
Jeff Washington (jwash) 2022-07-12 17:51:33 -05:00 committed by GitHub
parent 8217848388
commit 6b0eb5a42b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 269 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
runtime/src/accounts_db.rs
View File

@ -51,6 +51,7 @@ use {
pubkey_bins::PubkeyBinCalculator24,
read_only_accounts_cache::ReadOnlyAccountsCache,
rent_collector::RentCollector,
rent_paying_accounts_by_partition::RentPayingAccountsByPartition,
sorted_storages::SortedStorages,
storable_accounts::StorableAccounts,
},
@ -254,18 +255,20 @@ pub enum ScanStorageResult<R, B> {
Stored(B),
}
#[derive(Debug, Default, Clone, Copy)]
#[derive(Debug, Default)]
pub struct IndexGenerationInfo {
pub accounts_data_len: u64,
pub rent_paying_accounts_by_partition: RentPayingAccountsByPartition,
}
#[derive(Debug, Default, Clone, Copy)]
#[derive(Debug, Default)]
struct SlotIndexGenerationInfo {
insert_time_us: u64,
num_accounts: u64,
num_accounts_rent_paying: usize,
accounts_data_len: u64,
amount_to_top_off_rent: u64,
rent_paying_accounts_by_partition: Vec<Pubkey>,
}
#[derive(Default, Debug)]
@ -2016,11 +2019,11 @@ impl AccountsDb {
account_indexes: AccountSecondaryIndexes,
caching_enabled: bool,
shrink_ratio: AccountShrinkThreshold,
accounts_db_config: Option<AccountsDbConfig>,
mut accounts_db_config: Option<AccountsDbConfig>,
accounts_update_notifier: Option<AccountsUpdateNotifier>,
) -> Self {
let accounts_index =
AccountsIndex::new(accounts_db_config.as_ref().and_then(|x| x.index.clone()));
AccountsIndex::new(accounts_db_config.as_mut().and_then(|x| x.index.take()));
let accounts_hash_cache_path = accounts_db_config
.as_ref()
.and_then(|x| x.accounts_hash_cache_path.clone());
@ -7990,6 +7993,7 @@ impl AccountsDb {
let secondary = !self.account_indexes.is_empty();
let mut rent_paying_accounts_by_partition = Vec::default();
let mut accounts_data_len = 0;
let mut num_accounts_rent_paying = 0;
let num_accounts = accounts_map.len();
@ -8019,6 +8023,8 @@ impl AccountsDb {
{
amount_to_top_off_rent += amount_to_top_off_rent_this_account;
num_accounts_rent_paying += 1;
// remember this rent-paying account pubkey
rent_paying_accounts_by_partition.push(pubkey);
}
(
@ -8048,6 +8054,7 @@ impl AccountsDb {
num_accounts_rent_paying,
accounts_data_len,
amount_to_top_off_rent,
rent_paying_accounts_by_partition,
}
}
@ -8251,6 +8258,9 @@ impl AccountsDb {
);
let accounts_data_len = AtomicU64::new(0);
let rent_paying_accounts_by_partition =
Mutex::new(RentPayingAccountsByPartition::new(&schedule));
// pass == 0 always runs and generates the index
// pass == 1 only runs if verify == true.
// verify checks that all the expected items are in the accounts index and measures how long it takes to look them all up
@ -8311,6 +8321,8 @@ impl AccountsDb {
num_accounts_rent_paying: rent_paying_this_slot,
accounts_data_len: accounts_data_len_this_slot,
amount_to_top_off_rent: amount_to_top_off_rent_this_slot,
rent_paying_accounts_by_partition:
rent_paying_accounts_by_partition_this_slot,
} = self.generate_index_for_slot(accounts_map, slot, &rent_collector);
rent_paying.fetch_add(rent_paying_this_slot, Ordering::Relaxed);
amount_to_top_off_rent
@ -8318,6 +8330,14 @@ impl AccountsDb {
total_duplicates.fetch_add(total_this_slot, Ordering::Relaxed);
accounts_data_len
.fetch_add(accounts_data_len_this_slot, Ordering::Relaxed);
let mut rent_paying_accounts_by_partition =
rent_paying_accounts_by_partition.lock().unwrap();
rent_paying_accounts_by_partition_this_slot
.iter()
.for_each(|k| {
rent_paying_accounts_by_partition.add_account(k);
});
insert_us
} else {
// verify index matches expected and measure the time to get all items
@ -8471,6 +8491,9 @@ impl AccountsDb {
IndexGenerationInfo {
accounts_data_len: accounts_data_len.load(Ordering::Relaxed),
rent_paying_accounts_by_partition: rent_paying_accounts_by_partition
.into_inner()
.unwrap(),
}
}

5
runtime/src/accounts_index.rs
View File

@ -8,6 +8,7 @@ use {
inline_spl_token::{self, GenericTokenAccount},
inline_spl_token_2022,
pubkey_bins::PubkeyBinCalculator24,
rent_paying_accounts_by_partition::RentPayingAccountsByPartition,
rolling_bit_field::RollingBitField,
secondary_index::*,
},
@ -694,6 +695,9 @@ pub struct AccountsIndex<T: IndexValue> {
pub active_scans: AtomicUsize,
/// # of slots between latest max and latest scan
pub max_distance_to_min_scan_slot: AtomicU64,
/// populated at generate_index time - accounts that could possibly be rent paying
pub rent_paying_accounts_by_partition: RwLock<RentPayingAccountsByPartition>,
}
impl<T: IndexValue> AccountsIndex<T> {
@ -727,6 +731,7 @@ impl<T: IndexValue> AccountsIndex<T> {
roots_removed: AtomicUsize::default(),
active_scans: AtomicUsize::default(),
max_distance_to_min_scan_slot: AtomicU64::default(),
rent_paying_accounts_by_partition: RwLock::default(),
}
}

162
runtime/src/bank.rs
View File

@ -268,7 +268,7 @@ pub type BankSlotDelta = SlotDelta<Result<()>>;
// Each cycle is composed of <partition_count> number of tiny pubkey subranges
// to scan, which is always multiple of the number of slots in epoch.
pub(crate) type PartitionIndex = u64;
type PartitionsPerCycle = u64;
pub type PartitionsPerCycle = u64;
type Partition = (PartitionIndex, PartitionIndex, PartitionsPerCycle);
type RentCollectionCycleParams = (
Epoch,
@ -5217,6 +5217,28 @@ impl Bank {
});
}
/// return all end partition indexes for the given partition
/// partition could be (0, 1, N). In this case we only return [1]
/// the single 'end_index' that covers this partition.
/// partition could be (0, 2, N). In this case, we return [1, 2], which are all
/// the 'end_index' values contained in that range.
/// (0, 0, N) returns [0] as a special case.
/// There is a relationship between
/// 1. 'pubkey_range_from_partition'
/// 2. 'partition_from_pubkey'
/// 3. this function
fn get_partition_end_indexes(partition: &Partition) -> Vec<PartitionIndex> {
if partition.0 == partition.1 && partition.0 == 0 {
// special case for start=end=0. ie. (0, 0, N). This returns [0]
vec![0]
} else {
// normal case of (start, end, N)
// so, we want [start+1, start+2, ..=end]
// if start == end, then return []
(partition.0..partition.1).map(|index| index + 1).collect()
}
}
fn collect_rent_eagerly(&self, just_rewrites: bool) {
if self.lazy_rent_collection.load(Relaxed) {
return;
@ -5231,7 +5253,7 @@ impl Bank {
if parallel {
let ranges = partitions
.iter()
.map(|partition| Self::pubkey_range_from_partition(*partition))
.map(|partition| (*partition, Self::pubkey_range_from_partition(*partition)))
.collect::<Vec<_>>();
// test every range to make sure ranges are not overlapping
// some tests collect rent from overlapping ranges
@ -5243,8 +5265,8 @@ impl Bank {
continue;
}
let i = &ranges[i];
let j = &ranges[j];
let i = &ranges[i].1;
let j = &ranges[j].1;
// make sure i doesn't contain j
if i.contains(j.start()) || i.contains(j.end()) {
parallel = false;
@ -5257,7 +5279,7 @@ impl Bank {
let thread_pool = &self.rc.accounts.accounts_db.thread_pool;
thread_pool.install(|| {
ranges.into_par_iter().for_each(|range| {
self.collect_rent_in_range(range, just_rewrites, &rent_metrics)
self.collect_rent_in_range(range.0, range.1, just_rewrites, &rent_metrics)
});
});
}
@ -5329,6 +5351,8 @@ impl Bank {
&self,
mut accounts: Vec<(Pubkey, AccountSharedData, Slot)>,
just_rewrites: bool,
rent_paying_pubkeys: Option<&HashSet<Pubkey>>,
partition_index: PartitionIndex,
) -> CollectRentFromAccountsInfo {
let mut rent_debits = RentDebits::default();
let mut total_rent_collected_info = CollectedInfo::default();
@ -5379,6 +5403,22 @@ impl Bank {
rewrites_skipped.push((*pubkey, hash));
assert_eq!(rent_collected_info, CollectedInfo::default());
} else if !just_rewrites {
if rent_collected_info.rent_amount > 0 {
if let Some(rent_paying_pubkeys) = rent_paying_pubkeys {
if !rent_paying_pubkeys.contains(pubkey) {
// inc counter instead of assert while we verify this is correct
inc_new_counter_info!("unexpected-rent-paying-pubkey", 1);
warn!(
"Collecting rent from unexpected pubkey: {}, slot: {}, parent_slot: {:?}, partition_index: {}, partition_from_pubkey: {}",
pubkey,
self.slot(),
self.parent().map(|bank| bank.slot()),
partition_index,
Bank::partition_from_pubkey(pubkey, self.epoch_schedule.slots_per_epoch),
);
}
}
}
total_rent_collected_info += rent_collected_info;
accounts_to_store.push((pubkey, account));
}
@ -5411,7 +5451,28 @@ impl Bank {
metrics: &RentMetrics,
) {
let subrange_full = Self::pubkey_range_from_partition(partition);
self.collect_rent_in_range(subrange_full, just_rewrites, metrics)
self.collect_rent_in_range(partition, subrange_full, just_rewrites, metrics)
}
/// get all pubkeys that we expect to be rent-paying or None, if this was not initialized at load time (that should only exist in test cases)
fn get_rent_paying_pubkeys(&self, partition: &Partition) -> Option<HashSet<Pubkey>> {
let rent_paying_accounts = &self
.rc
.accounts
.accounts_db
.accounts_index
.rent_paying_accounts_by_partition
.read()
.unwrap();
rent_paying_accounts.is_initialized().then(|| {
Self::get_partition_end_indexes(partition)
.into_iter()
.flat_map(|end_index| {
rent_paying_accounts.get_pubkeys_in_partition_index(end_index)
})
.cloned()
.collect::<HashSet<_>>()
})
}
/// load accounts with pubkeys in 'subrange_full'
@ -5422,6 +5483,7 @@ impl Bank {
/// This flag is used when restoring from a snapshot to calculate and verify the initial bank's delta hash.
fn collect_rent_in_range(
&self,
partition: Partition,
subrange_full: RangeInclusive<Pubkey>,
just_rewrites: bool,
metrics: &RentMetrics,
@ -5435,6 +5497,9 @@ impl Bank {
hold_range.stop();
metrics.hold_range_us.fetch_add(hold_range.as_us(), Relaxed);
let rent_paying_pubkeys_ = self.get_rent_paying_pubkeys(&partition);
let rent_paying_pubkeys = rent_paying_pubkeys_.as_ref();
// divide the range into num_threads smaller ranges and process in parallel
// Note that 'pubkey_range_from_partition' cannot easily be re-used here to break the range smaller.
// It has special handling of 0..0 and partition_count changes affect all ranges unevenly.
@ -5469,7 +5534,12 @@ impl Bank {
.load_to_collect_rent_eagerly(&self.ancestors, subrange)
});
CollectRentInPartitionInfo::new(
self.collect_rent_from_accounts(accounts, just_rewrites),
self.collect_rent_from_accounts(
accounts,
just_rewrites,
rent_paying_pubkeys,
partition.1,
),
Duration::from_nanos(measure_load_accounts.as_ns()),
)
})
@ -5478,6 +5548,9 @@ impl Bank {
CollectRentInPartitionInfo::reduce,
);
// We cannot assert here that we collected from all expected keys.
// Some accounts may have been topped off or may have had all funds removed and gone to 0 lamports.
self.rc
.accounts
.hold_range_in_memory(&subrange_full, false, thread_pool);
@ -7819,6 +7892,7 @@ pub(crate) mod tests {
genesis_sysvar_and_builtin_program_lamports, GenesisConfigInfo,
ValidatorVoteKeypairs,
},
rent_paying_accounts_by_partition::RentPayingAccountsByPartition,
status_cache::MAX_CACHE_ENTRIES,
},
crossbeam_channel::{bounded, unbounded},
@ -9863,12 +9937,16 @@ pub(crate) mod tests {
let result = later_bank.collect_rent_from_accounts(
vec![(zero_lamport_pubkey, account.clone(), later_slot)],
just_rewrites,
None,
PartitionIndex::default(),
);
assert!(result.rewrites_skipped.is_empty());
// loaded from previous slot, so we skip rent collection on it
let result = later_bank.collect_rent_from_accounts(
vec![(zero_lamport_pubkey, account, later_slot - 1)],
just_rewrites,
None,
PartitionIndex::default(),
);
assert!(result.rewrites_skipped[0].0 == zero_lamport_pubkey);
}
@ -19209,6 +19287,76 @@ pub(crate) mod tests {
}
}
#[test]
fn test_get_partition_end_indexes() {
for n in 5..7 {
assert_eq!(vec![0], Bank::get_partition_end_indexes(&(0, 0, n)));
assert!(Bank::get_partition_end_indexes(&(1, 1, n)).is_empty());
assert_eq!(vec![1], Bank::get_partition_end_indexes(&(0, 1, n)));
assert_eq!(vec![1, 2], Bank::get_partition_end_indexes(&(0, 2, n)));
assert_eq!(vec![3, 4], Bank::get_partition_end_indexes(&(2, 4, n)));
}
}
#[test]
fn test_get_rent_paying_pubkeys() {
let lamports = 1;
let bank = create_simple_test_bank(lamports);
let n = 432_000;
assert!(bank.get_rent_paying_pubkeys(&(0, 1, n)).is_none());
assert!(bank.get_rent_paying_pubkeys(&(0, 2, n)).is_none());
assert!(bank.get_rent_paying_pubkeys(&(0, 0, n)).is_none());
let pk1 = Pubkey::new(&[2; 32]);
let pk2 = Pubkey::new(&[3; 32]);
let index1 = Bank::partition_from_pubkey(&pk1, n);
let index2 = Bank::partition_from_pubkey(&pk2, n);
assert!(index1 > 0, "{}", index1);
assert!(index2 > index1, "{}, {}", index2, index1);
let epoch_schedule = EpochSchedule::custom(n, 0, false);
let mut rent_paying_accounts_by_partition =
RentPayingAccountsByPartition::new(&epoch_schedule);
rent_paying_accounts_by_partition.add_account(&pk1);
rent_paying_accounts_by_partition.add_account(&pk2);
*bank
.rc
.accounts
.accounts_db
.accounts_index
.rent_paying_accounts_by_partition
.write()
.unwrap() = rent_paying_accounts_by_partition;
assert_eq!(
bank.get_rent_paying_pubkeys(&(0, 1, n)),
Some(HashSet::default())
);
assert_eq!(
bank.get_rent_paying_pubkeys(&(0, 2, n)),
Some(HashSet::default())
);
assert_eq!(
bank.get_rent_paying_pubkeys(&(index1.saturating_sub(1), index1, n)),
Some(HashSet::from([pk1]))
);
assert_eq!(
bank.get_rent_paying_pubkeys(&(index2.saturating_sub(1), index2, n)),
Some(HashSet::from([pk2]))
);
assert_eq!(
bank.get_rent_paying_pubkeys(&(index1.saturating_sub(1), index2, n)),
Some(HashSet::from([pk2, pk1]))
);
assert_eq!(
bank.get_rent_paying_pubkeys(&(0, 0, n)),
Some(HashSet::default())
);
}
/// Ensure that accounts data size is updated correctly by rent collection
#[test]
fn test_accounts_data_size_and_rent_collection() {

1
runtime/src/lib.rs
View File

@ -50,6 +50,7 @@ mod nonce_keyed_account;
mod pubkey_bins;
mod read_only_accounts_cache;
pub mod rent_collector;
mod rent_paying_accounts_by_partition;
mod rolling_bit_field;
pub mod runtime_config;
pub mod secondary_index;

72
runtime/src/rent_paying_accounts_by_partition.rs Normal file
View File

@ -0,0 +1,72 @@
//! Provide fast iteration of all pubkeys which could possibly be rent paying, grouped by rent collection partition
use {
crate::bank::{Bank, PartitionIndex, PartitionsPerCycle},
solana_sdk::{epoch_schedule::EpochSchedule, pubkey::Pubkey},
std::collections::HashSet,
};
/// populated at startup with the accounts that were found that are rent paying.
/// These are the 'possible' rent paying accounts.
/// This set can never grow during runtime since it is not possible to create rent paying accounts now.
/// It can shrink during execution if a rent paying account is dropped to lamports=0 or is topped off.
/// The next time the validator restarts, it will remove the account from this list.
#[derive(Debug, Default)]
pub struct RentPayingAccountsByPartition {
/// 1st index is partition end index, 0..=432_000
/// 2nd dimension is list of pubkeys which were identified at startup to be rent paying
/// At the moment, we use this data structure to verify all rent paying accounts are expected.
/// When we stop iterating the accounts index to FIND rent paying accounts, we will no longer need this to be a hashset.
/// It can just be a vec.
pub accounts: Vec<HashSet<Pubkey>>,
partition_count: PartitionsPerCycle,
}
impl RentPayingAccountsByPartition {
/// create new struct. Need slots per epoch from 'epoch_schedule'
pub fn new(epoch_schedule: &EpochSchedule) -> Self {
let partition_count = epoch_schedule.slots_per_epoch;
Self {
partition_count,
accounts: (0..=partition_count)
.into_iter()
.map(|_| HashSet::<Pubkey>::default())
.collect(),
}
}
/// Remember that 'pubkey' can possibly be rent paying.
pub fn add_account(&mut self, pubkey: &Pubkey) {
let partition_end_index = Bank::partition_from_pubkey(pubkey, self.partition_count);
let list = &mut self.accounts[partition_end_index as usize];
list.insert(*pubkey);
}
/// return all pubkeys that can possibly be rent paying with this partition end_index
pub fn get_pubkeys_in_partition_index(
&self,
partition_end_index: PartitionIndex,
) -> &HashSet<Pubkey> {
&self.accounts[partition_end_index as usize]
}
pub fn is_initialized(&self) -> bool {
self.partition_count != 0
}
}
#[cfg(test)]
pub(crate) mod tests {
use super::*;
#[test]
fn test_add() {
let mut test = RentPayingAccountsByPartition::new(&EpochSchedule::custom(32, 0, false));
let pk = Pubkey::new(&[1; 32]);
test.add_account(&pk);
// make sure duplicate adds only result in a single item
test.add_account(&pk);
assert_eq!(test.get_pubkeys_in_partition_index(0).len(), 1);
assert!(test.get_pubkeys_in_partition_index(1).is_empty());
assert!(test.is_initialized());
let test = RentPayingAccountsByPartition::default();
assert!(!test.is_initialized());
}
}

10
runtime/src/serde_snapshot.rs
View File

@ -714,11 +714,19 @@ where
})
.unwrap();
let IndexGenerationInfo { accounts_data_len } = accounts_db.generate_index(
let IndexGenerationInfo {
accounts_data_len,
rent_paying_accounts_by_partition,
} = accounts_db.generate_index(
limit_load_slot_count_from_snapshot,
verify_index,
genesis_config,
);
*accounts_db
.accounts_index
.rent_paying_accounts_by_partition
.write()
.unwrap() = rent_paying_accounts_by_partition;
accounts_db.maybe_add_filler_accounts(
&genesis_config.epoch_schedule,