accounts hash calls maybe_rehash_skipped_rewrite (#24316)
This commit is contained in:
parent
16222553b8
commit
dc98510d6d
|
@ -40,6 +40,7 @@ use {
|
||||||
append_vec::{AppendVec, StoredAccountMeta, StoredMeta, StoredMetaWriteVersion},
|
append_vec::{AppendVec, StoredAccountMeta, StoredMeta, StoredMetaWriteVersion},
|
||||||
cache_hash_data::CacheHashData,
|
cache_hash_data::CacheHashData,
|
||||||
contains::Contains,
|
contains::Contains,
|
||||||
|
expected_rent_collection::ExpectedRentCollection,
|
||||||
pubkey_bins::PubkeyBinCalculator24,
|
pubkey_bins::PubkeyBinCalculator24,
|
||||||
read_only_accounts_cache::ReadOnlyAccountsCache,
|
read_only_accounts_cache::ReadOnlyAccountsCache,
|
||||||
rent_collector::RentCollector,
|
rent_collector::RentCollector,
|
||||||
|
@ -5137,6 +5138,11 @@ impl AccountsDb {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// find slot >= 'slot' which is a root or in 'ancestors'
|
||||||
|
pub fn find_unskipped_slot(&self, slot: Slot, ancestors: Option<&Ancestors>) -> Option<Slot> {
|
||||||
|
self.accounts_index.get_next_original_root(slot, ancestors)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn checked_iterative_sum_for_capitalization(total_cap: u64, new_cap: u64) -> u64 {
|
pub fn checked_iterative_sum_for_capitalization(total_cap: u64, new_cap: u64) -> u64 {
|
||||||
let new_total = total_cap as u128 + new_cap as u128;
|
let new_total = total_cap as u128 + new_cap as u128;
|
||||||
AccountsHash::checked_cast_for_capitalization(new_total)
|
AccountsHash::checked_cast_for_capitalization(new_total)
|
||||||
|
@ -5171,6 +5177,8 @@ impl AccountsDb {
|
||||||
// We'll also accumulate the lamports within each chunk and fewer chunks results in less contention to accumulate the sum.
|
// We'll also accumulate the lamports within each chunk and fewer chunks results in less contention to accumulate the sum.
|
||||||
let chunks = crate::accounts_hash::MERKLE_FANOUT.pow(4);
|
let chunks = crate::accounts_hash::MERKLE_FANOUT.pow(4);
|
||||||
let total_lamports = Mutex::<u64>::new(0);
|
let total_lamports = Mutex::<u64>::new(0);
|
||||||
|
let stats = HashStats::default();
|
||||||
|
|
||||||
let get_hashes = || {
|
let get_hashes = || {
|
||||||
keys.par_chunks(chunks)
|
keys.par_chunks(chunks)
|
||||||
.map(|pubkeys| {
|
.map(|pubkeys| {
|
||||||
|
@ -5203,7 +5211,22 @@ impl AccountsDb {
|
||||||
.get_loaded_account()
|
.get_loaded_account()
|
||||||
.and_then(
|
.and_then(
|
||||||
|loaded_account| {
|
|loaded_account| {
|
||||||
|
let find_unskipped_slot = |slot: Slot| {
|
||||||
|
self.find_unskipped_slot(slot, config.ancestors)
|
||||||
|
};
|
||||||
let loaded_hash = loaded_account.loaded_hash();
|
let loaded_hash = loaded_account.loaded_hash();
|
||||||
|
let new_hash = ExpectedRentCollection::maybe_rehash_skipped_rewrite(
|
||||||
|
&loaded_account,
|
||||||
|
&loaded_hash,
|
||||||
|
pubkey,
|
||||||
|
*slot,
|
||||||
|
config.rent_collector,
|
||||||
|
&stats,
|
||||||
|
max_slot + 1, // this wants an 'exclusive' number
|
||||||
|
find_unskipped_slot,
|
||||||
|
self.filler_account_suffix.as_ref(),
|
||||||
|
);
|
||||||
|
let loaded_hash = new_hash.unwrap_or(loaded_hash);
|
||||||
let balance = loaded_account.lamports();
|
let balance = loaded_account.lamports();
|
||||||
if config.check_hash && !self.is_filler_account(pubkey) { // this will not be supported anymore
|
if config.check_hash && !self.is_filler_account(pubkey) { // this will not be supported anymore
|
||||||
let computed_hash =
|
let computed_hash =
|
||||||
|
@ -5260,6 +5283,16 @@ impl AccountsDb {
|
||||||
("hash", hash_time.as_us(), i64),
|
("hash", hash_time.as_us(), i64),
|
||||||
("hash_total", hash_total, i64),
|
("hash_total", hash_total, i64),
|
||||||
("collect", collect.as_us(), i64),
|
("collect", collect.as_us(), i64),
|
||||||
|
(
|
||||||
|
"rehashed_rewrites",
|
||||||
|
stats.rehash_required.load(Ordering::Relaxed),
|
||||||
|
i64
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"rehashed_rewrites_unnecessary",
|
||||||
|
stats.rehash_unnecessary.load(Ordering::Relaxed),
|
||||||
|
i64
|
||||||
|
),
|
||||||
);
|
);
|
||||||
Ok((accumulated_hash, total_lamports))
|
Ok((accumulated_hash, total_lamports))
|
||||||
}
|
}
|
||||||
|
@ -5663,6 +5696,8 @@ impl AccountsDb {
|
||||||
let range = bin_range.end - bin_range.start;
|
let range = bin_range.end - bin_range.start;
|
||||||
let sort_time = AtomicU64::new(0);
|
let sort_time = AtomicU64::new(0);
|
||||||
|
|
||||||
|
let find_unskipped_slot = |slot: Slot| self.find_unskipped_slot(slot, config.ancestors);
|
||||||
|
|
||||||
let result: Vec<BinnedHashData> = self.scan_account_storage_no_bank(
|
let result: Vec<BinnedHashData> = self.scan_account_storage_no_bank(
|
||||||
cache_hash_data,
|
cache_hash_data,
|
||||||
config,
|
config,
|
||||||
|
@ -5685,8 +5720,21 @@ impl AccountsDb {
|
||||||
raw_lamports
|
raw_lamports
|
||||||
};
|
};
|
||||||
|
|
||||||
let source_item =
|
let loaded_hash = loaded_account.loaded_hash();
|
||||||
CalculateHashIntermediate::new(loaded_account.loaded_hash(), balance, *pubkey);
|
let new_hash = ExpectedRentCollection::maybe_rehash_skipped_rewrite(
|
||||||
|
&loaded_account,
|
||||||
|
&loaded_hash,
|
||||||
|
pubkey,
|
||||||
|
slot,
|
||||||
|
config.rent_collector,
|
||||||
|
stats,
|
||||||
|
storage.range().end,
|
||||||
|
find_unskipped_slot,
|
||||||
|
filler_account_suffix,
|
||||||
|
);
|
||||||
|
let loaded_hash = new_hash.unwrap_or(loaded_hash);
|
||||||
|
|
||||||
|
let source_item = CalculateHashIntermediate::new(loaded_hash, balance, *pubkey);
|
||||||
|
|
||||||
if config.check_hash
|
if config.check_hash
|
||||||
&& !Self::is_filler_account_helper(pubkey, filler_account_suffix)
|
&& !Self::is_filler_account_helper(pubkey, filler_account_suffix)
|
||||||
|
|
|
@ -11,7 +11,7 @@ use {
|
||||||
borrow::Borrow,
|
borrow::Borrow,
|
||||||
convert::TryInto,
|
convert::TryInto,
|
||||||
sync::{
|
sync::{
|
||||||
atomic::{AtomicUsize, Ordering},
|
atomic::{AtomicU64, AtomicUsize, Ordering},
|
||||||
Mutex,
|
Mutex,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
@ -70,6 +70,10 @@ pub struct HashStats {
|
||||||
pub min_bin_size: usize,
|
pub min_bin_size: usize,
|
||||||
pub max_bin_size: usize,
|
pub max_bin_size: usize,
|
||||||
pub storage_size_quartiles: StorageSizeQuartileStats,
|
pub storage_size_quartiles: StorageSizeQuartileStats,
|
||||||
|
/// time spent hashing during rehash calls
|
||||||
|
pub rehash_hash_us: AtomicU64,
|
||||||
|
/// time spent determining whether to rehash during rehash calls
|
||||||
|
pub rehash_calc_us: AtomicU64,
|
||||||
/// # rehashes that took place and were necessary
|
/// # rehashes that took place and were necessary
|
||||||
pub rehash_required: AtomicUsize,
|
pub rehash_required: AtomicUsize,
|
||||||
/// # rehashes that took place and were UNnecessary
|
/// # rehashes that took place and were UNnecessary
|
||||||
|
@ -173,6 +177,16 @@ impl HashStats {
|
||||||
self.rehash_required.load(Ordering::Relaxed) as i64,
|
self.rehash_required.load(Ordering::Relaxed) as i64,
|
||||||
i64
|
i64
|
||||||
),
|
),
|
||||||
|
(
|
||||||
|
"rehash_hash_us",
|
||||||
|
self.rehash_hash_us.load(Ordering::Relaxed) as i64,
|
||||||
|
i64
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"rehash_calc_us",
|
||||||
|
self.rehash_calc_us.load(Ordering::Relaxed) as i64,
|
||||||
|
i64
|
||||||
|
),
|
||||||
(
|
(
|
||||||
"rehashed_rewrites_unnecessary",
|
"rehashed_rewrites_unnecessary",
|
||||||
self.rehash_unnecessary.load(Ordering::Relaxed) as i64,
|
self.rehash_unnecessary.load(Ordering::Relaxed) as i64,
|
||||||
|
|
|
@ -396,7 +396,9 @@ impl ExpectedRentCollection {
|
||||||
find_unskipped_slot: impl Fn(Slot) -> Option<Slot>,
|
find_unskipped_slot: impl Fn(Slot) -> Option<Slot>,
|
||||||
filler_account_suffix: Option<&Pubkey>,
|
filler_account_suffix: Option<&Pubkey>,
|
||||||
) -> Option<Hash> {
|
) -> Option<Hash> {
|
||||||
let expected = match ExpectedRentCollection::new(
|
use solana_measure::measure::Measure;
|
||||||
|
let mut m = Measure::start("rehash_calc_us");
|
||||||
|
let expected = ExpectedRentCollection::new(
|
||||||
pubkey,
|
pubkey,
|
||||||
loaded_account,
|
loaded_account,
|
||||||
storage_slot,
|
storage_slot,
|
||||||
|
@ -404,20 +406,26 @@ impl ExpectedRentCollection {
|
||||||
max_slot_in_storages_exclusive,
|
max_slot_in_storages_exclusive,
|
||||||
find_unskipped_slot,
|
find_unskipped_slot,
|
||||||
filler_account_suffix,
|
filler_account_suffix,
|
||||||
) {
|
);
|
||||||
|
|
||||||
|
m.stop();
|
||||||
|
stats.rehash_calc_us.fetch_add(m.as_us(), Ordering::Relaxed);
|
||||||
|
let expected = match expected {
|
||||||
None => {
|
None => {
|
||||||
// use the previously calculated hash
|
// use the previously calculated hash
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
Some(expected) => expected,
|
Some(expected) => expected,
|
||||||
};
|
};
|
||||||
|
let mut m = Measure::start("rehash_hash_us");
|
||||||
let recalc_hash = AccountsDb::hash_account_with_rent_epoch(
|
let recalc_hash = AccountsDb::hash_account_with_rent_epoch(
|
||||||
expected.expected_rent_collection_slot_max_epoch,
|
expected.expected_rent_collection_slot_max_epoch,
|
||||||
loaded_account,
|
loaded_account,
|
||||||
pubkey,
|
pubkey,
|
||||||
expected.rent_epoch,
|
expected.rent_epoch,
|
||||||
);
|
);
|
||||||
|
m.stop();
|
||||||
|
stats.rehash_hash_us.fetch_add(m.as_us(), Ordering::Relaxed);
|
||||||
if &recalc_hash == loaded_hash {
|
if &recalc_hash == loaded_hash {
|
||||||
// unnecessary calculation occurred
|
// unnecessary calculation occurred
|
||||||
stats.rehash_unnecessary.fetch_add(1, Ordering::Relaxed);
|
stats.rehash_unnecessary.fetch_add(1, Ordering::Relaxed);
|
||||||
|
|
Loading…
Reference in New Issue