add bank progress stats (#30405)
* add bank progress stats * handle more bank creation code paths * use wrapping sub
This commit is contained in:
parent
9ecc4462f8
commit
c032dc2754
|
@ -1491,6 +1491,43 @@ pub struct AccountsDb {
|
||||||
/// Some time later (to allow for slow calculation time), the bank hash at a slot calculated using 'M' includes the full accounts hash.
|
/// Some time later (to allow for slow calculation time), the bank hash at a slot calculated using 'M' includes the full accounts hash.
|
||||||
/// Thus, the state of all accounts on a validator is known to be correct at least once per epoch.
|
/// Thus, the state of all accounts on a validator is known to be correct at least once per epoch.
|
||||||
pub epoch_accounts_hash_manager: EpochAccountsHashManager,
|
pub epoch_accounts_hash_manager: EpochAccountsHashManager,
|
||||||
|
|
||||||
|
pub bank_progress: BankCreationFreezingProgress,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BankCreationFreezingProgress {
|
||||||
|
fn report(&self) {
|
||||||
|
if self.last_report.should_update(60_000) {
|
||||||
|
datapoint_info!(
|
||||||
|
"bank_progress",
|
||||||
|
(
|
||||||
|
"difference",
|
||||||
|
self.bank_creation_count
|
||||||
|
.load(Ordering::Acquire)
|
||||||
|
.wrapping_sub(
|
||||||
|
self.bank_freeze_or_destruction_count
|
||||||
|
.load(Ordering::Acquire)
|
||||||
|
),
|
||||||
|
i64
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
/// Keeps track of when all banks that were started as of a known point in time have been frozen or otherwise destroyed.
|
||||||
|
/// When 'bank_freeze_or_destruction_count' exceeds a prior value of 'bank_creation_count',
|
||||||
|
/// this means that we can know all banks that began loading accounts have completed as of the prior value of 'bank_creation_count'.
|
||||||
|
pub struct BankCreationFreezingProgress {
|
||||||
|
/// Incremented each time a bank is created.
|
||||||
|
/// Starting now, this bank could be finding accounts in the index and loading them from accounts db.
|
||||||
|
pub bank_creation_count: AtomicU32,
|
||||||
|
/// Incremented each time a bank is frozen or destroyed.
|
||||||
|
/// At this point, this bank has completed all account loading.
|
||||||
|
pub bank_freeze_or_destruction_count: AtomicU32,
|
||||||
|
|
||||||
|
last_report: AtomicInterval,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
|
@ -2401,6 +2438,7 @@ impl AccountsDb {
|
||||||
|
|
||||||
AccountsDb {
|
AccountsDb {
|
||||||
assert_stakes_cache_consistency: false,
|
assert_stakes_cache_consistency: false,
|
||||||
|
bank_progress: BankCreationFreezingProgress::default(),
|
||||||
create_ancient_storage: CreateAncientStorage::Append,
|
create_ancient_storage: CreateAncientStorage::Append,
|
||||||
verify_accounts_hash_in_bg: VerifyAccountsHashInBackground::default(),
|
verify_accounts_hash_in_bg: VerifyAccountsHashInBackground::default(),
|
||||||
filler_accounts_per_slot: AtomicU64::default(),
|
filler_accounts_per_slot: AtomicU64::default(),
|
||||||
|
@ -4046,6 +4084,7 @@ impl AccountsDb {
|
||||||
|
|
||||||
Self::update_shrink_stats(&self.shrink_stats, stats_sub);
|
Self::update_shrink_stats(&self.shrink_stats, stats_sub);
|
||||||
self.shrink_stats.report();
|
self.shrink_stats.report();
|
||||||
|
self.bank_progress.report();
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn update_shrink_stats(shrink_stats: &ShrinkStats, stats_sub: ShrinkStatsSub) {
|
pub(crate) fn update_shrink_stats(shrink_stats: &ShrinkStats, stats_sub: ShrinkStatsSub) {
|
||||||
|
|
|
@ -171,7 +171,7 @@ use {
|
||||||
sync::{
|
sync::{
|
||||||
atomic::{
|
atomic::{
|
||||||
AtomicBool, AtomicI64, AtomicU64, AtomicUsize,
|
AtomicBool, AtomicI64, AtomicU64, AtomicUsize,
|
||||||
Ordering::{AcqRel, Acquire, Relaxed},
|
Ordering::{AcqRel, Acquire, Relaxed, Release},
|
||||||
},
|
},
|
||||||
Arc, LockResult, RwLock, RwLockReadGuard, RwLockWriteGuard,
|
Arc, LockResult, RwLock, RwLockReadGuard, RwLockWriteGuard,
|
||||||
},
|
},
|
||||||
|
@ -800,6 +800,7 @@ impl PartialEq for Bank {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
let Self {
|
let Self {
|
||||||
|
bank_freeze_or_destruction_incremented: _,
|
||||||
rc: _,
|
rc: _,
|
||||||
status_cache: _,
|
status_cache: _,
|
||||||
blockhash_queue,
|
blockhash_queue,
|
||||||
|
@ -1120,6 +1121,9 @@ pub struct Bank {
|
||||||
pub incremental_snapshot_persistence: Option<BankIncrementalSnapshotPersistence>,
|
pub incremental_snapshot_persistence: Option<BankIncrementalSnapshotPersistence>,
|
||||||
|
|
||||||
pub loaded_programs_cache: Arc<RwLock<LoadedPrograms>>,
|
pub loaded_programs_cache: Arc<RwLock<LoadedPrograms>>,
|
||||||
|
|
||||||
|
/// true when the bank's freezing or destruction has completed
|
||||||
|
bank_freeze_or_destruction_incremented: AtomicBool,
|
||||||
}
|
}
|
||||||
|
|
||||||
struct VoteWithStakeDelegations {
|
struct VoteWithStakeDelegations {
|
||||||
|
@ -1269,6 +1273,7 @@ impl Bank {
|
||||||
|
|
||||||
fn default_with_accounts(accounts: Accounts) -> Self {
|
fn default_with_accounts(accounts: Accounts) -> Self {
|
||||||
let mut bank = Self {
|
let mut bank = Self {
|
||||||
|
bank_freeze_or_destruction_incremented: AtomicBool::default(),
|
||||||
incremental_snapshot_persistence: None,
|
incremental_snapshot_persistence: None,
|
||||||
rc: BankRc::new(accounts, Slot::default()),
|
rc: BankRc::new(accounts, Slot::default()),
|
||||||
status_cache: Arc::<RwLock<BankStatusCache>>::default(),
|
status_cache: Arc::<RwLock<BankStatusCache>>::default(),
|
||||||
|
@ -1332,6 +1337,7 @@ impl Bank {
|
||||||
loaded_programs_cache: Arc::<RwLock<LoadedPrograms>>::default(),
|
loaded_programs_cache: Arc::<RwLock<LoadedPrograms>>::default(),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
bank.bank_created();
|
||||||
let accounts_data_size_initial = bank.get_total_accounts_stats().unwrap().data_len as u64;
|
let accounts_data_size_initial = bank.get_total_accounts_stats().unwrap().data_len as u64;
|
||||||
bank.accounts_data_size_initial = accounts_data_size_initial;
|
bank.accounts_data_size_initial = accounts_data_size_initial;
|
||||||
|
|
||||||
|
@ -1543,7 +1549,9 @@ impl Bank {
|
||||||
let (feature_set, feature_set_time_us) = measure_us!(parent.feature_set.clone());
|
let (feature_set, feature_set_time_us) = measure_us!(parent.feature_set.clone());
|
||||||
|
|
||||||
let accounts_data_size_initial = parent.load_accounts_data_size();
|
let accounts_data_size_initial = parent.load_accounts_data_size();
|
||||||
let mut new = Bank {
|
parent.bank_created();
|
||||||
|
let mut new = Self {
|
||||||
|
bank_freeze_or_destruction_incremented: AtomicBool::default(),
|
||||||
incremental_snapshot_persistence: None,
|
incremental_snapshot_persistence: None,
|
||||||
rc,
|
rc,
|
||||||
status_cache,
|
status_cache,
|
||||||
|
@ -1777,6 +1785,29 @@ impl Bank {
|
||||||
self.vote_only_bank
|
self.vote_only_bank
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn bank_created(&self) {
|
||||||
|
self.rc
|
||||||
|
.accounts
|
||||||
|
.accounts_db
|
||||||
|
.bank_progress
|
||||||
|
.bank_creation_count
|
||||||
|
.fetch_add(1, Release);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bank_frozen_or_destroyed(&self) {
|
||||||
|
if !self
|
||||||
|
.bank_freeze_or_destruction_incremented
|
||||||
|
.swap(true, AcqRel)
|
||||||
|
{
|
||||||
|
self.rc
|
||||||
|
.accounts
|
||||||
|
.accounts_db
|
||||||
|
.bank_progress
|
||||||
|
.bank_freeze_or_destruction_count
|
||||||
|
.fetch_add(1, Release);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Like `new_from_parent` but additionally:
|
/// Like `new_from_parent` but additionally:
|
||||||
/// * Doesn't assume that the parent is anywhere near `slot`, parent could be millions of slots
|
/// * Doesn't assume that the parent is anywhere near `slot`, parent could be millions of slots
|
||||||
/// in the past
|
/// in the past
|
||||||
|
@ -1860,6 +1891,7 @@ impl Bank {
|
||||||
let feature_set = new();
|
let feature_set = new();
|
||||||
let mut bank = Self {
|
let mut bank = Self {
|
||||||
incremental_snapshot_persistence: fields.incremental_snapshot_persistence,
|
incremental_snapshot_persistence: fields.incremental_snapshot_persistence,
|
||||||
|
bank_freeze_or_destruction_incremented: AtomicBool::default(),
|
||||||
rc: bank_rc,
|
rc: bank_rc,
|
||||||
status_cache: new(),
|
status_cache: new(),
|
||||||
blockhash_queue: RwLock::new(fields.blockhash_queue),
|
blockhash_queue: RwLock::new(fields.blockhash_queue),
|
||||||
|
@ -1921,6 +1953,8 @@ impl Bank {
|
||||||
fee_structure: FeeStructure::default(),
|
fee_structure: FeeStructure::default(),
|
||||||
loaded_programs_cache: Arc::<RwLock<LoadedPrograms>>::default(),
|
loaded_programs_cache: Arc::<RwLock<LoadedPrograms>>::default(),
|
||||||
};
|
};
|
||||||
|
bank.bank_created();
|
||||||
|
|
||||||
bank.finish_init(
|
bank.finish_init(
|
||||||
genesis_config,
|
genesis_config,
|
||||||
additional_builtins,
|
additional_builtins,
|
||||||
|
@ -3166,6 +3200,7 @@ impl Bank {
|
||||||
self.freeze_started.store(true, Relaxed);
|
self.freeze_started.store(true, Relaxed);
|
||||||
*hash = self.hash_internal_state();
|
*hash = self.hash_internal_state();
|
||||||
self.rc.accounts.accounts_db.mark_slot_frozen(self.slot());
|
self.rc.accounts.accounts_db.mark_slot_frozen(self.slot());
|
||||||
|
self.bank_frozen_or_destroyed();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -8089,6 +8124,7 @@ impl TotalAccountsStats {
|
||||||
|
|
||||||
impl Drop for Bank {
|
impl Drop for Bank {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
|
self.bank_frozen_or_destroyed();
|
||||||
if let Some(drop_callback) = self.drop_callback.read().unwrap().0.as_ref() {
|
if let Some(drop_callback) = self.drop_callback.read().unwrap().0.as_ref() {
|
||||||
drop_callback.callback(self);
|
drop_callback.callback(self);
|
||||||
} else {
|
} else {
|
||||||
|
|
Loading…
Reference in New Issue