blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Uses enum for data source with calc_accounts_hash() (#28584)

This commit is contained in:
Brooks Prumo 2022-10-26 13:04:01 -04:00 committed by GitHub
parent 252d7f68da
commit 27269d833c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 78 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
accounts-bench/src/main.rs
View File

@ -10,7 +10,7 @@ use {
test_utils::{create_test_accounts, update_accounts_bench}, test_utils::{create_test_accounts, update_accounts_bench},
Accounts, Accounts,
}, },
accounts_db::AccountShrinkThreshold, accounts_db::{AccountShrinkThreshold, CalcAccountsHashDataSource},
accounts_index::AccountSecondaryIndexes, accounts_index::AccountSecondaryIndexes,
ancestors::Ancestors, ancestors::Ancestors,
rent_collector::RentCollector, rent_collector::RentCollector,
@ -126,7 +126,7 @@ fn main() {
time.stop(); time.stop();
let mut time_store = Measure::start("hash using store"); let mut time_store = Measure::start("hash using store");
let results_store = accounts.accounts_db.update_accounts_hash( let results_store = accounts.accounts_db.update_accounts_hash(
false, CalcAccountsHashDataSource::Storages,
false, false,
solana_sdk::clock::Slot::default(), solana_sdk::clock::Slot::default(),
&ancestors, &ancestors,

7
runtime/src/accounts.rs
View File

@ -4,8 +4,8 @@ use {
account_rent_state::{check_rent_state_with_account, RentState}, account_rent_state::{check_rent_state_with_account, RentState},
accounts_db::{ accounts_db::{
AccountShrinkThreshold, AccountsAddRootTiming, AccountsDb, AccountsDbConfig, AccountShrinkThreshold, AccountsAddRootTiming, AccountsDb, AccountsDbConfig,
BankHashInfo, IncludeSlotInHash, LoadHint, LoadedAccount, ScanStorageResult, BankHashInfo, CalcAccountsHashDataSource, IncludeSlotInHash, LoadHint, LoadedAccount,
ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS, ACCOUNTS_DB_CONFIG_FOR_TESTING, ScanStorageResult, ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS, ACCOUNTS_DB_CONFIG_FOR_TESTING,
}, },
accounts_index::{ accounts_index::{
AccountSecondaryIndexes, IndexKey, ScanConfig, ScanError, ScanResult, ZeroLamport, AccountSecondaryIndexes, IndexKey, ScanConfig, ScanError, ScanResult, ZeroLamport,
@ -789,14 +789,13 @@ impl Accounts {
epoch_schedule: &EpochSchedule, epoch_schedule: &EpochSchedule,
rent_collector: &RentCollector, rent_collector: &RentCollector,
) -> u64 { ) -> u64 {
let use_index = false;
let is_startup = true; let is_startup = true;
self.accounts_db self.accounts_db
.verify_accounts_hash_in_bg .verify_accounts_hash_in_bg
.wait_for_complete(); .wait_for_complete();
self.accounts_db self.accounts_db
.update_accounts_hash( .update_accounts_hash(
use_index, CalcAccountsHashDataSource::Storages,
debug_verify, debug_verify,
slot, slot,
ancestors, ancestors,

10
runtime/src/accounts_background_service.rs
View File

@ -5,6 +5,7 @@
mod stats; mod stats;
use { use {
crate::{ crate::{
accounts_db::CalcAccountsHashDataSource,
accounts_hash::CalcAccountsHashConfig, accounts_hash::CalcAccountsHashConfig,
bank::{Bank, BankSlotDelta, DropCallback}, bank::{Bank, BankSlotDelta, DropCallback},
bank_forks::BankForks, bank_forks::BankForks,
@ -296,7 +297,11 @@ impl SnapshotRequestHandler {
let previous_hash = if test_hash_calculation { let previous_hash = if test_hash_calculation {
// We have to use the index version here. // We have to use the index version here.
// We cannot calculate the non-index way because cache has not been flushed and stores don't match reality. This comment is out of date and can be re-evaluated. // We cannot calculate the non-index way because cache has not been flushed and stores don't match reality. This comment is out of date and can be re-evaluated.
snapshot_root_bank.update_accounts_hash_with_index_option(true, false, false) snapshot_root_bank.update_accounts_hash_with_index_option(
CalcAccountsHashDataSource::Index,
false,
false,
)
} else { } else {
Hash::default() Hash::default()
}; };
@ -330,14 +335,13 @@ impl SnapshotRequestHandler {
flush_accounts_cache_time.stop(); flush_accounts_cache_time.stop();
let hash_for_testing = if test_hash_calculation { let hash_for_testing = if test_hash_calculation {
let use_index_hash_calculation = false;
let check_hash = false; let check_hash = false;
let (this_hash, capitalization) = snapshot_root_bank let (this_hash, capitalization) = snapshot_root_bank
.accounts() .accounts()
.accounts_db .accounts_db
.calculate_accounts_hash( .calculate_accounts_hash(
use_index_hash_calculation, CalcAccountsHashDataSource::Storages,
snapshot_root_bank.slot(), snapshot_root_bank.slot(),
&CalcAccountsHashConfig { &CalcAccountsHashConfig {
use_bg_thread_pool: true, use_bg_thread_pool: true,

101
runtime/src/accounts_db.rs
View File

@ -6923,7 +6923,7 @@ impl AccountsDb {
is_startup: bool, is_startup: bool,
) -> (Hash, u64) { ) -> (Hash, u64) {
self.update_accounts_hash( self.update_accounts_hash(
true, CalcAccountsHashDataSource::Index,
debug_verify, debug_verify,
slot, slot,
ancestors, ancestors,
@ -7227,61 +7227,69 @@ impl AccountsDb {
pub(crate) fn calculate_accounts_hash( pub(crate) fn calculate_accounts_hash(
&self, &self,
use_index: bool, data_source: CalcAccountsHashDataSource,
slot: Slot, slot: Slot,
config: &CalcAccountsHashConfig<'_>, config: &CalcAccountsHashConfig<'_>,
) -> Result<(Hash, u64), BankHashVerificationError> { ) -> Result<(Hash, u64), BankHashVerificationError> {
if !use_index { match data_source {
if self.accounts_cache.contains_any_slots(slot) { CalcAccountsHashDataSource::Storages => {
// this indicates a race condition if self.accounts_cache.contains_any_slots(slot) {
inc_new_counter_info!("accounts_hash_items_in_write_cache", 1); // this indicates a race condition
inc_new_counter_info!("accounts_hash_items_in_write_cache", 1);
}
let mut collect_time = Measure::start("collect");
let (combined_maps, slots) =
self.get_snapshot_storages(slot, None, config.ancestors);
collect_time.stop();
let mut sort_time = Measure::start("sort_storages");
let min_root = self.accounts_index.min_alive_root();
let storages = SortedStorages::new_with_slots(
combined_maps.iter().zip(slots.into_iter()),
min_root,
Some(slot),
);
sort_time.stop();
let mut timings = HashStats {
collect_snapshots_us: collect_time.as_us(),
storage_sort_us: sort_time.as_us(),
..HashStats::default()
};
timings.calc_storage_size_quartiles(&combined_maps);
self.calculate_accounts_hash_from_storages(config, &storages, timings)
}
CalcAccountsHashDataSource::Index => {
self.calculate_accounts_hash_from_index(slot, config)
} }
let mut collect_time = Measure::start("collect");
let (combined_maps, slots) = self.get_snapshot_storages(slot, None, config.ancestors);
collect_time.stop();
let mut sort_time = Measure::start("sort_storages");
let min_root = self.accounts_index.min_alive_root();
let storages = SortedStorages::new_with_slots(
combined_maps.iter().zip(slots.into_iter()),
min_root,
Some(slot),
);
sort_time.stop();
let mut timings = HashStats {
collect_snapshots_us: collect_time.as_us(),
storage_sort_us: sort_time.as_us(),
..HashStats::default()
};
timings.calc_storage_size_quartiles(&combined_maps);
self.calculate_accounts_hash_from_storages(config, &storages, timings)
} else {
self.calculate_accounts_hash_from_index(slot, config)
} }
} }
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
fn calculate_accounts_hash_with_verify( fn calculate_accounts_hash_with_verify(
&self, &self,
use_index: bool, data_source: CalcAccountsHashDataSource,
debug_verify: bool, debug_verify: bool,
slot: Slot, slot: Slot,
config: CalcAccountsHashConfig<'_>, config: CalcAccountsHashConfig<'_>,
expected_capitalization: Option<u64>, expected_capitalization: Option<u64>,
) -> Result<(Hash, u64), BankHashVerificationError> { ) -> Result<(Hash, u64), BankHashVerificationError> {
let (hash, total_lamports) = self.calculate_accounts_hash(use_index, slot, &config)?; let (hash, total_lamports) = self.calculate_accounts_hash(data_source, slot, &config)?;
if debug_verify { if debug_verify {
// calculate the other way (store or non-store) and verify results match. // calculate the other way (store or non-store) and verify results match.
let data_source_other = match data_source {
CalcAccountsHashDataSource::Index => CalcAccountsHashDataSource::Storages,
CalcAccountsHashDataSource::Storages => CalcAccountsHashDataSource::Index,
};
let (hash_other, total_lamports_other) = let (hash_other, total_lamports_other) =
self.calculate_accounts_hash(!use_index, slot, &config)?; self.calculate_accounts_hash(data_source_other, slot, &config)?;
let success = hash == hash_other let success = hash == hash_other
&& total_lamports == total_lamports_other && total_lamports == total_lamports_other
&& total_lamports == expected_capitalization.unwrap_or(total_lamports); && total_lamports == expected_capitalization.unwrap_or(total_lamports);
assert!(success, "calculate_accounts_hash_with_verify mismatch. hashes: {}, {}; lamports: {}, {}; expected lamports: {:?}, using index: {}, slot: {}", hash, hash_other, total_lamports, total_lamports_other, expected_capitalization, use_index, slot); assert!(success, "calculate_accounts_hash_with_verify mismatch. hashes: {}, {}; lamports: {}, {}; expected lamports: {:?}, data source: {:?}, slot: {}", hash, hash_other, total_lamports, total_lamports_other, expected_capitalization, data_source, slot);
} }
Ok((hash, total_lamports)) Ok((hash, total_lamports))
} }
@ -7289,7 +7297,7 @@ impl AccountsDb {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn update_accounts_hash( pub fn update_accounts_hash(
&self, &self,
use_index: bool, data_source: CalcAccountsHashDataSource,
debug_verify: bool, debug_verify: bool,
slot: Slot, slot: Slot,
ancestors: &Ancestors, ancestors: &Ancestors,
@ -7301,7 +7309,7 @@ impl AccountsDb {
let check_hash = false; let check_hash = false;
let (hash, total_lamports) = self let (hash, total_lamports) = self
.calculate_accounts_hash_with_verify( .calculate_accounts_hash_with_verify(
use_index, data_source,
debug_verify, debug_verify,
slot, slot,
CalcAccountsHashConfig { CalcAccountsHashConfig {
@ -7595,10 +7603,9 @@ impl AccountsDb {
) -> Result<(), BankHashVerificationError> { ) -> Result<(), BankHashVerificationError> {
use BankHashVerificationError::*; use BankHashVerificationError::*;
let use_index = false;
let check_hash = false; // this will not be supported anymore let check_hash = false; // this will not be supported anymore
let (calculated_hash, calculated_lamports) = self.calculate_accounts_hash_with_verify( let (calculated_hash, calculated_lamports) = self.calculate_accounts_hash_with_verify(
use_index, CalcAccountsHashDataSource::Storages,
test_hash_calculation, test_hash_calculation,
slot, slot,
CalcAccountsHashConfig { CalcAccountsHashConfig {
@ -9390,6 +9397,13 @@ impl AccountsDb {
} }
} }
/// Specify the source of the accounts data when calculating the accounts hash
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum CalcAccountsHashDataSource {
Index,
Storages,
}
#[cfg(test)] #[cfg(test)]
impl AccountsDb { impl AccountsDb {
pub fn new(paths: Vec<PathBuf>, cluster_type: &ClusterType) -> Self { pub fn new(paths: Vec<PathBuf>, cluster_type: &ClusterType) -> Self {
@ -12344,10 +12358,13 @@ pub mod tests {
); );
db.add_root(some_slot); db.add_root(some_slot);
let check_hash = true; let check_hash = true;
for use_index in [true, false] { for data_source in [
CalcAccountsHashDataSource::Index,
CalcAccountsHashDataSource::Storages,
] {
assert!(db assert!(db
.calculate_accounts_hash( .calculate_accounts_hash(
use_index, data_source,
some_slot, some_slot,
&CalcAccountsHashConfig { &CalcAccountsHashConfig {
use_bg_thread_pool: true, // is_startup used to be false use_bg_thread_pool: true, // is_startup used to be false
@ -12399,7 +12416,7 @@ pub mod tests {
let check_hash = true; let check_hash = true;
assert_eq!( assert_eq!(
db.calculate_accounts_hash( db.calculate_accounts_hash(
false, CalcAccountsHashDataSource::Storages,
some_slot, some_slot,
&CalcAccountsHashConfig { &CalcAccountsHashConfig {
use_bg_thread_pool: true, // is_startup used to be false use_bg_thread_pool: true, // is_startup used to be false
@ -12410,7 +12427,7 @@ pub mod tests {
) )
.unwrap(), .unwrap(),
db.calculate_accounts_hash( db.calculate_accounts_hash(
true, CalcAccountsHashDataSource::Index,
some_slot, some_slot,
&CalcAccountsHashConfig { &CalcAccountsHashConfig {
use_bg_thread_pool: true, // is_startup used to be false use_bg_thread_pool: true, // is_startup used to be false

13
runtime/src/bank.rs
View File

@ -44,8 +44,9 @@ use {
TransactionLoadResult, TransactionLoadResult,
}, },
accounts_db::{ accounts_db::{
AccountShrinkThreshold, AccountsDbConfig, IncludeSlotInHash, SnapshotStorages, AccountShrinkThreshold, AccountsDbConfig, CalcAccountsHashDataSource,
ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS, ACCOUNTS_DB_CONFIG_FOR_TESTING, IncludeSlotInHash, SnapshotStorages, ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS,
ACCOUNTS_DB_CONFIG_FOR_TESTING,
}, },
accounts_index::{AccountSecondaryIndexes, IndexKey, ScanConfig, ScanResult, ZeroLamport}, accounts_index::{AccountSecondaryIndexes, IndexKey, ScanConfig, ScanResult, ZeroLamport},
accounts_update_notifier_interface::AccountsUpdateNotifier, accounts_update_notifier_interface::AccountsUpdateNotifier,
@ -6980,12 +6981,12 @@ impl Bank {
pub fn update_accounts_hash_with_index_option( pub fn update_accounts_hash_with_index_option(
&self, &self,
use_index: bool, data_source: CalcAccountsHashDataSource,
mut debug_verify: bool, mut debug_verify: bool,
is_startup: bool, is_startup: bool,
) -> Hash { ) -> Hash {
let (hash, total_lamports) = self.rc.accounts.accounts_db.update_accounts_hash( let (hash, total_lamports) = self.rc.accounts.accounts_db.update_accounts_hash(
use_index, data_source,
debug_verify, debug_verify,
self.slot(), self.slot(),
&self.ancestors, &self.ancestors,
@ -7007,7 +7008,7 @@ impl Bank {
// Run both versions of the calculation to attempt to get more info. // Run both versions of the calculation to attempt to get more info.
debug_verify = true; debug_verify = true;
self.rc.accounts.accounts_db.update_accounts_hash( self.rc.accounts.accounts_db.update_accounts_hash(
use_index, data_source,
debug_verify, debug_verify,
self.slot(), self.slot(),
&self.ancestors, &self.ancestors,
@ -7029,7 +7030,7 @@ impl Bank {
} }
pub fn update_accounts_hash(&self) -> Hash { pub fn update_accounts_hash(&self) -> Hash {
self.update_accounts_hash_with_index_option(true, false, false) self.update_accounts_hash_with_index_option(CalcAccountsHashDataSource::Index, false, false)
} }
/// A snapshot bank should be purged of 0 lamport accounts which are not part of the hash /// A snapshot bank should be purged of 0 lamport accounts which are not part of the hash