blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

add epoch_schedule and rent_collector to hash calc (#24012)

This commit is contained in:
Jeff Washington (jwash) 2022-03-31 10:51:18 -05:00 committed by GitHub
parent da001d54e5
commit 9c8dad33c7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 217 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
accounts-bench/src/main.rs
View File

@ -10,8 +10,11 @@ use {
accounts_db::AccountShrinkThreshold,
accounts_index::AccountSecondaryIndexes,
ancestors::Ancestors,
rent_collector::RentCollector,
},
solana_sdk::{
genesis_config::ClusterType, pubkey::Pubkey, sysvar::epoch_schedule::EpochSchedule,
},
solana_sdk::{genesis_config::ClusterType, pubkey::Pubkey},
std::{env, fs, path::PathBuf},
};
@ -114,7 +117,12 @@ fn main() {
} else {
let mut pubkeys: Vec<Pubkey> = vec![];
let mut time = Measure::start("hash");
let results = accounts.accounts_db.update_accounts_hash(0, &ancestors);
let results = accounts.accounts_db.update_accounts_hash(
0,
&ancestors,
&EpochSchedule::default(),
&RentCollector::default(),
);
time.stop();
let mut time_store = Measure::start("hash using store");
let results_store = accounts.accounts_db.update_accounts_hash_with_index_option(
@ -124,8 +132,8 @@ fn main() {
&ancestors,
None,
false,
None,
None,
&EpochSchedule::default(),
&RentCollector::default(),
false,
);
time_store.stop();

10
core/src/accounts_hash_verifier.rs
View File

@ -129,6 +129,8 @@ impl AccountsHashVerifier {
check_hash: false,
ancestors: None,
use_write_cache: false,
epoch_schedule: &accounts_package.epoch_schedule,
rent_collector: &accounts_package.rent_collector,
},
&sorted_storages,
timings,
@ -278,11 +280,15 @@ mod tests {
use {
super::*,
solana_gossip::{cluster_info::make_accounts_hashes_message, contact_info::ContactInfo},
solana_runtime::snapshot_utils::{ArchiveFormat, SnapshotVersion},
solana_runtime::{
rent_collector::RentCollector,
snapshot_utils::{ArchiveFormat, SnapshotVersion},
},
solana_sdk::{
genesis_config::ClusterType,
hash::hash,
signature::{Keypair, Signer},
sysvar::epoch_schedule::EpochSchedule,
},
solana_streamer::socket::SocketAddrSpace,
};
@ -364,6 +370,8 @@ mod tests {
cluster_type: ClusterType::MainnetBeta,
snapshot_type: None,
accounts: Arc::clone(&accounts),
epoch_schedule: EpochSchedule::default(),
rent_collector: RentCollector::default(),
};
AccountsHashVerifier::process_accounts_package(

20
runtime/benches/accounts.rs
View File

@ -13,6 +13,7 @@ use {
accounts_index::{AccountSecondaryIndexes, ScanConfig},
ancestors::Ancestors,
bank::*,
rent_collector::RentCollector,
},
solana_sdk::{
account::{AccountSharedData, ReadableAccount},
@ -20,6 +21,7 @@ use {
hash::Hash,
lamports::LamportsError,
pubkey::Pubkey,
sysvar::epoch_schedule::EpochSchedule,
},
std::{
collections::{HashMap, HashSet},
@ -107,7 +109,12 @@ fn test_accounts_hash_bank_hash(bencher: &mut Bencher) {
let slot = 0;
create_test_accounts(&accounts, &mut pubkeys, num_accounts, slot);
let ancestors = Ancestors::from(vec![0]);
let (_, total_lamports) = accounts.accounts_db.update_accounts_hash(0, &ancestors);
let (_, total_lamports) = accounts.accounts_db.update_accounts_hash(
0,
&ancestors,
&EpochSchedule::default(),
&RentCollector::default(),
);
let test_hash_calculation = false;
bencher.iter(|| {
assert!(accounts.verify_bank_hash_and_lamports(
@ -115,8 +122,8 @@ fn test_accounts_hash_bank_hash(bencher: &mut Bencher) {
&ancestors,
total_lamports,
test_hash_calculation,
None,
None,
&EpochSchedule::default(),
&RentCollector::default()
))
});
}
@ -135,7 +142,12 @@ fn test_update_accounts_hash(bencher: &mut Bencher) {
create_test_accounts(&accounts, &mut pubkeys, 50_000, 0);
let ancestors = Ancestors::from(vec![0]);
bencher.iter(|| {
accounts.accounts_db.update_accounts_hash(0, &ancestors);
accounts.accounts_db.update_accounts_hash(
0,
&ancestors,
&EpochSchedule::default(),
&RentCollector::default(),
);
});
}

8
runtime/src/accounts.rs
View File

@ -747,8 +747,8 @@ impl Accounts {
slot: Slot,
can_cached_slot_be_unflushed: bool,
debug_verify: bool,
epoch_schedule: Option<&EpochSchedule>,
rent_collector: Option<&RentCollector>,
epoch_schedule: &EpochSchedule,
rent_collector: &RentCollector,
) -> u64 {
let use_index = false;
let is_startup = false; // there may be conditions where this is called at startup.
@ -775,8 +775,8 @@ impl Accounts {
ancestors: &Ancestors,
total_lamports: u64,
test_hash_calculation: bool,
epoch_schedule: Option<&EpochSchedule>,
rent_collector: Option<&RentCollector>,
epoch_schedule: &EpochSchedule,
rent_collector: &RentCollector,
) -> bool {
if let Err(err) = self.accounts_db.verify_bank_hash_and_lamports_new(
slot,

190
runtime/src/accounts_db.rs
View File

@ -5234,15 +5234,37 @@ impl AccountsDb {
bank_hash_info.snapshot_hash
}
pub fn update_accounts_hash(&self, slot: Slot, ancestors: &Ancestors) -> (Hash, u64) {
pub fn update_accounts_hash(
&self,
slot: Slot,
ancestors: &Ancestors,
epoch_schedule: &EpochSchedule,
rent_collector: &RentCollector,
) -> (Hash, u64) {
self.update_accounts_hash_with_index_option(
true, false, slot, ancestors, None, false, None, None, false,
true,
false,
slot,
ancestors,
None,
false,
epoch_schedule,
rent_collector,
false,
)
}
pub fn update_accounts_hash_test(&self, slot: Slot, ancestors: &Ancestors) -> (Hash, u64) {
self.update_accounts_hash_with_index_option(
true, true, slot, ancestors, None, false, None, None, false,
true,
true,
slot,
ancestors,
None,
false,
&EpochSchedule::default(),
&RentCollector::default(),
false,
)
}
@ -5484,7 +5506,6 @@ impl AccountsDb {
use_index: bool,
slot: Slot,
config: &CalcAccountsHashConfig<'_>,
slots_per_epoch: Option<Slot>,
) -> Result<(Hash, u64), BankHashVerificationError> {
if !use_index {
let mut collect_time = Measure::start("collect");
@ -5499,7 +5520,7 @@ impl AccountsDb {
Some(slot),
);
self.mark_old_slots_as_dirty(&storages, slots_per_epoch);
self.mark_old_slots_as_dirty(&storages, Some(config.epoch_schedule.slots_per_epoch));
sort_time.stop();
let mut timings = HashStats {
@ -5528,15 +5549,14 @@ impl AccountsDb {
slot: Slot,
config: CalcAccountsHashConfig<'_>,
expected_capitalization: Option<u64>,
slots_per_epoch: Option<Slot>,
) -> Result<(Hash, u64), BankHashVerificationError> {
let _guard = self.active_stats.activate(ActiveStatItem::Hash);
let (hash, total_lamports) =
self.calculate_accounts_hash_helper(use_index, slot, &config, slots_per_epoch)?;
self.calculate_accounts_hash_helper(use_index, slot, &config)?;
if debug_verify {
// calculate the other way (store or non-store) and verify results match.
let (hash_other, total_lamports_other) =
self.calculate_accounts_hash_helper(!use_index, slot, &config, None)?;
self.calculate_accounts_hash_helper(!use_index, slot, &config)?;
let success = hash == hash_other
&& total_lamports == total_lamports_other
@ -5555,12 +5575,11 @@ impl AccountsDb {
ancestors: &Ancestors,
expected_capitalization: Option<u64>,
can_cached_slot_be_unflushed: bool,
epoch_schedule: Option<&EpochSchedule>,
_rent_collector: Option<&RentCollector>,
epoch_schedule: &EpochSchedule,
rent_collector: &RentCollector,
is_startup: bool,
) -> (Hash, u64) {
let check_hash = false;
let slots_per_epoch = epoch_schedule.map(|epoch_schedule| epoch_schedule.slots_per_epoch);
let (hash, total_lamports) = self
.calculate_accounts_hash_helper_with_verify(
use_index,
@ -5571,9 +5590,10 @@ impl AccountsDb {
check_hash,
ancestors: Some(ancestors),
use_write_cache: can_cached_slot_be_unflushed,
epoch_schedule,
rent_collector,
},
expected_capitalization,
slots_per_epoch,
)
.unwrap(); // unwrap here will never fail since check_hash = false
let mut bank_hashes = self.bank_hashes.write().unwrap();
@ -5784,14 +5804,16 @@ impl AccountsDb {
ancestors: &Ancestors,
total_lamports: u64,
test_hash_calculation: bool,
epoch_schedule: &EpochSchedule,
rent_collector: &RentCollector,
) -> Result<(), BankHashVerificationError> {
self.verify_bank_hash_and_lamports_new(
slot,
ancestors,
total_lamports,
test_hash_calculation,
None,
None,
epoch_schedule,
rent_collector,
)
}
@ -5802,8 +5824,8 @@ impl AccountsDb {
ancestors: &Ancestors,
total_lamports: u64,
test_hash_calculation: bool,
_epoch_schedule: Option<&EpochSchedule>,
_rent_collector: Option<&RentCollector>,
epoch_schedule: &EpochSchedule,
rent_collector: &RentCollector,
) -> Result<(), BankHashVerificationError> {
use BankHashVerificationError::*;
@ -5821,9 +5843,10 @@ impl AccountsDb {
check_hash,
ancestors: Some(ancestors),
use_write_cache: can_cached_slot_be_unflushed,
epoch_schedule,
rent_collector,
},
None,
None, // could use epoch_schedule.slots_per_epoch here
)?;
if calculated_lamports != total_lamports {
@ -7925,6 +7948,8 @@ pub mod tests {
check_hash: false,
ancestors: None,
use_write_cache: false,
epoch_schedule: &EpochSchedule::default(),
rent_collector: &RentCollector::default(),
},
&get_storage_refs(&storages),
HashStats::default(),
@ -7952,6 +7977,8 @@ pub mod tests {
check_hash: false,
ancestors: None,
use_write_cache: false,
epoch_schedule: &EpochSchedule::default(),
rent_collector: &RentCollector::default(),
},
&get_storage_refs(&storages),
HashStats::default(),
@ -9371,8 +9398,18 @@ pub mod tests {
let ancestors = linear_ancestors(latest_slot);
assert_eq!(
daccounts.update_accounts_hash(latest_slot, &ancestors),
accounts.update_accounts_hash(latest_slot, &ancestors)
daccounts.update_accounts_hash(
latest_slot,
&ancestors,
&EpochSchedule::default(),
&RentCollector::default()
),
accounts.update_accounts_hash(
latest_slot,
&ancestors,
&EpochSchedule::default(),
&RentCollector::default()
)
);
}
@ -9651,7 +9688,12 @@ pub mod tests {
accounts.add_root(current_slot);
accounts.print_accounts_stats("pre_f");
accounts.update_accounts_hash(4, &Ancestors::default());
accounts.update_accounts_hash(
4,
&Ancestors::default(),
&EpochSchedule::default(),
&RentCollector::default(),
);
let accounts = f(accounts, current_slot);
@ -9663,7 +9705,14 @@ pub mod tests {
assert_load_account(&accounts, current_slot, dummy_pubkey, dummy_lamport);
accounts
.verify_bank_hash_and_lamports(4, &Ancestors::default(), 1222, true)
.verify_bank_hash_and_lamports(
4,
&Ancestors::default(),
1222,
true,
&EpochSchedule::default(),
&RentCollector::default(),
)
.unwrap();
}
@ -9973,8 +10022,9 @@ pub mod tests {
check_hash,
ancestors: Some(&ancestors),
use_write_cache: false,
epoch_schedule: &EpochSchedule::default(),
rent_collector: &RentCollector::default(),
},
None
)
.is_err());
}
@ -10004,8 +10054,9 @@ pub mod tests {
check_hash,
ancestors: Some(&ancestors),
use_write_cache: false,
epoch_schedule: &EpochSchedule::default(),
rent_collector: &RentCollector::default(),
},
None,
)
.unwrap(),
db.calculate_accounts_hash_helper(
@ -10016,8 +10067,9 @@ pub mod tests {
check_hash,
ancestors: Some(&ancestors),
use_write_cache: false,
epoch_schedule: &EpochSchedule::default(),
rent_collector: &RentCollector::default(),
},
None,
)
.unwrap(),
);
@ -10039,13 +10091,27 @@ pub mod tests {
db.add_root(some_slot);
db.update_accounts_hash_test(some_slot, &ancestors);
assert_matches!(
db.verify_bank_hash_and_lamports(some_slot, &ancestors, 1, true),
db.verify_bank_hash_and_lamports(
some_slot,
&ancestors,
1,
true,
&EpochSchedule::default(),
&RentCollector::default()
),
Ok(_)
);
db.bank_hashes.write().unwrap().remove(&some_slot).unwrap();
assert_matches!(
db.verify_bank_hash_and_lamports(some_slot, &ancestors, 1, true),
db.verify_bank_hash_and_lamports(
some_slot,
&ancestors,
1,
true,
&EpochSchedule::default(),
&RentCollector::default()
),
Err(MissingBankHash)
);
@ -10060,7 +10126,14 @@ pub mod tests {
.unwrap()
.insert(some_slot, bank_hash_info);
assert_matches!(
db.verify_bank_hash_and_lamports(some_slot, &ancestors, 1, true),
db.verify_bank_hash_and_lamports(
some_slot,
&ancestors,
1,
true,
&EpochSchedule::default(),
&RentCollector::default()
),
Err(MismatchedBankHash)
);
}
@ -10081,7 +10154,14 @@ pub mod tests {
db.add_root(some_slot);
db.update_accounts_hash_test(some_slot, &ancestors);
assert_matches!(
db.verify_bank_hash_and_lamports(some_slot, &ancestors, 1, true),
db.verify_bank_hash_and_lamports(
some_slot,
&ancestors,
1,
true,
&EpochSchedule::default(),
&RentCollector::default()
),
Ok(_)
);
@ -10095,12 +10175,19 @@ pub mod tests {
);
db.update_accounts_hash_test(some_slot, &ancestors);
assert_matches!(
db.verify_bank_hash_and_lamports(some_slot, &ancestors, 2, true),
db.verify_bank_hash_and_lamports(
some_slot,
&ancestors,
2,
true,
&EpochSchedule::default(),
&RentCollector::default()
),
Ok(_)
);
assert_matches!(
db.verify_bank_hash_and_lamports(some_slot, &ancestors, 10, true),
db.verify_bank_hash_and_lamports(some_slot, &ancestors, 10, true, &EpochSchedule::default(), &RentCollector::default()),
Err(MismatchedTotalLamports(expected, actual)) if expected == 2 && actual == 10
);
}
@ -10120,7 +10207,14 @@ pub mod tests {
db.add_root(some_slot);
db.update_accounts_hash_test(some_slot, &ancestors);
assert_matches!(
db.verify_bank_hash_and_lamports(some_slot, &ancestors, 0, true),
db.verify_bank_hash_and_lamports(
some_slot,
&ancestors,
0,
true,
&EpochSchedule::default(),
&RentCollector::default()
),
Ok(_)
);
}
@ -10150,7 +10244,14 @@ pub mod tests {
db.store_accounts_unfrozen((some_slot, accounts), Some(&[&some_hash]), false);
db.add_root(some_slot);
assert_matches!(
db.verify_bank_hash_and_lamports(some_slot, &ancestors, 1, true),
db.verify_bank_hash_and_lamports(
some_slot,
&ancestors,
1,
true,
&EpochSchedule::default(),
&RentCollector::default()
),
Err(MismatchedBankHash)
);
}
@ -10754,14 +10855,33 @@ pub mod tests {
);
let no_ancestors = Ancestors::default();
accounts.update_accounts_hash(current_slot, &no_ancestors);
accounts.update_accounts_hash(
current_slot,
&no_ancestors,
&EpochSchedule::default(),
&RentCollector::default(),
);
accounts
.verify_bank_hash_and_lamports(current_slot, &no_ancestors, 22300, true)
.verify_bank_hash_and_lamports(
current_slot,
&no_ancestors,
22300,
true,
&EpochSchedule::default(),
&RentCollector::default(),
)
.unwrap();
let accounts = reconstruct_accounts_db_via_serialization(&accounts, current_slot);
accounts
.verify_bank_hash_and_lamports(current_slot, &no_ancestors, 22300, true)
.verify_bank_hash_and_lamports(
current_slot,
&no_ancestors,
22300,
true,
&EpochSchedule::default(),
&RentCollector::default(),
)
.unwrap();
// repeating should be no-op

5
runtime/src/accounts_hash.rs
View File

@ -1,11 +1,12 @@
use {
crate::{accounts_db::SnapshotStorages, ancestors::Ancestors},
crate::{accounts_db::SnapshotStorages, ancestors::Ancestors, rent_collector::RentCollector},
log::*,
rayon::prelude::*,
solana_measure::measure::Measure,
solana_sdk::{
hash::{Hash, Hasher},
pubkey::Pubkey,
sysvar::epoch_schedule::EpochSchedule,
},
std::{borrow::Borrow, convert::TryInto, sync::Mutex},
};
@ -33,6 +34,8 @@ pub struct CalcAccountsHashConfig<'a> {
/// does hash calc need to consider account data that exists in the write cache?
/// if so, 'ancestors' will be used for this purpose as well as storages.
pub use_write_cache: bool,
pub epoch_schedule: &'a EpochSchedule,
pub rent_collector: &'a RentCollector,
}
// smallest, 3 quartiles, largest, average

16
runtime/src/bank.rs
View File

@ -5857,8 +5857,8 @@ impl Bank {
&self.ancestors,
self.capitalization(),
test_hash_calculation,
Some(self.epoch_schedule()),
Some(&self.rent_collector),
self.epoch_schedule(),
&self.rent_collector,
)
}
@ -5926,8 +5926,8 @@ impl Bank {
self.slot(),
can_cached_slot_be_unflushed,
debug_verify,
Some(self.epoch_schedule()),
Some(&self.rent_collector),
self.epoch_schedule(),
&self.rent_collector,
)
}
@ -5980,8 +5980,8 @@ impl Bank {
&self.ancestors,
Some(self.capitalization()),
false,
Some(self.epoch_schedule()),
Some(&self.rent_collector),
self.epoch_schedule(),
&self.rent_collector,
is_startup,
);
if total_lamports != self.capitalization() {
@ -6006,8 +6006,8 @@ impl Bank {
&self.ancestors,
Some(self.capitalization()),
false,
Some(self.epoch_schedule()),
Some(&self.rent_collector),
self.epoch_schedule(),
&self.rent_collector,
is_startup,
);
}

9
runtime/src/snapshot_package.rs
View File

@ -3,6 +3,7 @@ use {
accounts::Accounts,
accounts_db::SnapshotStorages,
bank::{Bank, BankSlotDelta},
rent_collector::RentCollector,
snapshot_archive_info::{SnapshotArchiveInfo, SnapshotArchiveInfoGetter},
snapshot_utils::{
self, ArchiveFormat, BankSnapshotInfo, Result, SnapshotVersion,
@ -11,7 +12,9 @@ use {
},
crossbeam_channel::{Receiver, SendError, Sender},
log::*,
solana_sdk::{clock::Slot, genesis_config::ClusterType, hash::Hash},
solana_sdk::{
clock::Slot, genesis_config::ClusterType, hash::Hash, sysvar::epoch_schedule::EpochSchedule,
},
std::{
fs,
path::{Path, PathBuf},
@ -49,6 +52,8 @@ pub struct AccountsPackage {
pub cluster_type: ClusterType,
pub snapshot_type: Option<SnapshotType>,
pub accounts: Arc<Accounts>,
pub epoch_schedule: EpochSchedule,
pub rent_collector: RentCollector,
}
impl AccountsPackage {
@ -114,6 +119,8 @@ impl AccountsPackage {
cluster_type: bank.cluster_type(),
snapshot_type,
accounts: bank.accounts(),
epoch_schedule: *bank.epoch_schedule(),
rent_collector: bank.rent_collector(),
})
}
}

1
runtime/src/snapshot_utils.rs
View File

@ -162,6 +162,7 @@ struct UnpackedSnapshotsDirAndVersion {
}
#[derive(Error, Debug)]
#[allow(clippy::large_enum_variant)]
pub enum SnapshotError {
#[error("I/O error: {0}")]
Io(#[from] std::io::Error),