blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Rename AccountsPacakge to SnapshotPackage and AccountsPackagePre to AccountsPackage (#19231) 

Renaming these types to better communicate their usages, which will
further diverge as incremental snapshot support is added.

With the new names, AccountsPacakge now refers to the type between
AccountsBackgroundProcess and AccountsHashVerifier, and SnapshotPackage
refers to the type between AccountsHashVerifier and
SnapshotPackagerService.
This commit is contained in:
Brooks Prumo 2021-08-13 16:08:09 -05:00 committed by GitHub
parent 32501866b7
commit 176036aa58
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 78 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
core/src/accounts_hash_verifier.rs
View File

@ -11,7 +11,8 @@ use solana_runtime::{
accounts_db, accounts_db,
snapshot_archive_info::SnapshotArchiveInfoGetter, snapshot_archive_info::SnapshotArchiveInfoGetter,
snapshot_config::SnapshotConfig, snapshot_config::SnapshotConfig,
snapshot_package::{AccountsPackage, AccountsPackagePre, AccountsPackageReceiver}, snapshot_package::{AccountsPackage, AccountsPackageReceiver, SnapshotPackage},
snapshot_utils,
}; };
use solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey}; use solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey};
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet};
@ -61,7 +62,7 @@ impl AccountsHashVerifier {
Some(accounts_db::make_min_priority_thread_pool()); Some(accounts_db::make_min_priority_thread_pool());
} }
Self::process_accounts_package_pre( Self::process_accounts_package(
accounts_package, accounts_package,
&cluster_info, &cluster_info,
trusted_validators.as_ref(), trusted_validators.as_ref(),
@ -86,8 +87,8 @@ impl AccountsHashVerifier {
} }
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
fn process_accounts_package_pre( fn process_accounts_package(
accounts_package: AccountsPackagePre, accounts_package: AccountsPackage,
cluster_info: &ClusterInfo, cluster_info: &ClusterInfo,
trusted_validators: Option<&HashSet<Pubkey>>, trusted_validators: Option<&HashSet<Pubkey>>,
halt_on_trusted_validator_accounts_hash_mismatch: bool, halt_on_trusted_validator_accounts_hash_mismatch: bool,
@ -98,13 +99,10 @@ impl AccountsHashVerifier {
snapshot_config: Option<&SnapshotConfig>, snapshot_config: Option<&SnapshotConfig>,
thread_pool: Option<&ThreadPool>, thread_pool: Option<&ThreadPool>,
) { ) {
let accounts_package = solana_runtime::snapshot_utils::process_accounts_package_pre( let snapshot_package =
accounts_package, snapshot_utils::process_accounts_package(accounts_package, thread_pool, None);
thread_pool, Self::process_snapshot_package(
None, snapshot_package,
);
Self::process_accounts_package(
accounts_package,
cluster_info, cluster_info,
trusted_validators, trusted_validators,
halt_on_trusted_validator_accounts_hash_mismatch, halt_on_trusted_validator_accounts_hash_mismatch,
@ -116,8 +114,8 @@ impl AccountsHashVerifier {
); );
} }
fn process_accounts_package( fn process_snapshot_package(
accounts_package: AccountsPackage, snapshot_package: SnapshotPackage,
cluster_info: &ClusterInfo, cluster_info: &ClusterInfo,
trusted_validators: Option<&HashSet<Pubkey>>, trusted_validators: Option<&HashSet<Pubkey>>,
halt_on_trusted_validator_accounts_hash_mismatch: bool, halt_on_trusted_validator_accounts_hash_mismatch: bool,
@ -127,19 +125,19 @@ impl AccountsHashVerifier {
fault_injection_rate_slots: u64, fault_injection_rate_slots: u64,
snapshot_config: Option<&SnapshotConfig>, snapshot_config: Option<&SnapshotConfig>,
) { ) {
let hash = *accounts_package.hash(); let hash = *snapshot_package.hash();
if fault_injection_rate_slots != 0 if fault_injection_rate_slots != 0
&& accounts_package.slot() % fault_injection_rate_slots == 0 && snapshot_package.slot() % fault_injection_rate_slots == 0
{ {
// For testing, publish an invalid hash to gossip. // For testing, publish an invalid hash to gossip.
use rand::{thread_rng, Rng}; use rand::{thread_rng, Rng};
use solana_sdk::hash::extend_and_hash; use solana_sdk::hash::extend_and_hash;
warn!("inserting fault at slot: {}", accounts_package.slot()); warn!("inserting fault at slot: {}", snapshot_package.slot());
let rand = thread_rng().gen_range(0, 10); let rand = thread_rng().gen_range(0, 10);
let hash = extend_and_hash(&hash, &[rand]); let hash = extend_and_hash(&hash, &[rand]);
hashes.push((accounts_package.slot(), hash)); hashes.push((snapshot_package.slot(), hash));
} else { } else {
hashes.push((accounts_package.slot(), hash)); hashes.push((snapshot_package.slot(), hash));
} }
while hashes.len() > MAX_SNAPSHOT_HASHES { while hashes.len() > MAX_SNAPSHOT_HASHES {
@ -157,11 +155,11 @@ impl AccountsHashVerifier {
} }
if let Some(snapshot_config) = snapshot_config { if let Some(snapshot_config) = snapshot_config {
if accounts_package.block_height % snapshot_config.full_snapshot_archive_interval_slots if snapshot_package.block_height % snapshot_config.full_snapshot_archive_interval_slots
== 0 == 0
{ {
if let Some(pending_snapshot_package) = pending_snapshot_package { if let Some(pending_snapshot_package) = pending_snapshot_package {
*pending_snapshot_package.lock().unwrap() = Some(accounts_package); *pending_snapshot_package.lock().unwrap() = Some(snapshot_package);
} }
} }
} }
@ -306,7 +304,7 @@ mod tests {
let hash = hash(&[i as u8]); let hash = hash(&[i as u8]);
let archive_format = ArchiveFormat::TarBzip2; let archive_format = ArchiveFormat::TarBzip2;
let snapshot_version = SnapshotVersion::default(); let snapshot_version = SnapshotVersion::default();
let accounts_package = AccountsPackage::new( let snapshot_package = SnapshotPackage::new(
slot, slot,
block_height, block_height,
slot_deltas, slot_deltas,
@ -318,8 +316,8 @@ mod tests {
snapshot_version, snapshot_version,
); );
AccountsHashVerifier::process_accounts_package( AccountsHashVerifier::process_snapshot_package(
accounts_package, snapshot_package,
&cluster_info, &cluster_info,
Some(&trusted_validators), Some(&trusted_validators),
false, false,

8
core/src/snapshot_packager_service.rs
View File

@ -1,6 +1,6 @@
use solana_gossip::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES}; use solana_gossip::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES};
use solana_runtime::{ use solana_runtime::{
snapshot_archive_info::SnapshotArchiveInfoGetter, snapshot_package::AccountsPackage, snapshot_archive_info::SnapshotArchiveInfoGetter, snapshot_package::SnapshotPackage,
snapshot_utils, snapshot_utils,
}; };
use solana_sdk::{clock::Slot, hash::Hash}; use solana_sdk::{clock::Slot, hash::Hash};
@ -13,7 +13,7 @@ use std::{
time::Duration, time::Duration,
}; };
pub type PendingSnapshotPackage = Arc<Mutex<Option<AccountsPackage>>>; pub type PendingSnapshotPackage = Arc<Mutex<Option<SnapshotPackage>>>;
pub struct SnapshotPackagerService { pub struct SnapshotPackagerService {
t_snapshot_packager: JoinHandle<()>, t_snapshot_packager: JoinHandle<()>,
@ -81,7 +81,7 @@ mod tests {
use solana_runtime::{ use solana_runtime::{
accounts_db::AccountStorageEntry, accounts_db::AccountStorageEntry,
bank::BankSlotDelta, bank::BankSlotDelta,
snapshot_package::AccountsPackage, snapshot_package::SnapshotPackage,
snapshot_utils::{self, ArchiveFormat, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILE_NAME}, snapshot_utils::{self, ArchiveFormat, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILE_NAME},
}; };
use solana_sdk::hash::Hash; use solana_sdk::hash::Hash;
@ -165,7 +165,7 @@ mod tests {
&Hash::default(), &Hash::default(),
ArchiveFormat::TarBzip2, ArchiveFormat::TarBzip2,
); );
let snapshot_package = AccountsPackage::new( let snapshot_package = SnapshotPackage::new(
5, 5,
5, 5,
vec![], vec![],

19
core/tests/snapshots.rs
View File

@ -66,7 +66,7 @@ mod tests {
genesis_utils::{create_genesis_config, GenesisConfigInfo}, genesis_utils::{create_genesis_config, GenesisConfigInfo},
snapshot_archive_info::FullSnapshotArchiveInfo, snapshot_archive_info::FullSnapshotArchiveInfo,
snapshot_config::SnapshotConfig, snapshot_config::SnapshotConfig,
snapshot_package::AccountsPackagePre, snapshot_package::AccountsPackage,
snapshot_utils::{ snapshot_utils::{
self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
}, },
@ -275,7 +275,7 @@ mod tests {
let snapshot_path = &snapshot_config.snapshot_path; let snapshot_path = &snapshot_config.snapshot_path;
let last_bank_snapshot_info = snapshot_utils::get_highest_bank_snapshot_info(snapshot_path) let last_bank_snapshot_info = snapshot_utils::get_highest_bank_snapshot_info(snapshot_path)
.expect("no snapshots found in path"); .expect("no snapshots found in path");
let snapshot_package = AccountsPackagePre::new_full_snapshot_package( let accounts_package = AccountsPackage::new_for_full_snapshot(
last_bank, last_bank,
&last_bank_snapshot_info, &last_bank_snapshot_info,
snapshot_path, snapshot_path,
@ -287,8 +287,8 @@ mod tests {
None, None,
) )
.unwrap(); .unwrap();
let snapshot_package = snapshot_utils::process_accounts_package_pre( let snapshot_package = snapshot_utils::process_accounts_package(
snapshot_package, accounts_package,
Some(last_bank.get_thread_pool()), Some(last_bank.get_thread_pool()),
None, None,
); );
@ -508,15 +508,14 @@ mod tests {
let _package_receiver = std::thread::Builder::new() let _package_receiver = std::thread::Builder::new()
.name("package-receiver".to_string()) .name("package-receiver".to_string())
.spawn(move || { .spawn(move || {
while let Ok(mut snapshot_package) = receiver.recv() { while let Ok(mut accounts_package) = receiver.recv() {
// Only package the latest // Only package the latest
while let Ok(new_snapshot_package) = receiver.try_recv() { while let Ok(new_accounts_package) = receiver.try_recv() {
snapshot_package = new_snapshot_package; accounts_package = new_accounts_package;
} }
let snapshot_package = let snapshot_package = solana_runtime::snapshot_utils::process_accounts_package(
solana_runtime::snapshot_utils::process_accounts_package_pre( accounts_package,
snapshot_package,
Some(&thread_pool), Some(&thread_pool),
None, None,
); );

22
runtime/src/snapshot_package.rs
View File

@ -20,12 +20,12 @@ use std::{
}; };
use tempfile::TempDir; use tempfile::TempDir;
pub type AccountsPackageSender = Sender<AccountsPackagePre>; pub type AccountsPackageSender = Sender<AccountsPackage>;
pub type AccountsPackageReceiver = Receiver<AccountsPackagePre>; pub type AccountsPackageReceiver = Receiver<AccountsPackage>;
pub type AccountsPackageSendError = SendError<AccountsPackagePre>; pub type AccountsPackageSendError = SendError<AccountsPackage>;
#[derive(Debug)] #[derive(Debug)]
pub struct AccountsPackagePre { pub struct AccountsPackage {
pub slot: Slot, pub slot: Slot,
pub block_height: Slot, pub block_height: Slot,
pub slot_deltas: Vec<BankSlotDelta>, pub slot_deltas: Vec<BankSlotDelta>,
@ -40,8 +40,8 @@ pub struct AccountsPackagePre {
pub cluster_type: ClusterType, pub cluster_type: ClusterType,
} }
impl AccountsPackagePre { impl AccountsPackage {
/// Create a snapshot package /// Create an accounts package
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
fn new( fn new(
bank: &Bank, bank: &Bank,
@ -84,7 +84,7 @@ impl AccountsPackagePre {
/// Package up bank snapshot files, snapshot storages, and slot deltas for a full snapshot. /// Package up bank snapshot files, snapshot storages, and slot deltas for a full snapshot.
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn new_full_snapshot_package( pub fn new_for_full_snapshot(
bank: &Bank, bank: &Bank,
bank_snapshot_info: &BankSnapshotInfo, bank_snapshot_info: &BankSnapshotInfo,
snapshots_dir: impl AsRef<Path>, snapshots_dir: impl AsRef<Path>,
@ -120,7 +120,7 @@ impl AccountsPackagePre {
/// Package up bank snapshot files, snapshot storages, and slot deltas for an incremental snapshot. /// Package up bank snapshot files, snapshot storages, and slot deltas for an incremental snapshot.
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn new_incremental_snapshot_package( pub fn new_for_incremental_snapshot(
bank: &Bank, bank: &Bank,
incremental_snapshot_base_slot: Slot, incremental_snapshot_base_slot: Slot,
bank_snapshot_info: &BankSnapshotInfo, bank_snapshot_info: &BankSnapshotInfo,
@ -169,7 +169,7 @@ impl AccountsPackagePre {
} }
} }
pub struct AccountsPackage { pub struct SnapshotPackage {
pub snapshot_archive_info: SnapshotArchiveInfo, pub snapshot_archive_info: SnapshotArchiveInfo,
pub block_height: Slot, pub block_height: Slot,
pub slot_deltas: Vec<BankSlotDelta>, pub slot_deltas: Vec<BankSlotDelta>,
@ -178,7 +178,7 @@ pub struct AccountsPackage {
pub snapshot_version: SnapshotVersion, pub snapshot_version: SnapshotVersion,
} }
impl AccountsPackage { impl SnapshotPackage {
pub fn new( pub fn new(
slot: Slot, slot: Slot,
block_height: u64, block_height: u64,
@ -206,7 +206,7 @@ impl AccountsPackage {
} }
} }
impl SnapshotArchiveInfoGetter for AccountsPackage { impl SnapshotArchiveInfoGetter for SnapshotPackage {
fn snapshot_archive_info(&self) -> &SnapshotArchiveInfo { fn snapshot_archive_info(&self) -> &SnapshotArchiveInfo {
&self.snapshot_archive_info &self.snapshot_archive_info
} }

55
runtime/src/snapshot_utils.rs
View File

@ -13,7 +13,7 @@ use {
FullSnapshotArchiveInfo, IncrementalSnapshotArchiveInfo, SnapshotArchiveInfoGetter, FullSnapshotArchiveInfo, IncrementalSnapshotArchiveInfo, SnapshotArchiveInfoGetter,
}, },
snapshot_package::{ snapshot_package::{
AccountsPackage, AccountsPackagePre, AccountsPackageSendError, AccountsPackageSender, AccountsPackage, AccountsPackageSendError, AccountsPackageSender, SnapshotPackage,
}, },
sorted_storages::SortedStorages, sorted_storages::SortedStorages,
}, },
@ -239,9 +239,9 @@ pub fn remove_tmp_snapshot_archives(snapshot_archives_dir: &Path) {
} }
} }
/// Make a full snapshot archive out of the AccountsPackage /// Make a full snapshot archive out of the snapshot package
pub fn archive_snapshot_package( pub fn archive_snapshot_package(
snapshot_package: &AccountsPackage, snapshot_package: &SnapshotPackage,
maximum_snapshots_to_retain: usize, maximum_snapshots_to_retain: usize,
) -> Result<()> { ) -> Result<()> {
info!( info!(
@ -1549,7 +1549,7 @@ pub fn snapshot_bank(
let highest_bank_snapshot_info = get_highest_bank_snapshot_info(snapshots_dir) let highest_bank_snapshot_info = get_highest_bank_snapshot_info(snapshots_dir)
.expect("no snapshots found in config snapshots_dir"); .expect("no snapshots found in config snapshots_dir");
let package = AccountsPackagePre::new_full_snapshot_package( let accounts_package = AccountsPackage::new_for_full_snapshot(
root_bank, root_bank,
&highest_bank_snapshot_info, &highest_bank_snapshot_info,
snapshots_dir, snapshots_dir,
@ -1561,7 +1561,7 @@ pub fn snapshot_bank(
hash_for_testing, hash_for_testing,
)?; )?;
accounts_package_sender.send(package)?; accounts_package_sender.send(accounts_package)?;
Ok(()) Ok(())
} }
@ -1666,7 +1666,7 @@ pub fn package_process_and_archive_full_snapshot(
thread_pool: Option<&ThreadPool>, thread_pool: Option<&ThreadPool>,
maximum_snapshots_to_retain: usize, maximum_snapshots_to_retain: usize,
) -> Result<FullSnapshotArchiveInfo> { ) -> Result<FullSnapshotArchiveInfo> {
let package = AccountsPackagePre::new_full_snapshot_package( let accounts_package = AccountsPackage::new_for_full_snapshot(
bank, bank,
bank_snapshot_info, bank_snapshot_info,
snapshots_dir, snapshots_dir,
@ -1678,14 +1678,16 @@ pub fn package_process_and_archive_full_snapshot(
None, None,
)?; )?;
let package = process_and_archive_snapshot_package_pre( let snapshot_package = process_and_archive_accounts_package(
package, accounts_package,
thread_pool, thread_pool,
None, None,
maximum_snapshots_to_retain, maximum_snapshots_to_retain,
)?; )?;
Ok(FullSnapshotArchiveInfo::new(package.snapshot_archive_info)) Ok(FullSnapshotArchiveInfo::new(
snapshot_package.snapshot_archive_info,
))
} }
/// Helper function to hold shared code to package, process, and archive incremental snapshots /// Helper function to hold shared code to package, process, and archive incremental snapshots
@ -1702,7 +1704,7 @@ pub fn package_process_and_archive_incremental_snapshot(
thread_pool: Option<&ThreadPool>, thread_pool: Option<&ThreadPool>,
maximum_snapshots_to_retain: usize, maximum_snapshots_to_retain: usize,
) -> Result<IncrementalSnapshotArchiveInfo> { ) -> Result<IncrementalSnapshotArchiveInfo> {
let package = AccountsPackagePre::new_incremental_snapshot_package( let accounts_package = AccountsPackage::new_for_incremental_snapshot(
bank, bank,
incremental_snapshot_base_slot, incremental_snapshot_base_slot,
bank_snapshot_info, bank_snapshot_info,
@ -1715,8 +1717,8 @@ pub fn package_process_and_archive_incremental_snapshot(
None, None,
)?; )?;
let package = process_and_archive_snapshot_package_pre( let snapshot_package = process_and_archive_accounts_package(
package, accounts_package,
thread_pool, thread_pool,
Some(incremental_snapshot_base_slot), Some(incremental_snapshot_base_slot),
maximum_snapshots_to_retain, maximum_snapshots_to_retain,
@ -1724,30 +1726,33 @@ pub fn package_process_and_archive_incremental_snapshot(
Ok(IncrementalSnapshotArchiveInfo::new( Ok(IncrementalSnapshotArchiveInfo::new(
incremental_snapshot_base_slot, incremental_snapshot_base_slot,
package.snapshot_archive_info, snapshot_package.snapshot_archive_info,
)) ))
} }
/// Helper function to hold shared code to process and archive snapshot packages /// Helper function to hold shared code to process and archive accounts packages
fn process_and_archive_snapshot_package_pre( fn process_and_archive_accounts_package(
package_pre: AccountsPackagePre, accounts_package: AccountsPackage,
thread_pool: Option<&ThreadPool>, thread_pool: Option<&ThreadPool>,
incremental_snapshot_base_slot: Option<Slot>, incremental_snapshot_base_slot: Option<Slot>,
maximum_snapshots_to_retain: usize, maximum_snapshots_to_retain: usize,
) -> Result<AccountsPackage> { ) -> Result<SnapshotPackage> {
let package = let snapshot_package = process_accounts_package(
process_accounts_package_pre(package_pre, thread_pool, incremental_snapshot_base_slot); accounts_package,
thread_pool,
incremental_snapshot_base_slot,
);
archive_snapshot_package(&package, maximum_snapshots_to_retain)?; archive_snapshot_package(&snapshot_package, maximum_snapshots_to_retain)?;
Ok(package) Ok(snapshot_package)
} }
pub fn process_accounts_package_pre( pub fn process_accounts_package(
accounts_package: AccountsPackagePre, accounts_package: AccountsPackage,
thread_pool: Option<&ThreadPool>, thread_pool: Option<&ThreadPool>,
incremental_snapshot_base_slot: Option<Slot>, incremental_snapshot_base_slot: Option<Slot>,
) -> AccountsPackage { ) -> SnapshotPackage {
let mut time = Measure::start("hash"); let mut time = Measure::start("hash");
let hash = accounts_package.hash; // temporarily remaining here let hash = accounts_package.hash; // temporarily remaining here
@ -1789,7 +1794,7 @@ pub fn process_accounts_package_pre(
), ),
}; };
AccountsPackage::new( SnapshotPackage::new(
accounts_package.slot, accounts_package.slot,
accounts_package.block_height, accounts_package.block_height,
accounts_package.slot_deltas, accounts_package.slot_deltas,