blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Rename AccountsPacakge to SnapshotPackage and AccountsPackagePre to AccountsPackage (#19231) 

Renaming these types to better communicate their usages, which will
further diverge as incremental snapshot support is added.

With the new names, AccountsPacakge now refers to the type between
AccountsBackgroundProcess and AccountsHashVerifier, and SnapshotPackage
refers to the type between AccountsHashVerifier and
SnapshotPackagerService.
This commit is contained in:
Brooks Prumo 2021-08-13 16:08:09 -05:00 committed by GitHub
parent 32501866b7
commit 176036aa58
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 78 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
core/src/accounts_hash_verifier.rs
View File

@ -11,7 +11,8 @@ use solana_runtime::{
accounts_db,
snapshot_archive_info::SnapshotArchiveInfoGetter,
snapshot_config::SnapshotConfig,
snapshot_package::{AccountsPackage, AccountsPackagePre, AccountsPackageReceiver},
snapshot_package::{AccountsPackage, AccountsPackageReceiver, SnapshotPackage},
snapshot_utils,
};
use solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey};
use std::collections::{HashMap, HashSet};
@ -61,7 +62,7 @@ impl AccountsHashVerifier {
Some(accounts_db::make_min_priority_thread_pool());
}
Self::process_accounts_package_pre(
Self::process_accounts_package(
accounts_package,
&cluster_info,
trusted_validators.as_ref(),
@ -86,8 +87,8 @@ impl AccountsHashVerifier {
}
#[allow(clippy::too_many_arguments)]
fn process_accounts_package_pre(
accounts_package: AccountsPackagePre,
fn process_accounts_package(
accounts_package: AccountsPackage,
cluster_info: &ClusterInfo,
trusted_validators: Option<&HashSet<Pubkey>>,
halt_on_trusted_validator_accounts_hash_mismatch: bool,
@ -98,13 +99,10 @@ impl AccountsHashVerifier {
snapshot_config: Option<&SnapshotConfig>,
thread_pool: Option<&ThreadPool>,
) {
let accounts_package = solana_runtime::snapshot_utils::process_accounts_package_pre(
accounts_package,
thread_pool,
None,
);
Self::process_accounts_package(
accounts_package,
let snapshot_package =
snapshot_utils::process_accounts_package(accounts_package, thread_pool, None);
Self::process_snapshot_package(
snapshot_package,
cluster_info,
trusted_validators,
halt_on_trusted_validator_accounts_hash_mismatch,
@ -116,8 +114,8 @@ impl AccountsHashVerifier {
);
}
fn process_accounts_package(
accounts_package: AccountsPackage,
fn process_snapshot_package(
snapshot_package: SnapshotPackage,
cluster_info: &ClusterInfo,
trusted_validators: Option<&HashSet<Pubkey>>,
halt_on_trusted_validator_accounts_hash_mismatch: bool,
@ -127,19 +125,19 @@ impl AccountsHashVerifier {
fault_injection_rate_slots: u64,
snapshot_config: Option<&SnapshotConfig>,
) {
let hash = *accounts_package.hash();
let hash = *snapshot_package.hash();
if fault_injection_rate_slots != 0
&& accounts_package.slot() % fault_injection_rate_slots == 0
&& snapshot_package.slot() % fault_injection_rate_slots == 0
{
// For testing, publish an invalid hash to gossip.
use rand::{thread_rng, Rng};
use solana_sdk::hash::extend_and_hash;
warn!("inserting fault at slot: {}", accounts_package.slot());
warn!("inserting fault at slot: {}", snapshot_package.slot());
let rand = thread_rng().gen_range(0, 10);
let hash = extend_and_hash(&hash, &[rand]);
hashes.push((accounts_package.slot(), hash));
hashes.push((snapshot_package.slot(), hash));
} else {
hashes.push((accounts_package.slot(), hash));
hashes.push((snapshot_package.slot(), hash));
}
while hashes.len() > MAX_SNAPSHOT_HASHES {
@ -157,11 +155,11 @@ impl AccountsHashVerifier {
}
if let Some(snapshot_config) = snapshot_config {
if accounts_package.block_height % snapshot_config.full_snapshot_archive_interval_slots
if snapshot_package.block_height % snapshot_config.full_snapshot_archive_interval_slots
== 0
{
if let Some(pending_snapshot_package) = pending_snapshot_package {
*pending_snapshot_package.lock().unwrap() = Some(accounts_package);
*pending_snapshot_package.lock().unwrap() = Some(snapshot_package);
}
}
}
@ -306,7 +304,7 @@ mod tests {
let hash = hash(&[i as u8]);
let archive_format = ArchiveFormat::TarBzip2;
let snapshot_version = SnapshotVersion::default();
let accounts_package = AccountsPackage::new(
let snapshot_package = SnapshotPackage::new(
slot,
block_height,
slot_deltas,
@ -318,8 +316,8 @@ mod tests {
snapshot_version,
);
AccountsHashVerifier::process_accounts_package(
accounts_package,
AccountsHashVerifier::process_snapshot_package(
snapshot_package,
&cluster_info,
Some(&trusted_validators),
false,

8
core/src/snapshot_packager_service.rs
View File

@ -1,6 +1,6 @@
use solana_gossip::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES};
use solana_runtime::{
snapshot_archive_info::SnapshotArchiveInfoGetter, snapshot_package::AccountsPackage,
snapshot_archive_info::SnapshotArchiveInfoGetter, snapshot_package::SnapshotPackage,
snapshot_utils,
};
use solana_sdk::{clock::Slot, hash::Hash};
@ -13,7 +13,7 @@ use std::{
time::Duration,
};
pub type PendingSnapshotPackage = Arc<Mutex<Option<AccountsPackage>>>;
pub type PendingSnapshotPackage = Arc<Mutex<Option<SnapshotPackage>>>;
pub struct SnapshotPackagerService {
t_snapshot_packager: JoinHandle<()>,
@ -81,7 +81,7 @@ mod tests {
use solana_runtime::{
accounts_db::AccountStorageEntry,
bank::BankSlotDelta,
snapshot_package::AccountsPackage,
snapshot_package::SnapshotPackage,
snapshot_utils::{self, ArchiveFormat, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILE_NAME},
};
use solana_sdk::hash::Hash;
@ -165,7 +165,7 @@ mod tests {
&Hash::default(),
ArchiveFormat::TarBzip2,
);
let snapshot_package = AccountsPackage::new(
let snapshot_package = SnapshotPackage::new(
5,
5,
vec![],

25
core/tests/snapshots.rs
View File

@ -66,7 +66,7 @@ mod tests {
genesis_utils::{create_genesis_config, GenesisConfigInfo},
snapshot_archive_info::FullSnapshotArchiveInfo,
snapshot_config::SnapshotConfig,
snapshot_package::AccountsPackagePre,
snapshot_package::AccountsPackage,
snapshot_utils::{
self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
},
@ -275,7 +275,7 @@ mod tests {
let snapshot_path = &snapshot_config.snapshot_path;
let last_bank_snapshot_info = snapshot_utils::get_highest_bank_snapshot_info(snapshot_path)
.expect("no snapshots found in path");
let snapshot_package = AccountsPackagePre::new_full_snapshot_package(
let accounts_package = AccountsPackage::new_for_full_snapshot(
last_bank,
&last_bank_snapshot_info,
snapshot_path,
@ -287,8 +287,8 @@ mod tests {
None,
)
.unwrap();
let snapshot_package = snapshot_utils::process_accounts_package_pre(
snapshot_package,
let snapshot_package = snapshot_utils::process_accounts_package(
accounts_package,
Some(last_bank.get_thread_pool()),
None,
);
@ -508,18 +508,17 @@ mod tests {
let _package_receiver = std::thread::Builder::new()
.name("package-receiver".to_string())
.spawn(move || {
while let Ok(mut snapshot_package) = receiver.recv() {
while let Ok(mut accounts_package) = receiver.recv() {
// Only package the latest
while let Ok(new_snapshot_package) = receiver.try_recv() {
snapshot_package = new_snapshot_package;
while let Ok(new_accounts_package) = receiver.try_recv() {
accounts_package = new_accounts_package;
}
let snapshot_package =
solana_runtime::snapshot_utils::process_accounts_package_pre(
snapshot_package,
Some(&thread_pool),
None,
);
let snapshot_package = solana_runtime::snapshot_utils::process_accounts_package(
accounts_package,
Some(&thread_pool),
None,
);
*pending_snapshot_package.lock().unwrap() = Some(snapshot_package);
}

22
runtime/src/snapshot_package.rs
View File

@ -20,12 +20,12 @@ use std::{
};
use tempfile::TempDir;
pub type AccountsPackageSender = Sender<AccountsPackagePre>;
pub type AccountsPackageReceiver = Receiver<AccountsPackagePre>;
pub type AccountsPackageSendError = SendError<AccountsPackagePre>;
pub type AccountsPackageSender = Sender<AccountsPackage>;
pub type AccountsPackageReceiver = Receiver<AccountsPackage>;
pub type AccountsPackageSendError = SendError<AccountsPackage>;
#[derive(Debug)]
pub struct AccountsPackagePre {
pub struct AccountsPackage {
pub slot: Slot,
pub block_height: Slot,
pub slot_deltas: Vec<BankSlotDelta>,
@ -40,8 +40,8 @@ pub struct AccountsPackagePre {
pub cluster_type: ClusterType,
}
impl AccountsPackagePre {
/// Create a snapshot package
impl AccountsPackage {
/// Create an accounts package
#[allow(clippy::too_many_arguments)]
fn new(
bank: &Bank,
@ -84,7 +84,7 @@ impl AccountsPackagePre {
/// Package up bank snapshot files, snapshot storages, and slot deltas for a full snapshot.
#[allow(clippy::too_many_arguments)]
pub fn new_full_snapshot_package(
pub fn new_for_full_snapshot(
bank: &Bank,
bank_snapshot_info: &BankSnapshotInfo,
snapshots_dir: impl AsRef<Path>,
@ -120,7 +120,7 @@ impl AccountsPackagePre {
/// Package up bank snapshot files, snapshot storages, and slot deltas for an incremental snapshot.
#[allow(clippy::too_many_arguments)]
pub fn new_incremental_snapshot_package(
pub fn new_for_incremental_snapshot(
bank: &Bank,
incremental_snapshot_base_slot: Slot,
bank_snapshot_info: &BankSnapshotInfo,
@ -169,7 +169,7 @@ impl AccountsPackagePre {
}
}
pub struct AccountsPackage {
pub struct SnapshotPackage {
pub snapshot_archive_info: SnapshotArchiveInfo,
pub block_height: Slot,
pub slot_deltas: Vec<BankSlotDelta>,
@ -178,7 +178,7 @@ pub struct AccountsPackage {
pub snapshot_version: SnapshotVersion,
}
impl AccountsPackage {
impl SnapshotPackage {
pub fn new(
slot: Slot,
block_height: u64,
@ -206,7 +206,7 @@ impl AccountsPackage {
}
}
impl SnapshotArchiveInfoGetter for AccountsPackage {
impl SnapshotArchiveInfoGetter for SnapshotPackage {
fn snapshot_archive_info(&self) -> &SnapshotArchiveInfo {
&self.snapshot_archive_info
}

55
runtime/src/snapshot_utils.rs
View File

@ -13,7 +13,7 @@ use {
FullSnapshotArchiveInfo, IncrementalSnapshotArchiveInfo, SnapshotArchiveInfoGetter,
},
snapshot_package::{
AccountsPackage, AccountsPackagePre, AccountsPackageSendError, AccountsPackageSender,
AccountsPackage, AccountsPackageSendError, AccountsPackageSender, SnapshotPackage,
},
sorted_storages::SortedStorages,
},
@ -239,9 +239,9 @@ pub fn remove_tmp_snapshot_archives(snapshot_archives_dir: &Path) {
}
}
/// Make a full snapshot archive out of the AccountsPackage
/// Make a full snapshot archive out of the snapshot package
pub fn archive_snapshot_package(
snapshot_package: &AccountsPackage,
snapshot_package: &SnapshotPackage,
maximum_snapshots_to_retain: usize,
) -> Result<()> {
info!(
@ -1549,7 +1549,7 @@ pub fn snapshot_bank(
let highest_bank_snapshot_info = get_highest_bank_snapshot_info(snapshots_dir)
.expect("no snapshots found in config snapshots_dir");
let package = AccountsPackagePre::new_full_snapshot_package(
let accounts_package = AccountsPackage::new_for_full_snapshot(
root_bank,
&highest_bank_snapshot_info,
snapshots_dir,
@ -1561,7 +1561,7 @@ pub fn snapshot_bank(
hash_for_testing,
)?;
accounts_package_sender.send(package)?;
accounts_package_sender.send(accounts_package)?;
Ok(())
}
@ -1666,7 +1666,7 @@ pub fn package_process_and_archive_full_snapshot(
thread_pool: Option<&ThreadPool>,
maximum_snapshots_to_retain: usize,
) -> Result<FullSnapshotArchiveInfo> {
let package = AccountsPackagePre::new_full_snapshot_package(
let accounts_package = AccountsPackage::new_for_full_snapshot(
bank,
bank_snapshot_info,
snapshots_dir,
@ -1678,14 +1678,16 @@ pub fn package_process_and_archive_full_snapshot(
None,
)?;
let package = process_and_archive_snapshot_package_pre(
package,
let snapshot_package = process_and_archive_accounts_package(
accounts_package,
thread_pool,
None,
maximum_snapshots_to_retain,
)?;
Ok(FullSnapshotArchiveInfo::new(package.snapshot_archive_info))
Ok(FullSnapshotArchiveInfo::new(
snapshot_package.snapshot_archive_info,
))
}
/// Helper function to hold shared code to package, process, and archive incremental snapshots
@ -1702,7 +1704,7 @@ pub fn package_process_and_archive_incremental_snapshot(
thread_pool: Option<&ThreadPool>,
maximum_snapshots_to_retain: usize,
) -> Result<IncrementalSnapshotArchiveInfo> {
let package = AccountsPackagePre::new_incremental_snapshot_package(
let accounts_package = AccountsPackage::new_for_incremental_snapshot(
bank,
incremental_snapshot_base_slot,
bank_snapshot_info,
@ -1715,8 +1717,8 @@ pub fn package_process_and_archive_incremental_snapshot(
None,
)?;
let package = process_and_archive_snapshot_package_pre(
package,
let snapshot_package = process_and_archive_accounts_package(
accounts_package,
thread_pool,
Some(incremental_snapshot_base_slot),
maximum_snapshots_to_retain,
@ -1724,30 +1726,33 @@ pub fn package_process_and_archive_incremental_snapshot(
Ok(IncrementalSnapshotArchiveInfo::new(
incremental_snapshot_base_slot,
package.snapshot_archive_info,
snapshot_package.snapshot_archive_info,
))
}
/// Helper function to hold shared code to process and archive snapshot packages
fn process_and_archive_snapshot_package_pre(
package_pre: AccountsPackagePre,
/// Helper function to hold shared code to process and archive accounts packages
fn process_and_archive_accounts_package(
accounts_package: AccountsPackage,
thread_pool: Option<&ThreadPool>,
incremental_snapshot_base_slot: Option<Slot>,
maximum_snapshots_to_retain: usize,
) -> Result<AccountsPackage> {
let package =
process_accounts_package_pre(package_pre, thread_pool, incremental_snapshot_base_slot);
) -> Result<SnapshotPackage> {
let snapshot_package = process_accounts_package(
accounts_package,
thread_pool,
incremental_snapshot_base_slot,
);
archive_snapshot_package(&package, maximum_snapshots_to_retain)?;
archive_snapshot_package(&snapshot_package, maximum_snapshots_to_retain)?;
Ok(package)
Ok(snapshot_package)
}
pub fn process_accounts_package_pre(
accounts_package: AccountsPackagePre,
pub fn process_accounts_package(
accounts_package: AccountsPackage,
thread_pool: Option<&ThreadPool>,
incremental_snapshot_base_slot: Option<Slot>,
) -> AccountsPackage {
) -> SnapshotPackage {
let mut time = Measure::start("hash");
let hash = accounts_package.hash; // temporarily remaining here
@ -1789,7 +1794,7 @@ pub fn process_accounts_package_pre(
),
};
AccountsPackage::new(
SnapshotPackage::new(
accounts_package.slot,
accounts_package.block_height,
accounts_package.slot_deltas,