blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Uses SnapshotHash type in snapshot archive fields (#28681)

This commit is contained in:
Brooks Prumo 2022-10-31 14:28:35 -04:00 committed by GitHub
parent 34865d032c
commit 59bf1809fe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 100 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
core/src/snapshot_packager_service.rs
View File

@ -94,7 +94,7 @@ impl SnapshotPackagerService {
if let Some(snapshot_gossip_manager) = snapshot_gossip_manager.as_mut() {
snapshot_gossip_manager.push_snapshot_hash(
snapshot_package.snapshot_type,
(snapshot_package.slot(), *snapshot_package.hash()),
(snapshot_package.slot(), snapshot_package.hash().0),
);
}
}
@ -225,6 +225,7 @@ mod tests {
accounts_db::AccountStorageEntry,
bank::BankSlotDelta,
snapshot_archive_info::SnapshotArchiveInfo,
snapshot_hash::SnapshotHash,
snapshot_package::{SnapshotPackage, SnapshotType},
snapshot_utils::{
self, ArchiveFormat, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILENAME,
@ -309,7 +310,7 @@ mod tests {
// Create a packageable snapshot
let slot = 42;
let hash = Hash::default();
let hash = SnapshotHash(Hash::default());
let archive_format = ArchiveFormat::TarBzip2;
let output_tar_path = snapshot_utils::build_full_snapshot_archive_path(
&full_snapshot_archives_dir,

5
core/tests/snapshots.rs
View File

@ -24,6 +24,7 @@ use {
runtime_config::RuntimeConfig,
snapshot_archive_info::FullSnapshotArchiveInfo,
snapshot_config::SnapshotConfig,
snapshot_hash::SnapshotHash,
snapshot_package::{
AccountsPackage, AccountsPackageType, PendingSnapshotPackage, SnapshotPackage,
SnapshotType,
@ -149,7 +150,7 @@ fn restore_from_snapshot(
let full_snapshot_archive_path = snapshot_utils::build_full_snapshot_archive_path(
full_snapshot_archives_dir,
old_last_bank.slot(),
&old_last_bank.get_accounts_hash(),
&old_last_bank.get_snapshot_hash(),
ArchiveFormat::TarBzip2,
);
let full_snapshot_archive_info =
@ -467,7 +468,7 @@ fn test_concurrent_snapshot_packaging(
slot,
// this needs to match the hash value that we reserialize with later. It is complicated, so just use default.
// This hash value is just used to build the file name. Since this is mocked up test code, it is sufficient to pass default here.
&Hash::default(),
&SnapshotHash(Hash::default()),
ArchiveFormat::TarBzip2,
));
}

5
download-utils/src/lib.rs
View File

@ -4,10 +4,11 @@ use {
indicatif::{ProgressBar, ProgressStyle},
log::*,
solana_runtime::{
snapshot_hash::SnapshotHash,
snapshot_package::SnapshotType,
snapshot_utils::{self, ArchiveFormat},
},
solana_sdk::{clock::Slot, genesis_config::DEFAULT_GENESIS_ARCHIVE, hash::Hash},
solana_sdk::{clock::Slot, genesis_config::DEFAULT_GENESIS_ARCHIVE},
std::{
fs::{self, File},
io::{self, Read},
@ -260,7 +261,7 @@ pub fn download_snapshot_archive<'a, 'b>(
rpc_addr: &SocketAddr,
full_snapshot_archives_dir: &Path,
incremental_snapshot_archives_dir: &Path,
desired_snapshot_hash: (Slot, Hash),
desired_snapshot_hash: (Slot, SnapshotHash),
snapshot_type: SnapshotType,
maximum_full_snapshot_archives_to_retain: usize,
maximum_incremental_snapshot_archives_to_retain: usize,

4
ledger/src/bank_forks_utils.rs
View File

@ -231,7 +231,7 @@ fn bank_forks_from_snapshot(
let full_snapshot_hash = FullSnapshotHash {
hash: (
full_snapshot_archive_info.slot(),
*full_snapshot_archive_info.hash(),
full_snapshot_archive_info.hash().0,
),
};
let starting_incremental_snapshot_hash =
@ -240,7 +240,7 @@ fn bank_forks_from_snapshot(
base: full_snapshot_hash.hash,
hash: (
incremental_snapshot_archive_info.slot(),
*incremental_snapshot_archive_info.hash(),
incremental_snapshot_archive_info.hash().0,
),
}
});

6
runtime/src/bank.rs
View File

@ -60,6 +60,7 @@ use {
message_processor::MessageProcessor,
rent_collector::{CollectedInfo, RentCollector},
runtime_config::RuntimeConfig,
snapshot_hash::SnapshotHash,
stake_account::{self, StakeAccount},
stake_weighted_timestamp::{
calculate_stake_weighted_timestamp, MaxAllowableDrift,
@ -6972,6 +6973,11 @@ impl Bank {
self.rc.accounts.accounts_db.get_accounts_hash(self.slot)
}
pub fn get_snapshot_hash(&self) -> SnapshotHash {
let accounts_hash = self.get_accounts_hash();
SnapshotHash::new(&accounts_hash)
}
pub fn get_thread_pool(&self) -> &ThreadPool {
&self.rc.accounts.accounts_db.thread_pool_clean
}

13
runtime/src/snapshot_archive_info.rs
View File

@ -1,8 +1,11 @@
//! Information about snapshot archives
use {
crate::snapshot_utils::{self, ArchiveFormat, Result},
solana_sdk::{clock::Slot, hash::Hash},
crate::{
snapshot_hash::SnapshotHash,
snapshot_utils::{self, ArchiveFormat, Result},
},
solana_sdk::clock::Slot,
std::{cmp::Ordering, path::PathBuf},
};
@ -18,7 +21,7 @@ pub trait SnapshotArchiveInfoGetter {
self.snapshot_archive_info().slot
}
fn hash(&self) -> &Hash {
fn hash(&self) -> &SnapshotHash {
&self.snapshot_archive_info().hash
}
@ -45,8 +48,8 @@ pub struct SnapshotArchiveInfo {
/// Slot that the snapshot was made
pub slot: Slot,
/// Hash of the accounts at this slot
pub hash: Hash,
/// Hash for the snapshot
pub hash: SnapshotHash,
/// Archive format for the snapshot file
pub archive_format: ArchiveFormat,

14
runtime/src/snapshot_hash.rs
View File

@ -42,3 +42,17 @@ pub struct IncrementalSnapshotHashes {
pub base: (Slot, Hash),
pub hashes: Vec<(Slot, Hash)>,
}
/// The hash used for snapshot archives
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
pub struct SnapshotHash(pub Hash);
impl SnapshotHash {
/// Make a snapshot hash from an accounts hash
///
/// Will soon also incorporate the epoch accounts hash
#[must_use]
pub fn new(accounts_hash: &Hash) -> Self {
Self(*accounts_hash)
}
}

8
runtime/src/snapshot_package.rs
View File

@ -5,6 +5,7 @@ use {
bank::{Bank, BankSlotDelta},
rent_collector::RentCollector,
snapshot_archive_info::{SnapshotArchiveInfo, SnapshotArchiveInfoGetter},
snapshot_hash::SnapshotHash,
snapshot_utils::{
self, ArchiveFormat, BankSnapshotInfo, Result, SnapshotVersion,
TMP_BANK_SNAPSHOT_PREFIX,
@ -182,6 +183,7 @@ pub struct SnapshotPackage {
impl SnapshotPackage {
pub fn new(accounts_package: AccountsPackage, accounts_hash: Hash) -> Self {
let snapshot_hash = SnapshotHash::new(&accounts_hash);
let mut snapshot_storages = accounts_package.snapshot_storages;
let (snapshot_type, snapshot_archive_path) = match accounts_package.package_type {
AccountsPackageType::Snapshot(snapshot_type) => match snapshot_type {
@ -190,7 +192,7 @@ impl SnapshotPackage {
snapshot_utils::build_full_snapshot_archive_path(
accounts_package.full_snapshot_archives_dir,
accounts_package.slot,
&accounts_hash,
&snapshot_hash,
accounts_package.archive_format,
),
),
@ -213,7 +215,7 @@ impl SnapshotPackage {
accounts_package.incremental_snapshot_archives_dir,
incremental_snapshot_base_slot,
accounts_package.slot,
&accounts_hash,
&snapshot_hash,
accounts_package.archive_format,
),
)
@ -228,7 +230,7 @@ impl SnapshotPackage {
snapshot_archive_info: SnapshotArchiveInfo {
path: snapshot_archive_path,
slot: accounts_package.slot,
hash: accounts_hash,
hash: snapshot_hash,
archive_format: accounts_package.archive_format,
},
block_height: accounts_package.block_height,

78
runtime/src/snapshot_utils.rs
View File

@ -1,14 +1,17 @@
use {
crate::{
accounts_db::{
AccountShrinkThreshold, AccountsDbConfig, CalcAccountsHashDataSource, SnapshotStorage,
SnapshotStorages,
AccountShrinkThreshold, AccountStorageMap, AccountsDbConfig, AtomicAppendVecId,
CalcAccountsHashDataSource, SnapshotStorage, SnapshotStorages,
},
accounts_index::AccountSecondaryIndexes,
accounts_update_notifier_interface::AccountsUpdateNotifier,
bank::{Bank, BankFieldsToDeserialize, BankSlotDelta},
builtins::Builtins,
hardened_unpack::{unpack_snapshot, ParallelSelector, UnpackError, UnpackedAppendVecMap},
hardened_unpack::{
streaming_unpack_snapshot, unpack_snapshot, ParallelSelector, UnpackError,
UnpackedAppendVecMap,
},
runtime_config::RuntimeConfig,
serde_snapshot::{
bank_from_streams, bank_to_stream, fields_from_streams, SerdeStyle, SnapshotStreams,
@ -17,12 +20,16 @@ use {
snapshot_archive_info::{
FullSnapshotArchiveInfo, IncrementalSnapshotArchiveInfo, SnapshotArchiveInfoGetter,
},
snapshot_hash::SnapshotHash,
snapshot_package::{AccountsPackage, AccountsPackageType, SnapshotPackage, SnapshotType},
snapshot_utils::snapshot_storage_rebuilder::SnapshotStorageRebuilder,
snapshot_utils::snapshot_storage_rebuilder::{
RebuiltSnapshotStorage, SnapshotStorageRebuilder,
},
status_cache,
},
bincode::{config::Options, serialize_into},
bzip2::bufread::BzDecoder,
crossbeam_channel::Sender,
flate2::read::GzDecoder,
lazy_static::lazy_static,
log::*,
@ -49,6 +56,7 @@ use {
atomic::{AtomicBool, AtomicU32},
Arc,
},
thread::{Builder, JoinHandle},
},
tar::{self, Archive},
tempfile::TempDir,
@ -58,15 +66,6 @@ use {
mod archive_format;
mod snapshot_storage_rebuilder;
pub use archive_format::*;
use {
crate::{
accounts_db::{AccountStorageMap, AtomicAppendVecId},
hardened_unpack::streaming_unpack_snapshot,
snapshot_utils::snapshot_storage_rebuilder::RebuiltSnapshotStorage,
},
crossbeam_channel::Sender,
std::thread::{Builder, JoinHandle},
};
pub const SNAPSHOT_STATUS_CACHE_FILENAME: &str = "status_cache";
pub const SNAPSHOT_ARCHIVE_DOWNLOAD_DIR: &str = "remote";
@ -245,7 +244,7 @@ pub enum SnapshotError {
NoSnapshotArchives,
#[error("snapshot has mismatch: deserialized bank: {:?}, snapshot archive info: {:?}", .0, .1)]
MismatchedSlotHash((Slot, Hash), (Slot, Hash)),
MismatchedSlotHash((Slot, SnapshotHash), (Slot, SnapshotHash)),
#[error("snapshot slot deltas are invalid: {0}")]
VerifySlotDeltas(#[from] VerifySlotDeltasError),
@ -1152,10 +1151,10 @@ pub fn bank_from_latest_snapshot_archives(
fn verify_bank_against_expected_slot_hash(
bank: &Bank,
expected_slot: Slot,
expected_hash: Hash,
expected_hash: SnapshotHash,
) -> Result<()> {
let bank_slot = bank.slot();
let bank_hash = bank.get_accounts_hash();
let bank_hash = bank.get_snapshot_hash();
if bank_slot != expected_slot || bank_hash != expected_hash {
return Err(SnapshotError::MismatchedSlotHash(
@ -1357,13 +1356,13 @@ pub fn build_snapshot_archives_remote_dir(snapshot_archives_dir: impl AsRef<Path
pub fn build_full_snapshot_archive_path(
full_snapshot_archives_dir: impl AsRef<Path>,
slot: Slot,
hash: &Hash,
hash: &SnapshotHash,
archive_format: ArchiveFormat,
) -> PathBuf {
full_snapshot_archives_dir.as_ref().join(format!(
"snapshot-{}-{}.{}",
slot,
hash,
hash.0,
archive_format.extension(),
))
}
@ -1375,14 +1374,14 @@ pub fn build_incremental_snapshot_archive_path(
incremental_snapshot_archives_dir: impl AsRef<Path>,
base_slot: Slot,
slot: Slot,
hash: &Hash,
hash: &SnapshotHash,
archive_format: ArchiveFormat,
) -> PathBuf {
incremental_snapshot_archives_dir.as_ref().join(format!(
"incremental-snapshot-{}-{}-{}.{}",
base_slot,
slot,
hash,
hash.0,
archive_format.extension(),
))
}
@ -1390,7 +1389,7 @@ pub fn build_incremental_snapshot_archive_path(
/// Parse a full snapshot archive filename into its Slot, Hash, and Archive Format
pub(crate) fn parse_full_snapshot_archive_filename(
archive_filename: &str,
) -> Result<(Slot, Hash, ArchiveFormat)> {
) -> Result<(Slot, SnapshotHash, ArchiveFormat)> {
lazy_static! {
static ref RE: Regex = Regex::new(FULL_SNAPSHOT_ARCHIVE_FILENAME_REGEX).unwrap();
}
@ -1410,7 +1409,7 @@ pub(crate) fn parse_full_snapshot_archive_filename(
.map(|x| x.as_str().parse::<ArchiveFormat>())?
.ok()?;
Some((slot, hash, archive_format))
Some((slot, SnapshotHash(hash), archive_format))
})
};
@ -1422,7 +1421,7 @@ pub(crate) fn parse_full_snapshot_archive_filename(
/// Parse an incremental snapshot archive filename into its base Slot, actual Slot, Hash, and Archive Format
pub(crate) fn parse_incremental_snapshot_archive_filename(
archive_filename: &str,
) -> Result<(Slot, Slot, Hash, ArchiveFormat)> {
) -> Result<(Slot, Slot, SnapshotHash, ArchiveFormat)> {
lazy_static! {
static ref RE: Regex = Regex::new(INCREMENTAL_SNAPSHOT_ARCHIVE_FILENAME_REGEX).unwrap();
}
@ -1446,7 +1445,7 @@ pub(crate) fn parse_incremental_snapshot_archive_filename(
.map(|x| x.as_str().parse::<ArchiveFormat>())?
.ok()?;
Some((base_slot, slot, hash, archive_format))
Some((base_slot, slot, SnapshotHash(hash), archive_format))
})
};
@ -2542,7 +2541,7 @@ mod tests {
Hash::default()
))
.unwrap(),
(42, Hash::default(), ArchiveFormat::TarBzip2)
(42, SnapshotHash(Hash::default()), ArchiveFormat::TarBzip2)
);
assert_eq!(
parse_full_snapshot_archive_filename(&format!(
@ -2550,12 +2549,12 @@ mod tests {
Hash::default()
))
.unwrap(),
(43, Hash::default(), ArchiveFormat::TarZstd)
(43, SnapshotHash(Hash::default()), ArchiveFormat::TarZstd)
);
assert_eq!(
parse_full_snapshot_archive_filename(&format!("snapshot-44-{}.tar", Hash::default()))
.unwrap(),
(44, Hash::default(), ArchiveFormat::Tar)
(44, SnapshotHash(Hash::default()), ArchiveFormat::Tar)
);
assert_eq!(
parse_full_snapshot_archive_filename(&format!(
@ -2563,7 +2562,7 @@ mod tests {
Hash::default()
))
.unwrap(),
(45, Hash::default(), ArchiveFormat::TarLz4)
(45, SnapshotHash(Hash::default()), ArchiveFormat::TarLz4)
);
assert!(parse_full_snapshot_archive_filename("invalid").is_err());
@ -2615,7 +2614,12 @@ mod tests {
Hash::default()
))
.unwrap(),
(42, 123, Hash::default(), ArchiveFormat::TarBzip2)
(
42,
123,
SnapshotHash(Hash::default()),
ArchiveFormat::TarBzip2
)
);
assert_eq!(
parse_incremental_snapshot_archive_filename(&format!(
@ -2623,7 +2627,12 @@ mod tests {
Hash::default()
))
.unwrap(),
(43, 234, Hash::default(), ArchiveFormat::TarZstd)
(
43,
234,
SnapshotHash(Hash::default()),
ArchiveFormat::TarZstd
)
);
assert_eq!(
parse_incremental_snapshot_archive_filename(&format!(
@ -2631,7 +2640,7 @@ mod tests {
Hash::default()
))
.unwrap(),
(44, 345, Hash::default(), ArchiveFormat::Tar)
(44, 345, SnapshotHash(Hash::default()), ArchiveFormat::Tar)
);
assert_eq!(
parse_incremental_snapshot_archive_filename(&format!(
@ -2639,7 +2648,12 @@ mod tests {
Hash::default()
))
.unwrap(),
(45, 456, Hash::default(), ArchiveFormat::TarLz4)
(
45,
456,
SnapshotHash(Hash::default()),
ArchiveFormat::TarLz4
)
);
assert!(parse_incremental_snapshot_archive_filename("invalid").is_err());

16
validator/src/bootstrap.rs
View File

@ -717,8 +717,8 @@ fn get_highest_local_snapshot_hash(
incremental_snapshot_archives_dir: impl AsRef<Path>,
incremental_snapshot_fetch: bool,
) -> Option<(Slot, Hash)> {
snapshot_utils::get_highest_full_snapshot_archive_info(full_snapshot_archives_dir).and_then(
|full_snapshot_info| {
snapshot_utils::get_highest_full_snapshot_archive_info(full_snapshot_archives_dir)
.and_then(|full_snapshot_info| {
if incremental_snapshot_fetch {
snapshot_utils::get_highest_incremental_snapshot_archive_info(
incremental_snapshot_archives_dir,
@ -734,8 +734,8 @@ fn get_highest_local_snapshot_hash(
None
}
.or_else(|| Some((full_snapshot_info.slot(), *full_snapshot_info.hash())))
},
)
})
.map(|(slot, snapshot_hash)| (slot, snapshot_hash.0))
}
/// Get peer snapshot hashes
@ -1171,7 +1171,7 @@ fn download_snapshots(
.into_iter()
.any(|snapshot_archive| {
snapshot_archive.slot() == full_snapshot_hash.0
&& snapshot_archive.hash() == &full_snapshot_hash.1
&& snapshot_archive.hash().0 == full_snapshot_hash.1
})
{
info!(
@ -1201,7 +1201,7 @@ fn download_snapshots(
.into_iter()
.any(|snapshot_archive| {
snapshot_archive.slot() == incremental_snapshot_hash.0
&& snapshot_archive.hash() == &incremental_snapshot_hash.1
&& snapshot_archive.hash().0 == incremental_snapshot_hash.1
&& snapshot_archive.base_slot() == full_snapshot_hash.0
})
{
@ -1262,6 +1262,10 @@ fn download_snapshot(
slot: desired_snapshot_hash.0,
rpc_addr: rpc_contact_info.rpc,
};
let desired_snapshot_hash = (
desired_snapshot_hash.0,
solana_runtime::snapshot_hash::SnapshotHash(desired_snapshot_hash.1),
);
download_snapshot_archive(
&rpc_contact_info.rpc,
full_snapshot_archives_dir,