Rename CompressionType to ArchiveFormat
This commit is contained in:
parent
141e6706e6
commit
7be6770808
|
@ -175,7 +175,7 @@ mod tests {
|
|||
use super::*;
|
||||
use crate::cluster_info::make_accounts_hashes_message;
|
||||
use crate::contact_info::ContactInfo;
|
||||
use solana_runtime::bank_forks::CompressionType;
|
||||
use solana_runtime::bank_forks::ArchiveFormat;
|
||||
use solana_runtime::snapshot_utils::SnapshotVersion;
|
||||
use solana_sdk::{
|
||||
hash::hash,
|
||||
|
@ -239,7 +239,7 @@ mod tests {
|
|||
snapshot_links,
|
||||
tar_output_file: PathBuf::from("."),
|
||||
storages: vec![],
|
||||
compression: CompressionType::Bzip2,
|
||||
archive_format: ArchiveFormat::TarBzip2,
|
||||
snapshot_version: SnapshotVersion::default(),
|
||||
};
|
||||
|
||||
|
|
|
@ -439,9 +439,7 @@ mod tests {
|
|||
genesis_utils::{create_genesis_config, GenesisConfigInfo},
|
||||
get_tmp_ledger_path,
|
||||
};
|
||||
use solana_runtime::{
|
||||
bank::Bank, bank_forks::CompressionType, snapshot_utils::SnapshotVersion,
|
||||
};
|
||||
use solana_runtime::{bank::Bank, bank_forks::ArchiveFormat, snapshot_utils::SnapshotVersion};
|
||||
use solana_sdk::{genesis_config::ClusterType, signature::Signer};
|
||||
use std::net::{IpAddr, Ipv4Addr};
|
||||
|
||||
|
@ -534,7 +532,7 @@ mod tests {
|
|||
snapshot_interval_slots: 0,
|
||||
snapshot_package_output_path: PathBuf::from("/"),
|
||||
snapshot_path: PathBuf::from("/"),
|
||||
compression: CompressionType::Bzip2,
|
||||
archive_format: ArchiveFormat::TarBzip2,
|
||||
snapshot_version: SnapshotVersion::default(),
|
||||
}),
|
||||
bank_forks,
|
||||
|
|
|
@ -81,7 +81,7 @@ mod tests {
|
|||
use solana_runtime::{
|
||||
accounts_db::AccountStorageEntry,
|
||||
bank::BankSlotDelta,
|
||||
bank_forks::CompressionType,
|
||||
bank_forks::ArchiveFormat,
|
||||
snapshot_package::AccountsPackage,
|
||||
snapshot_utils::{self, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILE_NAME},
|
||||
};
|
||||
|
@ -163,7 +163,7 @@ mod tests {
|
|||
let output_tar_path = snapshot_utils::get_snapshot_archive_path(
|
||||
&snapshot_package_output_path,
|
||||
&(42, Hash::default()),
|
||||
&CompressionType::Bzip2,
|
||||
&ArchiveFormat::TarBzip2,
|
||||
);
|
||||
let snapshot_package = AccountsPackage::new(
|
||||
5,
|
||||
|
@ -173,7 +173,7 @@ mod tests {
|
|||
vec![storage_entries],
|
||||
output_tar_path.clone(),
|
||||
Hash::default(),
|
||||
CompressionType::Bzip2,
|
||||
ArchiveFormat::TarBzip2,
|
||||
SnapshotVersion::default(),
|
||||
);
|
||||
|
||||
|
@ -198,7 +198,7 @@ mod tests {
|
|||
output_tar_path,
|
||||
snapshots_dir,
|
||||
accounts_dir,
|
||||
CompressionType::Bzip2,
|
||||
ArchiveFormat::TarBzip2,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -8,7 +8,7 @@ use {
|
|||
solana_client::rpc_client::RpcClient,
|
||||
solana_ledger::{blockstore::create_new_ledger, create_new_tmp_ledger},
|
||||
solana_runtime::{
|
||||
bank_forks::{CompressionType, SnapshotConfig, SnapshotVersion},
|
||||
bank_forks::{ArchiveFormat, SnapshotConfig, SnapshotVersion},
|
||||
genesis_utils::create_genesis_config_with_leader_ex,
|
||||
hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
|
||||
},
|
||||
|
@ -354,7 +354,7 @@ impl TestValidator {
|
|||
snapshot_interval_slots: 100,
|
||||
snapshot_path: ledger_path.join("snapshot"),
|
||||
snapshot_package_output_path: ledger_path.to_path_buf(),
|
||||
compression: CompressionType::NoCompression,
|
||||
archive_format: ArchiveFormat::Tar,
|
||||
snapshot_version: SnapshotVersion::default(),
|
||||
}),
|
||||
enforce_ulimit_nofile: false,
|
||||
|
|
|
@ -45,7 +45,7 @@ mod tests {
|
|||
use solana_runtime::{
|
||||
accounts_background_service::{ABSRequestSender, SnapshotRequestHandler},
|
||||
bank::{Bank, BankSlotDelta},
|
||||
bank_forks::{BankForks, CompressionType, SnapshotConfig},
|
||||
bank_forks::{ArchiveFormat, BankForks, SnapshotConfig},
|
||||
genesis_utils::{create_genesis_config, GenesisConfigInfo},
|
||||
snapshot_utils,
|
||||
snapshot_utils::SnapshotVersion,
|
||||
|
@ -106,7 +106,7 @@ mod tests {
|
|||
snapshot_interval_slots,
|
||||
snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
|
||||
snapshot_path: PathBuf::from(snapshot_dir.path()),
|
||||
compression: CompressionType::Bzip2,
|
||||
archive_format: ArchiveFormat::TarBzip2,
|
||||
snapshot_version,
|
||||
};
|
||||
bank_forks.set_snapshot_config(Some(snapshot_config.clone()));
|
||||
|
@ -146,9 +146,9 @@ mod tests {
|
|||
snapshot_utils::get_snapshot_archive_path(
|
||||
snapshot_package_output_path,
|
||||
&(old_last_bank.slot(), old_last_bank.get_accounts_hash()),
|
||||
&CompressionType::Bzip2,
|
||||
&ArchiveFormat::TarBzip2,
|
||||
),
|
||||
CompressionType::Bzip2,
|
||||
ArchiveFormat::TarBzip2,
|
||||
old_genesis_config,
|
||||
None,
|
||||
None,
|
||||
|
@ -226,7 +226,7 @@ mod tests {
|
|||
last_bank.src.slot_deltas(&last_bank.src.roots()),
|
||||
&snapshot_config.snapshot_package_output_path,
|
||||
last_bank.get_snapshot_storages(),
|
||||
CompressionType::Bzip2,
|
||||
ArchiveFormat::TarBzip2,
|
||||
snapshot_version,
|
||||
)
|
||||
.unwrap();
|
||||
|
@ -347,7 +347,7 @@ mod tests {
|
|||
&snapshot_path,
|
||||
&snapshot_package_output_path,
|
||||
snapshot_config.snapshot_version,
|
||||
&snapshot_config.compression,
|
||||
&snapshot_config.archive_format,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
|
@ -375,7 +375,7 @@ mod tests {
|
|||
saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path(
|
||||
snapshot_package_output_path,
|
||||
&(slot, accounts_hash),
|
||||
&CompressionType::Bzip2,
|
||||
&ArchiveFormat::TarBzip2,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
@ -432,7 +432,7 @@ mod tests {
|
|||
saved_accounts_dir
|
||||
.path()
|
||||
.join(accounts_dir.path().file_name().unwrap()),
|
||||
CompressionType::Bzip2,
|
||||
ArchiveFormat::TarBzip2,
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
use console::Emoji;
|
||||
use indicatif::{ProgressBar, ProgressStyle};
|
||||
use log::*;
|
||||
use solana_runtime::{bank_forks::CompressionType, snapshot_utils};
|
||||
use solana_runtime::{bank_forks::ArchiveFormat, snapshot_utils};
|
||||
use solana_sdk::clock::Slot;
|
||||
use solana_sdk::hash::Hash;
|
||||
use std::fs::{self, File};
|
||||
|
@ -175,9 +175,9 @@ pub fn download_snapshot(
|
|||
snapshot_utils::purge_old_snapshot_archives(ledger_path);
|
||||
|
||||
for compression in &[
|
||||
CompressionType::Zstd,
|
||||
CompressionType::Gzip,
|
||||
CompressionType::Bzip2,
|
||||
ArchiveFormat::TarZstd,
|
||||
ArchiveFormat::TarGzip,
|
||||
ArchiveFormat::TarBzip2,
|
||||
] {
|
||||
let desired_snapshot_package = snapshot_utils::get_snapshot_archive_path(
|
||||
ledger_path,
|
||||
|
|
|
@ -25,7 +25,7 @@ use solana_ledger::{
|
|||
};
|
||||
use solana_runtime::{
|
||||
bank::{Bank, RewardCalculationEvent},
|
||||
bank_forks::{BankForks, CompressionType, SnapshotConfig},
|
||||
bank_forks::{ArchiveFormat, BankForks, SnapshotConfig},
|
||||
hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
|
||||
snapshot_utils,
|
||||
snapshot_utils::SnapshotVersion,
|
||||
|
@ -667,7 +667,7 @@ fn load_bank_forks(
|
|||
snapshot_interval_slots: 0, // Value doesn't matter
|
||||
snapshot_package_output_path,
|
||||
snapshot_path,
|
||||
compression: CompressionType::Bzip2,
|
||||
archive_format: ArchiveFormat::TarBzip2,
|
||||
snapshot_version: SnapshotVersion::default(),
|
||||
})
|
||||
};
|
||||
|
@ -1947,7 +1947,7 @@ fn main() {
|
|||
bank.src.slot_deltas(&bank.src.roots()),
|
||||
output_directory,
|
||||
storages,
|
||||
CompressionType::Zstd,
|
||||
ArchiveFormat::TarZstd,
|
||||
snapshot_version,
|
||||
)
|
||||
})
|
||||
|
|
|
@ -32,7 +32,7 @@ use solana_local_cluster::{
|
|||
local_cluster::{ClusterConfig, LocalCluster},
|
||||
};
|
||||
use solana_runtime::{
|
||||
bank_forks::{CompressionType, SnapshotConfig},
|
||||
bank_forks::{ArchiveFormat, SnapshotConfig},
|
||||
snapshot_utils,
|
||||
};
|
||||
use solana_sdk::{
|
||||
|
@ -1054,7 +1054,7 @@ fn test_snapshot_download() {
|
|||
let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
|
||||
&validator_snapshot_test_config.snapshot_output_path,
|
||||
&archive_snapshot_hash,
|
||||
&CompressionType::Bzip2,
|
||||
&ArchiveFormat::TarBzip2,
|
||||
);
|
||||
|
||||
// Download the snapshot, then boot a validator from it.
|
||||
|
@ -1124,7 +1124,7 @@ fn test_snapshot_restart_tower() {
|
|||
let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
|
||||
&validator_snapshot_test_config.snapshot_output_path,
|
||||
&archive_snapshot_hash,
|
||||
&CompressionType::Bzip2,
|
||||
&ArchiveFormat::TarBzip2,
|
||||
);
|
||||
fs::hard_link(archive_filename, &validator_archive_path).unwrap();
|
||||
|
||||
|
@ -1189,7 +1189,7 @@ fn test_snapshots_blockstore_floor() {
|
|||
let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
|
||||
&validator_snapshot_test_config.snapshot_output_path,
|
||||
&(archive_slot, archive_hash),
|
||||
&CompressionType::Bzip2,
|
||||
&ArchiveFormat::TarBzip2,
|
||||
);
|
||||
fs::hard_link(archive_filename, &validator_archive_path).unwrap();
|
||||
let slot_floor = archive_slot;
|
||||
|
@ -2446,7 +2446,7 @@ fn setup_snapshot_validator_config(
|
|||
snapshot_interval_slots,
|
||||
snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
|
||||
snapshot_path: PathBuf::from(snapshot_dir.path()),
|
||||
compression: CompressionType::Bzip2,
|
||||
archive_format: ArchiveFormat::TarBzip2,
|
||||
snapshot_version: snapshot_utils::SnapshotVersion::default(),
|
||||
};
|
||||
|
||||
|
|
|
@ -112,7 +112,7 @@ impl SnapshotRequestHandler {
|
|||
&self.snapshot_config.snapshot_path,
|
||||
&self.snapshot_config.snapshot_package_output_path,
|
||||
self.snapshot_config.snapshot_version,
|
||||
&self.snapshot_config.compression,
|
||||
&self.snapshot_config.archive_format,
|
||||
);
|
||||
if r.is_err() {
|
||||
warn!(
|
||||
|
|
|
@ -18,11 +18,11 @@ use std::{
|
|||
pub use crate::snapshot_utils::SnapshotVersion;
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub enum CompressionType {
|
||||
Bzip2,
|
||||
Gzip,
|
||||
Zstd,
|
||||
NoCompression,
|
||||
pub enum ArchiveFormat {
|
||||
TarBzip2,
|
||||
TarGzip,
|
||||
TarZstd,
|
||||
Tar,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
|
@ -36,7 +36,7 @@ pub struct SnapshotConfig {
|
|||
// Where to place the snapshots for recent slots
|
||||
pub snapshot_path: PathBuf,
|
||||
|
||||
pub compression: CompressionType,
|
||||
pub archive_format: ArchiveFormat,
|
||||
|
||||
// Snapshot version to generate
|
||||
pub snapshot_version: SnapshotVersion,
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
use crate::bank_forks::CompressionType;
|
||||
use crate::bank_forks::ArchiveFormat;
|
||||
use crate::snapshot_utils::SnapshotVersion;
|
||||
use crate::{accounts_db::SnapshotStorages, bank::BankSlotDelta};
|
||||
use solana_sdk::clock::Slot;
|
||||
|
@ -22,7 +22,7 @@ pub struct AccountsPackage {
|
|||
pub storages: SnapshotStorages,
|
||||
pub tar_output_file: PathBuf,
|
||||
pub hash: Hash,
|
||||
pub compression: CompressionType,
|
||||
pub archive_format: ArchiveFormat,
|
||||
pub snapshot_version: SnapshotVersion,
|
||||
}
|
||||
|
||||
|
@ -35,7 +35,7 @@ impl AccountsPackage {
|
|||
storages: SnapshotStorages,
|
||||
tar_output_file: PathBuf,
|
||||
hash: Hash,
|
||||
compression: CompressionType,
|
||||
archive_format: ArchiveFormat,
|
||||
snapshot_version: SnapshotVersion,
|
||||
) -> Self {
|
||||
Self {
|
||||
|
@ -46,7 +46,7 @@ impl AccountsPackage {
|
|||
storages,
|
||||
tar_output_file,
|
||||
hash,
|
||||
compression,
|
||||
archive_format,
|
||||
snapshot_version,
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
use crate::{
|
||||
accounts_index::AccountIndex,
|
||||
bank::{Bank, BankSlotDelta, Builtins},
|
||||
bank_forks::CompressionType,
|
||||
bank_forks::ArchiveFormat,
|
||||
hardened_unpack::{unpack_snapshot, UnpackError},
|
||||
serde_snapshot::{
|
||||
bank_from_stream, bank_to_stream, SerdeStyle, SnapshotStorage, SnapshotStorages,
|
||||
|
@ -158,7 +158,7 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
|
|||
status_cache_slot_deltas: Vec<BankSlotDelta>,
|
||||
snapshot_package_output_path: P,
|
||||
snapshot_storages: SnapshotStorages,
|
||||
compression: CompressionType,
|
||||
archive_format: ArchiveFormat,
|
||||
snapshot_version: SnapshotVersion,
|
||||
) -> Result<AccountsPackage> {
|
||||
// Hard link all the snapshots we need for this package
|
||||
|
@ -180,7 +180,7 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
|
|||
let snapshot_package_output_file = get_snapshot_archive_path(
|
||||
&snapshot_package_output_path,
|
||||
&(bank.slot(), bank.get_accounts_hash()),
|
||||
&compression,
|
||||
&archive_format,
|
||||
);
|
||||
|
||||
let package = AccountsPackage::new(
|
||||
|
@ -191,19 +191,19 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
|
|||
snapshot_storages,
|
||||
snapshot_package_output_file,
|
||||
bank.get_accounts_hash(),
|
||||
compression,
|
||||
archive_format,
|
||||
snapshot_version,
|
||||
);
|
||||
|
||||
Ok(package)
|
||||
}
|
||||
|
||||
fn get_compression_ext(compression: &CompressionType) -> &'static str {
|
||||
match compression {
|
||||
CompressionType::Bzip2 => ".tar.bz2",
|
||||
CompressionType::Gzip => ".tar.gz",
|
||||
CompressionType::Zstd => ".tar.zst",
|
||||
CompressionType::NoCompression => ".tar",
|
||||
fn get_archive_ext(archive_format: &ArchiveFormat) -> &'static str {
|
||||
match archive_format {
|
||||
ArchiveFormat::TarBzip2 => ".tar.bz2",
|
||||
ArchiveFormat::TarGzip => ".tar.gz",
|
||||
ArchiveFormat::TarZstd => ".tar.zst",
|
||||
ArchiveFormat::Tar => ".tar",
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -235,7 +235,10 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<()
|
|||
serialize_status_cache(
|
||||
snapshot_package.slot,
|
||||
&snapshot_package.slot_deltas,
|
||||
&snapshot_package.snapshot_links.path().join(SNAPSHOT_STATUS_CACHE_FILE_NAME),
|
||||
&snapshot_package
|
||||
.snapshot_links
|
||||
.path()
|
||||
.join(SNAPSHOT_STATUS_CACHE_FILE_NAME),
|
||||
)?;
|
||||
|
||||
let mut timer = Measure::start("snapshot_package-package_snapshots");
|
||||
|
@ -288,7 +291,7 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<()
|
|||
f.write_all(snapshot_package.snapshot_version.as_str().as_bytes())?;
|
||||
}
|
||||
|
||||
let file_ext = get_compression_ext(&snapshot_package.compression);
|
||||
let file_ext = get_archive_ext(&snapshot_package.archive_format);
|
||||
|
||||
// Tar the staging directory into the archive at `archive_path`
|
||||
//
|
||||
|
@ -319,23 +322,23 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<()
|
|||
Some(tar_output) => {
|
||||
let mut archive_file = fs::File::create(&archive_path)?;
|
||||
|
||||
match snapshot_package.compression {
|
||||
CompressionType::Bzip2 => {
|
||||
match snapshot_package.archive_format {
|
||||
ArchiveFormat::TarBzip2 => {
|
||||
let mut encoder =
|
||||
bzip2::write::BzEncoder::new(archive_file, bzip2::Compression::Best);
|
||||
io::copy(tar_output, &mut encoder)?;
|
||||
let _ = encoder.finish()?;
|
||||
}
|
||||
CompressionType::Gzip => {
|
||||
ArchiveFormat::TarGzip => {
|
||||
let mut encoder =
|
||||
flate2::write::GzEncoder::new(archive_file, flate2::Compression::default());
|
||||
io::copy(tar_output, &mut encoder)?;
|
||||
let _ = encoder.finish()?;
|
||||
}
|
||||
CompressionType::NoCompression => {
|
||||
ArchiveFormat::Tar => {
|
||||
io::copy(tar_output, &mut archive_file)?;
|
||||
}
|
||||
CompressionType::Zstd => {
|
||||
ArchiveFormat::TarZstd => {
|
||||
let mut encoder = zstd::stream::Encoder::new(archive_file, 0)?;
|
||||
io::copy(tar_output, &mut encoder)?;
|
||||
let _ = encoder.finish()?;
|
||||
|
@ -580,7 +583,7 @@ pub fn bank_from_archive<P: AsRef<Path>>(
|
|||
frozen_account_pubkeys: &[Pubkey],
|
||||
snapshot_path: &PathBuf,
|
||||
snapshot_tar: P,
|
||||
compression: CompressionType,
|
||||
archive_format: ArchiveFormat,
|
||||
genesis_config: &GenesisConfig,
|
||||
debug_keys: Option<Arc<HashSet<Pubkey>>>,
|
||||
additional_builtins: Option<&Builtins>,
|
||||
|
@ -590,7 +593,7 @@ pub fn bank_from_archive<P: AsRef<Path>>(
|
|||
let unpack_dir = tempfile::Builder::new()
|
||||
.prefix(TMP_SNAPSHOT_DIR_PREFIX)
|
||||
.tempdir_in(snapshot_path)?;
|
||||
untar_snapshot_in(&snapshot_tar, &unpack_dir, compression)?;
|
||||
untar_snapshot_in(&snapshot_tar, &unpack_dir, archive_format)?;
|
||||
|
||||
let mut measure = Measure::start("bank rebuild from snapshot");
|
||||
let unpacked_accounts_dir = unpack_dir.as_ref().join(TAR_ACCOUNTS_DIR);
|
||||
|
@ -624,27 +627,27 @@ pub fn bank_from_archive<P: AsRef<Path>>(
|
|||
pub fn get_snapshot_archive_path<P: AsRef<Path>>(
|
||||
snapshot_output_dir: P,
|
||||
snapshot_hash: &(Slot, Hash),
|
||||
compression: &CompressionType,
|
||||
archive_format: &ArchiveFormat,
|
||||
) -> PathBuf {
|
||||
snapshot_output_dir.as_ref().join(format!(
|
||||
"snapshot-{}-{}{}",
|
||||
snapshot_hash.0,
|
||||
snapshot_hash.1,
|
||||
get_compression_ext(compression),
|
||||
get_archive_ext(archive_format),
|
||||
))
|
||||
}
|
||||
|
||||
fn compression_type_from_str(compress: &str) -> Option<CompressionType> {
|
||||
match compress {
|
||||
"tar.bz2" => Some(CompressionType::Bzip2),
|
||||
"tar.gz" => Some(CompressionType::Gzip),
|
||||
"tar.zst" => Some(CompressionType::Zstd),
|
||||
"tar" => Some(CompressionType::NoCompression),
|
||||
fn archive_format_from_str(archive_format: &str) -> Option<ArchiveFormat> {
|
||||
match archive_format {
|
||||
"tar.bz2" => Some(ArchiveFormat::TarBzip2),
|
||||
"tar.gz" => Some(ArchiveFormat::TarGzip),
|
||||
"tar.zst" => Some(ArchiveFormat::TarZstd),
|
||||
"tar" => Some(ArchiveFormat::Tar),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash, CompressionType)> {
|
||||
fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash, ArchiveFormat)> {
|
||||
let snapshot_filename_regex =
|
||||
Regex::new(r"snapshot-(\d+)-([[:alnum:]]+)\.(tar|tar\.bz2|tar\.zst|tar\.gz)$").unwrap();
|
||||
|
||||
|
@ -653,12 +656,12 @@ fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash, CompressionTy
|
|||
let hash_str = captures.get(2).unwrap().as_str();
|
||||
let ext = captures.get(3).unwrap().as_str();
|
||||
|
||||
if let (Ok(slot), Ok(hash), Some(compression)) = (
|
||||
if let (Ok(slot), Ok(hash), Some(archive_format)) = (
|
||||
slot_str.parse::<Slot>(),
|
||||
hash_str.parse::<Hash>(),
|
||||
compression_type_from_str(ext),
|
||||
archive_format_from_str(ext),
|
||||
) {
|
||||
return Some((slot, hash, compression));
|
||||
return Some((slot, hash, archive_format));
|
||||
}
|
||||
}
|
||||
None
|
||||
|
@ -666,7 +669,7 @@ fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash, CompressionTy
|
|||
|
||||
pub fn get_snapshot_archives<P: AsRef<Path>>(
|
||||
snapshot_output_dir: P,
|
||||
) -> Vec<(PathBuf, (Slot, Hash, CompressionType))> {
|
||||
) -> Vec<(PathBuf, (Slot, Hash, ArchiveFormat))> {
|
||||
match fs::read_dir(&snapshot_output_dir) {
|
||||
Err(err) => {
|
||||
info!("Unable to read snapshot directory: {}", err);
|
||||
|
@ -697,7 +700,7 @@ pub fn get_snapshot_archives<P: AsRef<Path>>(
|
|||
|
||||
pub fn get_highest_snapshot_archive_path<P: AsRef<Path>>(
|
||||
snapshot_output_dir: P,
|
||||
) -> Option<(PathBuf, (Slot, Hash, CompressionType))> {
|
||||
) -> Option<(PathBuf, (Slot, Hash, ArchiveFormat))> {
|
||||
let archives = get_snapshot_archives(snapshot_output_dir);
|
||||
archives.into_iter().next()
|
||||
}
|
||||
|
@ -715,27 +718,27 @@ pub fn purge_old_snapshot_archives<P: AsRef<Path>>(snapshot_output_dir: P) {
|
|||
pub fn untar_snapshot_in<P: AsRef<Path>, Q: AsRef<Path>>(
|
||||
snapshot_tar: P,
|
||||
unpack_dir: Q,
|
||||
compression: CompressionType,
|
||||
archive_format: ArchiveFormat,
|
||||
) -> Result<()> {
|
||||
let mut measure = Measure::start("snapshot untar");
|
||||
let tar_name = File::open(&snapshot_tar)?;
|
||||
match compression {
|
||||
CompressionType::Bzip2 => {
|
||||
match archive_format {
|
||||
ArchiveFormat::TarBzip2 => {
|
||||
let tar = BzDecoder::new(BufReader::new(tar_name));
|
||||
let mut archive = Archive::new(tar);
|
||||
unpack_snapshot(&mut archive, unpack_dir)?;
|
||||
}
|
||||
CompressionType::Gzip => {
|
||||
ArchiveFormat::TarGzip => {
|
||||
let tar = GzDecoder::new(BufReader::new(tar_name));
|
||||
let mut archive = Archive::new(tar);
|
||||
unpack_snapshot(&mut archive, unpack_dir)?;
|
||||
}
|
||||
CompressionType::Zstd => {
|
||||
ArchiveFormat::TarZstd => {
|
||||
let tar = zstd::stream::read::Decoder::new(BufReader::new(tar_name))?;
|
||||
let mut archive = Archive::new(tar);
|
||||
unpack_snapshot(&mut archive, unpack_dir)?;
|
||||
}
|
||||
CompressionType::NoCompression => {
|
||||
ArchiveFormat::Tar => {
|
||||
let tar = BufReader::new(tar_name);
|
||||
let mut archive = Archive::new(tar);
|
||||
unpack_snapshot(&mut archive, unpack_dir)?;
|
||||
|
@ -834,7 +837,7 @@ pub fn verify_snapshot_archive<P, Q, R>(
|
|||
snapshot_archive: P,
|
||||
snapshots_to_verify: Q,
|
||||
storages_to_verify: R,
|
||||
compression: CompressionType,
|
||||
archive_format: ArchiveFormat,
|
||||
) where
|
||||
P: AsRef<Path>,
|
||||
Q: AsRef<Path>,
|
||||
|
@ -842,7 +845,7 @@ pub fn verify_snapshot_archive<P, Q, R>(
|
|||
{
|
||||
let temp_dir = tempfile::TempDir::new().unwrap();
|
||||
let unpack_dir = temp_dir.path();
|
||||
untar_snapshot_in(snapshot_archive, &unpack_dir, compression).unwrap();
|
||||
untar_snapshot_in(snapshot_archive, &unpack_dir, archive_format).unwrap();
|
||||
|
||||
// Check snapshots are the same
|
||||
let unpacked_snapshots = unpack_dir.join(&TAR_SNAPSHOTS_DIR);
|
||||
|
@ -873,7 +876,7 @@ pub fn snapshot_bank(
|
|||
snapshot_path: &Path,
|
||||
snapshot_package_output_path: &Path,
|
||||
snapshot_version: SnapshotVersion,
|
||||
compression: &CompressionType,
|
||||
archive_format: &ArchiveFormat,
|
||||
) -> Result<()> {
|
||||
let storages: Vec<_> = root_bank.get_snapshot_storages();
|
||||
let mut add_snapshot_time = Measure::start("add-snapshot-ms");
|
||||
|
@ -894,7 +897,7 @@ pub fn snapshot_bank(
|
|||
status_cache_slot_deltas,
|
||||
snapshot_package_output_path,
|
||||
storages,
|
||||
compression.clone(),
|
||||
archive_format.clone(),
|
||||
snapshot_version,
|
||||
)?;
|
||||
|
||||
|
@ -1019,15 +1022,15 @@ mod tests {
|
|||
fn test_snapshot_hash_of() {
|
||||
assert_eq!(
|
||||
snapshot_hash_of(&format!("snapshot-42-{}.tar.bz2", Hash::default())),
|
||||
Some((42, Hash::default(), CompressionType::Bzip2))
|
||||
Some((42, Hash::default(), ArchiveFormat::TarBzip2))
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot_hash_of(&format!("snapshot-43-{}.tar.zst", Hash::default())),
|
||||
Some((43, Hash::default(), CompressionType::Zstd))
|
||||
Some((43, Hash::default(), ArchiveFormat::TarZstd))
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot_hash_of(&format!("snapshot-42-{}.tar", Hash::default())),
|
||||
Some((42, Hash::default(), CompressionType::NoCompression))
|
||||
Some((42, Hash::default(), ArchiveFormat::Tar))
|
||||
);
|
||||
|
||||
assert!(snapshot_hash_of("invalid").is_none());
|
||||
|
|
|
@ -29,7 +29,7 @@ use solana_ledger::blockstore_db::BlockstoreRecoveryMode;
|
|||
use solana_perf::recycler::enable_recycler_warming;
|
||||
use solana_runtime::{
|
||||
accounts_index::AccountIndex,
|
||||
bank_forks::{CompressionType, SnapshotConfig, SnapshotVersion},
|
||||
bank_forks::{ArchiveFormat, SnapshotConfig, SnapshotVersion},
|
||||
hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
|
||||
snapshot_utils::get_highest_snapshot_archive_path,
|
||||
};
|
||||
|
@ -1359,13 +1359,14 @@ pub fn main() {
|
|||
other than increasing the account balance"),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("snapshot_compression")
|
||||
.long("snapshot-compression")
|
||||
.possible_values(&["bz2", "gzip", "zstd", "none"])
|
||||
Arg::with_name("snapshot_archive_format")
|
||||
.long("snapshot-archive-format")
|
||||
.alias("snapshot-compression") // Legacy name used by Solana v1.5.x and older
|
||||
.possible_values(&["bz2", "gzip", "zstd", "tar", "none"])
|
||||
.default_value("zstd")
|
||||
.value_name("COMPRESSION_TYPE")
|
||||
.value_name("ARCHIVE_TYPE")
|
||||
.takes_value(true)
|
||||
.help("Type of snapshot compression to use."),
|
||||
.help("Snapshot archive format to use."),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("max_genesis_archive_unpacked_size")
|
||||
|
@ -1670,14 +1671,14 @@ pub fn main() {
|
|||
exit(1);
|
||||
});
|
||||
|
||||
let snapshot_compression = {
|
||||
let compression_str = value_t_or_exit!(matches, "snapshot_compression", String);
|
||||
match compression_str.as_str() {
|
||||
"bz2" => CompressionType::Bzip2,
|
||||
"gzip" => CompressionType::Gzip,
|
||||
"zstd" => CompressionType::Zstd,
|
||||
"none" => CompressionType::NoCompression,
|
||||
_ => panic!("Compression type not recognized: {}", compression_str),
|
||||
let archive_format = {
|
||||
let archive_format_str = value_t_or_exit!(matches, "snapshot_archive_format", String);
|
||||
match archive_format_str.as_str() {
|
||||
"bz2" => ArchiveFormat::TarBzip2,
|
||||
"gzip" => ArchiveFormat::TarGzip,
|
||||
"zstd" => ArchiveFormat::TarZstd,
|
||||
"tar" | "none" => ArchiveFormat::Tar,
|
||||
_ => panic!("Archive format not recognized: {}", archive_format_str),
|
||||
}
|
||||
};
|
||||
|
||||
|
@ -1698,7 +1699,7 @@ pub fn main() {
|
|||
},
|
||||
snapshot_path,
|
||||
snapshot_package_output_path: ledger_path.clone(),
|
||||
compression: snapshot_compression,
|
||||
archive_format,
|
||||
snapshot_version,
|
||||
});
|
||||
|
||||
|
|
Loading…
Reference in New Issue