Add non-bz2 snapshot for faster creation for dev. (#8350)
* Add non-bz2 snapshot for faster creation for dev. * Fix tests.. * Revert and always just use snapshot.tar.bz2
This commit is contained in:
parent
f4622d67e9
commit
5ef06a9d36
|
@ -10,6 +10,7 @@ use solana_runtime::bank::{
|
||||||
use solana_sdk::clock::Slot;
|
use solana_sdk::clock::Slot;
|
||||||
use std::{
|
use std::{
|
||||||
cmp::Ordering,
|
cmp::Ordering,
|
||||||
|
env,
|
||||||
fs::{self, File},
|
fs::{self, File},
|
||||||
io::{BufReader, BufWriter, Error as IOError, ErrorKind, Read, Seek, SeekFrom, Write},
|
io::{BufReader, BufWriter, Error as IOError, ErrorKind, Read, Seek, SeekFrom, Write},
|
||||||
path::{Path, PathBuf},
|
path::{Path, PathBuf},
|
||||||
|
@ -173,10 +174,17 @@ pub fn archive_snapshot_package(snapshot_package: &SnapshotPackage) -> Result<()
|
||||||
f.write_all(&snapshot_version.into_bytes())?;
|
f.write_all(&snapshot_version.into_bytes())?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let archive_compress_options = if is_snapshot_compression_disabled() {
|
||||||
|
""
|
||||||
|
} else {
|
||||||
|
"j"
|
||||||
|
};
|
||||||
|
let archive_options = format!("{}cfhS", archive_compress_options);
|
||||||
|
|
||||||
// Tar the staging directory into the archive at `archive_path`
|
// Tar the staging directory into the archive at `archive_path`
|
||||||
let archive_path = tar_dir.join("new_state.tar.bz2");
|
let archive_path = tar_dir.join("new_state.tar.bz2");
|
||||||
let args = vec![
|
let args = vec![
|
||||||
"jcfhS",
|
archive_options.as_str(),
|
||||||
archive_path.to_str().unwrap(),
|
archive_path.to_str().unwrap(),
|
||||||
"-C",
|
"-C",
|
||||||
staging_dir.path().to_str().unwrap(),
|
staging_dir.path().to_str().unwrap(),
|
||||||
|
@ -480,6 +488,14 @@ pub fn bank_from_archive<P: AsRef<Path>>(
|
||||||
Ok(bank)
|
Ok(bank)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn is_snapshot_compression_disabled() -> bool {
|
||||||
|
if let Ok(flag) = env::var("SOLANA_DISABLE_SNAPSHOT_COMPRESSION") {
|
||||||
|
!(flag == "0" || flag == "false")
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn get_snapshot_archive_path<P: AsRef<Path>>(snapshot_output_dir: P) -> PathBuf {
|
pub fn get_snapshot_archive_path<P: AsRef<Path>>(snapshot_output_dir: P) -> PathBuf {
|
||||||
snapshot_output_dir.as_ref().join("snapshot.tar.bz2")
|
snapshot_output_dir.as_ref().join("snapshot.tar.bz2")
|
||||||
}
|
}
|
||||||
|
@ -489,10 +505,21 @@ pub fn untar_snapshot_in<P: AsRef<Path>, Q: AsRef<Path>>(
|
||||||
unpack_dir: Q,
|
unpack_dir: Q,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let mut measure = Measure::start("snapshot untar");
|
let mut measure = Measure::start("snapshot untar");
|
||||||
let tar_bz2 = File::open(snapshot_tar)?;
|
let tar_bz2 = File::open(&snapshot_tar)?;
|
||||||
let tar = BzDecoder::new(BufReader::new(tar_bz2));
|
let tar = BzDecoder::new(BufReader::new(tar_bz2));
|
||||||
let mut archive = Archive::new(tar);
|
let mut archive = Archive::new(tar);
|
||||||
|
if !is_snapshot_compression_disabled() {
|
||||||
archive.unpack(&unpack_dir)?;
|
archive.unpack(&unpack_dir)?;
|
||||||
|
} else if let Err(e) = archive.unpack(&unpack_dir) {
|
||||||
|
warn!(
|
||||||
|
"Trying to unpack as uncompressed tar because an error occured: {:?}",
|
||||||
|
e
|
||||||
|
);
|
||||||
|
let tar_bz2 = File::open(snapshot_tar)?;
|
||||||
|
let tar = BufReader::new(tar_bz2);
|
||||||
|
let mut archive = Archive::new(tar);
|
||||||
|
archive.unpack(&unpack_dir)?;
|
||||||
|
}
|
||||||
measure.stop();
|
measure.stop();
|
||||||
info!("{}", measure);
|
info!("{}", measure);
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|
Loading…
Reference in New Issue