blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Do not shell out for tar (#19043) 

When making a snapshot archive, we used to shell out and call `tar -S`
for sparse file support.  The tar crate supports sparse files, so no
need to do this anymore.

Fixes #10860
This commit is contained in:
Brooks Prumo 2021-08-04 17:07:55 -05:00 committed by GitHub
parent a1112254a5
commit 68cc71409e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 38 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
runtime/src/snapshot_utils.rs
View File

@ -28,13 +28,13 @@ use {
collections::HashSet, collections::HashSet,
fmt, fmt,
fs::{self, File}, fs::{self, File},
io::{self, BufReader, BufWriter, Error as IoError, ErrorKind, Read, Seek, Write}, io::{BufReader, BufWriter, Error as IoError, ErrorKind, Read, Seek, Write},
path::{Path, PathBuf}, path::{Path, PathBuf},
process::{self, ExitStatus}, process::ExitStatus,
str::FromStr, str::FromStr,
sync::Arc, sync::Arc,
}, },
tar::Archive, tar::{self, Archive},
tempfile::TempDir, tempfile::TempDir,
thiserror::Error, thiserror::Error,
}; };
@ -442,69 +442,46 @@ pub fn archive_snapshot_package(
let file_ext = get_archive_ext(snapshot_package.archive_format); let file_ext = get_archive_ext(snapshot_package.archive_format);
// Tar the staging directory into the archive at `archive_path` // Tar the staging directory into the archive at `archive_path`
//
// system `tar` program is used for -S (sparse file support)
let archive_path = tar_dir.join(format!( let archive_path = tar_dir.join(format!(
"{}{}.{}", "{}{}.{}",
TMP_FULL_SNAPSHOT_PREFIX, snapshot_package.slot, file_ext TMP_FULL_SNAPSHOT_PREFIX, snapshot_package.slot, file_ext
)); ));
let mut tar = process::Command::new("tar") {
.args(&[ let mut archive_file = fs::File::create(&archive_path)?;
"chS",
"-C",
staging_dir.path().to_str().unwrap(),
"accounts",
"snapshots",
"version",
])
.stdin(process::Stdio::null())
.stdout(process::Stdio::piped())
.stderr(process::Stdio::inherit())
.spawn()
.map_err(|e| SnapshotError::IoWithSource(e, "tar process spawn"))?;
match &mut tar.stdout { let do_archive_files = |encoder: &mut dyn Write| -> Result<()> {
None => { let mut archive = tar::Builder::new(encoder);
return Err(SnapshotError::Io(IoError::new( for dir in ["accounts", "snapshots"] {
ErrorKind::Other, archive.append_dir_all(dir, staging_dir.as_ref().join(dir))?;
"tar stdout unavailable".to_string(), }
))); archive.append_path_with_name(staging_dir.as_ref().join("version"), "version")?;
} archive.into_inner()?;
Some(tar_output) => { Ok(())
let mut archive_file = fs::File::create(&archive_path)?; };
match snapshot_package.archive_format { match snapshot_package.archive_format {
ArchiveFormat::TarBzip2 => { ArchiveFormat::TarBzip2 => {
let mut encoder = let mut encoder =
bzip2::write::BzEncoder::new(archive_file, bzip2::Compression::best()); bzip2::write::BzEncoder::new(archive_file, bzip2::Compression::best());
io::copy(tar_output, &mut encoder)?; do_archive_files(&mut encoder)?;
let _ = encoder.finish()?; encoder.finish()?;
} }
ArchiveFormat::TarGzip => { ArchiveFormat::TarGzip => {
let mut encoder = let mut encoder =
flate2::write::GzEncoder::new(archive_file, flate2::Compression::default()); flate2::write::GzEncoder::new(archive_file, flate2::Compression::default());
io::copy(tar_output, &mut encoder)?; do_archive_files(&mut encoder)?;
let _ = encoder.finish()?; encoder.finish()?;
} }
ArchiveFormat::Tar => { ArchiveFormat::TarZstd => {
io::copy(tar_output, &mut archive_file)?; let mut encoder = zstd::stream::Encoder::new(archive_file, 0)?;
} do_archive_files(&mut encoder)?;
ArchiveFormat::TarZstd => { encoder.finish()?;
let mut encoder = zstd::stream::Encoder::new(archive_file, 0)?; }
io::copy(tar_output, &mut encoder)?; ArchiveFormat::Tar => {
let _ = encoder.finish()?; do_archive_files(&mut archive_file)?;
} }
}; };
}
}
let tar_exit_status = tar
.wait()
.map_err(|e| SnapshotError::IoWithSource(e, "tar process wait"))?;
if !tar_exit_status.success() {
warn!("tar command failed with exit code: {}", tar_exit_status);
return Err(SnapshotError::ArchiveGenerationFailure(tar_exit_status));
} }
// Atomically move the archive into position for other validators to find // Atomically move the archive into position for other validators to find
@ -2581,7 +2558,7 @@ mod tests {
let accounts_dir = tempfile::TempDir::new().unwrap(); let accounts_dir = tempfile::TempDir::new().unwrap();
let snapshots_dir = tempfile::TempDir::new().unwrap(); let snapshots_dir = tempfile::TempDir::new().unwrap();
let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let snapshot_archive_format = ArchiveFormat::Tar; let snapshot_archive_format = ArchiveFormat::TarGzip;
let full_snapshot_archive_path = bank_to_full_snapshot_archive( let full_snapshot_archive_path = bank_to_full_snapshot_archive(
snapshots_dir.path(), snapshots_dir.path(),
@ -2656,7 +2633,7 @@ mod tests {
let accounts_dir = tempfile::TempDir::new().unwrap(); let accounts_dir = tempfile::TempDir::new().unwrap();
let snapshots_dir = tempfile::TempDir::new().unwrap(); let snapshots_dir = tempfile::TempDir::new().unwrap();
let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let snapshot_archive_format = ArchiveFormat::Tar; let snapshot_archive_format = ArchiveFormat::TarZstd;
let full_snapshot_slot = slot; let full_snapshot_slot = slot;
let full_snapshot_archive_path = bank_to_full_snapshot_archive( let full_snapshot_archive_path = bank_to_full_snapshot_archive(