Do not shell out for tar (#19043)
When making a snapshot archive, we used to shell out and call `tar -S` for sparse file support. The tar crate supports sparse files, so no need to do this anymore. Fixes #10860
This commit is contained in:
parent
a1112254a5
commit
68cc71409e
|
@ -28,13 +28,13 @@ use {
|
||||||
collections::HashSet,
|
collections::HashSet,
|
||||||
fmt,
|
fmt,
|
||||||
fs::{self, File},
|
fs::{self, File},
|
||||||
io::{self, BufReader, BufWriter, Error as IoError, ErrorKind, Read, Seek, Write},
|
io::{BufReader, BufWriter, Error as IoError, ErrorKind, Read, Seek, Write},
|
||||||
path::{Path, PathBuf},
|
path::{Path, PathBuf},
|
||||||
process::{self, ExitStatus},
|
process::ExitStatus,
|
||||||
str::FromStr,
|
str::FromStr,
|
||||||
sync::Arc,
|
sync::Arc,
|
||||||
},
|
},
|
||||||
tar::Archive,
|
tar::{self, Archive},
|
||||||
tempfile::TempDir,
|
tempfile::TempDir,
|
||||||
thiserror::Error,
|
thiserror::Error,
|
||||||
};
|
};
|
||||||
|
@ -442,70 +442,47 @@ pub fn archive_snapshot_package(
|
||||||
let file_ext = get_archive_ext(snapshot_package.archive_format);
|
let file_ext = get_archive_ext(snapshot_package.archive_format);
|
||||||
|
|
||||||
// Tar the staging directory into the archive at `archive_path`
|
// Tar the staging directory into the archive at `archive_path`
|
||||||
//
|
|
||||||
// system `tar` program is used for -S (sparse file support)
|
|
||||||
let archive_path = tar_dir.join(format!(
|
let archive_path = tar_dir.join(format!(
|
||||||
"{}{}.{}",
|
"{}{}.{}",
|
||||||
TMP_FULL_SNAPSHOT_PREFIX, snapshot_package.slot, file_ext
|
TMP_FULL_SNAPSHOT_PREFIX, snapshot_package.slot, file_ext
|
||||||
));
|
));
|
||||||
|
|
||||||
let mut tar = process::Command::new("tar")
|
{
|
||||||
.args(&[
|
|
||||||
"chS",
|
|
||||||
"-C",
|
|
||||||
staging_dir.path().to_str().unwrap(),
|
|
||||||
"accounts",
|
|
||||||
"snapshots",
|
|
||||||
"version",
|
|
||||||
])
|
|
||||||
.stdin(process::Stdio::null())
|
|
||||||
.stdout(process::Stdio::piped())
|
|
||||||
.stderr(process::Stdio::inherit())
|
|
||||||
.spawn()
|
|
||||||
.map_err(|e| SnapshotError::IoWithSource(e, "tar process spawn"))?;
|
|
||||||
|
|
||||||
match &mut tar.stdout {
|
|
||||||
None => {
|
|
||||||
return Err(SnapshotError::Io(IoError::new(
|
|
||||||
ErrorKind::Other,
|
|
||||||
"tar stdout unavailable".to_string(),
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
Some(tar_output) => {
|
|
||||||
let mut archive_file = fs::File::create(&archive_path)?;
|
let mut archive_file = fs::File::create(&archive_path)?;
|
||||||
|
|
||||||
|
let do_archive_files = |encoder: &mut dyn Write| -> Result<()> {
|
||||||
|
let mut archive = tar::Builder::new(encoder);
|
||||||
|
for dir in ["accounts", "snapshots"] {
|
||||||
|
archive.append_dir_all(dir, staging_dir.as_ref().join(dir))?;
|
||||||
|
}
|
||||||
|
archive.append_path_with_name(staging_dir.as_ref().join("version"), "version")?;
|
||||||
|
archive.into_inner()?;
|
||||||
|
Ok(())
|
||||||
|
};
|
||||||
|
|
||||||
match snapshot_package.archive_format {
|
match snapshot_package.archive_format {
|
||||||
ArchiveFormat::TarBzip2 => {
|
ArchiveFormat::TarBzip2 => {
|
||||||
let mut encoder =
|
let mut encoder =
|
||||||
bzip2::write::BzEncoder::new(archive_file, bzip2::Compression::best());
|
bzip2::write::BzEncoder::new(archive_file, bzip2::Compression::best());
|
||||||
io::copy(tar_output, &mut encoder)?;
|
do_archive_files(&mut encoder)?;
|
||||||
let _ = encoder.finish()?;
|
encoder.finish()?;
|
||||||
}
|
}
|
||||||
ArchiveFormat::TarGzip => {
|
ArchiveFormat::TarGzip => {
|
||||||
let mut encoder =
|
let mut encoder =
|
||||||
flate2::write::GzEncoder::new(archive_file, flate2::Compression::default());
|
flate2::write::GzEncoder::new(archive_file, flate2::Compression::default());
|
||||||
io::copy(tar_output, &mut encoder)?;
|
do_archive_files(&mut encoder)?;
|
||||||
let _ = encoder.finish()?;
|
encoder.finish()?;
|
||||||
}
|
|
||||||
ArchiveFormat::Tar => {
|
|
||||||
io::copy(tar_output, &mut archive_file)?;
|
|
||||||
}
|
}
|
||||||
ArchiveFormat::TarZstd => {
|
ArchiveFormat::TarZstd => {
|
||||||
let mut encoder = zstd::stream::Encoder::new(archive_file, 0)?;
|
let mut encoder = zstd::stream::Encoder::new(archive_file, 0)?;
|
||||||
io::copy(tar_output, &mut encoder)?;
|
do_archive_files(&mut encoder)?;
|
||||||
let _ = encoder.finish()?;
|
encoder.finish()?;
|
||||||
|
}
|
||||||
|
ArchiveFormat::Tar => {
|
||||||
|
do_archive_files(&mut archive_file)?;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let tar_exit_status = tar
|
|
||||||
.wait()
|
|
||||||
.map_err(|e| SnapshotError::IoWithSource(e, "tar process wait"))?;
|
|
||||||
if !tar_exit_status.success() {
|
|
||||||
warn!("tar command failed with exit code: {}", tar_exit_status);
|
|
||||||
return Err(SnapshotError::ArchiveGenerationFailure(tar_exit_status));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Atomically move the archive into position for other validators to find
|
// Atomically move the archive into position for other validators to find
|
||||||
let metadata = fs::metadata(&archive_path)
|
let metadata = fs::metadata(&archive_path)
|
||||||
|
@ -2581,7 +2558,7 @@ mod tests {
|
||||||
let accounts_dir = tempfile::TempDir::new().unwrap();
|
let accounts_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshots_dir = tempfile::TempDir::new().unwrap();
|
let snapshots_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archive_format = ArchiveFormat::Tar;
|
let snapshot_archive_format = ArchiveFormat::TarGzip;
|
||||||
|
|
||||||
let full_snapshot_archive_path = bank_to_full_snapshot_archive(
|
let full_snapshot_archive_path = bank_to_full_snapshot_archive(
|
||||||
snapshots_dir.path(),
|
snapshots_dir.path(),
|
||||||
|
@ -2656,7 +2633,7 @@ mod tests {
|
||||||
let accounts_dir = tempfile::TempDir::new().unwrap();
|
let accounts_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshots_dir = tempfile::TempDir::new().unwrap();
|
let snapshots_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archive_format = ArchiveFormat::Tar;
|
let snapshot_archive_format = ArchiveFormat::TarZstd;
|
||||||
|
|
||||||
let full_snapshot_slot = slot;
|
let full_snapshot_slot = slot;
|
||||||
let full_snapshot_archive_path = bank_to_full_snapshot_archive(
|
let full_snapshot_archive_path = bank_to_full_snapshot_archive(
|
||||||
|
|
Loading…
Reference in New Issue