2021-03-10 09:49:10 -08:00
|
|
|
use {
|
|
|
|
bzip2::bufread::BzDecoder,
|
|
|
|
log::*,
|
|
|
|
rand::{thread_rng, Rng},
|
2021-12-03 09:00:31 -08:00
|
|
|
solana_sdk::genesis_config::{GenesisConfig, DEFAULT_GENESIS_ARCHIVE, DEFAULT_GENESIS_FILE},
|
2021-03-10 09:49:10 -08:00
|
|
|
std::{
|
|
|
|
collections::HashMap,
|
|
|
|
fs::{self, File},
|
|
|
|
io::{BufReader, Read},
|
|
|
|
path::{
|
2021-08-18 13:49:02 -07:00
|
|
|
Component::{self, CurDir, Normal},
|
2021-03-10 09:49:10 -08:00
|
|
|
Path, PathBuf,
|
|
|
|
},
|
|
|
|
time::Instant,
|
2020-03-25 02:46:41 -07:00
|
|
|
},
|
2021-03-10 09:49:10 -08:00
|
|
|
tar::{
|
|
|
|
Archive,
|
|
|
|
EntryType::{Directory, GNUSparse, Regular},
|
|
|
|
},
|
|
|
|
thiserror::Error,
|
2020-03-25 02:46:41 -07:00
|
|
|
};
|
|
|
|
|
|
|
|
#[derive(Error, Debug)]
|
|
|
|
pub enum UnpackError {
|
2020-04-29 18:53:34 -07:00
|
|
|
#[error("IO error: {0}")]
|
2021-02-18 23:42:09 -08:00
|
|
|
Io(#[from] std::io::Error),
|
2020-04-29 18:53:34 -07:00
|
|
|
#[error("Archive error: {0}")]
|
2020-03-25 02:46:41 -07:00
|
|
|
Archive(String),
|
|
|
|
}
|
|
|
|
|
|
|
|
pub type Result<T> = std::result::Result<T, UnpackError>;
|
|
|
|
|
2020-12-09 09:46:33 -08:00
|
|
|
// 64 TiB; some safe margin to the max 128 TiB in amd64 linux userspace VmSize
|
|
|
|
// (ref: https://unix.stackexchange.com/a/386555/364236)
|
|
|
|
// note that this is directly related to the mmaped data size
|
|
|
|
// so protect against insane value
|
|
|
|
// This is the file size including holes for sparse files
|
|
|
|
const MAX_SNAPSHOT_ARCHIVE_UNPACKED_APPARENT_SIZE: u64 = 64 * 1024 * 1024 * 1024 * 1024;
|
|
|
|
|
|
|
|
// 4 TiB;
|
|
|
|
// This is the actually consumed disk usage for sparse files
|
|
|
|
const MAX_SNAPSHOT_ARCHIVE_UNPACKED_ACTUAL_SIZE: u64 = 4 * 1024 * 1024 * 1024 * 1024;
|
|
|
|
|
2020-12-11 03:01:22 -08:00
|
|
|
const MAX_SNAPSHOT_ARCHIVE_UNPACKED_COUNT: u64 = 5_000_000;
|
2020-04-29 18:53:34 -07:00
|
|
|
pub const MAX_GENESIS_ARCHIVE_UNPACKED_SIZE: u64 = 10 * 1024 * 1024; // 10 MiB
|
2020-03-25 02:46:41 -07:00
|
|
|
const MAX_GENESIS_ARCHIVE_UNPACKED_COUNT: u64 = 100;
|
|
|
|
|
|
|
|
fn checked_total_size_sum(total_size: u64, entry_size: u64, limit_size: u64) -> Result<u64> {
|
2020-12-09 09:46:33 -08:00
|
|
|
trace!(
|
|
|
|
"checked_total_size_sum: {} + {} < {}",
|
|
|
|
total_size,
|
|
|
|
entry_size,
|
|
|
|
limit_size,
|
|
|
|
);
|
2020-03-25 02:46:41 -07:00
|
|
|
let total_size = total_size.saturating_add(entry_size);
|
|
|
|
if total_size > limit_size {
|
|
|
|
return Err(UnpackError::Archive(format!(
|
2022-12-06 06:30:06 -08:00
|
|
|
"too large archive: {total_size} than limit: {limit_size}",
|
2020-03-25 02:46:41 -07:00
|
|
|
)));
|
|
|
|
}
|
|
|
|
Ok(total_size)
|
|
|
|
}
|
|
|
|
|
|
|
|
fn checked_total_count_increment(total_count: u64, limit_count: u64) -> Result<u64> {
|
|
|
|
let total_count = total_count + 1;
|
|
|
|
if total_count > limit_count {
|
|
|
|
return Err(UnpackError::Archive(format!(
|
2022-12-06 06:30:06 -08:00
|
|
|
"too many files in snapshot: {total_count:?}"
|
2020-03-25 02:46:41 -07:00
|
|
|
)));
|
|
|
|
}
|
|
|
|
Ok(total_count)
|
|
|
|
}
|
|
|
|
|
|
|
|
fn check_unpack_result(unpack_result: bool, path: String) -> Result<()> {
|
|
|
|
if !unpack_result {
|
2022-12-06 06:30:06 -08:00
|
|
|
return Err(UnpackError::Archive(format!("failed to unpack: {path:?}")));
|
2020-03-25 02:46:41 -07:00
|
|
|
}
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2022-06-21 13:26:51 -07:00
|
|
|
#[derive(Debug, PartialEq, Eq)]
|
2021-06-24 15:29:49 -07:00
|
|
|
pub enum UnpackPath<'a> {
|
|
|
|
Valid(&'a Path),
|
|
|
|
Ignore,
|
|
|
|
Invalid,
|
|
|
|
}
|
|
|
|
|
2022-07-19 12:48:57 -07:00
|
|
|
fn unpack_archive<'a, A, C, D>(
|
2020-03-25 02:46:41 -07:00
|
|
|
archive: &mut Archive<A>,
|
2020-12-09 09:46:33 -08:00
|
|
|
apparent_limit_size: u64,
|
|
|
|
actual_limit_size: u64,
|
2020-03-25 02:46:41 -07:00
|
|
|
limit_count: u64,
|
2022-07-19 12:48:57 -07:00
|
|
|
mut entry_checker: C, // checks if entry is valid
|
|
|
|
entry_processor: D, // processes entry after setting permissions
|
2020-03-25 02:46:41 -07:00
|
|
|
) -> Result<()>
|
|
|
|
where
|
2022-07-19 12:48:57 -07:00
|
|
|
A: Read,
|
2021-06-24 15:29:49 -07:00
|
|
|
C: FnMut(&[&str], tar::EntryType) -> UnpackPath<'a>,
|
2022-07-19 12:48:57 -07:00
|
|
|
D: Fn(PathBuf),
|
2020-03-25 02:46:41 -07:00
|
|
|
{
|
2020-12-09 09:46:33 -08:00
|
|
|
let mut apparent_total_size: u64 = 0;
|
|
|
|
let mut actual_total_size: u64 = 0;
|
2020-03-25 02:46:41 -07:00
|
|
|
let mut total_count: u64 = 0;
|
|
|
|
|
2020-05-27 10:41:05 -07:00
|
|
|
let mut total_entries = 0;
|
|
|
|
let mut last_log_update = Instant::now();
|
2020-03-25 02:46:41 -07:00
|
|
|
for entry in archive.entries()? {
|
|
|
|
let mut entry = entry?;
|
|
|
|
let path = entry.path()?;
|
|
|
|
let path_str = path.display().to_string();
|
|
|
|
|
|
|
|
// Although the `tar` crate safely skips at the actual unpacking, fail
|
|
|
|
// first by ourselves when there are odd paths like including `..` or /
|
|
|
|
// for our clearer pattern matching reasoning:
|
|
|
|
// https://docs.rs/tar/0.4.26/src/tar/entry.rs.html#371
|
|
|
|
let parts = path.components().map(|p| match p {
|
|
|
|
CurDir => Some("."),
|
|
|
|
Normal(c) => c.to_str(),
|
|
|
|
_ => None, // Prefix (for Windows) and RootDir are forbidden
|
|
|
|
});
|
2021-04-09 23:57:32 -07:00
|
|
|
|
|
|
|
// Reject old-style BSD directory entries that aren't explicitly tagged as directories
|
|
|
|
let legacy_dir_entry =
|
|
|
|
entry.header().as_ustar().is_none() && entry.path_bytes().ends_with(b"/");
|
|
|
|
let kind = entry.header().entry_type();
|
|
|
|
let reject_legacy_dir_entry = legacy_dir_entry && (kind != Directory);
|
|
|
|
|
|
|
|
if parts.clone().any(|p| p.is_none()) || reject_legacy_dir_entry {
|
2020-03-25 02:46:41 -07:00
|
|
|
return Err(UnpackError::Archive(format!(
|
2022-12-06 06:30:06 -08:00
|
|
|
"invalid path found: {path_str:?}"
|
2020-03-25 02:46:41 -07:00
|
|
|
)));
|
|
|
|
}
|
|
|
|
|
|
|
|
let parts: Vec<_> = parts.map(|p| p.unwrap()).collect();
|
2023-01-24 16:44:35 -08:00
|
|
|
let account_filename =
|
|
|
|
(parts.len() == 2 && parts[0] == "accounts").then(|| PathBuf::from(parts[1]));
|
2021-04-09 23:57:32 -07:00
|
|
|
let unpack_dir = match entry_checker(parts.as_slice(), kind) {
|
2021-06-24 15:29:49 -07:00
|
|
|
UnpackPath::Invalid => {
|
2021-03-10 09:49:10 -08:00
|
|
|
return Err(UnpackError::Archive(format!(
|
|
|
|
"extra entry found: {:?} {:?}",
|
|
|
|
path_str,
|
|
|
|
entry.header().entry_type(),
|
|
|
|
)));
|
|
|
|
}
|
2021-06-24 15:29:49 -07:00
|
|
|
UnpackPath::Ignore => {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
UnpackPath::Valid(unpack_dir) => unpack_dir,
|
2021-03-10 09:49:10 -08:00
|
|
|
};
|
|
|
|
|
2020-12-09 09:46:33 -08:00
|
|
|
apparent_total_size = checked_total_size_sum(
|
|
|
|
apparent_total_size,
|
|
|
|
entry.header().size()?,
|
|
|
|
apparent_limit_size,
|
|
|
|
)?;
|
|
|
|
actual_total_size = checked_total_size_sum(
|
|
|
|
actual_total_size,
|
|
|
|
entry.header().entry_size()?,
|
|
|
|
actual_limit_size,
|
|
|
|
)?;
|
2020-03-25 02:46:41 -07:00
|
|
|
total_count = checked_total_count_increment(total_count, limit_count)?;
|
|
|
|
|
2021-08-18 13:49:02 -07:00
|
|
|
let target = sanitize_path(&entry.path()?, unpack_dir)?; // ? handles file system errors
|
|
|
|
if target.is_none() {
|
|
|
|
continue; // skip it
|
|
|
|
}
|
|
|
|
let target = target.unwrap();
|
|
|
|
|
|
|
|
let unpack = entry.unpack(target);
|
|
|
|
check_unpack_result(unpack.map(|_unpack| true)?, path_str)?;
|
2021-04-09 23:57:32 -07:00
|
|
|
|
|
|
|
// Sanitize permissions.
|
|
|
|
let mode = match entry.header().entry_type() {
|
|
|
|
GNUSparse | Regular => 0o644,
|
|
|
|
_ => 0o755,
|
|
|
|
};
|
2022-07-19 12:48:57 -07:00
|
|
|
let entry_path_buf = unpack_dir.join(entry.path()?);
|
|
|
|
set_perms(&entry_path_buf, mode)?;
|
|
|
|
|
2023-01-24 16:44:35 -08:00
|
|
|
let entry_path = if let Some(account_filename) = account_filename {
|
|
|
|
let stripped_path = unpack_dir.join(account_filename); // strip away "accounts"
|
|
|
|
fs::rename(&entry_path_buf, &stripped_path)?;
|
|
|
|
stripped_path
|
|
|
|
} else {
|
|
|
|
entry_path_buf
|
|
|
|
};
|
|
|
|
|
2022-07-19 12:48:57 -07:00
|
|
|
// Process entry after setting permissions
|
2023-01-24 16:44:35 -08:00
|
|
|
entry_processor(entry_path);
|
2021-04-09 23:57:32 -07:00
|
|
|
|
2020-05-27 10:41:05 -07:00
|
|
|
total_entries += 1;
|
|
|
|
let now = Instant::now();
|
|
|
|
if now.duration_since(last_log_update).as_secs() >= 10 {
|
|
|
|
info!("unpacked {} entries so far...", total_entries);
|
|
|
|
last_log_update = now;
|
|
|
|
}
|
2020-03-25 02:46:41 -07:00
|
|
|
}
|
2020-05-27 10:41:05 -07:00
|
|
|
info!("unpacked {} entries total", total_entries);
|
2020-03-25 02:46:41 -07:00
|
|
|
|
2021-04-09 23:57:32 -07:00
|
|
|
return Ok(());
|
|
|
|
|
|
|
|
#[cfg(unix)]
|
|
|
|
fn set_perms(dst: &Path, mode: u32) -> std::io::Result<()> {
|
|
|
|
use std::os::unix::fs::PermissionsExt;
|
|
|
|
|
|
|
|
let perm = fs::Permissions::from_mode(mode as _);
|
|
|
|
fs::set_permissions(dst, perm)
|
|
|
|
}
|
|
|
|
|
|
|
|
#[cfg(windows)]
|
|
|
|
fn set_perms(dst: &Path, _mode: u32) -> std::io::Result<()> {
|
|
|
|
let mut perm = fs::metadata(dst)?.permissions();
|
|
|
|
perm.set_readonly(false);
|
|
|
|
fs::set_permissions(dst, perm)
|
|
|
|
}
|
2020-03-25 02:46:41 -07:00
|
|
|
}
|
|
|
|
|
2021-08-18 13:49:02 -07:00
|
|
|
// return Err on file system error
|
|
|
|
// return Some(path) if path is good
|
|
|
|
// return None if we should skip this file
|
|
|
|
fn sanitize_path(entry_path: &Path, dst: &Path) -> Result<Option<PathBuf>> {
|
|
|
|
// We cannot call unpack_in because it errors if we try to use 2 account paths.
|
|
|
|
// So, this code is borrowed from unpack_in
|
|
|
|
// ref: https://docs.rs/tar/*/tar/struct.Entry.html#method.unpack_in
|
|
|
|
let mut file_dst = dst.to_path_buf();
|
|
|
|
const SKIP: Result<Option<PathBuf>> = Ok(None);
|
|
|
|
{
|
|
|
|
let path = entry_path;
|
|
|
|
for part in path.components() {
|
|
|
|
match part {
|
|
|
|
// Leading '/' characters, root paths, and '.'
|
|
|
|
// components are just ignored and treated as "empty
|
|
|
|
// components"
|
|
|
|
Component::Prefix(..) | Component::RootDir | Component::CurDir => continue,
|
|
|
|
|
|
|
|
// If any part of the filename is '..', then skip over
|
|
|
|
// unpacking the file to prevent directory traversal
|
|
|
|
// security issues. See, e.g.: CVE-2001-1267,
|
|
|
|
// CVE-2002-0399, CVE-2005-1918, CVE-2007-4131
|
|
|
|
Component::ParentDir => return SKIP,
|
|
|
|
|
|
|
|
Component::Normal(part) => file_dst.push(part),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Skip cases where only slashes or '.' parts were seen, because
|
|
|
|
// this is effectively an empty filename.
|
|
|
|
if *dst == *file_dst {
|
|
|
|
return SKIP;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Skip entries without a parent (i.e. outside of FS root)
|
|
|
|
let parent = match file_dst.parent() {
|
|
|
|
Some(p) => p,
|
|
|
|
None => return SKIP,
|
|
|
|
};
|
|
|
|
|
|
|
|
fs::create_dir_all(parent)?;
|
|
|
|
|
|
|
|
// Here we are different than untar_in. The code for tar::unpack_in internally calling unpack is a little different.
|
|
|
|
// ignore return value here
|
|
|
|
validate_inside_dst(dst, parent)?;
|
|
|
|
let target = parent.join(entry_path.file_name().unwrap());
|
|
|
|
|
|
|
|
Ok(Some(target))
|
|
|
|
}
|
|
|
|
|
|
|
|
// copied from:
|
|
|
|
// https://github.com/alexcrichton/tar-rs/blob/d90a02f582c03dfa0fd11c78d608d0974625ae5d/src/entry.rs#L781
|
|
|
|
fn validate_inside_dst(dst: &Path, file_dst: &Path) -> Result<PathBuf> {
|
|
|
|
// Abort if target (canonical) parent is outside of `dst`
|
|
|
|
let canon_parent = file_dst.canonicalize().map_err(|err| {
|
|
|
|
UnpackError::Archive(format!(
|
|
|
|
"{} while canonicalizing {}",
|
|
|
|
err,
|
|
|
|
file_dst.display()
|
|
|
|
))
|
|
|
|
})?;
|
|
|
|
let canon_target = dst.canonicalize().map_err(|err| {
|
|
|
|
UnpackError::Archive(format!("{} while canonicalizing {}", err, dst.display()))
|
|
|
|
})?;
|
|
|
|
if !canon_parent.starts_with(&canon_target) {
|
|
|
|
return Err(UnpackError::Archive(format!(
|
|
|
|
"trying to unpack outside of destination path: {}",
|
|
|
|
canon_target.display()
|
|
|
|
)));
|
|
|
|
}
|
|
|
|
Ok(canon_target)
|
|
|
|
}
|
|
|
|
|
2021-05-27 10:00:27 -07:00
|
|
|
/// Map from AppendVec file name to unpacked file system location
|
2021-03-10 09:49:10 -08:00
|
|
|
pub type UnpackedAppendVecMap = HashMap<String, PathBuf>;
|
|
|
|
|
2021-06-24 15:29:49 -07:00
|
|
|
// select/choose only 'index' out of each # of 'divisions' of total items.
|
|
|
|
pub struct ParallelSelector {
|
|
|
|
pub index: usize,
|
|
|
|
pub divisions: usize,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl ParallelSelector {
|
|
|
|
pub fn select_index(&self, index: usize) -> bool {
|
|
|
|
index % self.divisions == self.index
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-07-26 12:09:28 -07:00
|
|
|
/// Unpacks snapshot and collects AppendVec file names & paths
|
2021-03-10 09:49:10 -08:00
|
|
|
pub fn unpack_snapshot<A: Read>(
|
2020-03-25 02:46:41 -07:00
|
|
|
archive: &mut Archive<A>,
|
2021-03-10 09:49:10 -08:00
|
|
|
ledger_dir: &Path,
|
|
|
|
account_paths: &[PathBuf],
|
2021-06-24 15:29:49 -07:00
|
|
|
parallel_selector: Option<ParallelSelector>,
|
2021-03-10 09:49:10 -08:00
|
|
|
) -> Result<UnpackedAppendVecMap> {
|
|
|
|
let mut unpacked_append_vec_map = UnpackedAppendVecMap::new();
|
2022-07-19 12:48:57 -07:00
|
|
|
|
|
|
|
unpack_snapshot_with_processors(
|
|
|
|
archive,
|
|
|
|
ledger_dir,
|
|
|
|
account_paths,
|
|
|
|
parallel_selector,
|
|
|
|
|file, path| {
|
|
|
|
unpacked_append_vec_map.insert(file.to_string(), path.join("accounts").join(file));
|
|
|
|
},
|
|
|
|
|_| {},
|
|
|
|
)
|
|
|
|
.map(|_| unpacked_append_vec_map)
|
|
|
|
}
|
|
|
|
|
2022-07-26 12:09:28 -07:00
|
|
|
/// Unpacks snapshots and sends entry file paths through the `sender` channel
|
|
|
|
pub fn streaming_unpack_snapshot<A: Read>(
|
|
|
|
archive: &mut Archive<A>,
|
|
|
|
ledger_dir: &Path,
|
|
|
|
account_paths: &[PathBuf],
|
|
|
|
parallel_selector: Option<ParallelSelector>,
|
|
|
|
sender: &crossbeam_channel::Sender<PathBuf>,
|
|
|
|
) -> Result<()> {
|
|
|
|
unpack_snapshot_with_processors(
|
|
|
|
archive,
|
|
|
|
ledger_dir,
|
|
|
|
account_paths,
|
|
|
|
parallel_selector,
|
|
|
|
|_, _| {},
|
|
|
|
|entry_path_buf| {
|
|
|
|
if entry_path_buf.is_file() {
|
|
|
|
sender.send(entry_path_buf).unwrap();
|
|
|
|
}
|
|
|
|
},
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
2022-07-19 12:48:57 -07:00
|
|
|
fn unpack_snapshot_with_processors<A, F, G>(
|
|
|
|
archive: &mut Archive<A>,
|
|
|
|
ledger_dir: &Path,
|
|
|
|
account_paths: &[PathBuf],
|
|
|
|
parallel_selector: Option<ParallelSelector>,
|
|
|
|
mut accounts_path_processor: F,
|
|
|
|
entry_processor: G,
|
|
|
|
) -> Result<()>
|
|
|
|
where
|
|
|
|
A: Read,
|
|
|
|
F: FnMut(&str, &Path),
|
|
|
|
G: Fn(PathBuf),
|
|
|
|
{
|
|
|
|
assert!(!account_paths.is_empty());
|
2021-06-24 15:29:49 -07:00
|
|
|
let mut i = 0;
|
2021-03-10 09:49:10 -08:00
|
|
|
|
2020-03-25 02:46:41 -07:00
|
|
|
unpack_archive(
|
|
|
|
archive,
|
2020-12-09 09:46:33 -08:00
|
|
|
MAX_SNAPSHOT_ARCHIVE_UNPACKED_APPARENT_SIZE,
|
|
|
|
MAX_SNAPSHOT_ARCHIVE_UNPACKED_ACTUAL_SIZE,
|
2020-03-25 02:46:41 -07:00
|
|
|
MAX_SNAPSHOT_ARCHIVE_UNPACKED_COUNT,
|
2021-03-10 09:49:10 -08:00
|
|
|
|parts, kind| {
|
|
|
|
if is_valid_snapshot_archive_entry(parts, kind) {
|
2021-06-24 15:29:49 -07:00
|
|
|
i += 1;
|
|
|
|
match ¶llel_selector {
|
|
|
|
Some(parallel_selector) => {
|
|
|
|
if !parallel_selector.select_index(i - 1) {
|
|
|
|
return UnpackPath::Ignore;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
None => {}
|
|
|
|
};
|
2021-03-10 09:49:10 -08:00
|
|
|
if let ["accounts", file] = parts {
|
|
|
|
// Randomly distribute the accounts files about the available `account_paths`,
|
|
|
|
let path_index = thread_rng().gen_range(0, account_paths.len());
|
2022-07-19 12:48:57 -07:00
|
|
|
match account_paths
|
|
|
|
.get(path_index)
|
|
|
|
.map(|path_buf| path_buf.as_path())
|
|
|
|
{
|
|
|
|
Some(path) => {
|
2022-08-22 18:01:03 -07:00
|
|
|
accounts_path_processor(file, path);
|
2022-07-19 12:48:57 -07:00
|
|
|
UnpackPath::Valid(path)
|
|
|
|
}
|
2021-06-24 15:29:49 -07:00
|
|
|
None => UnpackPath::Invalid,
|
|
|
|
}
|
2021-03-10 09:49:10 -08:00
|
|
|
} else {
|
2021-06-24 15:29:49 -07:00
|
|
|
UnpackPath::Valid(ledger_dir)
|
2021-03-10 09:49:10 -08:00
|
|
|
}
|
|
|
|
} else {
|
2021-06-24 15:29:49 -07:00
|
|
|
UnpackPath::Invalid
|
2021-03-10 09:49:10 -08:00
|
|
|
}
|
|
|
|
},
|
2022-07-19 12:48:57 -07:00
|
|
|
entry_processor,
|
2020-03-25 02:46:41 -07:00
|
|
|
)
|
|
|
|
}
|
|
|
|
|
2020-12-15 08:48:21 -08:00
|
|
|
fn all_digits(v: &str) -> bool {
|
|
|
|
if v.is_empty() {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
for x in v.chars() {
|
2022-05-21 05:03:11 -07:00
|
|
|
if !x.is_ascii_digit() {
|
2020-12-15 08:48:21 -08:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
true
|
|
|
|
}
|
2020-03-25 02:46:41 -07:00
|
|
|
|
2020-12-15 08:48:21 -08:00
|
|
|
fn like_storage(v: &str) -> bool {
|
|
|
|
let mut periods = 0;
|
|
|
|
let mut saw_numbers = false;
|
|
|
|
for x in v.chars() {
|
2022-05-21 05:03:11 -07:00
|
|
|
if !x.is_ascii_digit() {
|
2020-12-15 08:48:21 -08:00
|
|
|
if x == '.' {
|
|
|
|
if periods > 0 || !saw_numbers {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
saw_numbers = false;
|
|
|
|
periods += 1;
|
|
|
|
} else {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
saw_numbers = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
saw_numbers && periods == 1
|
|
|
|
}
|
|
|
|
|
|
|
|
fn is_valid_snapshot_archive_entry(parts: &[&str], kind: tar::EntryType) -> bool {
|
2020-03-25 02:46:41 -07:00
|
|
|
match (parts, kind) {
|
|
|
|
(["version"], Regular) => true,
|
|
|
|
(["accounts"], Directory) => true,
|
2020-12-15 08:48:21 -08:00
|
|
|
(["accounts", file], GNUSparse) if like_storage(file) => true,
|
|
|
|
(["accounts", file], Regular) if like_storage(file) => true,
|
2020-03-25 02:46:41 -07:00
|
|
|
(["snapshots"], Directory) => true,
|
2021-03-04 10:09:10 -08:00
|
|
|
(["snapshots", "status_cache"], GNUSparse) => true,
|
2020-03-25 02:46:41 -07:00
|
|
|
(["snapshots", "status_cache"], Regular) => true,
|
2021-03-04 10:09:10 -08:00
|
|
|
(["snapshots", dir, file], GNUSparse) if all_digits(dir) && all_digits(file) => true,
|
2020-12-15 08:48:21 -08:00
|
|
|
(["snapshots", dir, file], Regular) if all_digits(dir) && all_digits(file) => true,
|
|
|
|
(["snapshots", dir], Directory) if all_digits(dir) => true,
|
2020-03-25 02:46:41 -07:00
|
|
|
_ => false,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-04-29 18:53:34 -07:00
|
|
|
pub fn open_genesis_config(
|
|
|
|
ledger_path: &Path,
|
|
|
|
max_genesis_archive_unpacked_size: u64,
|
|
|
|
) -> GenesisConfig {
|
2021-06-18 06:34:46 -07:00
|
|
|
GenesisConfig::load(ledger_path).unwrap_or_else(|load_err| {
|
2021-05-24 07:45:36 -07:00
|
|
|
let genesis_package = ledger_path.join(DEFAULT_GENESIS_ARCHIVE);
|
2020-04-29 18:53:34 -07:00
|
|
|
unpack_genesis_archive(
|
|
|
|
&genesis_package,
|
|
|
|
ledger_path,
|
|
|
|
max_genesis_archive_unpacked_size,
|
|
|
|
)
|
|
|
|
.unwrap_or_else(|unpack_err| {
|
2020-03-25 02:46:41 -07:00
|
|
|
warn!(
|
|
|
|
"Failed to open ledger genesis_config at {:?}: {}, {}",
|
|
|
|
ledger_path, load_err, unpack_err,
|
|
|
|
);
|
|
|
|
std::process::exit(1);
|
|
|
|
});
|
|
|
|
|
|
|
|
// loading must succeed at this moment
|
2021-06-18 06:34:46 -07:00
|
|
|
GenesisConfig::load(ledger_path).unwrap()
|
2020-03-25 02:46:41 -07:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn unpack_genesis_archive(
|
|
|
|
archive_filename: &Path,
|
|
|
|
destination_dir: &Path,
|
2020-04-29 18:53:34 -07:00
|
|
|
max_genesis_archive_unpacked_size: u64,
|
|
|
|
) -> std::result::Result<(), UnpackError> {
|
2020-03-25 02:46:41 -07:00
|
|
|
info!("Extracting {:?}...", archive_filename);
|
|
|
|
let extract_start = Instant::now();
|
|
|
|
|
2020-04-29 18:53:34 -07:00
|
|
|
fs::create_dir_all(destination_dir)?;
|
2022-09-22 15:23:03 -07:00
|
|
|
let tar_bz2 = File::open(archive_filename)?;
|
2020-03-25 02:46:41 -07:00
|
|
|
let tar = BzDecoder::new(BufReader::new(tar_bz2));
|
|
|
|
let mut archive = Archive::new(tar);
|
2020-04-29 18:53:34 -07:00
|
|
|
unpack_genesis(
|
|
|
|
&mut archive,
|
|
|
|
destination_dir,
|
|
|
|
max_genesis_archive_unpacked_size,
|
|
|
|
)?;
|
2020-03-25 02:46:41 -07:00
|
|
|
info!(
|
|
|
|
"Extracted {:?} in {:?}",
|
|
|
|
archive_filename,
|
|
|
|
Instant::now().duration_since(extract_start)
|
|
|
|
);
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2021-03-10 09:49:10 -08:00
|
|
|
fn unpack_genesis<A: Read>(
|
2020-04-29 18:53:34 -07:00
|
|
|
archive: &mut Archive<A>,
|
2021-03-10 09:49:10 -08:00
|
|
|
unpack_dir: &Path,
|
2020-04-29 18:53:34 -07:00
|
|
|
max_genesis_archive_unpacked_size: u64,
|
|
|
|
) -> Result<()> {
|
2020-03-25 02:46:41 -07:00
|
|
|
unpack_archive(
|
|
|
|
archive,
|
2020-04-29 18:53:34 -07:00
|
|
|
max_genesis_archive_unpacked_size,
|
2020-12-09 09:46:33 -08:00
|
|
|
max_genesis_archive_unpacked_size,
|
2020-03-25 02:46:41 -07:00
|
|
|
MAX_GENESIS_ARCHIVE_UNPACKED_COUNT,
|
2022-05-30 22:38:45 -07:00
|
|
|
|p, k| is_valid_genesis_archive_entry(unpack_dir, p, k),
|
2022-07-19 12:48:57 -07:00
|
|
|
|_| {},
|
2020-03-25 02:46:41 -07:00
|
|
|
)
|
|
|
|
}
|
|
|
|
|
2022-05-30 22:38:45 -07:00
|
|
|
fn is_valid_genesis_archive_entry<'a>(
|
|
|
|
unpack_dir: &'a Path,
|
|
|
|
parts: &[&str],
|
|
|
|
kind: tar::EntryType,
|
|
|
|
) -> UnpackPath<'a> {
|
2020-03-25 02:46:41 -07:00
|
|
|
trace!("validating: {:?} {:?}", parts, kind);
|
2020-12-13 17:26:34 -08:00
|
|
|
#[allow(clippy::match_like_matches_macro)]
|
2020-03-25 02:46:41 -07:00
|
|
|
match (parts, kind) {
|
2022-05-30 22:38:45 -07:00
|
|
|
([DEFAULT_GENESIS_FILE], GNUSparse) => UnpackPath::Valid(unpack_dir),
|
|
|
|
([DEFAULT_GENESIS_FILE], Regular) => UnpackPath::Valid(unpack_dir),
|
|
|
|
(["rocksdb"], Directory) => UnpackPath::Ignore,
|
|
|
|
(["rocksdb", _], GNUSparse) => UnpackPath::Ignore,
|
|
|
|
(["rocksdb", _], Regular) => UnpackPath::Ignore,
|
|
|
|
(["rocksdb_fifo"], Directory) => UnpackPath::Ignore,
|
|
|
|
(["rocksdb_fifo", _], GNUSparse) => UnpackPath::Ignore,
|
|
|
|
(["rocksdb_fifo", _], Regular) => UnpackPath::Ignore,
|
|
|
|
_ => UnpackPath::Invalid,
|
2020-03-25 02:46:41 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
mod tests {
|
2021-12-03 09:00:31 -08:00
|
|
|
use {
|
|
|
|
super::*,
|
|
|
|
assert_matches::assert_matches,
|
|
|
|
tar::{Builder, Header},
|
|
|
|
};
|
2020-03-25 02:46:41 -07:00
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_archive_is_valid_entry() {
|
|
|
|
assert!(is_valid_snapshot_archive_entry(
|
|
|
|
&["snapshots"],
|
|
|
|
tar::EntryType::Directory
|
|
|
|
));
|
2020-12-15 08:48:21 -08:00
|
|
|
assert!(!is_valid_snapshot_archive_entry(
|
|
|
|
&["snapshots", ""],
|
|
|
|
tar::EntryType::Directory
|
|
|
|
));
|
2020-03-25 02:46:41 -07:00
|
|
|
assert!(is_valid_snapshot_archive_entry(
|
|
|
|
&["snapshots", "3"],
|
|
|
|
tar::EntryType::Directory
|
|
|
|
));
|
|
|
|
assert!(is_valid_snapshot_archive_entry(
|
|
|
|
&["snapshots", "3", "3"],
|
|
|
|
tar::EntryType::Regular
|
|
|
|
));
|
|
|
|
assert!(is_valid_snapshot_archive_entry(
|
|
|
|
&["version"],
|
|
|
|
tar::EntryType::Regular
|
|
|
|
));
|
|
|
|
assert!(is_valid_snapshot_archive_entry(
|
|
|
|
&["accounts"],
|
|
|
|
tar::EntryType::Directory
|
|
|
|
));
|
|
|
|
assert!(!is_valid_snapshot_archive_entry(
|
2020-12-15 08:48:21 -08:00
|
|
|
&["accounts", ""],
|
2020-03-25 02:46:41 -07:00
|
|
|
tar::EntryType::Regular
|
|
|
|
));
|
2020-12-15 08:48:21 -08:00
|
|
|
|
2020-03-25 02:46:41 -07:00
|
|
|
assert!(!is_valid_snapshot_archive_entry(
|
|
|
|
&["snapshots"],
|
|
|
|
tar::EntryType::Regular
|
|
|
|
));
|
|
|
|
assert!(!is_valid_snapshot_archive_entry(
|
|
|
|
&["snapshots", "x0"],
|
|
|
|
tar::EntryType::Directory
|
|
|
|
));
|
|
|
|
assert!(!is_valid_snapshot_archive_entry(
|
|
|
|
&["snapshots", "0x"],
|
|
|
|
tar::EntryType::Directory
|
|
|
|
));
|
2021-11-15 09:02:15 -08:00
|
|
|
assert!(!is_valid_snapshot_archive_entry(
|
|
|
|
&["snapshots", "①"],
|
|
|
|
tar::EntryType::Directory
|
|
|
|
));
|
2020-03-25 02:46:41 -07:00
|
|
|
assert!(!is_valid_snapshot_archive_entry(
|
|
|
|
&["snapshots", "0", "aa"],
|
|
|
|
tar::EntryType::Regular
|
|
|
|
));
|
|
|
|
assert!(!is_valid_snapshot_archive_entry(
|
|
|
|
&["aaaa"],
|
|
|
|
tar::EntryType::Regular
|
|
|
|
));
|
|
|
|
}
|
|
|
|
|
2020-12-15 08:48:21 -08:00
|
|
|
#[test]
|
|
|
|
fn test_valid_snapshot_accounts() {
|
|
|
|
solana_logger::setup();
|
|
|
|
assert!(is_valid_snapshot_archive_entry(
|
|
|
|
&["accounts", "0.0"],
|
|
|
|
tar::EntryType::Regular
|
|
|
|
));
|
|
|
|
assert!(is_valid_snapshot_archive_entry(
|
|
|
|
&["accounts", "01829.077"],
|
|
|
|
tar::EntryType::Regular
|
|
|
|
));
|
|
|
|
|
|
|
|
assert!(!is_valid_snapshot_archive_entry(
|
|
|
|
&["accounts", "1.2.34"],
|
|
|
|
tar::EntryType::Regular
|
|
|
|
));
|
|
|
|
assert!(!is_valid_snapshot_archive_entry(
|
|
|
|
&["accounts", "12."],
|
|
|
|
tar::EntryType::Regular
|
|
|
|
));
|
|
|
|
assert!(!is_valid_snapshot_archive_entry(
|
|
|
|
&["accounts", ".12"],
|
|
|
|
tar::EntryType::Regular
|
|
|
|
));
|
|
|
|
assert!(!is_valid_snapshot_archive_entry(
|
|
|
|
&["accounts", "0x0"],
|
|
|
|
tar::EntryType::Regular
|
|
|
|
));
|
|
|
|
assert!(!is_valid_snapshot_archive_entry(
|
|
|
|
&["accounts", "abc"],
|
|
|
|
tar::EntryType::Regular
|
|
|
|
));
|
|
|
|
assert!(!is_valid_snapshot_archive_entry(
|
|
|
|
&["accounts", "232323"],
|
|
|
|
tar::EntryType::Regular
|
|
|
|
));
|
2021-11-15 09:02:15 -08:00
|
|
|
assert!(!is_valid_snapshot_archive_entry(
|
|
|
|
&["accounts", "৬.¾"],
|
|
|
|
tar::EntryType::Regular
|
|
|
|
));
|
2020-12-15 08:48:21 -08:00
|
|
|
}
|
|
|
|
|
2020-03-25 02:46:41 -07:00
|
|
|
#[test]
|
|
|
|
fn test_archive_is_valid_archive_entry() {
|
2022-05-30 22:38:45 -07:00
|
|
|
let path = Path::new("");
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(path, &["genesis.bin"], tar::EntryType::Regular),
|
|
|
|
UnpackPath::Valid(path)
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(path, &["genesis.bin"], tar::EntryType::GNUSparse,),
|
|
|
|
UnpackPath::Valid(path)
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(path, &["rocksdb"], tar::EntryType::Directory),
|
|
|
|
UnpackPath::Ignore
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(path, &["rocksdb", "foo"], tar::EntryType::Regular),
|
|
|
|
UnpackPath::Ignore
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(path, &["rocksdb", "foo"], tar::EntryType::GNUSparse,),
|
|
|
|
UnpackPath::Ignore
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(path, &["rocksdb_fifo"], tar::EntryType::Directory),
|
|
|
|
UnpackPath::Ignore
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(path, &["rocksdb_fifo", "foo"], tar::EntryType::Regular),
|
|
|
|
UnpackPath::Ignore
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(
|
|
|
|
path,
|
|
|
|
&["rocksdb_fifo", "foo"],
|
|
|
|
tar::EntryType::GNUSparse,
|
|
|
|
),
|
|
|
|
UnpackPath::Ignore
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(path, &["aaaa"], tar::EntryType::Regular),
|
|
|
|
UnpackPath::Invalid
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(path, &["aaaa"], tar::EntryType::GNUSparse,),
|
|
|
|
UnpackPath::Invalid
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(path, &["rocksdb"], tar::EntryType::Regular),
|
|
|
|
UnpackPath::Invalid
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(path, &["rocksdb"], tar::EntryType::GNUSparse,),
|
|
|
|
UnpackPath::Invalid
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(path, &["rocksdb", "foo"], tar::EntryType::Directory,),
|
|
|
|
UnpackPath::Invalid
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(
|
|
|
|
path,
|
|
|
|
&["rocksdb", "foo", "bar"],
|
|
|
|
tar::EntryType::Directory,
|
|
|
|
),
|
|
|
|
UnpackPath::Invalid
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(
|
|
|
|
path,
|
|
|
|
&["rocksdb", "foo", "bar"],
|
|
|
|
tar::EntryType::Regular
|
|
|
|
),
|
|
|
|
UnpackPath::Invalid
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(
|
|
|
|
path,
|
|
|
|
&["rocksdb", "foo", "bar"],
|
|
|
|
tar::EntryType::GNUSparse
|
|
|
|
),
|
|
|
|
UnpackPath::Invalid
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(path, &["rocksdb_fifo"], tar::EntryType::Regular),
|
|
|
|
UnpackPath::Invalid
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(path, &["rocksdb_fifo"], tar::EntryType::GNUSparse,),
|
|
|
|
UnpackPath::Invalid
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(
|
|
|
|
path,
|
|
|
|
&["rocksdb_fifo", "foo"],
|
|
|
|
tar::EntryType::Directory,
|
|
|
|
),
|
|
|
|
UnpackPath::Invalid
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(
|
|
|
|
path,
|
|
|
|
&["rocksdb_fifo", "foo", "bar"],
|
|
|
|
tar::EntryType::Directory,
|
|
|
|
),
|
|
|
|
UnpackPath::Invalid
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(
|
|
|
|
path,
|
|
|
|
&["rocksdb_fifo", "foo", "bar"],
|
|
|
|
tar::EntryType::Regular
|
|
|
|
),
|
|
|
|
UnpackPath::Invalid
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
is_valid_genesis_archive_entry(
|
|
|
|
path,
|
|
|
|
&["rocksdb_fifo", "foo", "bar"],
|
|
|
|
tar::EntryType::GNUSparse
|
|
|
|
),
|
|
|
|
UnpackPath::Invalid
|
|
|
|
);
|
2020-03-25 02:46:41 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
fn with_finalize_and_unpack<C>(archive: tar::Builder<Vec<u8>>, checker: C) -> Result<()>
|
|
|
|
where
|
|
|
|
C: Fn(&mut Archive<BufReader<&[u8]>>, &Path) -> Result<()>,
|
|
|
|
{
|
|
|
|
let data = archive.into_inner().unwrap();
|
|
|
|
let reader = BufReader::new(&data[..]);
|
|
|
|
let mut archive: Archive<std::io::BufReader<&[u8]>> = Archive::new(reader);
|
|
|
|
let temp_dir = tempfile::TempDir::new().unwrap();
|
|
|
|
|
2021-04-09 23:57:32 -07:00
|
|
|
checker(&mut archive, temp_dir.path())?;
|
|
|
|
// Check that there is no bad permissions preventing deletion.
|
|
|
|
let result = temp_dir.close();
|
|
|
|
assert_matches!(result, Ok(()));
|
|
|
|
Ok(())
|
2020-03-25 02:46:41 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
fn finalize_and_unpack_snapshot(archive: tar::Builder<Vec<u8>>) -> Result<()> {
|
2021-03-10 09:49:10 -08:00
|
|
|
with_finalize_and_unpack(archive, |a, b| {
|
2022-07-26 12:09:28 -07:00
|
|
|
unpack_snapshot_with_processors(a, b, &[PathBuf::new()], None, |_, _| {}, |_| {})
|
2021-03-10 09:49:10 -08:00
|
|
|
})
|
2020-03-25 02:46:41 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
fn finalize_and_unpack_genesis(archive: tar::Builder<Vec<u8>>) -> Result<()> {
|
2020-04-29 18:53:34 -07:00
|
|
|
with_finalize_and_unpack(archive, |a, b| {
|
|
|
|
unpack_genesis(a, b, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE)
|
|
|
|
})
|
2020-03-25 02:46:41 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_archive_unpack_snapshot_ok() {
|
|
|
|
let mut header = Header::new_gnu();
|
|
|
|
header.set_path("version").unwrap();
|
|
|
|
header.set_size(4);
|
|
|
|
header.set_cksum();
|
|
|
|
|
|
|
|
let data: &[u8] = &[1, 2, 3, 4];
|
|
|
|
|
|
|
|
let mut archive = Builder::new(Vec::new());
|
|
|
|
archive.append(&header, data).unwrap();
|
|
|
|
|
|
|
|
let result = finalize_and_unpack_snapshot(archive);
|
|
|
|
assert_matches!(result, Ok(()));
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_archive_unpack_genesis_ok() {
|
|
|
|
let mut header = Header::new_gnu();
|
|
|
|
header.set_path("genesis.bin").unwrap();
|
|
|
|
header.set_size(4);
|
|
|
|
header.set_cksum();
|
|
|
|
|
|
|
|
let data: &[u8] = &[1, 2, 3, 4];
|
|
|
|
|
|
|
|
let mut archive = Builder::new(Vec::new());
|
|
|
|
archive.append(&header, data).unwrap();
|
|
|
|
|
|
|
|
let result = finalize_and_unpack_genesis(archive);
|
|
|
|
assert_matches!(result, Ok(()));
|
|
|
|
}
|
|
|
|
|
2021-04-09 23:57:32 -07:00
|
|
|
#[test]
|
|
|
|
fn test_archive_unpack_genesis_bad_perms() {
|
|
|
|
let mut archive = Builder::new(Vec::new());
|
|
|
|
|
|
|
|
let mut header = Header::new_gnu();
|
|
|
|
header.set_path("rocksdb").unwrap();
|
|
|
|
header.set_entry_type(Directory);
|
|
|
|
header.set_size(0);
|
|
|
|
header.set_cksum();
|
|
|
|
let data: &[u8] = &[];
|
|
|
|
archive.append(&header, data).unwrap();
|
|
|
|
|
|
|
|
let mut header = Header::new_gnu();
|
|
|
|
header.set_path("rocksdb/test").unwrap();
|
|
|
|
header.set_size(4);
|
|
|
|
header.set_cksum();
|
|
|
|
let data: &[u8] = &[1, 2, 3, 4];
|
|
|
|
archive.append(&header, data).unwrap();
|
|
|
|
|
|
|
|
// Removing all permissions makes it harder to delete this directory
|
|
|
|
// or work with files inside it.
|
|
|
|
let mut header = Header::new_gnu();
|
|
|
|
header.set_path("rocksdb").unwrap();
|
|
|
|
header.set_entry_type(Directory);
|
|
|
|
header.set_mode(0o000);
|
|
|
|
header.set_size(0);
|
|
|
|
header.set_cksum();
|
|
|
|
let data: &[u8] = &[];
|
|
|
|
archive.append(&header, data).unwrap();
|
|
|
|
|
|
|
|
let result = finalize_and_unpack_genesis(archive);
|
|
|
|
assert_matches!(result, Ok(()));
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_archive_unpack_genesis_bad_rocksdb_subdir() {
|
|
|
|
let mut archive = Builder::new(Vec::new());
|
|
|
|
|
|
|
|
let mut header = Header::new_gnu();
|
|
|
|
header.set_path("rocksdb").unwrap();
|
|
|
|
header.set_entry_type(Directory);
|
|
|
|
header.set_size(0);
|
|
|
|
header.set_cksum();
|
|
|
|
let data: &[u8] = &[];
|
|
|
|
archive.append(&header, data).unwrap();
|
|
|
|
|
|
|
|
// tar-rs treats following entry as a Directory to support old tar formats.
|
|
|
|
let mut header = Header::new_gnu();
|
|
|
|
header.set_path("rocksdb/test/").unwrap();
|
|
|
|
header.set_entry_type(Regular);
|
|
|
|
header.set_size(0);
|
|
|
|
header.set_cksum();
|
|
|
|
let data: &[u8] = &[];
|
|
|
|
archive.append(&header, data).unwrap();
|
|
|
|
|
|
|
|
let result = finalize_and_unpack_genesis(archive);
|
|
|
|
assert_matches!(result, Err(UnpackError::Archive(ref message)) if message == "invalid path found: \"rocksdb/test/\"");
|
|
|
|
}
|
|
|
|
|
2020-03-25 02:46:41 -07:00
|
|
|
#[test]
|
|
|
|
fn test_archive_unpack_snapshot_invalid_path() {
|
|
|
|
let mut header = Header::new_gnu();
|
|
|
|
// bypass the sanitization of the .set_path()
|
|
|
|
for (p, c) in header
|
|
|
|
.as_old_mut()
|
|
|
|
.name
|
|
|
|
.iter_mut()
|
|
|
|
.zip(b"foo/../../../dangerous".iter().chain(Some(&0)))
|
|
|
|
{
|
|
|
|
*p = *c;
|
|
|
|
}
|
|
|
|
header.set_size(4);
|
|
|
|
header.set_cksum();
|
|
|
|
|
|
|
|
let data: &[u8] = &[1, 2, 3, 4];
|
|
|
|
|
|
|
|
let mut archive = Builder::new(Vec::new());
|
|
|
|
archive.append(&header, data).unwrap();
|
|
|
|
let result = finalize_and_unpack_snapshot(archive);
|
2020-05-15 09:35:43 -07:00
|
|
|
assert_matches!(result, Err(UnpackError::Archive(ref message)) if message == "invalid path found: \"foo/../../../dangerous\"");
|
2020-03-25 02:46:41 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
fn with_archive_unpack_snapshot_invalid_path(path: &str) -> Result<()> {
|
|
|
|
let mut header = Header::new_gnu();
|
|
|
|
// bypass the sanitization of the .set_path()
|
|
|
|
for (p, c) in header
|
|
|
|
.as_old_mut()
|
|
|
|
.name
|
|
|
|
.iter_mut()
|
|
|
|
.zip(path.as_bytes().iter().chain(Some(&0)))
|
|
|
|
{
|
|
|
|
*p = *c;
|
|
|
|
}
|
|
|
|
header.set_size(4);
|
|
|
|
header.set_cksum();
|
|
|
|
|
|
|
|
let data: &[u8] = &[1, 2, 3, 4];
|
|
|
|
|
|
|
|
let mut archive = Builder::new(Vec::new());
|
|
|
|
archive.append(&header, data).unwrap();
|
|
|
|
with_finalize_and_unpack(archive, |unpacking_archive, path| {
|
|
|
|
for entry in unpacking_archive.entries()? {
|
2020-05-15 09:35:43 -07:00
|
|
|
if !entry?.unpack_in(path)? {
|
2020-03-25 02:46:41 -07:00
|
|
|
return Err(UnpackError::Archive("failed!".to_string()));
|
|
|
|
} else if !path.join(path).exists() {
|
|
|
|
return Err(UnpackError::Archive("not existing!".to_string()));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
Ok(())
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_archive_unpack_itself() {
|
|
|
|
assert_matches!(
|
|
|
|
with_archive_unpack_snapshot_invalid_path("ryoqun/work"),
|
|
|
|
Ok(())
|
|
|
|
);
|
|
|
|
// Absolute paths are neutralized as relative
|
|
|
|
assert_matches!(
|
|
|
|
with_archive_unpack_snapshot_invalid_path("/etc/passwd"),
|
|
|
|
Ok(())
|
|
|
|
);
|
2020-05-15 09:35:43 -07:00
|
|
|
assert_matches!(with_archive_unpack_snapshot_invalid_path("../../../dangerous"), Err(UnpackError::Archive(ref message)) if message == "failed!");
|
2020-03-25 02:46:41 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_archive_unpack_snapshot_invalid_entry() {
|
|
|
|
let mut header = Header::new_gnu();
|
|
|
|
header.set_path("foo").unwrap();
|
|
|
|
header.set_size(4);
|
|
|
|
header.set_cksum();
|
|
|
|
|
|
|
|
let data: &[u8] = &[1, 2, 3, 4];
|
|
|
|
|
|
|
|
let mut archive = Builder::new(Vec::new());
|
|
|
|
archive.append(&header, data).unwrap();
|
|
|
|
let result = finalize_and_unpack_snapshot(archive);
|
2020-10-30 09:36:12 -07:00
|
|
|
assert_matches!(result, Err(UnpackError::Archive(ref message)) if message == "extra entry found: \"foo\" Regular");
|
2020-03-25 02:46:41 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_archive_unpack_snapshot_too_large() {
|
|
|
|
let mut header = Header::new_gnu();
|
|
|
|
header.set_path("version").unwrap();
|
2020-05-15 09:35:43 -07:00
|
|
|
header.set_size(1024 * 1024 * 1024 * 1024 * 1024);
|
2020-03-25 02:46:41 -07:00
|
|
|
header.set_cksum();
|
|
|
|
|
|
|
|
let data: &[u8] = &[1, 2, 3, 4];
|
|
|
|
|
|
|
|
let mut archive = Builder::new(Vec::new());
|
|
|
|
archive.append(&header, data).unwrap();
|
|
|
|
let result = finalize_and_unpack_snapshot(archive);
|
2020-12-09 09:46:33 -08:00
|
|
|
assert_matches!(
|
|
|
|
result,
|
|
|
|
Err(UnpackError::Archive(ref message))
|
|
|
|
if message == &format!(
|
2022-12-06 06:30:06 -08:00
|
|
|
"too large archive: 1125899906842624 than limit: {MAX_SNAPSHOT_ARCHIVE_UNPACKED_APPARENT_SIZE}"
|
2020-12-09 09:46:33 -08:00
|
|
|
)
|
|
|
|
);
|
2020-03-25 02:46:41 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_archive_unpack_snapshot_bad_unpack() {
|
|
|
|
let result = check_unpack_result(false, "abc".to_string());
|
2020-05-15 09:35:43 -07:00
|
|
|
assert_matches!(result, Err(UnpackError::Archive(ref message)) if message == "failed to unpack: \"abc\"");
|
2020-03-25 02:46:41 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_archive_checked_total_size_sum() {
|
2020-12-09 09:46:33 -08:00
|
|
|
let result = checked_total_size_sum(500, 500, MAX_SNAPSHOT_ARCHIVE_UNPACKED_ACTUAL_SIZE);
|
2020-03-25 02:46:41 -07:00
|
|
|
assert_matches!(result, Ok(1000));
|
|
|
|
|
2020-12-09 09:46:33 -08:00
|
|
|
let result = checked_total_size_sum(
|
|
|
|
u64::max_value() - 2,
|
|
|
|
2,
|
|
|
|
MAX_SNAPSHOT_ARCHIVE_UNPACKED_ACTUAL_SIZE,
|
|
|
|
);
|
|
|
|
assert_matches!(
|
|
|
|
result,
|
|
|
|
Err(UnpackError::Archive(ref message))
|
|
|
|
if message == &format!(
|
2022-12-06 06:30:06 -08:00
|
|
|
"too large archive: 18446744073709551615 than limit: {MAX_SNAPSHOT_ARCHIVE_UNPACKED_ACTUAL_SIZE}"
|
2020-12-09 09:46:33 -08:00
|
|
|
)
|
|
|
|
);
|
2020-03-25 02:46:41 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_archive_checked_total_size_count() {
|
|
|
|
let result = checked_total_count_increment(101, MAX_SNAPSHOT_ARCHIVE_UNPACKED_COUNT);
|
|
|
|
assert_matches!(result, Ok(102));
|
|
|
|
|
|
|
|
let result =
|
|
|
|
checked_total_count_increment(999_999_999_999, MAX_SNAPSHOT_ARCHIVE_UNPACKED_COUNT);
|
2020-12-09 09:46:33 -08:00
|
|
|
assert_matches!(
|
|
|
|
result,
|
|
|
|
Err(UnpackError::Archive(ref message))
|
|
|
|
if message == "too many files in snapshot: 1000000000000"
|
|
|
|
);
|
2020-03-25 02:46:41 -07:00
|
|
|
}
|
|
|
|
}
|