2019-07-31 17:58:10 -07:00
|
|
|
use crate::snapshot_package::SnapshotPackage;
|
|
|
|
use bincode::{deserialize_from, serialize_into};
|
2019-08-14 19:25:22 -07:00
|
|
|
use bzip2::bufread::BzDecoder;
|
2019-08-13 17:20:14 -07:00
|
|
|
use fs_extra::dir::CopyOptions;
|
2019-09-25 18:07:41 -07:00
|
|
|
use solana_measure::measure::Measure;
|
2019-08-05 22:53:19 -07:00
|
|
|
use solana_runtime::bank::Bank;
|
2019-08-06 18:47:30 -07:00
|
|
|
use solana_runtime::status_cache::SlotDelta;
|
|
|
|
use solana_sdk::transaction;
|
|
|
|
use std::cmp::Ordering;
|
2019-07-31 17:58:10 -07:00
|
|
|
use std::fs;
|
|
|
|
use std::fs::File;
|
|
|
|
use std::io::{BufReader, BufWriter, Error as IOError, ErrorKind};
|
|
|
|
use std::path::{Path, PathBuf};
|
|
|
|
use tar::Archive;
|
|
|
|
|
2019-09-25 13:42:19 -07:00
|
|
|
pub const SNAPSHOT_STATUS_CACHE_FILE_NAME: &str = "status_cache";
|
2019-10-18 14:58:16 -07:00
|
|
|
pub const TAR_SNAPSHOTS_DIR: &str = "snapshots";
|
|
|
|
pub const TAR_ACCOUNTS_DIR: &str = "accounts";
|
2019-08-06 18:47:30 -07:00
|
|
|
|
|
|
|
#[derive(PartialEq, Ord, Eq, Debug)]
|
|
|
|
pub struct SlotSnapshotPaths {
|
|
|
|
pub slot: u64,
|
|
|
|
pub snapshot_file_path: PathBuf,
|
|
|
|
}
|
|
|
|
|
2019-10-18 18:16:06 -07:00
|
|
|
#[derive(Debug)]
|
|
|
|
pub enum SnapshotError {
|
|
|
|
IO(std::io::Error),
|
|
|
|
Serialize(std::boxed::Box<bincode::ErrorKind>),
|
|
|
|
FsExtra(fs_extra::error::Error),
|
|
|
|
}
|
|
|
|
pub type Result<T> = std::result::Result<T, SnapshotError>;
|
|
|
|
|
|
|
|
impl std::convert::From<std::io::Error> for SnapshotError {
|
|
|
|
fn from(e: std::io::Error) -> SnapshotError {
|
|
|
|
SnapshotError::IO(e)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl std::convert::From<std::boxed::Box<bincode::ErrorKind>> for SnapshotError {
|
|
|
|
fn from(e: std::boxed::Box<bincode::ErrorKind>) -> SnapshotError {
|
|
|
|
SnapshotError::Serialize(e)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl std::convert::From<fs_extra::error::Error> for SnapshotError {
|
|
|
|
fn from(e: fs_extra::error::Error) -> SnapshotError {
|
|
|
|
SnapshotError::FsExtra(e)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-08-06 18:47:30 -07:00
|
|
|
impl PartialOrd for SlotSnapshotPaths {
|
|
|
|
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
|
|
|
Some(self.slot.cmp(&other.slot))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl SlotSnapshotPaths {
|
2019-09-23 20:12:16 -07:00
|
|
|
fn copy_snapshot_directory<P: AsRef<Path>>(&self, snapshot_hardlink_dir: P) -> Result<()> {
|
2019-08-06 18:47:30 -07:00
|
|
|
// Create a new directory in snapshot_hardlink_dir
|
|
|
|
let new_slot_hardlink_dir = snapshot_hardlink_dir.as_ref().join(self.slot.to_string());
|
|
|
|
let _ = fs::remove_dir_all(&new_slot_hardlink_dir);
|
|
|
|
fs::create_dir_all(&new_slot_hardlink_dir)?;
|
|
|
|
|
2019-09-23 20:12:16 -07:00
|
|
|
// Copy the snapshot
|
|
|
|
fs::copy(
|
2019-08-06 18:47:30 -07:00
|
|
|
&self.snapshot_file_path,
|
|
|
|
&new_slot_hardlink_dir.join(self.slot.to_string()),
|
|
|
|
)?;
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-08-07 13:12:53 -07:00
|
|
|
pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
|
2019-07-31 17:58:10 -07:00
|
|
|
bank: &Bank,
|
2019-09-25 13:42:19 -07:00
|
|
|
snapshot_files: &SlotSnapshotPaths,
|
2019-08-07 13:12:53 -07:00
|
|
|
snapshot_package_output_file: P,
|
|
|
|
snapshot_path: Q,
|
2019-09-25 13:42:19 -07:00
|
|
|
slots_to_snapshot: &[u64],
|
2019-07-31 17:58:10 -07:00
|
|
|
) -> Result<SnapshotPackage> {
|
|
|
|
// Hard link all the snapshots we need for this package
|
2019-08-07 13:12:53 -07:00
|
|
|
let snapshot_hard_links_dir = tempfile::tempdir_in(snapshot_path)?;
|
2019-07-31 17:58:10 -07:00
|
|
|
|
|
|
|
// Get a reference to all the relevant AccountStorageEntries
|
2019-08-13 16:05:37 -07:00
|
|
|
let account_storage_entries: Vec<_> = bank
|
|
|
|
.rc
|
|
|
|
.get_storage_entries()
|
|
|
|
.into_iter()
|
|
|
|
.filter(|x| x.fork_id() <= bank.slot())
|
|
|
|
.collect();
|
2019-07-31 17:58:10 -07:00
|
|
|
|
|
|
|
// Create a snapshot package
|
2019-08-05 22:53:19 -07:00
|
|
|
info!(
|
2019-07-31 17:58:10 -07:00
|
|
|
"Snapshot for bank: {} has {} account storage entries",
|
2019-08-21 23:59:11 -07:00
|
|
|
bank.slot(),
|
2019-07-31 17:58:10 -07:00
|
|
|
account_storage_entries.len()
|
|
|
|
);
|
|
|
|
|
|
|
|
// Any errors from this point on will cause the above SnapshotPackage to drop, clearing
|
|
|
|
// any temporary state created for the SnapshotPackage (like the snapshot_hard_links_dir)
|
2019-09-25 13:42:19 -07:00
|
|
|
snapshot_files.copy_snapshot_directory(snapshot_hard_links_dir.path())?;
|
2019-07-31 17:58:10 -07:00
|
|
|
|
2019-08-07 13:12:53 -07:00
|
|
|
let package = SnapshotPackage::new(
|
2019-08-21 16:36:21 -07:00
|
|
|
bank.slot(),
|
2019-09-25 13:42:19 -07:00
|
|
|
bank.src.slot_deltas(slots_to_snapshot),
|
2019-08-07 13:12:53 -07:00
|
|
|
snapshot_hard_links_dir,
|
|
|
|
account_storage_entries,
|
|
|
|
snapshot_package_output_file.as_ref().to_path_buf(),
|
|
|
|
);
|
|
|
|
|
2019-07-31 17:58:10 -07:00
|
|
|
Ok(package)
|
|
|
|
}
|
|
|
|
|
2019-08-23 13:02:07 -07:00
|
|
|
pub fn get_snapshot_paths<P: AsRef<Path>>(snapshot_path: P) -> Vec<SlotSnapshotPaths>
|
|
|
|
where
|
|
|
|
P: std::fmt::Debug,
|
|
|
|
{
|
|
|
|
match fs::read_dir(&snapshot_path) {
|
|
|
|
Ok(paths) => {
|
|
|
|
let mut names = paths
|
|
|
|
.filter_map(|entry| {
|
|
|
|
entry.ok().and_then(|e| {
|
|
|
|
e.path()
|
|
|
|
.file_name()
|
|
|
|
.and_then(|n| n.to_str().map(|s| s.parse::<u64>().ok()))
|
|
|
|
.unwrap_or(None)
|
|
|
|
})
|
|
|
|
})
|
|
|
|
.map(|slot| {
|
|
|
|
let snapshot_path = snapshot_path.as_ref().join(slot.to_string());
|
|
|
|
SlotSnapshotPaths {
|
|
|
|
slot,
|
|
|
|
snapshot_file_path: snapshot_path.join(get_snapshot_file_name(slot)),
|
|
|
|
}
|
|
|
|
})
|
|
|
|
.collect::<Vec<SlotSnapshotPaths>>();
|
|
|
|
|
|
|
|
names.sort();
|
|
|
|
names
|
|
|
|
}
|
|
|
|
Err(err) => {
|
|
|
|
info!(
|
|
|
|
"Unable to read snapshot directory {:?}: {}",
|
|
|
|
snapshot_path, err
|
|
|
|
);
|
|
|
|
vec![]
|
|
|
|
}
|
|
|
|
}
|
2019-07-31 17:58:10 -07:00
|
|
|
}
|
|
|
|
|
2019-09-25 13:42:19 -07:00
|
|
|
pub fn add_snapshot<P: AsRef<Path>>(snapshot_path: P, bank: &Bank) -> Result<()> {
|
2019-07-31 17:58:10 -07:00
|
|
|
let slot = bank.slot();
|
2019-08-06 18:47:30 -07:00
|
|
|
// snapshot_path/slot
|
2019-07-31 17:58:10 -07:00
|
|
|
let slot_snapshot_dir = get_bank_snapshot_dir(snapshot_path, slot);
|
2019-10-18 18:16:06 -07:00
|
|
|
fs::create_dir_all(slot_snapshot_dir.clone())?;
|
2019-07-31 17:58:10 -07:00
|
|
|
|
2019-08-06 18:47:30 -07:00
|
|
|
// the snapshot is stored as snapshot_path/slot/slot
|
2019-07-31 17:58:10 -07:00
|
|
|
let snapshot_file_path = slot_snapshot_dir.join(get_snapshot_file_name(slot));
|
2019-08-05 22:53:19 -07:00
|
|
|
info!(
|
2019-09-25 13:42:19 -07:00
|
|
|
"creating snapshot {}, path: {:?}",
|
2019-07-31 17:58:10 -07:00
|
|
|
bank.slot(),
|
2019-08-06 18:47:30 -07:00
|
|
|
snapshot_file_path,
|
2019-07-31 17:58:10 -07:00
|
|
|
);
|
2019-10-03 19:44:23 -07:00
|
|
|
|
2019-08-06 18:47:30 -07:00
|
|
|
let snapshot_file = File::create(&snapshot_file_path)?;
|
|
|
|
// snapshot writer
|
|
|
|
let mut snapshot_stream = BufWriter::new(snapshot_file);
|
2019-07-31 17:58:10 -07:00
|
|
|
// Create the snapshot
|
2019-10-18 13:35:05 -07:00
|
|
|
serialize_into(&mut snapshot_stream, &*bank)?;
|
2019-09-25 18:07:41 -07:00
|
|
|
let mut bank_rc_serialize = Measure::start("bank_rc_serialize-ms");
|
2019-10-18 13:35:05 -07:00
|
|
|
serialize_into(&mut snapshot_stream, &bank.rc)?;
|
2019-09-25 18:07:41 -07:00
|
|
|
bank_rc_serialize.stop();
|
|
|
|
inc_new_counter_info!("bank-rc-serialize-ms", bank_rc_serialize.as_ms() as usize);
|
|
|
|
|
2019-08-05 22:53:19 -07:00
|
|
|
info!(
|
2019-09-25 13:42:19 -07:00
|
|
|
"successfully created snapshot {}, path: {:?}",
|
2019-07-31 17:58:10 -07:00
|
|
|
bank.slot(),
|
2019-08-06 18:47:30 -07:00
|
|
|
snapshot_file_path,
|
2019-07-31 17:58:10 -07:00
|
|
|
);
|
2019-08-06 18:47:30 -07:00
|
|
|
|
2019-07-31 17:58:10 -07:00
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn remove_snapshot<P: AsRef<Path>>(slot: u64, snapshot_path: P) -> Result<()> {
|
|
|
|
let slot_snapshot_dir = get_bank_snapshot_dir(&snapshot_path, slot);
|
|
|
|
// Remove the snapshot directory for this slot
|
|
|
|
fs::remove_dir_all(slot_snapshot_dir)?;
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2019-08-21 23:59:11 -07:00
|
|
|
pub fn bank_slot_from_archive<P: AsRef<Path>>(snapshot_tar: P) -> Result<u64> {
|
2019-08-23 13:02:07 -07:00
|
|
|
let tempdir = tempfile::TempDir::new()?;
|
2019-08-21 23:59:11 -07:00
|
|
|
untar_snapshot_in(&snapshot_tar, &tempdir)?;
|
|
|
|
let unpacked_snapshots_dir = tempdir.path().join(TAR_SNAPSHOTS_DIR);
|
|
|
|
let snapshot_paths = get_snapshot_paths(&unpacked_snapshots_dir);
|
|
|
|
let last_root_paths = snapshot_paths
|
|
|
|
.last()
|
|
|
|
.ok_or_else(|| get_io_error("No snapshots found in snapshots directory"))?;
|
|
|
|
let file = File::open(&last_root_paths.snapshot_file_path)?;
|
|
|
|
let mut stream = BufReader::new(file);
|
2019-10-18 13:35:05 -07:00
|
|
|
let bank: Bank = deserialize_from(&mut stream)?;
|
2019-08-21 23:59:11 -07:00
|
|
|
Ok(bank.slot())
|
|
|
|
}
|
|
|
|
|
2019-08-13 17:20:14 -07:00
|
|
|
pub fn bank_from_archive<P: AsRef<Path>>(
|
|
|
|
account_paths: String,
|
2019-10-18 14:58:16 -07:00
|
|
|
snapshot_path: &PathBuf,
|
2019-08-13 17:20:14 -07:00
|
|
|
snapshot_tar: P,
|
|
|
|
) -> Result<Bank> {
|
|
|
|
// Untar the snapshot into a temp directory under `snapshot_config.snapshot_path()`
|
2019-10-18 14:58:16 -07:00
|
|
|
let unpack_dir = tempfile::tempdir_in(snapshot_path)?;
|
2019-08-13 17:20:14 -07:00
|
|
|
untar_snapshot_in(&snapshot_tar, &unpack_dir)?;
|
|
|
|
|
|
|
|
let unpacked_accounts_dir = unpack_dir.as_ref().join(TAR_ACCOUNTS_DIR);
|
|
|
|
let unpacked_snapshots_dir = unpack_dir.as_ref().join(TAR_SNAPSHOTS_DIR);
|
2019-09-25 13:42:19 -07:00
|
|
|
let bank = rebuild_bank_from_snapshots(
|
|
|
|
account_paths,
|
|
|
|
&unpacked_snapshots_dir,
|
|
|
|
unpacked_accounts_dir,
|
|
|
|
)?;
|
2019-08-13 17:20:14 -07:00
|
|
|
|
2019-09-20 13:21:12 -07:00
|
|
|
if !bank.verify_hash_internal_state() {
|
2019-10-03 19:44:23 -07:00
|
|
|
panic!("Snapshot bank failed to verify");
|
2019-09-20 13:21:12 -07:00
|
|
|
}
|
|
|
|
|
2019-10-18 14:58:16 -07:00
|
|
|
// Move the unpacked snapshots into `snapshot_path`
|
2019-08-23 13:02:07 -07:00
|
|
|
let dir_files = fs::read_dir(&unpacked_snapshots_dir).unwrap_or_else(|err| {
|
|
|
|
panic!(
|
|
|
|
"Invalid snapshot path {:?}: {}",
|
|
|
|
unpacked_snapshots_dir, err
|
|
|
|
)
|
|
|
|
});
|
2019-08-13 17:20:14 -07:00
|
|
|
let paths: Vec<PathBuf> = dir_files
|
|
|
|
.filter_map(|entry| entry.ok().map(|e| e.path()))
|
|
|
|
.collect();
|
|
|
|
let mut copy_options = CopyOptions::new();
|
|
|
|
copy_options.overwrite = true;
|
2019-10-18 14:58:16 -07:00
|
|
|
fs_extra::move_items(&paths, &snapshot_path, ©_options)?;
|
2019-08-13 17:20:14 -07:00
|
|
|
|
|
|
|
Ok(bank)
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn get_snapshot_tar_path<P: AsRef<Path>>(snapshot_output_dir: P) -> PathBuf {
|
2019-08-14 19:25:22 -07:00
|
|
|
snapshot_output_dir.as_ref().join("snapshot.tar.bz2")
|
2019-08-13 17:20:14 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
pub fn untar_snapshot_in<P: AsRef<Path>, Q: AsRef<Path>>(
|
|
|
|
snapshot_tar: P,
|
|
|
|
unpack_dir: Q,
|
|
|
|
) -> Result<()> {
|
2019-08-14 19:25:22 -07:00
|
|
|
let tar_bz2 = File::open(snapshot_tar)?;
|
|
|
|
let tar = BzDecoder::new(BufReader::new(tar_bz2));
|
2019-08-13 17:20:14 -07:00
|
|
|
let mut archive = Archive::new(tar);
|
|
|
|
archive.unpack(&unpack_dir)?;
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
|
|
|
fn rebuild_bank_from_snapshots<P>(
|
2019-08-05 22:53:19 -07:00
|
|
|
local_account_paths: String,
|
2019-09-25 13:42:19 -07:00
|
|
|
unpacked_snapshots_dir: &PathBuf,
|
2019-08-06 18:47:30 -07:00
|
|
|
append_vecs_path: P,
|
2019-08-05 22:53:19 -07:00
|
|
|
) -> Result<Bank>
|
|
|
|
where
|
|
|
|
P: AsRef<Path>,
|
|
|
|
{
|
2019-09-25 13:42:19 -07:00
|
|
|
let mut snapshot_paths = get_snapshot_paths(&unpacked_snapshots_dir);
|
|
|
|
if snapshot_paths.len() > 1 {
|
|
|
|
return Err(get_io_error("invalid snapshot format"));
|
|
|
|
}
|
|
|
|
let root_paths = snapshot_paths
|
|
|
|
.pop()
|
2019-08-05 22:53:19 -07:00
|
|
|
.ok_or_else(|| get_io_error("No snapshots found in snapshots directory"))?;
|
2019-09-25 13:42:19 -07:00
|
|
|
|
|
|
|
// Rebuild the root bank
|
|
|
|
info!("Loading from {:?}", &root_paths.snapshot_file_path);
|
|
|
|
let file = File::open(&root_paths.snapshot_file_path)?;
|
2019-08-05 22:53:19 -07:00
|
|
|
let mut stream = BufReader::new(file);
|
2019-10-18 13:35:05 -07:00
|
|
|
let bank: Bank = deserialize_from(&mut stream)?;
|
2019-08-05 22:53:19 -07:00
|
|
|
|
|
|
|
// Rebuild accounts
|
|
|
|
bank.rc
|
|
|
|
.accounts_from_stream(&mut stream, local_account_paths, append_vecs_path)?;
|
|
|
|
|
2019-09-25 13:42:19 -07:00
|
|
|
// Rebuild status cache
|
|
|
|
let status_cache_path = unpacked_snapshots_dir.join(SNAPSHOT_STATUS_CACHE_FILE_NAME);
|
|
|
|
let status_cache = File::open(status_cache_path)?;
|
|
|
|
let mut stream = BufReader::new(status_cache);
|
2019-10-18 13:35:05 -07:00
|
|
|
let slot_deltas: Vec<SlotDelta<transaction::Result<()>>> =
|
|
|
|
deserialize_from(&mut stream).unwrap_or_default();
|
2019-08-06 18:47:30 -07:00
|
|
|
|
2019-09-25 13:42:19 -07:00
|
|
|
bank.src.append(&slot_deltas);
|
2019-07-31 17:58:10 -07:00
|
|
|
|
2019-08-05 22:53:19 -07:00
|
|
|
Ok(bank)
|
2019-07-31 17:58:10 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
fn get_snapshot_file_name(slot: u64) -> String {
|
|
|
|
slot.to_string()
|
|
|
|
}
|
|
|
|
|
|
|
|
fn get_bank_snapshot_dir<P: AsRef<Path>>(path: P, slot: u64) -> PathBuf {
|
|
|
|
path.as_ref().join(slot.to_string())
|
|
|
|
}
|
|
|
|
|
2019-10-18 18:16:06 -07:00
|
|
|
fn get_io_error(error: &str) -> SnapshotError {
|
2019-08-14 23:14:40 -07:00
|
|
|
warn!("Snapshot Error: {:?}", error);
|
2019-10-18 18:16:06 -07:00
|
|
|
SnapshotError::IO(IOError::new(ErrorKind::Other, error))
|
2019-07-31 17:58:10 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
pub mod tests {
|
|
|
|
use super::*;
|
|
|
|
use tempfile::TempDir;
|
|
|
|
|
|
|
|
pub fn verify_snapshot_tar<P, Q, R>(
|
|
|
|
snapshot_tar: P,
|
|
|
|
snapshots_to_verify: Q,
|
|
|
|
storages_to_verify: R,
|
|
|
|
) where
|
|
|
|
P: AsRef<Path>,
|
|
|
|
Q: AsRef<Path>,
|
|
|
|
R: AsRef<Path>,
|
|
|
|
{
|
|
|
|
let temp_dir = TempDir::new().unwrap();
|
|
|
|
let unpack_dir = temp_dir.path();
|
|
|
|
untar_snapshot_in(snapshot_tar, &unpack_dir).unwrap();
|
|
|
|
|
|
|
|
// Check snapshots are the same
|
2019-08-05 22:53:19 -07:00
|
|
|
let unpacked_snapshots = unpack_dir.join(&TAR_SNAPSHOTS_DIR);
|
2019-07-31 17:58:10 -07:00
|
|
|
assert!(!dir_diff::is_different(&snapshots_to_verify, unpacked_snapshots).unwrap());
|
|
|
|
|
|
|
|
// Check the account entries are the same
|
|
|
|
let unpacked_accounts = unpack_dir.join(&TAR_ACCOUNTS_DIR);
|
|
|
|
assert!(!dir_diff::is_different(&storages_to_verify, unpacked_accounts).unwrap());
|
|
|
|
}
|
|
|
|
}
|