diff --git a/download-utils/src/lib.rs b/download-utils/src/lib.rs
index be4ef95b9..f191b5e7d 100644
--- a/download-utils/src/lib.rs
+++ b/download-utils/src/lib.rs
@@ -268,6 +268,10 @@ pub fn download_snapshot_archive<'a, 'b>(
maximum_incremental_snapshot_archives_to_retain,
);
+ let snapshot_archives_remote_dir =
+ snapshot_utils::build_snapshot_archives_remote_dir(snapshot_archives_dir);
+ fs::create_dir_all(&snapshot_archives_remote_dir).unwrap();
+
for archive_format in [
ArchiveFormat::TarZstd,
ArchiveFormat::TarGzip,
@@ -276,14 +280,14 @@ pub fn download_snapshot_archive<'a, 'b>(
] {
let destination_path = match snapshot_type {
SnapshotType::FullSnapshot => snapshot_utils::build_full_snapshot_archive_path(
- snapshot_archives_dir,
+ &snapshot_archives_remote_dir,
desired_snapshot_hash.0,
&desired_snapshot_hash.1,
archive_format,
),
SnapshotType::IncrementalSnapshot(base_slot) => {
snapshot_utils::build_incremental_snapshot_archive_path(
- snapshot_archives_dir,
+ &snapshot_archives_remote_dir,
base_slot,
desired_snapshot_hash.0,
&desired_snapshot_hash.1,
diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs
index 55684c0a4..26488b4c7 100644
--- a/local-cluster/tests/local_cluster.rs
+++ b/local-cluster/tests/local_cluster.rs
@@ -1083,6 +1083,22 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
}
};
+ let copy_files_with_remote = |from: &Path, to: &Path| {
+ copy_files(from, to);
+ let remote_from = snapshot_utils::build_snapshot_archives_remote_dir(from);
+ let remote_to = snapshot_utils::build_snapshot_archives_remote_dir(to);
+ let _ = fs::create_dir_all(&remote_from);
+ let _ = fs::create_dir_all(&remote_to);
+ copy_files(&remote_from, &remote_to);
+ };
+
+ let delete_files_with_remote = |from: &Path| {
+ delete_files(from);
+ let remote_dir = snapshot_utils::build_snapshot_archives_remote_dir(from);
+ let _ = fs::create_dir_all(&remote_dir);
+ delete_files(&remote_dir);
+ };
+
// After downloading the snapshots, copy them over to a backup directory. Later we'll need to
// restart the node and guarantee that the only snapshots present are these initial ones. So,
// the easiest way to do that is create a backup now, delete the ones on the node before
@@ -1092,7 +1108,7 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
"Backing up validator snapshots to dir: {}...",
backup_validator_snapshot_archives_dir.path().display()
);
- copy_files(
+ copy_files_with_remote(
validator_snapshot_test_config.snapshot_archives_dir.path(),
backup_validator_snapshot_archives_dir.path(),
);
@@ -1170,8 +1186,8 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
trace!(
"Delete all the snapshots on the validator and restore the originals from the backup..."
);
- delete_files(validator_snapshot_test_config.snapshot_archives_dir.path());
- copy_files(
+ delete_files_with_remote(validator_snapshot_test_config.snapshot_archives_dir.path());
+ copy_files_with_remote(
backup_validator_snapshot_archives_dir.path(),
validator_snapshot_test_config.snapshot_archives_dir.path(),
);
diff --git a/rpc/src/rpc_service.rs b/rpc/src/rpc_service.rs
index f9586b87a..b26b24394 100644
--- a/rpc/src/rpc_service.rs
+++ b/rpc/src/rpc_service.rs
@@ -153,6 +153,20 @@ impl RpcRequestMiddleware {
tokio::fs::File::open(path).await
}
+ fn find_snapshot_file
(&self, stem: P) -> PathBuf
+ where
+ P: AsRef,
+ {
+ let root = &self.snapshot_config.as_ref().unwrap().snapshot_archives_dir;
+ let local_path = root.join(&stem);
+ if local_path.exists() {
+ local_path
+ } else {
+ // remote snapshot archive path
+ snapshot_utils::build_snapshot_archives_remote_dir(root).join(stem)
+ }
+ }
+
fn process_file_get(&self, path: &str) -> RequestMiddlewareAction {
let stem = path.split_at(1).1; // Drop leading '/' from path
let filename = {
@@ -163,11 +177,7 @@ impl RpcRequestMiddleware {
}
_ => {
inc_new_counter_info!("rpc-get_snapshot", 1);
- self.snapshot_config
- .as_ref()
- .unwrap()
- .snapshot_archives_dir
- .join(stem)
+ self.find_snapshot_file(stem)
}
}
};
diff --git a/runtime/src/snapshot_archive_info.rs b/runtime/src/snapshot_archive_info.rs
index 353370e42..457f828aa 100644
--- a/runtime/src/snapshot_archive_info.rs
+++ b/runtime/src/snapshot_archive_info.rs
@@ -25,6 +25,15 @@ pub trait SnapshotArchiveInfoGetter {
fn archive_format(&self) -> ArchiveFormat {
self.snapshot_archive_info().archive_format
}
+
+ fn is_remote(&self) -> bool {
+ self.snapshot_archive_info()
+ .path
+ .parent()
+ .map_or(false, |p| {
+ p.ends_with(snapshot_utils::SNAPSHOT_ARCHIVE_DOWNLOAD_DIR)
+ })
+ }
}
/// Common information about a snapshot archive
@@ -79,7 +88,7 @@ impl PartialOrd for FullSnapshotArchiveInfo {
}
}
-// Order `FullSnapshotArchiveInfo` by slot (ascending), which practially is sorting chronologically
+// Order `FullSnapshotArchiveInfo` by slot (ascending), which practically is sorting chronologically
impl Ord for FullSnapshotArchiveInfo {
fn cmp(&self, other: &Self) -> Ordering {
self.slot().cmp(&other.slot())
@@ -141,7 +150,7 @@ impl PartialOrd for IncrementalSnapshotArchiveInfo {
}
// Order `IncrementalSnapshotArchiveInfo` by base slot (ascending), then slot (ascending), which
-// practially is sorting chronologically
+// practically is sorting chronologically
impl Ord for IncrementalSnapshotArchiveInfo {
fn cmp(&self, other: &Self) -> Ordering {
self.base_slot()
diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs
index 649615c01..54121a66c 100644
--- a/runtime/src/snapshot_utils.rs
+++ b/runtime/src/snapshot_utils.rs
@@ -47,6 +47,7 @@ mod archive_format;
pub use archive_format::*;
pub const SNAPSHOT_STATUS_CACHE_FILENAME: &str = "status_cache";
+pub const SNAPSHOT_ARCHIVE_DOWNLOAD_DIR: &str = "remote";
pub const DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS: Slot = 25_000;
pub const DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS: Slot = 100;
const MAX_SNAPSHOT_DATA_FILE_SIZE: u64 = 32 * 1024 * 1024 * 1024; // 32 GiB
@@ -789,7 +790,7 @@ pub fn bank_from_snapshot_archives(
let mut measure_verify = Measure::start("verify");
if !bank.verify_snapshot_bank(
test_hash_calculation,
- accounts_db_skip_shrink,
+ accounts_db_skip_shrink || !full_snapshot_archive_info.is_remote(),
Some(full_snapshot_archive_info.slot()),
) && limit_load_slot_count_from_snapshot.is_none()
{
@@ -1033,6 +1034,12 @@ pub fn path_to_file_name_str(path: &Path) -> Result<&str> {
.ok_or_else(|| SnapshotError::FileNameToStrError(path.to_path_buf()))
}
+pub fn build_snapshot_archives_remote_dir(snapshot_archives_dir: impl AsRef) -> PathBuf {
+ snapshot_archives_dir
+ .as_ref()
+ .join(SNAPSHOT_ARCHIVE_DOWNLOAD_DIR)
+}
+
/// Build the full snapshot archive path from its components: the snapshot archives directory, the
/// snapshot slot, the accounts hash, and the archive format.
pub fn build_full_snapshot_archive_path(
@@ -1136,54 +1143,57 @@ pub(crate) fn parse_incremental_snapshot_archive_filename(
})
}
-/// Get a list of the full snapshot archives in a directory
+/// Walk down the snapshot archive to collect snapshot archive file info
+fn get_snapshot_archives(snapshot_archives_dir: &Path, cb: F) -> Vec
+where
+ F: Fn(PathBuf) -> Result,
+{
+ let walk_dir = |dir: &Path| -> Vec {
+ let entry_iter = fs::read_dir(dir);
+ match entry_iter {
+ Err(err) => {
+ info!(
+ "Unable to read snapshot archives directory: err: {}, path: {}",
+ err,
+ dir.display()
+ );
+ vec![]
+ }
+ Ok(entries) => entries
+ .filter_map(|entry| entry.map_or(None, |entry| cb(entry.path()).ok()))
+ .collect(),
+ }
+ };
+
+ let mut ret = walk_dir(snapshot_archives_dir);
+ ret.append(&mut walk_dir(
+ build_snapshot_archives_remote_dir(snapshot_archives_dir).as_ref(),
+ ));
+ ret
+}
+
+/// Get a list of the full snapshot archives from a directory
pub fn get_full_snapshot_archives(snapshot_archives_dir: P) -> Vec
where
P: AsRef,
{
- match fs::read_dir(&snapshot_archives_dir) {
- Err(err) => {
- info!(
- "Unable to read snapshot archives directory: err: {}, path: {}",
- err,
- snapshot_archives_dir.as_ref().display()
- );
- vec![]
- }
- Ok(files) => files
- .filter_map(|entry| {
- entry.map_or(None, |entry| {
- FullSnapshotArchiveInfo::new_from_path(entry.path()).ok()
- })
- })
- .collect(),
- }
+ get_snapshot_archives(
+ snapshot_archives_dir.as_ref(),
+ FullSnapshotArchiveInfo::new_from_path,
+ )
}
-/// Get a list of the incremental snapshot archives in a directory
+/// Get a list of the incremental snapshot archives from a directory
pub fn get_incremental_snapshot_archives(
snapshot_archives_dir: P,
) -> Vec
where
P: AsRef,
{
- match fs::read_dir(&snapshot_archives_dir) {
- Err(err) => {
- info!(
- "Unable to read snapshot archives directory: err: {}, path: {}",
- err,
- snapshot_archives_dir.as_ref().display()
- );
- vec![]
- }
- Ok(files) => files
- .filter_map(|entry| {
- entry.map_or(None, |entry| {
- IncrementalSnapshotArchiveInfo::new_from_path(entry.path()).ok()
- })
- })
- .collect(),
- }
+ get_snapshot_archives(
+ snapshot_archives_dir.as_ref(),
+ IncrementalSnapshotArchiveInfo::new_from_path,
+ )
}
/// Get the highest slot of the full snapshot archives in a directory
@@ -2274,6 +2284,7 @@ mod tests {
min_incremental_snapshot_slot: Slot,
max_incremental_snapshot_slot: Slot,
) {
+ fs::create_dir_all(snapshot_archives_dir).unwrap();
for full_snapshot_slot in min_full_snapshot_slot..max_full_snapshot_slot {
for incremental_snapshot_slot in
min_incremental_snapshot_slot..max_incremental_snapshot_slot
@@ -2328,6 +2339,25 @@ mod tests {
assert_eq!(snapshot_archives.len() as Slot, max_slot - min_slot);
}
+ #[test]
+ fn test_get_full_snapshot_archives_remote() {
+ solana_logger::setup();
+ let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
+ let min_slot = 123;
+ let max_slot = 456;
+ common_create_snapshot_archive_files(
+ &temp_snapshot_archives_dir.path().join("remote"),
+ min_slot,
+ max_slot,
+ 0,
+ 0,
+ );
+
+ let snapshot_archives = get_full_snapshot_archives(temp_snapshot_archives_dir);
+ assert_eq!(snapshot_archives.len() as Slot, max_slot - min_slot);
+ assert!(snapshot_archives.iter().all(|info| info.is_remote()));
+ }
+
#[test]
fn test_get_incremental_snapshot_archives() {
solana_logger::setup();
@@ -2353,6 +2383,34 @@ mod tests {
);
}
+ #[test]
+ fn test_get_incremental_snapshot_archives_remote() {
+ solana_logger::setup();
+ let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
+ let min_full_snapshot_slot = 12;
+ let max_full_snapshot_slot = 23;
+ let min_incremental_snapshot_slot = 34;
+ let max_incremental_snapshot_slot = 45;
+ common_create_snapshot_archive_files(
+ &temp_snapshot_archives_dir.path().join("remote"),
+ min_full_snapshot_slot,
+ max_full_snapshot_slot,
+ min_incremental_snapshot_slot,
+ max_incremental_snapshot_slot,
+ );
+
+ let incremental_snapshot_archives =
+ get_incremental_snapshot_archives(temp_snapshot_archives_dir);
+ assert_eq!(
+ incremental_snapshot_archives.len() as Slot,
+ (max_full_snapshot_slot - min_full_snapshot_slot)
+ * (max_incremental_snapshot_slot - min_incremental_snapshot_slot)
+ );
+ assert!(incremental_snapshot_archives
+ .iter()
+ .all(|info| info.is_remote()));
+ }
+
#[test]
fn test_get_highest_full_snapshot_archive_slot() {
solana_logger::setup();