Move async remove to snapshot_utils.rs (#29406)
This commit is contained in:
parent
71ba409bc2
commit
3363c08ac0
|
@ -86,7 +86,7 @@ use {
|
||||||
snapshot_config::SnapshotConfig,
|
snapshot_config::SnapshotConfig,
|
||||||
snapshot_hash::StartingSnapshotHashes,
|
snapshot_hash::StartingSnapshotHashes,
|
||||||
snapshot_package::PendingSnapshotPackage,
|
snapshot_package::PendingSnapshotPackage,
|
||||||
snapshot_utils,
|
snapshot_utils::{self, move_and_async_delete_path},
|
||||||
},
|
},
|
||||||
solana_sdk::{
|
solana_sdk::{
|
||||||
clock::Slot,
|
clock::Slot,
|
||||||
|
@ -2009,91 +2009,6 @@ fn get_stake_percent_in_gossip(bank: &Bank, cluster_info: &ClusterInfo, log: boo
|
||||||
online_stake_percentage as u64
|
online_stake_percentage as u64
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Delete directories/files asynchronously to avoid blocking on it.
|
|
||||||
/// Fist, in sync context, rename the original path to *_deleted,
|
|
||||||
/// then spawn a thread to delete the renamed path.
|
|
||||||
/// If the process is killed and the deleting process is not done,
|
|
||||||
/// the leftover path will be deleted in the next process life, so
|
|
||||||
/// there is no file space leaking.
|
|
||||||
pub fn move_and_async_delete_path(path: impl AsRef<Path> + Copy) {
|
|
||||||
let mut path_delete = PathBuf::new();
|
|
||||||
path_delete.push(path);
|
|
||||||
path_delete.set_file_name(format!(
|
|
||||||
"{}{}",
|
|
||||||
path_delete.file_name().unwrap().to_str().unwrap(),
|
|
||||||
"_to_be_deleted"
|
|
||||||
));
|
|
||||||
|
|
||||||
if path_delete.exists() {
|
|
||||||
std::fs::remove_dir_all(&path_delete).unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
if !path.as_ref().exists() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Err(err) = std::fs::rename(path, &path_delete) {
|
|
||||||
warn!(
|
|
||||||
"Path renaming failed: {}. Falling back to rm_dir in sync mode",
|
|
||||||
err.to_string()
|
|
||||||
);
|
|
||||||
delete_contents_of_path(path);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
Builder::new()
|
|
||||||
.name("solDeletePath".to_string())
|
|
||||||
.spawn(move || {
|
|
||||||
std::fs::remove_dir_all(path_delete).unwrap();
|
|
||||||
})
|
|
||||||
.unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Delete the files and subdirectories in a directory.
|
|
||||||
/// This is useful if the process does not have permission
|
|
||||||
/// to delete the top level directory it might be able to
|
|
||||||
/// delete the contents of that directory.
|
|
||||||
fn delete_contents_of_path(path: impl AsRef<Path> + Copy) {
|
|
||||||
if let Ok(dir_entries) = std::fs::read_dir(path) {
|
|
||||||
for entry in dir_entries.flatten() {
|
|
||||||
let sub_path = entry.path();
|
|
||||||
let metadata = match entry.metadata() {
|
|
||||||
Ok(metadata) => metadata,
|
|
||||||
Err(err) => {
|
|
||||||
warn!(
|
|
||||||
"Failed to get metadata for {}. Error: {}",
|
|
||||||
sub_path.display(),
|
|
||||||
err.to_string()
|
|
||||||
);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if metadata.is_dir() {
|
|
||||||
if let Err(err) = std::fs::remove_dir_all(&sub_path) {
|
|
||||||
warn!(
|
|
||||||
"Failed to remove sub directory {}. Error: {}",
|
|
||||||
sub_path.display(),
|
|
||||||
err.to_string()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} else if metadata.is_file() {
|
|
||||||
if let Err(err) = std::fs::remove_file(&sub_path) {
|
|
||||||
warn!(
|
|
||||||
"Failed to remove file {}. Error: {}",
|
|
||||||
sub_path.display(),
|
|
||||||
err.to_string()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
warn!(
|
|
||||||
"Failed to read the sub paths of {}",
|
|
||||||
path.as_ref().display()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn cleanup_accounts_paths(config: &ValidatorConfig) {
|
fn cleanup_accounts_paths(config: &ValidatorConfig) {
|
||||||
for accounts_path in &config.account_paths {
|
for accounts_path in &config.account_paths {
|
||||||
move_and_async_delete_path(accounts_path);
|
move_and_async_delete_path(accounts_path);
|
||||||
|
|
|
@ -24,10 +24,7 @@ use {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
solana_cli_output::{CliAccount, CliAccountNewConfig, OutputFormat},
|
solana_cli_output::{CliAccount, CliAccountNewConfig, OutputFormat},
|
||||||
solana_core::{
|
solana_core::system_monitor_service::{SystemMonitorService, SystemMonitorStatsReportConfig},
|
||||||
system_monitor_service::{SystemMonitorService, SystemMonitorStatsReportConfig},
|
|
||||||
validator::move_and_async_delete_path,
|
|
||||||
},
|
|
||||||
solana_entry::entry::Entry,
|
solana_entry::entry::Entry,
|
||||||
solana_geyser_plugin_manager::geyser_plugin_service::GeyserPluginService,
|
solana_geyser_plugin_manager::geyser_plugin_service::GeyserPluginService,
|
||||||
solana_ledger::{
|
solana_ledger::{
|
||||||
|
@ -63,8 +60,8 @@ use {
|
||||||
snapshot_hash::StartingSnapshotHashes,
|
snapshot_hash::StartingSnapshotHashes,
|
||||||
snapshot_minimizer::SnapshotMinimizer,
|
snapshot_minimizer::SnapshotMinimizer,
|
||||||
snapshot_utils::{
|
snapshot_utils::{
|
||||||
self, ArchiveFormat, SnapshotVersion, DEFAULT_ARCHIVE_COMPRESSION,
|
self, move_and_async_delete_path, ArchiveFormat, SnapshotVersion,
|
||||||
SUPPORTED_ARCHIVE_COMPRESSION,
|
DEFAULT_ARCHIVE_COMPRESSION, SUPPORTED_ARCHIVE_COMPRESSION,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
solana_sdk::{
|
solana_sdk::{
|
||||||
|
|
|
@ -277,6 +277,91 @@ pub enum VerifySlotDeltasError {
|
||||||
BadSlotHistory,
|
BadSlotHistory,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Delete the files and subdirectories in a directory.
|
||||||
|
/// This is useful if the process does not have permission
|
||||||
|
/// to delete the top level directory it might be able to
|
||||||
|
/// delete the contents of that directory.
|
||||||
|
fn delete_contents_of_path(path: impl AsRef<Path> + Copy) {
|
||||||
|
if let Ok(dir_entries) = std::fs::read_dir(path) {
|
||||||
|
for entry in dir_entries.flatten() {
|
||||||
|
let sub_path = entry.path();
|
||||||
|
let metadata = match entry.metadata() {
|
||||||
|
Ok(metadata) => metadata,
|
||||||
|
Err(err) => {
|
||||||
|
warn!(
|
||||||
|
"Failed to get metadata for {}. Error: {}",
|
||||||
|
sub_path.display(),
|
||||||
|
err.to_string()
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if metadata.is_dir() {
|
||||||
|
if let Err(err) = std::fs::remove_dir_all(&sub_path) {
|
||||||
|
warn!(
|
||||||
|
"Failed to remove sub directory {}. Error: {}",
|
||||||
|
sub_path.display(),
|
||||||
|
err.to_string()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} else if metadata.is_file() {
|
||||||
|
if let Err(err) = std::fs::remove_file(&sub_path) {
|
||||||
|
warn!(
|
||||||
|
"Failed to remove file {}. Error: {}",
|
||||||
|
sub_path.display(),
|
||||||
|
err.to_string()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
warn!(
|
||||||
|
"Failed to read the sub paths of {}",
|
||||||
|
path.as_ref().display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Delete directories/files asynchronously to avoid blocking on it.
|
||||||
|
/// Fist, in sync context, rename the original path to *_deleted,
|
||||||
|
/// then spawn a thread to delete the renamed path.
|
||||||
|
/// If the process is killed and the deleting process is not done,
|
||||||
|
/// the leftover path will be deleted in the next process life, so
|
||||||
|
/// there is no file space leaking.
|
||||||
|
pub fn move_and_async_delete_path(path: impl AsRef<Path> + Copy) {
|
||||||
|
let mut path_delete = PathBuf::new();
|
||||||
|
path_delete.push(path);
|
||||||
|
path_delete.set_file_name(format!(
|
||||||
|
"{}{}",
|
||||||
|
path_delete.file_name().unwrap().to_str().unwrap(),
|
||||||
|
"_to_be_deleted"
|
||||||
|
));
|
||||||
|
|
||||||
|
if path_delete.exists() {
|
||||||
|
std::fs::remove_dir_all(&path_delete).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
if !path.as_ref().exists() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Err(err) = std::fs::rename(path, &path_delete) {
|
||||||
|
warn!(
|
||||||
|
"Path renaming failed: {}. Falling back to rm_dir in sync mode",
|
||||||
|
err.to_string()
|
||||||
|
);
|
||||||
|
delete_contents_of_path(path);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Builder::new()
|
||||||
|
.name("solDeletePath".to_string())
|
||||||
|
.spawn(move || {
|
||||||
|
std::fs::remove_dir_all(path_delete).unwrap();
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
/// If the validator halts in the middle of `archive_snapshot_package()`, the temporary staging
|
/// If the validator halts in the middle of `archive_snapshot_package()`, the temporary staging
|
||||||
/// directory won't be cleaned up. Call this function to clean them up.
|
/// directory won't be cleaned up. Call this function to clean them up.
|
||||||
pub fn remove_tmp_snapshot_archives(snapshot_archives_dir: impl AsRef<Path>) {
|
pub fn remove_tmp_snapshot_archives(snapshot_archives_dir: impl AsRef<Path>) {
|
||||||
|
|
Loading…
Reference in New Issue