Revert "Add run parent directory for accounts files (#29794)" (#29899)

This PR is causing OOM on master.  Reverting it for now.

This reverts commit 74f89d1494.
This commit is contained in:
Xiang Zhu 2023-01-25 10:03:01 -08:00 committed by GitHub
parent 2194551f87
commit 4ebcacb4a3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 46 additions and 149 deletions

View File

@ -228,8 +228,7 @@ mod tests {
snapshot_hash::SnapshotHash, snapshot_hash::SnapshotHash,
snapshot_package::{SnapshotPackage, SnapshotType}, snapshot_package::{SnapshotPackage, SnapshotType},
snapshot_utils::{ snapshot_utils::{
self, create_accounts_run_and_snapshot_dirs, ArchiveFormat, SnapshotVersion, self, ArchiveFormat, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILENAME,
SNAPSHOT_STATUS_CACHE_FILENAME,
}, },
}, },
solana_sdk::hash::Hash, solana_sdk::hash::Hash,
@ -268,10 +267,6 @@ mod tests {
fn create_and_verify_snapshot(temp_dir: &Path) { fn create_and_verify_snapshot(temp_dir: &Path) {
let accounts_dir = temp_dir.join("accounts"); let accounts_dir = temp_dir.join("accounts");
let accounts_dir = create_accounts_run_and_snapshot_dirs(accounts_dir)
.unwrap()
.0;
let snapshots_dir = temp_dir.join("snapshots"); let snapshots_dir = temp_dir.join("snapshots");
let full_snapshot_archives_dir = temp_dir.join("full_snapshot_archives"); let full_snapshot_archives_dir = temp_dir.join("full_snapshot_archives");
let incremental_snapshot_archives_dir = temp_dir.join("incremental_snapshot_archives"); let incremental_snapshot_archives_dir = temp_dir.join("incremental_snapshot_archives");

View File

@ -1,6 +1,5 @@
#![allow(clippy::integer_arithmetic)] #![allow(clippy::integer_arithmetic)]
use { use {
crate::snapshot_utils::create_tmp_accounts_dir_for_tests,
log::*, log::*,
solana_core::{ solana_core::{
accounts_hash_verifier::AccountsHashVerifier, accounts_hash_verifier::AccountsHashVerifier,
@ -443,9 +442,9 @@ fn test_snapshots_have_expected_epoch_accounts_hash() {
std::thread::sleep(Duration::from_secs(1)); std::thread::sleep(Duration::from_secs(1));
}; };
let (_tmp_dir, accounts_dir) = create_tmp_accounts_dir_for_tests(); let accounts_dir = TempDir::new().unwrap();
let deserialized_bank = snapshot_utils::bank_from_snapshot_archives( let deserialized_bank = snapshot_utils::bank_from_snapshot_archives(
&[accounts_dir.as_path().to_path_buf()], &[accounts_dir.path().to_path_buf()],
&snapshot_config.bank_snapshots_dir, &snapshot_config.bank_snapshots_dir,
&full_snapshot_archive_info, &full_snapshot_archive_info,
None, None,

View File

@ -1,7 +1,6 @@
#![allow(clippy::integer_arithmetic)] #![allow(clippy::integer_arithmetic)]
use { use {
crate::snapshot_utils::create_tmp_accounts_dir_for_tests,
bincode::serialize_into, bincode::serialize_into,
crossbeam_channel::unbounded, crossbeam_channel::unbounded,
fs_extra::dir::CopyOptions, fs_extra::dir::CopyOptions,
@ -75,8 +74,7 @@ struct SnapshotTestConfig {
incremental_snapshot_archives_dir: TempDir, incremental_snapshot_archives_dir: TempDir,
full_snapshot_archives_dir: TempDir, full_snapshot_archives_dir: TempDir,
bank_snapshots_dir: TempDir, bank_snapshots_dir: TempDir,
accounts_dir: PathBuf, accounts_dir: TempDir,
_accounts_tmp_dir: TempDir,
} }
impl SnapshotTestConfig { impl SnapshotTestConfig {
@ -87,7 +85,7 @@ impl SnapshotTestConfig {
full_snapshot_archive_interval_slots: Slot, full_snapshot_archive_interval_slots: Slot,
incremental_snapshot_archive_interval_slots: Slot, incremental_snapshot_archive_interval_slots: Slot,
) -> SnapshotTestConfig { ) -> SnapshotTestConfig {
let (_accounts_tmp_dir, accounts_dir) = create_tmp_accounts_dir_for_tests(); let accounts_dir = TempDir::new().unwrap();
let bank_snapshots_dir = TempDir::new().unwrap(); let bank_snapshots_dir = TempDir::new().unwrap();
let full_snapshot_archives_dir = TempDir::new().unwrap(); let full_snapshot_archives_dir = TempDir::new().unwrap();
let incremental_snapshot_archives_dir = TempDir::new().unwrap(); let incremental_snapshot_archives_dir = TempDir::new().unwrap();
@ -104,7 +102,7 @@ impl SnapshotTestConfig {
let bank0 = Bank::new_with_paths_for_tests( let bank0 = Bank::new_with_paths_for_tests(
&genesis_config_info.genesis_config, &genesis_config_info.genesis_config,
Arc::<RuntimeConfig>::default(), Arc::<RuntimeConfig>::default(),
vec![accounts_dir.clone()], vec![accounts_dir.path().to_path_buf()],
AccountSecondaryIndexes::default(), AccountSecondaryIndexes::default(),
accounts_db::AccountShrinkThreshold::default(), accounts_db::AccountShrinkThreshold::default(),
); );
@ -133,7 +131,6 @@ impl SnapshotTestConfig {
full_snapshot_archives_dir, full_snapshot_archives_dir,
bank_snapshots_dir, bank_snapshots_dir,
accounts_dir, accounts_dir,
_accounts_tmp_dir,
} }
} }
} }
@ -299,8 +296,8 @@ fn run_bank_forks_snapshot_n<F>(
.unwrap(); .unwrap();
// Restore bank from snapshot // Restore bank from snapshot
let (_tmp_dir, temporary_accounts_dir) = create_tmp_accounts_dir_for_tests(); let temporary_accounts_dir = TempDir::new().unwrap();
let account_paths = &[temporary_accounts_dir]; let account_paths = &[temporary_accounts_dir.path().to_path_buf()];
let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config; let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config;
restore_from_snapshot(bank_forks, last_slot, genesis_config, account_paths); restore_from_snapshot(bank_forks, last_slot, genesis_config, account_paths);
@ -729,7 +726,7 @@ fn test_bank_forks_incremental_snapshot(
INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
); );
trace!("SnapshotTestConfig:\naccounts_dir: {}\nbank_snapshots_dir: {}\nfull_snapshot_archives_dir: {}\nincremental_snapshot_archives_dir: {}", trace!("SnapshotTestConfig:\naccounts_dir: {}\nbank_snapshots_dir: {}\nfull_snapshot_archives_dir: {}\nincremental_snapshot_archives_dir: {}",
snapshot_test_config.accounts_dir.display(), snapshot_test_config.bank_snapshots_dir.path().display(), snapshot_test_config.full_snapshot_archives_dir.path().display(), snapshot_test_config.incremental_snapshot_archives_dir.path().display()); snapshot_test_config.accounts_dir.path().display(), snapshot_test_config.bank_snapshots_dir.path().display(), snapshot_test_config.full_snapshot_archives_dir.path().display(), snapshot_test_config.incremental_snapshot_archives_dir.path().display());
let bank_forks = &mut snapshot_test_config.bank_forks; let bank_forks = &mut snapshot_test_config.bank_forks;
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair; let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
@ -824,11 +821,11 @@ fn test_bank_forks_incremental_snapshot(
// Accounts directory needs to be separate from the active accounts directory // Accounts directory needs to be separate from the active accounts directory
// so that dropping append vecs in the active accounts directory doesn't // so that dropping append vecs in the active accounts directory doesn't
// delete the unpacked appendvecs in the snapshot // delete the unpacked appendvecs in the snapshot
let (_tmp_dir, temporary_accounts_dir) = create_tmp_accounts_dir_for_tests(); let temporary_accounts_dir = TempDir::new().unwrap();
restore_from_snapshots_and_check_banks_are_equal( restore_from_snapshots_and_check_banks_are_equal(
&bank, &bank,
&snapshot_test_config.snapshot_config, &snapshot_test_config.snapshot_config,
temporary_accounts_dir, temporary_accounts_dir.path().to_path_buf(),
&snapshot_test_config.genesis_config_info.genesis_config, &snapshot_test_config.genesis_config_info.genesis_config,
) )
.unwrap(); .unwrap();
@ -1123,7 +1120,7 @@ fn test_snapshots_with_background_services(
} }
// Load the snapshot and ensure it matches what's in BankForks // Load the snapshot and ensure it matches what's in BankForks
let (_tmp_dir, temporary_accounts_dir) = create_tmp_accounts_dir_for_tests(); let temporary_accounts_dir = TempDir::new().unwrap();
let (deserialized_bank, ..) = snapshot_utils::bank_from_latest_snapshot_archives( let (deserialized_bank, ..) = snapshot_utils::bank_from_latest_snapshot_archives(
&snapshot_test_config.snapshot_config.bank_snapshots_dir, &snapshot_test_config.snapshot_config.bank_snapshots_dir,
&snapshot_test_config &snapshot_test_config
@ -1132,7 +1129,7 @@ fn test_snapshots_with_background_services(
&snapshot_test_config &snapshot_test_config
.snapshot_config .snapshot_config
.incremental_snapshot_archives_dir, .incremental_snapshot_archives_dir,
&[temporary_accounts_dir], &[temporary_accounts_dir.as_ref().to_path_buf()],
&snapshot_test_config.genesis_config_info.genesis_config, &snapshot_test_config.genesis_config_info.genesis_config,
&RuntimeConfig::default(), &RuntimeConfig::default(),
None, None,

View File

@ -62,8 +62,8 @@ use {
snapshot_hash::StartingSnapshotHashes, snapshot_hash::StartingSnapshotHashes,
snapshot_minimizer::SnapshotMinimizer, snapshot_minimizer::SnapshotMinimizer,
snapshot_utils::{ snapshot_utils::{
self, create_accounts_run_and_snapshot_dirs, move_and_async_delete_path, ArchiveFormat, self, move_and_async_delete_path, ArchiveFormat, SnapshotVersion,
SnapshotVersion, DEFAULT_ARCHIVE_COMPRESSION, SUPPORTED_ARCHIVE_COMPRESSION, DEFAULT_ARCHIVE_COMPRESSION, SUPPORTED_ARCHIVE_COMPRESSION,
}, },
}, },
solana_sdk::{ solana_sdk::{
@ -1106,22 +1106,6 @@ fn load_bank_forks(
); );
vec![non_primary_accounts_path] vec![non_primary_accounts_path]
}; };
// For all account_paths, set up the run/ and snapshot/ sub directories.
let account_run_paths: Vec<PathBuf> = account_paths.into_iter().map(
|account_path| {
match create_accounts_run_and_snapshot_dirs(&account_path) {
Ok((account_run_path, _account_snapshot_path)) => account_run_path,
Err(err) => {
eprintln!("Unable to create account run and snapshot sub directories: {}, err: {err:?}", account_path.display());
exit(1);
}
}
}).collect();
// From now on, use run/ paths in the same way as the previous account_paths.
let account_paths = account_run_paths;
info!("Cleaning contents of account paths: {:?}", account_paths); info!("Cleaning contents of account paths: {:?}", account_paths);
let mut measure = Measure::start("clean_accounts_paths"); let mut measure = Measure::start("clean_accounts_paths");
account_paths.iter().for_each(|path| { account_paths.iter().for_each(|path| {

View File

@ -22,7 +22,6 @@ use {
ValidatorVoteKeypairs, ValidatorVoteKeypairs,
}, },
snapshot_config::SnapshotConfig, snapshot_config::SnapshotConfig,
snapshot_utils::create_accounts_run_and_snapshot_dirs,
}, },
solana_sdk::{ solana_sdk::{
account::{Account, AccountSharedData}, account::{Account, AccountSharedData},
@ -148,11 +147,7 @@ impl LocalCluster {
config: &mut ValidatorConfig, config: &mut ValidatorConfig,
ledger_path: &Path, ledger_path: &Path,
) { ) {
config.account_paths = vec![ config.account_paths = vec![ledger_path.join("accounts")];
create_accounts_run_and_snapshot_dirs(ledger_path.join("accounts"))
.unwrap()
.0,
];
config.tower_storage = Arc::new(FileTowerStorage::new(ledger_path.to_path_buf())); config.tower_storage = Arc::new(FileTowerStorage::new(ledger_path.to_path_buf()));
let snapshot_config = &mut config.snapshot_config; let snapshot_config = &mut config.snapshot_config;

View File

@ -21,9 +21,7 @@ use {
validator_configs::*, validator_configs::*,
}, },
solana_rpc_client::rpc_client::RpcClient, solana_rpc_client::rpc_client::RpcClient,
solana_runtime::{ solana_runtime::snapshot_config::SnapshotConfig,
snapshot_config::SnapshotConfig, snapshot_utils::create_accounts_run_and_snapshot_dirs,
},
solana_sdk::{ solana_sdk::{
account::AccountSharedData, account::AccountSharedData,
clock::{self, Slot, DEFAULT_MS_PER_SLOT, DEFAULT_TICKS_PER_SLOT}, clock::{self, Slot, DEFAULT_MS_PER_SLOT, DEFAULT_TICKS_PER_SLOT},
@ -438,7 +436,7 @@ pub fn generate_account_paths(num_account_paths: usize) -> (Vec<TempDir>, Vec<Pa
.collect(); .collect();
let account_storage_paths: Vec<_> = account_storage_dirs let account_storage_paths: Vec<_> = account_storage_dirs
.iter() .iter()
.map(|a| create_accounts_run_and_snapshot_dirs(a.path()).unwrap().0) .map(|a| a.path().to_path_buf())
.collect(); .collect();
(account_storage_dirs, account_storage_paths) (account_storage_dirs, account_storage_paths)
} }

View File

@ -38,9 +38,7 @@ use {
snapshot_archive_info::SnapshotArchiveInfoGetter, snapshot_archive_info::SnapshotArchiveInfoGetter,
snapshot_config::SnapshotConfig, snapshot_config::SnapshotConfig,
snapshot_package::SnapshotType, snapshot_package::SnapshotType,
snapshot_utils::{ snapshot_utils::{self, ArchiveFormat, SnapshotVersion},
self, create_accounts_run_and_snapshot_dirs, ArchiveFormat, SnapshotVersion,
},
}, },
solana_sdk::{ solana_sdk::{
account::AccountSharedData, account::AccountSharedData,
@ -2154,11 +2152,7 @@ fn create_snapshot_to_hard_fork(
let (bank_forks, ..) = bank_forks_utils::load( let (bank_forks, ..) = bank_forks_utils::load(
&genesis_config, &genesis_config,
blockstore, blockstore,
vec![ vec![ledger_path.join("accounts")],
create_accounts_run_and_snapshot_dirs(ledger_path.join("accounts"))
.unwrap()
.0,
],
None, None,
snapshot_config.as_ref(), snapshot_config.as_ref(),
process_options, process_options,

View File

@ -53,7 +53,6 @@ use {
read_only_accounts_cache::ReadOnlyAccountsCache, read_only_accounts_cache::ReadOnlyAccountsCache,
rent_collector::RentCollector, rent_collector::RentCollector,
rent_paying_accounts_by_partition::RentPayingAccountsByPartition, rent_paying_accounts_by_partition::RentPayingAccountsByPartition,
snapshot_utils::create_accounts_run_and_snapshot_dirs,
sorted_storages::SortedStorages, sorted_storages::SortedStorages,
storable_accounts::StorableAccounts, storable_accounts::StorableAccounts,
verify_accounts_hash_in_background::VerifyAccountsHashInBackground, verify_accounts_hash_in_background::VerifyAccountsHashInBackground,
@ -1114,14 +1113,7 @@ impl AccountStorageEntry {
pub fn get_temp_accounts_paths(count: u32) -> IoResult<(Vec<TempDir>, Vec<PathBuf>)> { pub fn get_temp_accounts_paths(count: u32) -> IoResult<(Vec<TempDir>, Vec<PathBuf>)> {
let temp_dirs: IoResult<Vec<TempDir>> = (0..count).map(|_| TempDir::new()).collect(); let temp_dirs: IoResult<Vec<TempDir>> = (0..count).map(|_| TempDir::new()).collect();
let temp_dirs = temp_dirs?; let temp_dirs = temp_dirs?;
let paths: Vec<PathBuf> = temp_dirs.iter().map(|t| t.path().to_path_buf()).collect();
let paths: IoResult<Vec<_>> = temp_dirs
.iter()
.map(|temp_dir| {
create_accounts_run_and_snapshot_dirs(temp_dir).map(|(run_dir, _snapshot_dir)| run_dir)
})
.collect();
let paths = paths?;
Ok((temp_dirs, paths)) Ok((temp_dirs, paths))
} }

View File

@ -132,8 +132,6 @@ where
} }
let parts: Vec<_> = parts.map(|p| p.unwrap()).collect(); let parts: Vec<_> = parts.map(|p| p.unwrap()).collect();
let account_filename =
(parts.len() == 2 && parts[0] == "accounts").then(|| PathBuf::from(parts[1]));
let unpack_dir = match entry_checker(parts.as_slice(), kind) { let unpack_dir = match entry_checker(parts.as_slice(), kind) {
UnpackPath::Invalid => { UnpackPath::Invalid => {
return Err(UnpackError::Archive(format!( return Err(UnpackError::Archive(format!(
@ -177,16 +175,8 @@ where
let entry_path_buf = unpack_dir.join(entry.path()?); let entry_path_buf = unpack_dir.join(entry.path()?);
set_perms(&entry_path_buf, mode)?; set_perms(&entry_path_buf, mode)?;
let entry_path = if let Some(account_filename) = account_filename {
let stripped_path = unpack_dir.join(account_filename); // strip away "accounts"
fs::rename(&entry_path_buf, &stripped_path)?;
stripped_path
} else {
entry_path_buf
};
// Process entry after setting permissions // Process entry after setting permissions
entry_processor(entry_path); entry_processor(entry_path_buf);
total_entries += 1; total_entries += 1;
let now = Instant::now(); let now = Instant::now();

View File

@ -13,9 +13,7 @@ use {
bank::{Bank, BankTestConfig}, bank::{Bank, BankTestConfig},
epoch_accounts_hash, epoch_accounts_hash,
genesis_utils::{self, activate_all_features, activate_feature}, genesis_utils::{self, activate_all_features, activate_feature},
snapshot_utils::{ snapshot_utils::{get_storages_to_serialize, ArchiveFormat},
create_tmp_accounts_dir_for_tests, get_storages_to_serialize, ArchiveFormat,
},
status_cache::StatusCache, status_cache::StatusCache,
}, },
bincode::serialize_into, bincode::serialize_into,
@ -577,7 +575,7 @@ fn test_extra_fields_full_snapshot_archive() {
// Set extra field // Set extra field
bank.fee_rate_governor.lamports_per_signature = 7000; bank.fee_rate_governor.lamports_per_signature = 7000;
let (_tmp_dir, accounts_dir) = create_tmp_accounts_dir_for_tests(); let accounts_dir = TempDir::new().unwrap();
let bank_snapshots_dir = TempDir::new().unwrap(); let bank_snapshots_dir = TempDir::new().unwrap();
let full_snapshot_archives_dir = TempDir::new().unwrap(); let full_snapshot_archives_dir = TempDir::new().unwrap();
let incremental_snapshot_archives_dir = TempDir::new().unwrap(); let incremental_snapshot_archives_dir = TempDir::new().unwrap();
@ -597,7 +595,7 @@ fn test_extra_fields_full_snapshot_archive() {
// Deserialize // Deserialize
let (dbank, _) = snapshot_utils::bank_from_snapshot_archives( let (dbank, _) = snapshot_utils::bank_from_snapshot_archives(
&[accounts_dir], &[PathBuf::from(accounts_dir.path())],
bank_snapshots_dir.path(), bank_snapshots_dir.path(),
&snapshot_archive_info, &snapshot_archive_info,
None, None,

View File

@ -833,21 +833,6 @@ where
Ok(()) Ok(())
} }
/// To allow generating a bank snapshot directory with full state information, we need to
/// hardlink account appendvec files from the runtime operation directory to a snapshot
/// hardlink directory. This is to create the run/ and snapshot sub directories for an
/// account_path provided by the user. These two sub directories are on the same file
/// system partition to allow hard-linking.
pub fn create_accounts_run_and_snapshot_dirs(
account_dir: impl AsRef<Path>,
) -> std::io::Result<(PathBuf, PathBuf)> {
let run_path = account_dir.as_ref().join("run");
let snapshot_path = account_dir.as_ref().join("snapshot");
fs::create_dir_all(&run_path)?;
fs::create_dir_all(&snapshot_path)?;
Ok((run_path, snapshot_path))
}
/// Serialize a bank to a snapshot /// Serialize a bank to a snapshot
/// ///
/// **DEVELOPER NOTE** Any error that is returned from this function may bring down the node! This /// **DEVELOPER NOTE** Any error that is returned from this function may bring down the node! This
@ -2138,11 +2123,10 @@ pub fn verify_snapshot_archive<P, Q, R>(
{ {
let temp_dir = tempfile::TempDir::new().unwrap(); let temp_dir = tempfile::TempDir::new().unwrap();
let unpack_dir = temp_dir.path(); let unpack_dir = temp_dir.path();
let account_dir = create_accounts_run_and_snapshot_dirs(unpack_dir).unwrap().0;
untar_snapshot_in( untar_snapshot_in(
snapshot_archive, snapshot_archive,
unpack_dir, unpack_dir,
&[account_dir.clone()], &[unpack_dir.to_path_buf()],
archive_format, archive_format,
1, 1,
) )
@ -2163,14 +2147,9 @@ pub fn verify_snapshot_archive<P, Q, R>(
assert!(!dir_diff::is_different(&snapshots_to_verify, unpacked_snapshots).unwrap()); assert!(!dir_diff::is_different(&snapshots_to_verify, unpacked_snapshots).unwrap());
// In the unarchiving case, there is an extra empty "accounts" directory. The account
// files in the archive accounts/ have been expanded to [account_paths].
// Remove the empty "accounts" directory for the directory comparison below.
// In some test cases the directory to compare do not come from unarchiving.
// Ignore the error when this directory does not exist.
_ = std::fs::remove_dir(account_dir.join("accounts"));
// Check the account entries are the same // Check the account entries are the same
assert!(!dir_diff::is_different(&storages_to_verify, account_dir).unwrap()); let unpacked_accounts = unpack_dir.join("accounts");
assert!(!dir_diff::is_different(&storages_to_verify, unpacked_accounts).unwrap());
} }
/// Remove outdated bank snapshots /// Remove outdated bank snapshots
@ -2427,12 +2406,6 @@ pub fn should_take_incremental_snapshot(
&& last_full_snapshot_slot.is_some() && last_full_snapshot_slot.is_some()
} }
pub fn create_tmp_accounts_dir_for_tests() -> (TempDir, PathBuf) {
let tmp_dir = tempfile::TempDir::new().unwrap();
let account_dir = create_accounts_run_and_snapshot_dirs(&tmp_dir).unwrap().0;
(tmp_dir, account_dir)
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use { use {
@ -3351,7 +3324,7 @@ mod tests {
original_bank.register_tick(&Hash::new_unique()); original_bank.register_tick(&Hash::new_unique());
} }
let (_tmp_dir, accounts_dir) = create_tmp_accounts_dir_for_tests(); let accounts_dir = tempfile::TempDir::new().unwrap();
let bank_snapshots_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
@ -3370,7 +3343,7 @@ mod tests {
.unwrap(); .unwrap();
let (roundtrip_bank, _) = bank_from_snapshot_archives( let (roundtrip_bank, _) = bank_from_snapshot_archives(
&[accounts_dir], &[PathBuf::from(accounts_dir.path())],
bank_snapshots_dir.path(), bank_snapshots_dir.path(),
&snapshot_archive_info, &snapshot_archive_info,
None, None,
@ -3463,7 +3436,7 @@ mod tests {
bank4.register_tick(&Hash::new_unique()); bank4.register_tick(&Hash::new_unique());
} }
let (_tmp_dir, accounts_dir) = create_tmp_accounts_dir_for_tests(); let accounts_dir = tempfile::TempDir::new().unwrap();
let bank_snapshots_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
@ -3482,7 +3455,7 @@ mod tests {
.unwrap(); .unwrap();
let (roundtrip_bank, _) = bank_from_snapshot_archives( let (roundtrip_bank, _) = bank_from_snapshot_archives(
&[accounts_dir], &[PathBuf::from(accounts_dir.path())],
bank_snapshots_dir.path(), bank_snapshots_dir.path(),
&full_snapshot_archive_info, &full_snapshot_archive_info,
None, None,
@ -3554,7 +3527,7 @@ mod tests {
bank1.register_tick(&Hash::new_unique()); bank1.register_tick(&Hash::new_unique());
} }
let (_tmp_dir, accounts_dir) = create_tmp_accounts_dir_for_tests(); let accounts_dir = tempfile::TempDir::new().unwrap();
let bank_snapshots_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
@ -3614,7 +3587,7 @@ mod tests {
.unwrap(); .unwrap();
let (roundtrip_bank, _) = bank_from_snapshot_archives( let (roundtrip_bank, _) = bank_from_snapshot_archives(
&[accounts_dir], &[PathBuf::from(accounts_dir.path())],
bank_snapshots_dir.path(), bank_snapshots_dir.path(),
&full_snapshot_archive_info, &full_snapshot_archive_info,
Some(&incremental_snapshot_archive_info), Some(&incremental_snapshot_archive_info),
@ -3676,7 +3649,7 @@ mod tests {
bank1.register_tick(&Hash::new_unique()); bank1.register_tick(&Hash::new_unique());
} }
let (_tmp_dir, accounts_dir) = create_tmp_accounts_dir_for_tests(); let accounts_dir = tempfile::TempDir::new().unwrap();
let bank_snapshots_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
@ -3739,7 +3712,7 @@ mod tests {
&bank_snapshots_dir, &bank_snapshots_dir,
&full_snapshot_archives_dir, &full_snapshot_archives_dir,
&incremental_snapshot_archives_dir, &incremental_snapshot_archives_dir,
&[accounts_dir], &[accounts_dir.as_ref().to_path_buf()],
&genesis_config, &genesis_config,
&RuntimeConfig::default(), &RuntimeConfig::default(),
None, None,
@ -3789,7 +3762,7 @@ mod tests {
let key1 = Keypair::new(); let key1 = Keypair::new();
let key2 = Keypair::new(); let key2 = Keypair::new();
let (_tmp_dir, accounts_dir) = create_tmp_accounts_dir_for_tests(); let accounts_dir = tempfile::TempDir::new().unwrap();
let bank_snapshots_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
@ -3801,7 +3774,7 @@ mod tests {
let bank0 = Arc::new(Bank::new_with_paths_for_tests( let bank0 = Arc::new(Bank::new_with_paths_for_tests(
&genesis_config, &genesis_config,
Arc::<RuntimeConfig>::default(), Arc::<RuntimeConfig>::default(),
vec![accounts_dir.clone()], vec![accounts_dir.path().to_path_buf()],
AccountSecondaryIndexes::default(), AccountSecondaryIndexes::default(),
AccountShrinkThreshold::default(), AccountShrinkThreshold::default(),
)); ));
@ -3875,7 +3848,7 @@ mod tests {
) )
.unwrap(); .unwrap();
let (deserialized_bank, _) = bank_from_snapshot_archives( let (deserialized_bank, _) = bank_from_snapshot_archives(
&[accounts_dir.as_path().to_path_buf()], &[accounts_dir.path().to_path_buf()],
bank_snapshots_dir.path(), bank_snapshots_dir.path(),
&full_snapshot_archive_info, &full_snapshot_archive_info,
Some(&incremental_snapshot_archive_info), Some(&incremental_snapshot_archive_info),
@ -3940,7 +3913,7 @@ mod tests {
.unwrap(); .unwrap();
let (deserialized_bank, _) = bank_from_snapshot_archives( let (deserialized_bank, _) = bank_from_snapshot_archives(
&[accounts_dir.as_path().to_path_buf()], &[accounts_dir.path().to_path_buf()],
bank_snapshots_dir.path(), bank_snapshots_dir.path(),
&full_snapshot_archive_info, &full_snapshot_archive_info,
Some(&incremental_snapshot_archive_info), Some(&incremental_snapshot_archive_info),

View File

@ -25,7 +25,7 @@ use {
accounts_db::AccountsDbConfig, accounts_index::AccountsIndexConfig, bank_forks::BankForks, accounts_db::AccountsDbConfig, accounts_index::AccountsIndexConfig, bank_forks::BankForks,
genesis_utils::create_genesis_config_with_leader_ex, genesis_utils::create_genesis_config_with_leader_ex,
hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, runtime_config::RuntimeConfig, hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, runtime_config::RuntimeConfig,
snapshot_config::SnapshotConfig, snapshot_utils::create_accounts_run_and_snapshot_dirs, snapshot_config::SnapshotConfig,
}, },
solana_sdk::{ solana_sdk::{
account::{Account, AccountSharedData}, account::{Account, AccountSharedData},
@ -802,11 +802,7 @@ impl TestValidator {
rpc_config: config.rpc_config.clone(), rpc_config: config.rpc_config.clone(),
pubsub_config: config.pubsub_config.clone(), pubsub_config: config.pubsub_config.clone(),
accounts_hash_interval_slots: 100, accounts_hash_interval_slots: 100,
account_paths: vec![ account_paths: vec![ledger_path.join("accounts")],
create_accounts_run_and_snapshot_dirs(ledger_path.join("accounts"))
.unwrap()
.0,
],
poh_verify: false, // Skip PoH verification of ledger on startup for speed poh_verify: false, // Skip PoH verification of ledger on startup for speed
snapshot_config: SnapshotConfig { snapshot_config: SnapshotConfig {
full_snapshot_archive_interval_slots: 100, full_snapshot_archive_interval_slots: 100,

View File

@ -35,9 +35,7 @@ use {
}, },
runtime_config::RuntimeConfig, runtime_config::RuntimeConfig,
snapshot_config::{SnapshotConfig, SnapshotUsage}, snapshot_config::{SnapshotConfig, SnapshotUsage},
snapshot_utils::{ snapshot_utils::{self, ArchiveFormat, SnapshotVersion},
self, create_accounts_run_and_snapshot_dirs, ArchiveFormat, SnapshotVersion,
},
}, },
solana_sdk::{ solana_sdk::{
clock::{Slot, DEFAULT_S_PER_SLOT}, clock::{Slot, DEFAULT_S_PER_SLOT},
@ -1269,7 +1267,7 @@ pub fn main() {
.ok(); .ok();
// Create and canonicalize account paths to avoid issues with symlink creation // Create and canonicalize account paths to avoid issues with symlink creation
let account_run_paths: Vec<PathBuf> = account_paths validator_config.account_paths = account_paths
.into_iter() .into_iter()
.map(|account_path| { .map(|account_path| {
match fs::create_dir_all(&account_path).and_then(|_| fs::canonicalize(&account_path)) { match fs::create_dir_all(&account_path).and_then(|_| fs::canonicalize(&account_path)) {
@ -1279,20 +1277,8 @@ pub fn main() {
exit(1); exit(1);
} }
} }
}).map( })
|account_path| { .collect();
// For all account_paths, set up the run/ and snapshot/ sub directories.
match create_accounts_run_and_snapshot_dirs(&account_path) {
Ok((account_run_path, _account_snapshot_path)) => account_run_path,
Err(err) => {
eprintln!("Unable to create account run and snapshot sub directories: {}, err: {err:?}", account_path.display());
exit(1);
}
}
}).collect();
// From now on, use run/ paths in the same way as the previous account_paths.
validator_config.account_paths = account_run_paths;
validator_config.account_shrink_paths = account_shrink_paths.map(|paths| { validator_config.account_shrink_paths = account_shrink_paths.map(|paths| {
paths paths