diff --git a/core/src/retransmit_stage.rs b/core/src/retransmit_stage.rs index da31c374e7..4d827260bf 100644 --- a/core/src/retransmit_stage.rs +++ b/core/src/retransmit_stage.rs @@ -295,7 +295,7 @@ mod tests { ..ProcessOptions::default() }; let (bank_forks, _, cached_leader_schedule) = - process_blocktree(&genesis_config, &blocktree, None, opts).unwrap(); + process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap(); let leader_schedule_cache = Arc::new(cached_leader_schedule); let bank_forks = Arc::new(RwLock::new(bank_forks)); diff --git a/core/src/validator.rs b/core/src/validator.rs index ff864e5070..479b977941 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -60,7 +60,7 @@ pub struct ValidatorConfig { pub transaction_status_service_disabled: bool, pub blockstream_unix_socket: Option, pub storage_slots_per_turn: u64, - pub account_paths: Option, + pub account_paths: Vec, pub rpc_config: JsonRpcConfig, pub snapshot_config: Option, pub max_ledger_slots: Option, @@ -80,7 +80,7 @@ impl Default for ValidatorConfig { blockstream_unix_socket: None, storage_slots_per_turn: DEFAULT_SLOTS_PER_TURN, max_ledger_slots: None, - account_paths: None, + account_paths: Vec::new(), rpc_config: JsonRpcConfig::default(), snapshot_config: None, broadcast_stage_type: BroadcastStageType::Standard, @@ -469,7 +469,7 @@ impl Validator { pub fn new_banks_from_blocktree( expected_genesis_hash: Option, blocktree_path: &Path, - account_paths: Option, + account_paths: Vec, snapshot_config: Option, poh_verify: bool, dev_halt_at_slot: Option, diff --git a/core/tests/bank_forks.rs b/core/tests/bank_forks.rs index 6351f9c133..05ecd54ae6 100644 --- a/core/tests/bank_forks.rs +++ b/core/tests/bank_forks.rs @@ -52,7 +52,7 @@ mod tests { let genesis_config_info = create_genesis_config(10_000); let bank0 = Bank::new_with_paths( &genesis_config_info.genesis_config, - Some(accounts_dir.path().to_str().unwrap().to_string()), + vec![accounts_dir.path().to_path_buf()], ); bank0.freeze(); let mut bank_forks = BankForks::new(0, bank0); @@ -73,7 +73,7 @@ mod tests { } } - fn restore_from_snapshot(old_bank_forks: &BankForks, account_paths: String) { + fn restore_from_snapshot(old_bank_forks: &BankForks, account_paths: Vec) { let (snapshot_path, snapshot_package_output_path) = old_bank_forks .snapshot_config .as_ref() @@ -81,7 +81,7 @@ mod tests { .unwrap(); let deserialized_bank = snapshot_utils::bank_from_archive( - account_paths, + &account_paths, &old_bank_forks .snapshot_config .as_ref() @@ -151,10 +151,7 @@ mod tests { .unwrap(); SnapshotPackagerService::package_snapshots(&snapshot_package).unwrap(); - restore_from_snapshot( - bank_forks, - accounts_dir.path().to_str().unwrap().to_string(), - ); + restore_from_snapshot(bank_forks, vec![accounts_dir.path().to_path_buf()]); } #[test] diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 4470c40197..c42450b22e 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -582,9 +582,9 @@ fn main() { }) }; let account_paths = if let Some(account_paths) = matches.value_of("account_paths") { - Some(account_paths.to_string()) + account_paths.split(',').map(PathBuf::from).collect() } else { - Some(ledger_path.join("accounts").to_str().unwrap().to_string()) + vec![ledger_path.join("accounts")] }; let process_options = blocktree_processor::ProcessOptions { diff --git a/ledger/src/bank_forks_utils.rs b/ledger/src/bank_forks_utils.rs index 39141ee1bf..4e916b5d0f 100644 --- a/ledger/src/bank_forks_utils.rs +++ b/ledger/src/bank_forks_utils.rs @@ -7,12 +7,12 @@ use crate::{ }; use log::*; use solana_sdk::genesis_config::GenesisConfig; -use std::{fs, sync::Arc}; +use std::{fs, path::PathBuf, sync::Arc}; pub fn load( genesis_config: &GenesisConfig, blocktree: &Blocktree, - account_paths: Option, + account_paths: Vec, snapshot_config: Option<&SnapshotConfig>, process_options: ProcessOptions, ) -> Result<(BankForks, Vec, LeaderScheduleCache), BlocktreeProcessorError> { @@ -30,10 +30,13 @@ pub fn load( if tar.exists() { info!("Loading snapshot package: {:?}", tar); // Fail hard here if snapshot fails to load, don't silently continue + + if account_paths.is_empty() { + panic!("Account paths not present when booting from snapshot") + } + let deserialized_bank = snapshot_utils::bank_from_archive( - account_paths - .clone() - .expect("Account paths not present when booting from snapshot"), + &account_paths, &snapshot_config.snapshot_path, &tar, ) diff --git a/ledger/src/blocktree_processor.rs b/ledger/src/blocktree_processor.rs index 1a496c61ca..5cc34a3e1d 100644 --- a/ledger/src/blocktree_processor.rs +++ b/ledger/src/blocktree_processor.rs @@ -27,6 +27,7 @@ use solana_sdk::{ }; use std::{ cell::RefCell, + path::PathBuf, result, sync::Arc, time::{Duration, Instant}, @@ -255,7 +256,7 @@ pub struct ProcessOptions { pub fn process_blocktree( genesis_config: &GenesisConfig, blocktree: &Blocktree, - account_paths: Option, + account_paths: Vec, opts: ProcessOptions, ) -> result::Result<(BankForks, Vec, LeaderScheduleCache), BlocktreeProcessorError> { if let Some(num_threads) = opts.override_num_threads { @@ -664,7 +665,7 @@ pub mod tests { ..ProcessOptions::default() }; assert_eq!( - process_blocktree(&genesis_config, &blocktree, None, opts).err(), + process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).err(), Some(BlocktreeProcessorError::InvalidBlock( BlockError::InvalidTickHashCount )), @@ -705,7 +706,7 @@ pub mod tests { ..ProcessOptions::default() }; assert_eq!( - process_blocktree(&genesis_config, &blocktree, None, opts).err(), + process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).err(), Some(BlocktreeProcessorError::InvalidBlock( BlockError::InvalidTickCount )), @@ -757,7 +758,7 @@ pub mod tests { ..ProcessOptions::default() }; assert_eq!( - process_blocktree(&genesis_config, &blocktree, None, opts).err(), + process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).err(), Some(BlocktreeProcessorError::InvalidBlock( BlockError::TrailingEntry )), @@ -824,7 +825,7 @@ pub mod tests { ..ProcessOptions::default() }; let (mut _bank_forks, bank_forks_info, _) = - process_blocktree(&genesis_config, &blocktree, None, opts).unwrap(); + process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap(); assert_eq!(bank_forks_info.len(), 1); assert_eq!( @@ -886,7 +887,7 @@ pub mod tests { ..ProcessOptions::default() }; let (bank_forks, bank_forks_info, _) = - process_blocktree(&genesis_config, &blocktree, None, opts).unwrap(); + process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap(); assert_eq!(bank_forks_info.len(), 1); // One fork, other one is ignored b/c not a descendant of the root @@ -960,7 +961,7 @@ pub mod tests { ..ProcessOptions::default() }; let (bank_forks, bank_forks_info, _) = - process_blocktree(&genesis_config, &blocktree, None, opts).unwrap(); + process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap(); assert_eq!(bank_forks_info.len(), 2); // There are two forks assert_eq!( @@ -1024,9 +1025,13 @@ pub mod tests { blocktree.set_dead_slot(2).unwrap(); fill_blocktree_slot_with_ticks(&blocktree, ticks_per_slot, 3, 1, slot1_blockhash); - let (bank_forks, bank_forks_info, _) = - process_blocktree(&genesis_config, &blocktree, None, ProcessOptions::default()) - .unwrap(); + let (bank_forks, bank_forks_info, _) = process_blocktree( + &genesis_config, + &blocktree, + Vec::new(), + ProcessOptions::default(), + ) + .unwrap(); assert_eq!(bank_forks_info.len(), 1); assert_eq!(bank_forks_info[0], BankForksInfo { bank_slot: 3 }); @@ -1056,7 +1061,7 @@ pub mod tests { Blocktree::open(&ledger_path).expect("Expected to successfully open database ledger"); // Let last_slot be the number of slots in the first two epochs - let epoch_schedule = get_epoch_schedule(&genesis_config, None); + let epoch_schedule = get_epoch_schedule(&genesis_config, Vec::new()); let last_slot = epoch_schedule.get_last_slot_in_epoch(1); // Create a single chain of slots with all indexes in the range [0, last_slot + 1] @@ -1083,7 +1088,7 @@ pub mod tests { ..ProcessOptions::default() }; let (bank_forks, bank_forks_info, _) = - process_blocktree(&genesis_config, &blocktree, None, opts).unwrap(); + process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap(); assert_eq!(bank_forks_info.len(), 1); // There is one fork assert_eq!( @@ -1231,7 +1236,7 @@ pub mod tests { ..ProcessOptions::default() }; let (bank_forks, bank_forks_info, _) = - process_blocktree(&genesis_config, &blocktree, None, opts).unwrap(); + process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap(); assert_eq!(bank_forks_info.len(), 1); assert_eq!(bank_forks.root(), 0); @@ -1260,7 +1265,7 @@ pub mod tests { ..ProcessOptions::default() }; let (bank_forks, bank_forks_info, _) = - process_blocktree(&genesis_config, &blocktree, None, opts).unwrap(); + process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap(); assert_eq!(bank_forks_info.len(), 1); assert_eq!(bank_forks_info[0], BankForksInfo { bank_slot: 0 }); @@ -1278,7 +1283,7 @@ pub mod tests { override_num_threads: Some(1), ..ProcessOptions::default() }; - process_blocktree(&genesis_config, &blocktree, None, opts).unwrap(); + process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap(); PAR_THREAD_POOL.with(|pool| { assert_eq!(pool.borrow().current_num_threads(), 1); }); @@ -1295,7 +1300,7 @@ pub mod tests { ..ProcessOptions::default() }; let (_bank_forks, _bank_forks_info, cached_leader_schedule) = - process_blocktree(&genesis_config, &blocktree, None, opts).unwrap(); + process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap(); assert_eq!(cached_leader_schedule.max_schedules(), std::usize::MAX); } @@ -1355,7 +1360,7 @@ pub mod tests { entry_callback: Some(entry_callback), ..ProcessOptions::default() }; - process_blocktree(&genesis_config, &blocktree, None, opts).unwrap(); + process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap(); assert_eq!(*callback_counter.write().unwrap(), 2); } @@ -2190,7 +2195,7 @@ pub mod tests { fn get_epoch_schedule( genesis_config: &GenesisConfig, - account_paths: Option, + account_paths: Vec, ) -> EpochSchedule { let bank = Bank::new_with_paths(&genesis_config, account_paths); bank.epoch_schedule().clone() diff --git a/ledger/src/snapshot_utils.rs b/ledger/src/snapshot_utils.rs index 2f5d5993a6..ad1c65dc37 100644 --- a/ledger/src/snapshot_utils.rs +++ b/ledger/src/snapshot_utils.rs @@ -195,7 +195,7 @@ pub fn bank_slot_from_archive>(snapshot_tar: P) -> Result { } pub fn bank_from_archive>( - account_paths: String, + account_paths: &[PathBuf], snapshot_path: &PathBuf, snapshot_tar: P, ) -> Result { @@ -254,7 +254,7 @@ pub fn untar_snapshot_in, Q: AsRef>( } fn rebuild_bank_from_snapshots

( - local_account_paths: String, + local_account_paths: &[PathBuf], unpacked_snapshots_dir: &PathBuf, append_vecs_path: P, ) -> Result diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index ba613d406d..0f5cbe6e30 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -17,7 +17,6 @@ use solana_local_cluster::{ cluster_tests, local_cluster::{ClusterConfig, LocalCluster}, }; -use solana_runtime::accounts_db::AccountsDB; use solana_sdk::timing::timestamp; use solana_sdk::{ client::SyncClient, @@ -788,7 +787,7 @@ fn test_snapshots_restart_validity() { let (new_account_storage_dirs, new_account_storage_paths) = generate_account_paths(num_account_paths); all_account_storage_dirs.push(new_account_storage_dirs); - snapshot_test_config.validator_config.account_paths = Some(new_account_storage_paths); + snapshot_test_config.validator_config.account_paths = new_account_storage_paths; // Restart node trace!("Restarting cluster from snapshot"); @@ -1040,15 +1039,14 @@ fn wait_for_next_snapshot>(cluster: &LocalCluster, tar: P) { } } -fn generate_account_paths(num_account_paths: usize) -> (Vec, String) { +fn generate_account_paths(num_account_paths: usize) -> (Vec, Vec) { let account_storage_dirs: Vec = (0..num_account_paths) .map(|_| TempDir::new().unwrap()) .collect(); let account_storage_paths: Vec<_> = account_storage_dirs .iter() - .map(|a| a.path().to_str().unwrap().to_string()) + .map(|a| a.path().to_path_buf()) .collect(); - let account_storage_paths = AccountsDB::format_paths(account_storage_paths); (account_storage_dirs, account_storage_paths) } @@ -1079,7 +1077,7 @@ fn setup_snapshot_validator_config( let mut validator_config = ValidatorConfig::default(); validator_config.rpc_config.enable_validator_exit = true; validator_config.snapshot_config = Some(snapshot_config); - validator_config.account_paths = Some(account_storage_paths); + validator_config.account_paths = account_storage_paths; SnapshotValidatorConfig { _snapshot_dir: snapshot_dir, diff --git a/runtime/benches/accounts.rs b/runtime/benches/accounts.rs index c42b2f1ed8..eab2b5ef80 100644 --- a/runtime/benches/accounts.rs +++ b/runtime/benches/accounts.rs @@ -7,7 +7,7 @@ use solana_runtime::bank::*; use solana_sdk::account::Account; use solana_sdk::genesis_config::create_genesis_config; use solana_sdk::pubkey::Pubkey; -use std::sync::Arc; +use std::{path::PathBuf, sync::Arc}; use test::Bencher; fn deposit_many(bank: &Bank, pubkeys: &mut Vec, num: usize) { @@ -24,7 +24,7 @@ fn deposit_many(bank: &Bank, pubkeys: &mut Vec, num: usize) { #[bench] fn test_accounts_create(bencher: &mut Bencher) { let (genesis_config, _) = create_genesis_config(10_000); - let bank0 = Bank::new_with_paths(&genesis_config, Some("bench_a0".to_string())); + let bank0 = Bank::new_with_paths(&genesis_config, vec![PathBuf::from("bench_a0")]); bencher.iter(|| { let mut pubkeys: Vec = vec![]; deposit_many(&bank0, &mut pubkeys, 1000); @@ -37,7 +37,7 @@ fn test_accounts_squash(bencher: &mut Bencher) { let mut banks: Vec> = Vec::with_capacity(10); banks.push(Arc::new(Bank::new_with_paths( &genesis_config, - Some("bench_a1".to_string()), + vec![PathBuf::from("bench_a1")], ))); let mut pubkeys: Vec = vec![]; deposit_many(&banks[0], &mut pubkeys, 250000); @@ -60,7 +60,7 @@ fn test_accounts_squash(bencher: &mut Bencher) { #[bench] fn test_accounts_hash_internal_state(bencher: &mut Bencher) { - let accounts = Accounts::new(Some("bench_accounts_hash_internal".to_string())); + let accounts = Accounts::new(vec![PathBuf::from("bench_accounts_hash_internal")]); let mut pubkeys: Vec = vec![]; create_test_accounts(&accounts, &mut pubkeys, 60000, 0); let ancestors = vec![(0, 0)].into_iter().collect(); diff --git a/runtime/src/accounts.rs b/runtime/src/accounts.rs index 40c462ed44..1470fc3a31 100644 --- a/runtime/src/accounts.rs +++ b/runtime/src/accounts.rs @@ -18,7 +18,7 @@ use solana_sdk::transaction::Result; use solana_sdk::transaction::{Transaction, TransactionError}; use std::collections::{HashMap, HashSet}; use std::io::{BufReader, Error as IOError, Read}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, RwLock}; use crate::transaction_utils::OrderedIterator; @@ -52,7 +52,7 @@ pub type TransactionLoaders = Vec>; pub type TransactionLoadResult = (TransactionAccounts, TransactionLoaders, TransactionRent); impl Accounts { - pub fn new(paths: Option) -> Self { + pub fn new(paths: Vec) -> Self { let accounts_db = Arc::new(AccountsDB::new(paths)); Accounts { @@ -76,7 +76,7 @@ impl Accounts { pub fn accounts_from_stream>( &self, stream: &mut BufReader, - local_paths: String, + local_paths: &[PathBuf], append_vecs_path: P, ) -> std::result::Result<(), IOError> { self.accounts_db @@ -621,7 +621,7 @@ mod tests { ) -> Vec> { let mut hash_queue = BlockhashQueue::new(100); hash_queue.register_hash(&tx.message().recent_blockhash, &fee_calculator); - let accounts = Accounts::new(None); + let accounts = Accounts::new(Vec::new()); for ka in ka.iter() { accounts.store_slow(0, &ka.0, &ka.1); } @@ -1052,7 +1052,7 @@ mod tests { #[test] fn test_load_by_program_slot() { - let accounts = Accounts::new(None); + let accounts = Accounts::new(Vec::new()); // Load accounts owned by various programs into AccountsDB let pubkey0 = Pubkey::new_rand(); @@ -1075,7 +1075,7 @@ mod tests { #[test] fn test_accounts_account_not_found() { - let accounts = Accounts::new(None); + let accounts = Accounts::new(Vec::new()); let mut error_counters = ErrorCounters::default(); let ancestors = vec![(0, 0)].into_iter().collect(); @@ -1097,14 +1097,14 @@ mod tests { #[test] #[should_panic] fn test_accounts_empty_hash_internal_state() { - let accounts = Accounts::new(None); + let accounts = Accounts::new(Vec::new()); accounts.hash_internal_state(0); } #[test] #[should_panic] fn test_accounts_empty_account_hash_internal_state() { - let accounts = Accounts::new(None); + let accounts = Accounts::new(Vec::new()); accounts.store_slow(0, &Pubkey::default(), &Account::new(1, 0, &sysvar::id())); accounts.hash_internal_state(0); } @@ -1126,7 +1126,7 @@ mod tests { fn test_accounts_serialize() { solana_logger::setup(); let (_accounts_dir, paths) = get_temp_accounts_paths(4).unwrap(); - let accounts = Accounts::new(Some(paths)); + let accounts = Accounts::new(paths); let mut pubkeys: Vec = vec![]; create_test_accounts(&accounts, &mut pubkeys, 100, 0); @@ -1148,9 +1148,9 @@ mod tests { let buf = writer.into_inner(); let mut reader = BufReader::new(&buf[..]); let (_accounts_dir, daccounts_paths) = get_temp_accounts_paths(2).unwrap(); - let daccounts = Accounts::new(Some(daccounts_paths.clone())); + let daccounts = Accounts::new(daccounts_paths.clone()); assert!(daccounts - .accounts_from_stream(&mut reader, daccounts_paths, copied_accounts.path()) + .accounts_from_stream(&mut reader, &daccounts_paths, copied_accounts.path()) .is_ok()); check_accounts(&daccounts, &pubkeys, 100); assert_eq!( @@ -1171,7 +1171,7 @@ mod tests { let account2 = Account::new(3, 0, &Pubkey::default()); let account3 = Account::new(4, 0, &Pubkey::default()); - let accounts = Accounts::new(None); + let accounts = Accounts::new(Vec::new()); accounts.store_slow(0, &keypair0.pubkey(), &account0); accounts.store_slow(0, &keypair1.pubkey(), &account1); accounts.store_slow(0, &keypair2.pubkey(), &account2); @@ -1283,7 +1283,7 @@ mod tests { let account1 = Account::new(2, 0, &Pubkey::default()); let account2 = Account::new(3, 0, &Pubkey::default()); - let accounts = Accounts::new(None); + let accounts = Accounts::new(Vec::new()); accounts.store_slow(0, &keypair0.pubkey(), &account0); accounts.store_slow(0, &keypair1.pubkey(), &account1); accounts.store_slow(0, &keypair2.pubkey(), &account2); @@ -1404,7 +1404,7 @@ mod tests { let mut loaded = vec![loaded0, loaded1]; - let accounts = Accounts::new(None); + let accounts = Accounts::new(Vec::new()); { let mut readonly_locks = accounts.readonly_locks.write().unwrap(); let readonly_locks = readonly_locks.as_mut().unwrap(); diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs index a4ae59bc79..5c5a79c6e6 100644 --- a/runtime/src/accounts_db.rs +++ b/runtime/src/accounts_db.rs @@ -307,18 +307,11 @@ impl AccountStorageEntry { } } -pub fn get_paths_vec(paths: &str) -> Vec { - paths.split(',').map(PathBuf::from).collect() -} - -pub fn get_temp_accounts_paths(count: u32) -> IOResult<(Vec, String)> { +pub fn get_temp_accounts_paths(count: u32) -> IOResult<(Vec, Vec)> { let temp_dirs: IOResult> = (0..count).map(|_| TempDir::new()).collect(); let temp_dirs = temp_dirs?; - let paths: Vec = temp_dirs - .iter() - .map(|t| t.path().to_str().unwrap().to_owned()) - .collect(); - Ok((temp_dirs, paths.join(","))) + let paths: Vec = temp_dirs.iter().map(|t| t.path().to_path_buf()).collect(); + Ok((temp_dirs, paths)) } pub struct AccountsDBSerialize<'a> { @@ -413,10 +406,10 @@ impl Default for AccountsDB { } impl AccountsDB { - pub fn new(paths: Option) -> Self { - if let Some(paths) = paths { + pub fn new(paths: Vec) -> Self { + if !paths.is_empty() { Self { - paths: RwLock::new(get_paths_vec(&paths)), + paths: RwLock::new(paths), temp_paths: None, ..Self::default() } @@ -425,7 +418,7 @@ impl AccountsDB { // for testing let (temp_dirs, paths) = get_temp_accounts_paths(DEFAULT_NUM_DIRS).unwrap(); Self { - paths: RwLock::new(get_paths_vec(&paths)), + paths: RwLock::new(paths), temp_paths: Some(temp_dirs), ..Self::default() } @@ -436,29 +429,21 @@ impl AccountsDB { pub fn new_single() -> Self { AccountsDB { min_num_stores: 0, - ..AccountsDB::new(None) + ..AccountsDB::new(Vec::new()) } } #[cfg(test)] - pub fn new_sized(paths: Option, file_size: u64) -> Self { + pub fn new_sized(paths: Vec, file_size: u64) -> Self { AccountsDB { file_size, ..AccountsDB::new(paths) } } - pub fn format_paths>(paths: Vec

) -> String { - let paths: Vec = paths - .iter() - .map(|p| p.as_ref().to_str().unwrap().to_owned()) - .collect(); - paths.join(",") - } - pub fn accounts_from_stream>( &self, mut stream: &mut BufReader, - local_account_paths: String, + local_account_paths: &[PathBuf], append_vecs_path: P, ) -> Result<(), IOError> { let _len: usize = @@ -467,7 +452,6 @@ impl AccountsDB { deserialize_from(&mut stream).map_err(|e| AccountsDB::get_io_error(&e.to_string()))?; // Remap the deserialized AppendVec paths to point to correct local paths - let local_account_paths = get_paths_vec(&local_account_paths); let new_storage_map: Result, IOError> = storage .0 .into_iter() @@ -534,7 +518,7 @@ impl AccountsDB { .insert(slot_hash.0, slot_hash.1); // Process deserialized data, set necessary fields in self - *self.paths.write().unwrap() = local_account_paths; + *self.paths.write().unwrap() = local_account_paths.to_vec(); let max_id: usize = *storage .0 .values() @@ -1214,7 +1198,7 @@ pub mod tests { #[test] fn test_accountsdb_add_root() { solana_logger::setup(); - let db = AccountsDB::new(None); + let db = AccountsDB::new(Vec::new()); let key = Pubkey::default(); let account0 = Account::new(1, 0, &key); @@ -1227,7 +1211,7 @@ pub mod tests { #[test] fn test_accountsdb_latest_ancestor() { solana_logger::setup(); - let db = AccountsDB::new(None); + let db = AccountsDB::new(Vec::new()); let key = Pubkey::default(); let account0 = Account::new(1, 0, &key); @@ -1254,7 +1238,7 @@ pub mod tests { #[test] fn test_accountsdb_latest_ancestor_with_root() { solana_logger::setup(); - let db = AccountsDB::new(None); + let db = AccountsDB::new(Vec::new()); let key = Pubkey::default(); let account0 = Account::new(1, 0, &key); @@ -1274,7 +1258,7 @@ pub mod tests { #[test] fn test_accountsdb_root_one_slot() { solana_logger::setup(); - let db = AccountsDB::new(None); + let db = AccountsDB::new(Vec::new()); let key = Pubkey::default(); let account0 = Account::new(1, 0, &key); @@ -1315,7 +1299,7 @@ pub mod tests { #[test] fn test_accountsdb_add_root_many() { - let db = AccountsDB::new(None); + let db = AccountsDB::new(Vec::new()); let mut pubkeys: Vec = vec![]; create_account(&db, &mut pubkeys, 0, 100, 0, 0); @@ -1383,7 +1367,7 @@ pub mod tests { let key = Pubkey::default(); // 1 token in the "root", i.e. db zero - let db0 = AccountsDB::new(None); + let db0 = AccountsDB::new(Vec::new()); let account0 = Account::new(1, 0, &key); db0.store(0, &[(&key, &account0)]); @@ -1492,7 +1476,7 @@ pub mod tests { #[test] fn test_account_one() { let (_accounts_dirs, paths) = get_temp_accounts_paths(1).unwrap(); - let db = AccountsDB::new(Some(paths)); + let db = AccountsDB::new(paths); let mut pubkeys: Vec = vec![]; create_account(&db, &mut pubkeys, 0, 1, 0, 0); let ancestors = vec![(0, 0)].into_iter().collect(); @@ -1505,7 +1489,7 @@ pub mod tests { #[test] fn test_account_many() { let (_accounts_dirs, paths) = get_temp_accounts_paths(2).unwrap(); - let db = AccountsDB::new(Some(paths)); + let db = AccountsDB::new(paths); let mut pubkeys: Vec = vec![]; create_account(&db, &mut pubkeys, 0, 100, 0, 0); check_accounts(&db, &pubkeys, 0, 100, 1); @@ -1524,7 +1508,7 @@ pub mod tests { fn test_account_grow_many() { let (_accounts_dir, paths) = get_temp_accounts_paths(2).unwrap(); let size = 4096; - let accounts = AccountsDB::new_sized(Some(paths), size); + let accounts = AccountsDB::new_sized(paths, size); let mut keys = vec![]; for i in 0..9 { let key = Pubkey::new_rand(); @@ -1623,7 +1607,7 @@ pub mod tests { #[test] fn test_purge_slot_not_root() { - let accounts = AccountsDB::new(None); + let accounts = AccountsDB::new(Vec::new()); let mut pubkeys: Vec = vec![]; create_account(&accounts, &mut pubkeys, 0, 1, 0, 0); let ancestors = vec![(0, 0)].into_iter().collect(); @@ -1634,7 +1618,7 @@ pub mod tests { #[test] fn test_purge_slot_after_root() { - let accounts = AccountsDB::new(None); + let accounts = AccountsDB::new(Vec::new()); let mut pubkeys: Vec = vec![]; create_account(&accounts, &mut pubkeys, 0, 1, 0, 0); let ancestors = vec![(0, 0)].into_iter().collect(); @@ -1648,7 +1632,7 @@ pub mod tests { //This test is pedantic //A slot is purged when a non root bank is cleaned up. If a slot is behind root but it is //not root, it means we are retaining dead banks. - let accounts = AccountsDB::new(None); + let accounts = AccountsDB::new(Vec::new()); let pubkey = Pubkey::new_rand(); let account = Account::new(1, 0, &Account::default().owner); //store an account @@ -1811,18 +1795,13 @@ pub mod tests { let buf = writer.into_inner(); let mut reader = BufReader::new(&buf[..]); - let daccounts = AccountsDB::new(None); - - let local_paths = { - let paths = daccounts.paths.read().unwrap(); - AccountsDB::format_paths(paths.to_vec()) - }; - + let daccounts = AccountsDB::new(Vec::new()); + let local_paths = daccounts.paths.read().unwrap().clone(); let copied_accounts = TempDir::new().unwrap(); // Simulate obtaining a copy of the AppendVecs from a tarball copy_append_vecs(&accounts, copied_accounts.path()).unwrap(); daccounts - .accounts_from_stream(&mut reader, local_paths, copied_accounts.path()) + .accounts_from_stream(&mut reader, &local_paths, copied_accounts.path()) .unwrap(); print_count_and_status("daccounts", &daccounts); @@ -2023,7 +2002,7 @@ pub mod tests { let min_file_bytes = std::mem::size_of::() + std::mem::size_of::(); - let db = Arc::new(AccountsDB::new_sized(None, min_file_bytes as u64)); + let db = Arc::new(AccountsDB::new_sized(Vec::new(), min_file_bytes as u64)); db.add_root(slot_id); let thread_hdls: Vec<_> = (0..num_threads) @@ -2060,7 +2039,7 @@ pub mod tests { #[test] fn test_accountsdb_scan_accounts() { solana_logger::setup(); - let db = AccountsDB::new(None); + let db = AccountsDB::new(Vec::new()); let key = Pubkey::default(); let key0 = Pubkey::new_rand(); let account0 = Account::new(1, 0, &key); @@ -2093,7 +2072,7 @@ pub mod tests { #[test] fn test_store_large_account() { solana_logger::setup(); - let db = AccountsDB::new(None); + let db = AccountsDB::new(Vec::new()); let key = Pubkey::default(); let data_len = DEFAULT_FILE_SIZE as usize + 7; diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 768e44ddf3..6c2b598b0c 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -48,7 +48,7 @@ use solana_vote_program::vote_state::VoteState; use std::{ collections::HashMap, io::{BufReader, Cursor, Error as IOError, Read}, - path::Path, + path::{Path, PathBuf}, sync::atomic::{AtomicBool, AtomicU64, Ordering}, sync::{Arc, RwLock, RwLockReadGuard}, }; @@ -70,8 +70,8 @@ pub struct BankRc { } impl BankRc { - pub fn new(account_paths: String, id: AppendVecId, slot: Slot) -> Self { - let accounts = Accounts::new(Some(account_paths)); + pub fn new(account_paths: Vec, id: AppendVecId, slot: Slot) -> Self { + let accounts = Accounts::new(account_paths); accounts .accounts_db .next_id @@ -86,7 +86,7 @@ impl BankRc { pub fn accounts_from_stream>( &self, mut stream: &mut BufReader, - local_paths: String, + local_paths: &[PathBuf], append_vecs_path: P, ) -> std::result::Result<(), IOError> { let _len: usize = @@ -289,10 +289,10 @@ impl Default for BlockhashQueue { impl Bank { pub fn new(genesis_config: &GenesisConfig) -> Self { - Self::new_with_paths(&genesis_config, None) + Self::new_with_paths(&genesis_config, Vec::new()) } - pub fn new_with_paths(genesis_config: &GenesisConfig, paths: Option) -> Self { + pub fn new_with_paths(genesis_config: &GenesisConfig, paths: Vec) -> Self { let mut bank = Self::default(); bank.ancestors.insert(bank.slot(), 0); bank.rc.accounts = Arc::new(Accounts::new(paths)); @@ -417,7 +417,7 @@ impl Bank { pub fn create_with_genesis( genesis_config: &GenesisConfig, - account_paths: String, + account_paths: Vec, status_cache_rc: &StatusCacheRc, id: AppendVecId, ) -> Self { @@ -3921,7 +3921,7 @@ mod tests { copy_append_vecs(&bank2.rc.accounts.accounts_db, copied_accounts.path()).unwrap(); dbank .rc - .accounts_from_stream(&mut reader, dbank_paths, copied_accounts.path()) + .accounts_from_stream(&mut reader, &dbank_paths, copied_accounts.path()) .unwrap(); assert_eq!(dbank.get_balance(&key1.pubkey()), 0); assert_eq!(dbank.get_balance(&key2.pubkey()), 10); diff --git a/validator/src/main.rs b/validator/src/main.rs index fa4c14b8e7..59b1ee3f9f 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -591,12 +591,28 @@ pub fn main() { solana_net_utils::parse_port_range(matches.value_of("dynamic_port_range").unwrap()) .expect("invalid dynamic_port_range"); - if let Some(account_paths) = matches.value_of("account_paths") { - validator_config.account_paths = Some(account_paths.to_string()); + let account_paths = if let Some(account_paths) = matches.value_of("account_paths") { + account_paths.split(',').map(PathBuf::from).collect() } else { - validator_config.account_paths = - Some(ledger_path.join("accounts").to_str().unwrap().to_string()); - } + vec![ledger_path.join("accounts")] + }; + + // Create and canonicalize account paths to avoid issues with symlink creation + validator_config.account_paths = account_paths + .into_iter() + .map(|account_path| { + match fs::create_dir_all(&account_path).and_then(|_| fs::canonicalize(&account_path)) { + Ok(account_path) => account_path, + Err(err) => { + eprintln!( + "Unable to access account path: {:?}, err: {:?}", + account_path, err + ); + exit(1); + } + } + }) + .collect(); let snapshot_interval_slots = value_t_or_exit!(matches, "snapshot_interval_slots", usize); let snapshot_path = ledger_path.clone().join("snapshot");