Fail fast if account paths cannot be canonicalized (#7300)
* Canonicalize account paths to avoid symlink issues * fixes
This commit is contained in:
parent
7c3be2ec9a
commit
b7d4330dd4
|
@ -295,7 +295,7 @@ mod tests {
|
||||||
..ProcessOptions::default()
|
..ProcessOptions::default()
|
||||||
};
|
};
|
||||||
let (bank_forks, _, cached_leader_schedule) =
|
let (bank_forks, _, cached_leader_schedule) =
|
||||||
process_blocktree(&genesis_config, &blocktree, None, opts).unwrap();
|
process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap();
|
||||||
let leader_schedule_cache = Arc::new(cached_leader_schedule);
|
let leader_schedule_cache = Arc::new(cached_leader_schedule);
|
||||||
let bank_forks = Arc::new(RwLock::new(bank_forks));
|
let bank_forks = Arc::new(RwLock::new(bank_forks));
|
||||||
|
|
||||||
|
|
|
@ -60,7 +60,7 @@ pub struct ValidatorConfig {
|
||||||
pub transaction_status_service_disabled: bool,
|
pub transaction_status_service_disabled: bool,
|
||||||
pub blockstream_unix_socket: Option<PathBuf>,
|
pub blockstream_unix_socket: Option<PathBuf>,
|
||||||
pub storage_slots_per_turn: u64,
|
pub storage_slots_per_turn: u64,
|
||||||
pub account_paths: Option<String>,
|
pub account_paths: Vec<PathBuf>,
|
||||||
pub rpc_config: JsonRpcConfig,
|
pub rpc_config: JsonRpcConfig,
|
||||||
pub snapshot_config: Option<SnapshotConfig>,
|
pub snapshot_config: Option<SnapshotConfig>,
|
||||||
pub max_ledger_slots: Option<u64>,
|
pub max_ledger_slots: Option<u64>,
|
||||||
|
@ -80,7 +80,7 @@ impl Default for ValidatorConfig {
|
||||||
blockstream_unix_socket: None,
|
blockstream_unix_socket: None,
|
||||||
storage_slots_per_turn: DEFAULT_SLOTS_PER_TURN,
|
storage_slots_per_turn: DEFAULT_SLOTS_PER_TURN,
|
||||||
max_ledger_slots: None,
|
max_ledger_slots: None,
|
||||||
account_paths: None,
|
account_paths: Vec::new(),
|
||||||
rpc_config: JsonRpcConfig::default(),
|
rpc_config: JsonRpcConfig::default(),
|
||||||
snapshot_config: None,
|
snapshot_config: None,
|
||||||
broadcast_stage_type: BroadcastStageType::Standard,
|
broadcast_stage_type: BroadcastStageType::Standard,
|
||||||
|
@ -469,7 +469,7 @@ impl Validator {
|
||||||
pub fn new_banks_from_blocktree(
|
pub fn new_banks_from_blocktree(
|
||||||
expected_genesis_hash: Option<Hash>,
|
expected_genesis_hash: Option<Hash>,
|
||||||
blocktree_path: &Path,
|
blocktree_path: &Path,
|
||||||
account_paths: Option<String>,
|
account_paths: Vec<PathBuf>,
|
||||||
snapshot_config: Option<SnapshotConfig>,
|
snapshot_config: Option<SnapshotConfig>,
|
||||||
poh_verify: bool,
|
poh_verify: bool,
|
||||||
dev_halt_at_slot: Option<Slot>,
|
dev_halt_at_slot: Option<Slot>,
|
||||||
|
|
|
@ -52,7 +52,7 @@ mod tests {
|
||||||
let genesis_config_info = create_genesis_config(10_000);
|
let genesis_config_info = create_genesis_config(10_000);
|
||||||
let bank0 = Bank::new_with_paths(
|
let bank0 = Bank::new_with_paths(
|
||||||
&genesis_config_info.genesis_config,
|
&genesis_config_info.genesis_config,
|
||||||
Some(accounts_dir.path().to_str().unwrap().to_string()),
|
vec![accounts_dir.path().to_path_buf()],
|
||||||
);
|
);
|
||||||
bank0.freeze();
|
bank0.freeze();
|
||||||
let mut bank_forks = BankForks::new(0, bank0);
|
let mut bank_forks = BankForks::new(0, bank0);
|
||||||
|
@ -73,7 +73,7 @@ mod tests {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn restore_from_snapshot(old_bank_forks: &BankForks, account_paths: String) {
|
fn restore_from_snapshot(old_bank_forks: &BankForks, account_paths: Vec<PathBuf>) {
|
||||||
let (snapshot_path, snapshot_package_output_path) = old_bank_forks
|
let (snapshot_path, snapshot_package_output_path) = old_bank_forks
|
||||||
.snapshot_config
|
.snapshot_config
|
||||||
.as_ref()
|
.as_ref()
|
||||||
|
@ -81,7 +81,7 @@ mod tests {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let deserialized_bank = snapshot_utils::bank_from_archive(
|
let deserialized_bank = snapshot_utils::bank_from_archive(
|
||||||
account_paths,
|
&account_paths,
|
||||||
&old_bank_forks
|
&old_bank_forks
|
||||||
.snapshot_config
|
.snapshot_config
|
||||||
.as_ref()
|
.as_ref()
|
||||||
|
@ -151,10 +151,7 @@ mod tests {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
SnapshotPackagerService::package_snapshots(&snapshot_package).unwrap();
|
SnapshotPackagerService::package_snapshots(&snapshot_package).unwrap();
|
||||||
|
|
||||||
restore_from_snapshot(
|
restore_from_snapshot(bank_forks, vec![accounts_dir.path().to_path_buf()]);
|
||||||
bank_forks,
|
|
||||||
accounts_dir.path().to_str().unwrap().to_string(),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
|
@ -582,9 +582,9 @@ fn main() {
|
||||||
})
|
})
|
||||||
};
|
};
|
||||||
let account_paths = if let Some(account_paths) = matches.value_of("account_paths") {
|
let account_paths = if let Some(account_paths) = matches.value_of("account_paths") {
|
||||||
Some(account_paths.to_string())
|
account_paths.split(',').map(PathBuf::from).collect()
|
||||||
} else {
|
} else {
|
||||||
Some(ledger_path.join("accounts").to_str().unwrap().to_string())
|
vec![ledger_path.join("accounts")]
|
||||||
};
|
};
|
||||||
|
|
||||||
let process_options = blocktree_processor::ProcessOptions {
|
let process_options = blocktree_processor::ProcessOptions {
|
||||||
|
|
|
@ -7,12 +7,12 @@ use crate::{
|
||||||
};
|
};
|
||||||
use log::*;
|
use log::*;
|
||||||
use solana_sdk::genesis_config::GenesisConfig;
|
use solana_sdk::genesis_config::GenesisConfig;
|
||||||
use std::{fs, sync::Arc};
|
use std::{fs, path::PathBuf, sync::Arc};
|
||||||
|
|
||||||
pub fn load(
|
pub fn load(
|
||||||
genesis_config: &GenesisConfig,
|
genesis_config: &GenesisConfig,
|
||||||
blocktree: &Blocktree,
|
blocktree: &Blocktree,
|
||||||
account_paths: Option<String>,
|
account_paths: Vec<PathBuf>,
|
||||||
snapshot_config: Option<&SnapshotConfig>,
|
snapshot_config: Option<&SnapshotConfig>,
|
||||||
process_options: ProcessOptions,
|
process_options: ProcessOptions,
|
||||||
) -> Result<(BankForks, Vec<BankForksInfo>, LeaderScheduleCache), BlocktreeProcessorError> {
|
) -> Result<(BankForks, Vec<BankForksInfo>, LeaderScheduleCache), BlocktreeProcessorError> {
|
||||||
|
@ -30,10 +30,13 @@ pub fn load(
|
||||||
if tar.exists() {
|
if tar.exists() {
|
||||||
info!("Loading snapshot package: {:?}", tar);
|
info!("Loading snapshot package: {:?}", tar);
|
||||||
// Fail hard here if snapshot fails to load, don't silently continue
|
// Fail hard here if snapshot fails to load, don't silently continue
|
||||||
|
|
||||||
|
if account_paths.is_empty() {
|
||||||
|
panic!("Account paths not present when booting from snapshot")
|
||||||
|
}
|
||||||
|
|
||||||
let deserialized_bank = snapshot_utils::bank_from_archive(
|
let deserialized_bank = snapshot_utils::bank_from_archive(
|
||||||
account_paths
|
&account_paths,
|
||||||
.clone()
|
|
||||||
.expect("Account paths not present when booting from snapshot"),
|
|
||||||
&snapshot_config.snapshot_path,
|
&snapshot_config.snapshot_path,
|
||||||
&tar,
|
&tar,
|
||||||
)
|
)
|
||||||
|
|
|
@ -27,6 +27,7 @@ use solana_sdk::{
|
||||||
};
|
};
|
||||||
use std::{
|
use std::{
|
||||||
cell::RefCell,
|
cell::RefCell,
|
||||||
|
path::PathBuf,
|
||||||
result,
|
result,
|
||||||
sync::Arc,
|
sync::Arc,
|
||||||
time::{Duration, Instant},
|
time::{Duration, Instant},
|
||||||
|
@ -255,7 +256,7 @@ pub struct ProcessOptions {
|
||||||
pub fn process_blocktree(
|
pub fn process_blocktree(
|
||||||
genesis_config: &GenesisConfig,
|
genesis_config: &GenesisConfig,
|
||||||
blocktree: &Blocktree,
|
blocktree: &Blocktree,
|
||||||
account_paths: Option<String>,
|
account_paths: Vec<PathBuf>,
|
||||||
opts: ProcessOptions,
|
opts: ProcessOptions,
|
||||||
) -> result::Result<(BankForks, Vec<BankForksInfo>, LeaderScheduleCache), BlocktreeProcessorError> {
|
) -> result::Result<(BankForks, Vec<BankForksInfo>, LeaderScheduleCache), BlocktreeProcessorError> {
|
||||||
if let Some(num_threads) = opts.override_num_threads {
|
if let Some(num_threads) = opts.override_num_threads {
|
||||||
|
@ -664,7 +665,7 @@ pub mod tests {
|
||||||
..ProcessOptions::default()
|
..ProcessOptions::default()
|
||||||
};
|
};
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
process_blocktree(&genesis_config, &blocktree, None, opts).err(),
|
process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).err(),
|
||||||
Some(BlocktreeProcessorError::InvalidBlock(
|
Some(BlocktreeProcessorError::InvalidBlock(
|
||||||
BlockError::InvalidTickHashCount
|
BlockError::InvalidTickHashCount
|
||||||
)),
|
)),
|
||||||
|
@ -705,7 +706,7 @@ pub mod tests {
|
||||||
..ProcessOptions::default()
|
..ProcessOptions::default()
|
||||||
};
|
};
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
process_blocktree(&genesis_config, &blocktree, None, opts).err(),
|
process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).err(),
|
||||||
Some(BlocktreeProcessorError::InvalidBlock(
|
Some(BlocktreeProcessorError::InvalidBlock(
|
||||||
BlockError::InvalidTickCount
|
BlockError::InvalidTickCount
|
||||||
)),
|
)),
|
||||||
|
@ -757,7 +758,7 @@ pub mod tests {
|
||||||
..ProcessOptions::default()
|
..ProcessOptions::default()
|
||||||
};
|
};
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
process_blocktree(&genesis_config, &blocktree, None, opts).err(),
|
process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).err(),
|
||||||
Some(BlocktreeProcessorError::InvalidBlock(
|
Some(BlocktreeProcessorError::InvalidBlock(
|
||||||
BlockError::TrailingEntry
|
BlockError::TrailingEntry
|
||||||
)),
|
)),
|
||||||
|
@ -824,7 +825,7 @@ pub mod tests {
|
||||||
..ProcessOptions::default()
|
..ProcessOptions::default()
|
||||||
};
|
};
|
||||||
let (mut _bank_forks, bank_forks_info, _) =
|
let (mut _bank_forks, bank_forks_info, _) =
|
||||||
process_blocktree(&genesis_config, &blocktree, None, opts).unwrap();
|
process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap();
|
||||||
|
|
||||||
assert_eq!(bank_forks_info.len(), 1);
|
assert_eq!(bank_forks_info.len(), 1);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
@ -886,7 +887,7 @@ pub mod tests {
|
||||||
..ProcessOptions::default()
|
..ProcessOptions::default()
|
||||||
};
|
};
|
||||||
let (bank_forks, bank_forks_info, _) =
|
let (bank_forks, bank_forks_info, _) =
|
||||||
process_blocktree(&genesis_config, &blocktree, None, opts).unwrap();
|
process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap();
|
||||||
|
|
||||||
assert_eq!(bank_forks_info.len(), 1); // One fork, other one is ignored b/c not a descendant of the root
|
assert_eq!(bank_forks_info.len(), 1); // One fork, other one is ignored b/c not a descendant of the root
|
||||||
|
|
||||||
|
@ -960,7 +961,7 @@ pub mod tests {
|
||||||
..ProcessOptions::default()
|
..ProcessOptions::default()
|
||||||
};
|
};
|
||||||
let (bank_forks, bank_forks_info, _) =
|
let (bank_forks, bank_forks_info, _) =
|
||||||
process_blocktree(&genesis_config, &blocktree, None, opts).unwrap();
|
process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap();
|
||||||
|
|
||||||
assert_eq!(bank_forks_info.len(), 2); // There are two forks
|
assert_eq!(bank_forks_info.len(), 2); // There are two forks
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
@ -1024,8 +1025,12 @@ pub mod tests {
|
||||||
blocktree.set_dead_slot(2).unwrap();
|
blocktree.set_dead_slot(2).unwrap();
|
||||||
fill_blocktree_slot_with_ticks(&blocktree, ticks_per_slot, 3, 1, slot1_blockhash);
|
fill_blocktree_slot_with_ticks(&blocktree, ticks_per_slot, 3, 1, slot1_blockhash);
|
||||||
|
|
||||||
let (bank_forks, bank_forks_info, _) =
|
let (bank_forks, bank_forks_info, _) = process_blocktree(
|
||||||
process_blocktree(&genesis_config, &blocktree, None, ProcessOptions::default())
|
&genesis_config,
|
||||||
|
&blocktree,
|
||||||
|
Vec::new(),
|
||||||
|
ProcessOptions::default(),
|
||||||
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
assert_eq!(bank_forks_info.len(), 1);
|
assert_eq!(bank_forks_info.len(), 1);
|
||||||
|
@ -1056,7 +1061,7 @@ pub mod tests {
|
||||||
Blocktree::open(&ledger_path).expect("Expected to successfully open database ledger");
|
Blocktree::open(&ledger_path).expect("Expected to successfully open database ledger");
|
||||||
|
|
||||||
// Let last_slot be the number of slots in the first two epochs
|
// Let last_slot be the number of slots in the first two epochs
|
||||||
let epoch_schedule = get_epoch_schedule(&genesis_config, None);
|
let epoch_schedule = get_epoch_schedule(&genesis_config, Vec::new());
|
||||||
let last_slot = epoch_schedule.get_last_slot_in_epoch(1);
|
let last_slot = epoch_schedule.get_last_slot_in_epoch(1);
|
||||||
|
|
||||||
// Create a single chain of slots with all indexes in the range [0, last_slot + 1]
|
// Create a single chain of slots with all indexes in the range [0, last_slot + 1]
|
||||||
|
@ -1083,7 +1088,7 @@ pub mod tests {
|
||||||
..ProcessOptions::default()
|
..ProcessOptions::default()
|
||||||
};
|
};
|
||||||
let (bank_forks, bank_forks_info, _) =
|
let (bank_forks, bank_forks_info, _) =
|
||||||
process_blocktree(&genesis_config, &blocktree, None, opts).unwrap();
|
process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap();
|
||||||
|
|
||||||
assert_eq!(bank_forks_info.len(), 1); // There is one fork
|
assert_eq!(bank_forks_info.len(), 1); // There is one fork
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
@ -1231,7 +1236,7 @@ pub mod tests {
|
||||||
..ProcessOptions::default()
|
..ProcessOptions::default()
|
||||||
};
|
};
|
||||||
let (bank_forks, bank_forks_info, _) =
|
let (bank_forks, bank_forks_info, _) =
|
||||||
process_blocktree(&genesis_config, &blocktree, None, opts).unwrap();
|
process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap();
|
||||||
|
|
||||||
assert_eq!(bank_forks_info.len(), 1);
|
assert_eq!(bank_forks_info.len(), 1);
|
||||||
assert_eq!(bank_forks.root(), 0);
|
assert_eq!(bank_forks.root(), 0);
|
||||||
|
@ -1260,7 +1265,7 @@ pub mod tests {
|
||||||
..ProcessOptions::default()
|
..ProcessOptions::default()
|
||||||
};
|
};
|
||||||
let (bank_forks, bank_forks_info, _) =
|
let (bank_forks, bank_forks_info, _) =
|
||||||
process_blocktree(&genesis_config, &blocktree, None, opts).unwrap();
|
process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap();
|
||||||
|
|
||||||
assert_eq!(bank_forks_info.len(), 1);
|
assert_eq!(bank_forks_info.len(), 1);
|
||||||
assert_eq!(bank_forks_info[0], BankForksInfo { bank_slot: 0 });
|
assert_eq!(bank_forks_info[0], BankForksInfo { bank_slot: 0 });
|
||||||
|
@ -1278,7 +1283,7 @@ pub mod tests {
|
||||||
override_num_threads: Some(1),
|
override_num_threads: Some(1),
|
||||||
..ProcessOptions::default()
|
..ProcessOptions::default()
|
||||||
};
|
};
|
||||||
process_blocktree(&genesis_config, &blocktree, None, opts).unwrap();
|
process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap();
|
||||||
PAR_THREAD_POOL.with(|pool| {
|
PAR_THREAD_POOL.with(|pool| {
|
||||||
assert_eq!(pool.borrow().current_num_threads(), 1);
|
assert_eq!(pool.borrow().current_num_threads(), 1);
|
||||||
});
|
});
|
||||||
|
@ -1295,7 +1300,7 @@ pub mod tests {
|
||||||
..ProcessOptions::default()
|
..ProcessOptions::default()
|
||||||
};
|
};
|
||||||
let (_bank_forks, _bank_forks_info, cached_leader_schedule) =
|
let (_bank_forks, _bank_forks_info, cached_leader_schedule) =
|
||||||
process_blocktree(&genesis_config, &blocktree, None, opts).unwrap();
|
process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap();
|
||||||
assert_eq!(cached_leader_schedule.max_schedules(), std::usize::MAX);
|
assert_eq!(cached_leader_schedule.max_schedules(), std::usize::MAX);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1355,7 +1360,7 @@ pub mod tests {
|
||||||
entry_callback: Some(entry_callback),
|
entry_callback: Some(entry_callback),
|
||||||
..ProcessOptions::default()
|
..ProcessOptions::default()
|
||||||
};
|
};
|
||||||
process_blocktree(&genesis_config, &blocktree, None, opts).unwrap();
|
process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).unwrap();
|
||||||
assert_eq!(*callback_counter.write().unwrap(), 2);
|
assert_eq!(*callback_counter.write().unwrap(), 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2190,7 +2195,7 @@ pub mod tests {
|
||||||
|
|
||||||
fn get_epoch_schedule(
|
fn get_epoch_schedule(
|
||||||
genesis_config: &GenesisConfig,
|
genesis_config: &GenesisConfig,
|
||||||
account_paths: Option<String>,
|
account_paths: Vec<PathBuf>,
|
||||||
) -> EpochSchedule {
|
) -> EpochSchedule {
|
||||||
let bank = Bank::new_with_paths(&genesis_config, account_paths);
|
let bank = Bank::new_with_paths(&genesis_config, account_paths);
|
||||||
bank.epoch_schedule().clone()
|
bank.epoch_schedule().clone()
|
||||||
|
|
|
@ -195,7 +195,7 @@ pub fn bank_slot_from_archive<P: AsRef<Path>>(snapshot_tar: P) -> Result<u64> {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn bank_from_archive<P: AsRef<Path>>(
|
pub fn bank_from_archive<P: AsRef<Path>>(
|
||||||
account_paths: String,
|
account_paths: &[PathBuf],
|
||||||
snapshot_path: &PathBuf,
|
snapshot_path: &PathBuf,
|
||||||
snapshot_tar: P,
|
snapshot_tar: P,
|
||||||
) -> Result<Bank> {
|
) -> Result<Bank> {
|
||||||
|
@ -254,7 +254,7 @@ pub fn untar_snapshot_in<P: AsRef<Path>, Q: AsRef<Path>>(
|
||||||
}
|
}
|
||||||
|
|
||||||
fn rebuild_bank_from_snapshots<P>(
|
fn rebuild_bank_from_snapshots<P>(
|
||||||
local_account_paths: String,
|
local_account_paths: &[PathBuf],
|
||||||
unpacked_snapshots_dir: &PathBuf,
|
unpacked_snapshots_dir: &PathBuf,
|
||||||
append_vecs_path: P,
|
append_vecs_path: P,
|
||||||
) -> Result<Bank>
|
) -> Result<Bank>
|
||||||
|
|
|
@ -17,7 +17,6 @@ use solana_local_cluster::{
|
||||||
cluster_tests,
|
cluster_tests,
|
||||||
local_cluster::{ClusterConfig, LocalCluster},
|
local_cluster::{ClusterConfig, LocalCluster},
|
||||||
};
|
};
|
||||||
use solana_runtime::accounts_db::AccountsDB;
|
|
||||||
use solana_sdk::timing::timestamp;
|
use solana_sdk::timing::timestamp;
|
||||||
use solana_sdk::{
|
use solana_sdk::{
|
||||||
client::SyncClient,
|
client::SyncClient,
|
||||||
|
@ -788,7 +787,7 @@ fn test_snapshots_restart_validity() {
|
||||||
let (new_account_storage_dirs, new_account_storage_paths) =
|
let (new_account_storage_dirs, new_account_storage_paths) =
|
||||||
generate_account_paths(num_account_paths);
|
generate_account_paths(num_account_paths);
|
||||||
all_account_storage_dirs.push(new_account_storage_dirs);
|
all_account_storage_dirs.push(new_account_storage_dirs);
|
||||||
snapshot_test_config.validator_config.account_paths = Some(new_account_storage_paths);
|
snapshot_test_config.validator_config.account_paths = new_account_storage_paths;
|
||||||
|
|
||||||
// Restart node
|
// Restart node
|
||||||
trace!("Restarting cluster from snapshot");
|
trace!("Restarting cluster from snapshot");
|
||||||
|
@ -1040,15 +1039,14 @@ fn wait_for_next_snapshot<P: AsRef<Path>>(cluster: &LocalCluster, tar: P) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn generate_account_paths(num_account_paths: usize) -> (Vec<TempDir>, String) {
|
fn generate_account_paths(num_account_paths: usize) -> (Vec<TempDir>, Vec<PathBuf>) {
|
||||||
let account_storage_dirs: Vec<TempDir> = (0..num_account_paths)
|
let account_storage_dirs: Vec<TempDir> = (0..num_account_paths)
|
||||||
.map(|_| TempDir::new().unwrap())
|
.map(|_| TempDir::new().unwrap())
|
||||||
.collect();
|
.collect();
|
||||||
let account_storage_paths: Vec<_> = account_storage_dirs
|
let account_storage_paths: Vec<_> = account_storage_dirs
|
||||||
.iter()
|
.iter()
|
||||||
.map(|a| a.path().to_str().unwrap().to_string())
|
.map(|a| a.path().to_path_buf())
|
||||||
.collect();
|
.collect();
|
||||||
let account_storage_paths = AccountsDB::format_paths(account_storage_paths);
|
|
||||||
(account_storage_dirs, account_storage_paths)
|
(account_storage_dirs, account_storage_paths)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1079,7 +1077,7 @@ fn setup_snapshot_validator_config(
|
||||||
let mut validator_config = ValidatorConfig::default();
|
let mut validator_config = ValidatorConfig::default();
|
||||||
validator_config.rpc_config.enable_validator_exit = true;
|
validator_config.rpc_config.enable_validator_exit = true;
|
||||||
validator_config.snapshot_config = Some(snapshot_config);
|
validator_config.snapshot_config = Some(snapshot_config);
|
||||||
validator_config.account_paths = Some(account_storage_paths);
|
validator_config.account_paths = account_storage_paths;
|
||||||
|
|
||||||
SnapshotValidatorConfig {
|
SnapshotValidatorConfig {
|
||||||
_snapshot_dir: snapshot_dir,
|
_snapshot_dir: snapshot_dir,
|
||||||
|
|
|
@ -7,7 +7,7 @@ use solana_runtime::bank::*;
|
||||||
use solana_sdk::account::Account;
|
use solana_sdk::account::Account;
|
||||||
use solana_sdk::genesis_config::create_genesis_config;
|
use solana_sdk::genesis_config::create_genesis_config;
|
||||||
use solana_sdk::pubkey::Pubkey;
|
use solana_sdk::pubkey::Pubkey;
|
||||||
use std::sync::Arc;
|
use std::{path::PathBuf, sync::Arc};
|
||||||
use test::Bencher;
|
use test::Bencher;
|
||||||
|
|
||||||
fn deposit_many(bank: &Bank, pubkeys: &mut Vec<Pubkey>, num: usize) {
|
fn deposit_many(bank: &Bank, pubkeys: &mut Vec<Pubkey>, num: usize) {
|
||||||
|
@ -24,7 +24,7 @@ fn deposit_many(bank: &Bank, pubkeys: &mut Vec<Pubkey>, num: usize) {
|
||||||
#[bench]
|
#[bench]
|
||||||
fn test_accounts_create(bencher: &mut Bencher) {
|
fn test_accounts_create(bencher: &mut Bencher) {
|
||||||
let (genesis_config, _) = create_genesis_config(10_000);
|
let (genesis_config, _) = create_genesis_config(10_000);
|
||||||
let bank0 = Bank::new_with_paths(&genesis_config, Some("bench_a0".to_string()));
|
let bank0 = Bank::new_with_paths(&genesis_config, vec![PathBuf::from("bench_a0")]);
|
||||||
bencher.iter(|| {
|
bencher.iter(|| {
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
deposit_many(&bank0, &mut pubkeys, 1000);
|
deposit_many(&bank0, &mut pubkeys, 1000);
|
||||||
|
@ -37,7 +37,7 @@ fn test_accounts_squash(bencher: &mut Bencher) {
|
||||||
let mut banks: Vec<Arc<Bank>> = Vec::with_capacity(10);
|
let mut banks: Vec<Arc<Bank>> = Vec::with_capacity(10);
|
||||||
banks.push(Arc::new(Bank::new_with_paths(
|
banks.push(Arc::new(Bank::new_with_paths(
|
||||||
&genesis_config,
|
&genesis_config,
|
||||||
Some("bench_a1".to_string()),
|
vec![PathBuf::from("bench_a1")],
|
||||||
)));
|
)));
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
deposit_many(&banks[0], &mut pubkeys, 250000);
|
deposit_many(&banks[0], &mut pubkeys, 250000);
|
||||||
|
@ -60,7 +60,7 @@ fn test_accounts_squash(bencher: &mut Bencher) {
|
||||||
|
|
||||||
#[bench]
|
#[bench]
|
||||||
fn test_accounts_hash_internal_state(bencher: &mut Bencher) {
|
fn test_accounts_hash_internal_state(bencher: &mut Bencher) {
|
||||||
let accounts = Accounts::new(Some("bench_accounts_hash_internal".to_string()));
|
let accounts = Accounts::new(vec![PathBuf::from("bench_accounts_hash_internal")]);
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_test_accounts(&accounts, &mut pubkeys, 60000, 0);
|
create_test_accounts(&accounts, &mut pubkeys, 60000, 0);
|
||||||
let ancestors = vec![(0, 0)].into_iter().collect();
|
let ancestors = vec![(0, 0)].into_iter().collect();
|
||||||
|
|
|
@ -18,7 +18,7 @@ use solana_sdk::transaction::Result;
|
||||||
use solana_sdk::transaction::{Transaction, TransactionError};
|
use solana_sdk::transaction::{Transaction, TransactionError};
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::io::{BufReader, Error as IOError, Read};
|
use std::io::{BufReader, Error as IOError, Read};
|
||||||
use std::path::Path;
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::{Arc, Mutex, RwLock};
|
use std::sync::{Arc, Mutex, RwLock};
|
||||||
|
|
||||||
use crate::transaction_utils::OrderedIterator;
|
use crate::transaction_utils::OrderedIterator;
|
||||||
|
@ -52,7 +52,7 @@ pub type TransactionLoaders = Vec<Vec<(Pubkey, Account)>>;
|
||||||
pub type TransactionLoadResult = (TransactionAccounts, TransactionLoaders, TransactionRent);
|
pub type TransactionLoadResult = (TransactionAccounts, TransactionLoaders, TransactionRent);
|
||||||
|
|
||||||
impl Accounts {
|
impl Accounts {
|
||||||
pub fn new(paths: Option<String>) -> Self {
|
pub fn new(paths: Vec<PathBuf>) -> Self {
|
||||||
let accounts_db = Arc::new(AccountsDB::new(paths));
|
let accounts_db = Arc::new(AccountsDB::new(paths));
|
||||||
|
|
||||||
Accounts {
|
Accounts {
|
||||||
|
@ -76,7 +76,7 @@ impl Accounts {
|
||||||
pub fn accounts_from_stream<R: Read, P: AsRef<Path>>(
|
pub fn accounts_from_stream<R: Read, P: AsRef<Path>>(
|
||||||
&self,
|
&self,
|
||||||
stream: &mut BufReader<R>,
|
stream: &mut BufReader<R>,
|
||||||
local_paths: String,
|
local_paths: &[PathBuf],
|
||||||
append_vecs_path: P,
|
append_vecs_path: P,
|
||||||
) -> std::result::Result<(), IOError> {
|
) -> std::result::Result<(), IOError> {
|
||||||
self.accounts_db
|
self.accounts_db
|
||||||
|
@ -621,7 +621,7 @@ mod tests {
|
||||||
) -> Vec<Result<TransactionLoadResult>> {
|
) -> Vec<Result<TransactionLoadResult>> {
|
||||||
let mut hash_queue = BlockhashQueue::new(100);
|
let mut hash_queue = BlockhashQueue::new(100);
|
||||||
hash_queue.register_hash(&tx.message().recent_blockhash, &fee_calculator);
|
hash_queue.register_hash(&tx.message().recent_blockhash, &fee_calculator);
|
||||||
let accounts = Accounts::new(None);
|
let accounts = Accounts::new(Vec::new());
|
||||||
for ka in ka.iter() {
|
for ka in ka.iter() {
|
||||||
accounts.store_slow(0, &ka.0, &ka.1);
|
accounts.store_slow(0, &ka.0, &ka.1);
|
||||||
}
|
}
|
||||||
|
@ -1052,7 +1052,7 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_load_by_program_slot() {
|
fn test_load_by_program_slot() {
|
||||||
let accounts = Accounts::new(None);
|
let accounts = Accounts::new(Vec::new());
|
||||||
|
|
||||||
// Load accounts owned by various programs into AccountsDB
|
// Load accounts owned by various programs into AccountsDB
|
||||||
let pubkey0 = Pubkey::new_rand();
|
let pubkey0 = Pubkey::new_rand();
|
||||||
|
@ -1075,7 +1075,7 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accounts_account_not_found() {
|
fn test_accounts_account_not_found() {
|
||||||
let accounts = Accounts::new(None);
|
let accounts = Accounts::new(Vec::new());
|
||||||
let mut error_counters = ErrorCounters::default();
|
let mut error_counters = ErrorCounters::default();
|
||||||
let ancestors = vec![(0, 0)].into_iter().collect();
|
let ancestors = vec![(0, 0)].into_iter().collect();
|
||||||
|
|
||||||
|
@ -1097,14 +1097,14 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
#[should_panic]
|
#[should_panic]
|
||||||
fn test_accounts_empty_hash_internal_state() {
|
fn test_accounts_empty_hash_internal_state() {
|
||||||
let accounts = Accounts::new(None);
|
let accounts = Accounts::new(Vec::new());
|
||||||
accounts.hash_internal_state(0);
|
accounts.hash_internal_state(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[should_panic]
|
#[should_panic]
|
||||||
fn test_accounts_empty_account_hash_internal_state() {
|
fn test_accounts_empty_account_hash_internal_state() {
|
||||||
let accounts = Accounts::new(None);
|
let accounts = Accounts::new(Vec::new());
|
||||||
accounts.store_slow(0, &Pubkey::default(), &Account::new(1, 0, &sysvar::id()));
|
accounts.store_slow(0, &Pubkey::default(), &Account::new(1, 0, &sysvar::id()));
|
||||||
accounts.hash_internal_state(0);
|
accounts.hash_internal_state(0);
|
||||||
}
|
}
|
||||||
|
@ -1126,7 +1126,7 @@ mod tests {
|
||||||
fn test_accounts_serialize() {
|
fn test_accounts_serialize() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let (_accounts_dir, paths) = get_temp_accounts_paths(4).unwrap();
|
let (_accounts_dir, paths) = get_temp_accounts_paths(4).unwrap();
|
||||||
let accounts = Accounts::new(Some(paths));
|
let accounts = Accounts::new(paths);
|
||||||
|
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_test_accounts(&accounts, &mut pubkeys, 100, 0);
|
create_test_accounts(&accounts, &mut pubkeys, 100, 0);
|
||||||
|
@ -1148,9 +1148,9 @@ mod tests {
|
||||||
let buf = writer.into_inner();
|
let buf = writer.into_inner();
|
||||||
let mut reader = BufReader::new(&buf[..]);
|
let mut reader = BufReader::new(&buf[..]);
|
||||||
let (_accounts_dir, daccounts_paths) = get_temp_accounts_paths(2).unwrap();
|
let (_accounts_dir, daccounts_paths) = get_temp_accounts_paths(2).unwrap();
|
||||||
let daccounts = Accounts::new(Some(daccounts_paths.clone()));
|
let daccounts = Accounts::new(daccounts_paths.clone());
|
||||||
assert!(daccounts
|
assert!(daccounts
|
||||||
.accounts_from_stream(&mut reader, daccounts_paths, copied_accounts.path())
|
.accounts_from_stream(&mut reader, &daccounts_paths, copied_accounts.path())
|
||||||
.is_ok());
|
.is_ok());
|
||||||
check_accounts(&daccounts, &pubkeys, 100);
|
check_accounts(&daccounts, &pubkeys, 100);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
@ -1171,7 +1171,7 @@ mod tests {
|
||||||
let account2 = Account::new(3, 0, &Pubkey::default());
|
let account2 = Account::new(3, 0, &Pubkey::default());
|
||||||
let account3 = Account::new(4, 0, &Pubkey::default());
|
let account3 = Account::new(4, 0, &Pubkey::default());
|
||||||
|
|
||||||
let accounts = Accounts::new(None);
|
let accounts = Accounts::new(Vec::new());
|
||||||
accounts.store_slow(0, &keypair0.pubkey(), &account0);
|
accounts.store_slow(0, &keypair0.pubkey(), &account0);
|
||||||
accounts.store_slow(0, &keypair1.pubkey(), &account1);
|
accounts.store_slow(0, &keypair1.pubkey(), &account1);
|
||||||
accounts.store_slow(0, &keypair2.pubkey(), &account2);
|
accounts.store_slow(0, &keypair2.pubkey(), &account2);
|
||||||
|
@ -1283,7 +1283,7 @@ mod tests {
|
||||||
let account1 = Account::new(2, 0, &Pubkey::default());
|
let account1 = Account::new(2, 0, &Pubkey::default());
|
||||||
let account2 = Account::new(3, 0, &Pubkey::default());
|
let account2 = Account::new(3, 0, &Pubkey::default());
|
||||||
|
|
||||||
let accounts = Accounts::new(None);
|
let accounts = Accounts::new(Vec::new());
|
||||||
accounts.store_slow(0, &keypair0.pubkey(), &account0);
|
accounts.store_slow(0, &keypair0.pubkey(), &account0);
|
||||||
accounts.store_slow(0, &keypair1.pubkey(), &account1);
|
accounts.store_slow(0, &keypair1.pubkey(), &account1);
|
||||||
accounts.store_slow(0, &keypair2.pubkey(), &account2);
|
accounts.store_slow(0, &keypair2.pubkey(), &account2);
|
||||||
|
@ -1404,7 +1404,7 @@ mod tests {
|
||||||
|
|
||||||
let mut loaded = vec![loaded0, loaded1];
|
let mut loaded = vec![loaded0, loaded1];
|
||||||
|
|
||||||
let accounts = Accounts::new(None);
|
let accounts = Accounts::new(Vec::new());
|
||||||
{
|
{
|
||||||
let mut readonly_locks = accounts.readonly_locks.write().unwrap();
|
let mut readonly_locks = accounts.readonly_locks.write().unwrap();
|
||||||
let readonly_locks = readonly_locks.as_mut().unwrap();
|
let readonly_locks = readonly_locks.as_mut().unwrap();
|
||||||
|
|
|
@ -307,18 +307,11 @@ impl AccountStorageEntry {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_paths_vec(paths: &str) -> Vec<PathBuf> {
|
pub fn get_temp_accounts_paths(count: u32) -> IOResult<(Vec<TempDir>, Vec<PathBuf>)> {
|
||||||
paths.split(',').map(PathBuf::from).collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn get_temp_accounts_paths(count: u32) -> IOResult<(Vec<TempDir>, String)> {
|
|
||||||
let temp_dirs: IOResult<Vec<TempDir>> = (0..count).map(|_| TempDir::new()).collect();
|
let temp_dirs: IOResult<Vec<TempDir>> = (0..count).map(|_| TempDir::new()).collect();
|
||||||
let temp_dirs = temp_dirs?;
|
let temp_dirs = temp_dirs?;
|
||||||
let paths: Vec<String> = temp_dirs
|
let paths: Vec<PathBuf> = temp_dirs.iter().map(|t| t.path().to_path_buf()).collect();
|
||||||
.iter()
|
Ok((temp_dirs, paths))
|
||||||
.map(|t| t.path().to_str().unwrap().to_owned())
|
|
||||||
.collect();
|
|
||||||
Ok((temp_dirs, paths.join(",")))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct AccountsDBSerialize<'a> {
|
pub struct AccountsDBSerialize<'a> {
|
||||||
|
@ -413,10 +406,10 @@ impl Default for AccountsDB {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl AccountsDB {
|
impl AccountsDB {
|
||||||
pub fn new(paths: Option<String>) -> Self {
|
pub fn new(paths: Vec<PathBuf>) -> Self {
|
||||||
if let Some(paths) = paths {
|
if !paths.is_empty() {
|
||||||
Self {
|
Self {
|
||||||
paths: RwLock::new(get_paths_vec(&paths)),
|
paths: RwLock::new(paths),
|
||||||
temp_paths: None,
|
temp_paths: None,
|
||||||
..Self::default()
|
..Self::default()
|
||||||
}
|
}
|
||||||
|
@ -425,7 +418,7 @@ impl AccountsDB {
|
||||||
// for testing
|
// for testing
|
||||||
let (temp_dirs, paths) = get_temp_accounts_paths(DEFAULT_NUM_DIRS).unwrap();
|
let (temp_dirs, paths) = get_temp_accounts_paths(DEFAULT_NUM_DIRS).unwrap();
|
||||||
Self {
|
Self {
|
||||||
paths: RwLock::new(get_paths_vec(&paths)),
|
paths: RwLock::new(paths),
|
||||||
temp_paths: Some(temp_dirs),
|
temp_paths: Some(temp_dirs),
|
||||||
..Self::default()
|
..Self::default()
|
||||||
}
|
}
|
||||||
|
@ -436,29 +429,21 @@ impl AccountsDB {
|
||||||
pub fn new_single() -> Self {
|
pub fn new_single() -> Self {
|
||||||
AccountsDB {
|
AccountsDB {
|
||||||
min_num_stores: 0,
|
min_num_stores: 0,
|
||||||
..AccountsDB::new(None)
|
..AccountsDB::new(Vec::new())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub fn new_sized(paths: Option<String>, file_size: u64) -> Self {
|
pub fn new_sized(paths: Vec<PathBuf>, file_size: u64) -> Self {
|
||||||
AccountsDB {
|
AccountsDB {
|
||||||
file_size,
|
file_size,
|
||||||
..AccountsDB::new(paths)
|
..AccountsDB::new(paths)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn format_paths<P: AsRef<Path>>(paths: Vec<P>) -> String {
|
|
||||||
let paths: Vec<String> = paths
|
|
||||||
.iter()
|
|
||||||
.map(|p| p.as_ref().to_str().unwrap().to_owned())
|
|
||||||
.collect();
|
|
||||||
paths.join(",")
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn accounts_from_stream<R: Read, P: AsRef<Path>>(
|
pub fn accounts_from_stream<R: Read, P: AsRef<Path>>(
|
||||||
&self,
|
&self,
|
||||||
mut stream: &mut BufReader<R>,
|
mut stream: &mut BufReader<R>,
|
||||||
local_account_paths: String,
|
local_account_paths: &[PathBuf],
|
||||||
append_vecs_path: P,
|
append_vecs_path: P,
|
||||||
) -> Result<(), IOError> {
|
) -> Result<(), IOError> {
|
||||||
let _len: usize =
|
let _len: usize =
|
||||||
|
@ -467,7 +452,6 @@ impl AccountsDB {
|
||||||
deserialize_from(&mut stream).map_err(|e| AccountsDB::get_io_error(&e.to_string()))?;
|
deserialize_from(&mut stream).map_err(|e| AccountsDB::get_io_error(&e.to_string()))?;
|
||||||
|
|
||||||
// Remap the deserialized AppendVec paths to point to correct local paths
|
// Remap the deserialized AppendVec paths to point to correct local paths
|
||||||
let local_account_paths = get_paths_vec(&local_account_paths);
|
|
||||||
let new_storage_map: Result<HashMap<Slot, SlotStores>, IOError> = storage
|
let new_storage_map: Result<HashMap<Slot, SlotStores>, IOError> = storage
|
||||||
.0
|
.0
|
||||||
.into_iter()
|
.into_iter()
|
||||||
|
@ -534,7 +518,7 @@ impl AccountsDB {
|
||||||
.insert(slot_hash.0, slot_hash.1);
|
.insert(slot_hash.0, slot_hash.1);
|
||||||
|
|
||||||
// Process deserialized data, set necessary fields in self
|
// Process deserialized data, set necessary fields in self
|
||||||
*self.paths.write().unwrap() = local_account_paths;
|
*self.paths.write().unwrap() = local_account_paths.to_vec();
|
||||||
let max_id: usize = *storage
|
let max_id: usize = *storage
|
||||||
.0
|
.0
|
||||||
.values()
|
.values()
|
||||||
|
@ -1214,7 +1198,7 @@ pub mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accountsdb_add_root() {
|
fn test_accountsdb_add_root() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let db = AccountsDB::new(None);
|
let db = AccountsDB::new(Vec::new());
|
||||||
let key = Pubkey::default();
|
let key = Pubkey::default();
|
||||||
let account0 = Account::new(1, 0, &key);
|
let account0 = Account::new(1, 0, &key);
|
||||||
|
|
||||||
|
@ -1227,7 +1211,7 @@ pub mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accountsdb_latest_ancestor() {
|
fn test_accountsdb_latest_ancestor() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let db = AccountsDB::new(None);
|
let db = AccountsDB::new(Vec::new());
|
||||||
let key = Pubkey::default();
|
let key = Pubkey::default();
|
||||||
let account0 = Account::new(1, 0, &key);
|
let account0 = Account::new(1, 0, &key);
|
||||||
|
|
||||||
|
@ -1254,7 +1238,7 @@ pub mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accountsdb_latest_ancestor_with_root() {
|
fn test_accountsdb_latest_ancestor_with_root() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let db = AccountsDB::new(None);
|
let db = AccountsDB::new(Vec::new());
|
||||||
let key = Pubkey::default();
|
let key = Pubkey::default();
|
||||||
let account0 = Account::new(1, 0, &key);
|
let account0 = Account::new(1, 0, &key);
|
||||||
|
|
||||||
|
@ -1274,7 +1258,7 @@ pub mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accountsdb_root_one_slot() {
|
fn test_accountsdb_root_one_slot() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let db = AccountsDB::new(None);
|
let db = AccountsDB::new(Vec::new());
|
||||||
|
|
||||||
let key = Pubkey::default();
|
let key = Pubkey::default();
|
||||||
let account0 = Account::new(1, 0, &key);
|
let account0 = Account::new(1, 0, &key);
|
||||||
|
@ -1315,7 +1299,7 @@ pub mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accountsdb_add_root_many() {
|
fn test_accountsdb_add_root_many() {
|
||||||
let db = AccountsDB::new(None);
|
let db = AccountsDB::new(Vec::new());
|
||||||
|
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_account(&db, &mut pubkeys, 0, 100, 0, 0);
|
create_account(&db, &mut pubkeys, 0, 100, 0, 0);
|
||||||
|
@ -1383,7 +1367,7 @@ pub mod tests {
|
||||||
let key = Pubkey::default();
|
let key = Pubkey::default();
|
||||||
|
|
||||||
// 1 token in the "root", i.e. db zero
|
// 1 token in the "root", i.e. db zero
|
||||||
let db0 = AccountsDB::new(None);
|
let db0 = AccountsDB::new(Vec::new());
|
||||||
let account0 = Account::new(1, 0, &key);
|
let account0 = Account::new(1, 0, &key);
|
||||||
db0.store(0, &[(&key, &account0)]);
|
db0.store(0, &[(&key, &account0)]);
|
||||||
|
|
||||||
|
@ -1492,7 +1476,7 @@ pub mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_account_one() {
|
fn test_account_one() {
|
||||||
let (_accounts_dirs, paths) = get_temp_accounts_paths(1).unwrap();
|
let (_accounts_dirs, paths) = get_temp_accounts_paths(1).unwrap();
|
||||||
let db = AccountsDB::new(Some(paths));
|
let db = AccountsDB::new(paths);
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_account(&db, &mut pubkeys, 0, 1, 0, 0);
|
create_account(&db, &mut pubkeys, 0, 1, 0, 0);
|
||||||
let ancestors = vec![(0, 0)].into_iter().collect();
|
let ancestors = vec![(0, 0)].into_iter().collect();
|
||||||
|
@ -1505,7 +1489,7 @@ pub mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_account_many() {
|
fn test_account_many() {
|
||||||
let (_accounts_dirs, paths) = get_temp_accounts_paths(2).unwrap();
|
let (_accounts_dirs, paths) = get_temp_accounts_paths(2).unwrap();
|
||||||
let db = AccountsDB::new(Some(paths));
|
let db = AccountsDB::new(paths);
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_account(&db, &mut pubkeys, 0, 100, 0, 0);
|
create_account(&db, &mut pubkeys, 0, 100, 0, 0);
|
||||||
check_accounts(&db, &pubkeys, 0, 100, 1);
|
check_accounts(&db, &pubkeys, 0, 100, 1);
|
||||||
|
@ -1524,7 +1508,7 @@ pub mod tests {
|
||||||
fn test_account_grow_many() {
|
fn test_account_grow_many() {
|
||||||
let (_accounts_dir, paths) = get_temp_accounts_paths(2).unwrap();
|
let (_accounts_dir, paths) = get_temp_accounts_paths(2).unwrap();
|
||||||
let size = 4096;
|
let size = 4096;
|
||||||
let accounts = AccountsDB::new_sized(Some(paths), size);
|
let accounts = AccountsDB::new_sized(paths, size);
|
||||||
let mut keys = vec![];
|
let mut keys = vec![];
|
||||||
for i in 0..9 {
|
for i in 0..9 {
|
||||||
let key = Pubkey::new_rand();
|
let key = Pubkey::new_rand();
|
||||||
|
@ -1623,7 +1607,7 @@ pub mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_purge_slot_not_root() {
|
fn test_purge_slot_not_root() {
|
||||||
let accounts = AccountsDB::new(None);
|
let accounts = AccountsDB::new(Vec::new());
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_account(&accounts, &mut pubkeys, 0, 1, 0, 0);
|
create_account(&accounts, &mut pubkeys, 0, 1, 0, 0);
|
||||||
let ancestors = vec![(0, 0)].into_iter().collect();
|
let ancestors = vec![(0, 0)].into_iter().collect();
|
||||||
|
@ -1634,7 +1618,7 @@ pub mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_purge_slot_after_root() {
|
fn test_purge_slot_after_root() {
|
||||||
let accounts = AccountsDB::new(None);
|
let accounts = AccountsDB::new(Vec::new());
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_account(&accounts, &mut pubkeys, 0, 1, 0, 0);
|
create_account(&accounts, &mut pubkeys, 0, 1, 0, 0);
|
||||||
let ancestors = vec![(0, 0)].into_iter().collect();
|
let ancestors = vec![(0, 0)].into_iter().collect();
|
||||||
|
@ -1648,7 +1632,7 @@ pub mod tests {
|
||||||
//This test is pedantic
|
//This test is pedantic
|
||||||
//A slot is purged when a non root bank is cleaned up. If a slot is behind root but it is
|
//A slot is purged when a non root bank is cleaned up. If a slot is behind root but it is
|
||||||
//not root, it means we are retaining dead banks.
|
//not root, it means we are retaining dead banks.
|
||||||
let accounts = AccountsDB::new(None);
|
let accounts = AccountsDB::new(Vec::new());
|
||||||
let pubkey = Pubkey::new_rand();
|
let pubkey = Pubkey::new_rand();
|
||||||
let account = Account::new(1, 0, &Account::default().owner);
|
let account = Account::new(1, 0, &Account::default().owner);
|
||||||
//store an account
|
//store an account
|
||||||
|
@ -1811,18 +1795,13 @@ pub mod tests {
|
||||||
|
|
||||||
let buf = writer.into_inner();
|
let buf = writer.into_inner();
|
||||||
let mut reader = BufReader::new(&buf[..]);
|
let mut reader = BufReader::new(&buf[..]);
|
||||||
let daccounts = AccountsDB::new(None);
|
let daccounts = AccountsDB::new(Vec::new());
|
||||||
|
let local_paths = daccounts.paths.read().unwrap().clone();
|
||||||
let local_paths = {
|
|
||||||
let paths = daccounts.paths.read().unwrap();
|
|
||||||
AccountsDB::format_paths(paths.to_vec())
|
|
||||||
};
|
|
||||||
|
|
||||||
let copied_accounts = TempDir::new().unwrap();
|
let copied_accounts = TempDir::new().unwrap();
|
||||||
// Simulate obtaining a copy of the AppendVecs from a tarball
|
// Simulate obtaining a copy of the AppendVecs from a tarball
|
||||||
copy_append_vecs(&accounts, copied_accounts.path()).unwrap();
|
copy_append_vecs(&accounts, copied_accounts.path()).unwrap();
|
||||||
daccounts
|
daccounts
|
||||||
.accounts_from_stream(&mut reader, local_paths, copied_accounts.path())
|
.accounts_from_stream(&mut reader, &local_paths, copied_accounts.path())
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
print_count_and_status("daccounts", &daccounts);
|
print_count_and_status("daccounts", &daccounts);
|
||||||
|
@ -2023,7 +2002,7 @@ pub mod tests {
|
||||||
let min_file_bytes = std::mem::size_of::<StoredMeta>()
|
let min_file_bytes = std::mem::size_of::<StoredMeta>()
|
||||||
+ std::mem::size_of::<crate::append_vec::AccountMeta>();
|
+ std::mem::size_of::<crate::append_vec::AccountMeta>();
|
||||||
|
|
||||||
let db = Arc::new(AccountsDB::new_sized(None, min_file_bytes as u64));
|
let db = Arc::new(AccountsDB::new_sized(Vec::new(), min_file_bytes as u64));
|
||||||
|
|
||||||
db.add_root(slot_id);
|
db.add_root(slot_id);
|
||||||
let thread_hdls: Vec<_> = (0..num_threads)
|
let thread_hdls: Vec<_> = (0..num_threads)
|
||||||
|
@ -2060,7 +2039,7 @@ pub mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accountsdb_scan_accounts() {
|
fn test_accountsdb_scan_accounts() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let db = AccountsDB::new(None);
|
let db = AccountsDB::new(Vec::new());
|
||||||
let key = Pubkey::default();
|
let key = Pubkey::default();
|
||||||
let key0 = Pubkey::new_rand();
|
let key0 = Pubkey::new_rand();
|
||||||
let account0 = Account::new(1, 0, &key);
|
let account0 = Account::new(1, 0, &key);
|
||||||
|
@ -2093,7 +2072,7 @@ pub mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_store_large_account() {
|
fn test_store_large_account() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let db = AccountsDB::new(None);
|
let db = AccountsDB::new(Vec::new());
|
||||||
|
|
||||||
let key = Pubkey::default();
|
let key = Pubkey::default();
|
||||||
let data_len = DEFAULT_FILE_SIZE as usize + 7;
|
let data_len = DEFAULT_FILE_SIZE as usize + 7;
|
||||||
|
|
|
@ -48,7 +48,7 @@ use solana_vote_program::vote_state::VoteState;
|
||||||
use std::{
|
use std::{
|
||||||
collections::HashMap,
|
collections::HashMap,
|
||||||
io::{BufReader, Cursor, Error as IOError, Read},
|
io::{BufReader, Cursor, Error as IOError, Read},
|
||||||
path::Path,
|
path::{Path, PathBuf},
|
||||||
sync::atomic::{AtomicBool, AtomicU64, Ordering},
|
sync::atomic::{AtomicBool, AtomicU64, Ordering},
|
||||||
sync::{Arc, RwLock, RwLockReadGuard},
|
sync::{Arc, RwLock, RwLockReadGuard},
|
||||||
};
|
};
|
||||||
|
@ -70,8 +70,8 @@ pub struct BankRc {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl BankRc {
|
impl BankRc {
|
||||||
pub fn new(account_paths: String, id: AppendVecId, slot: Slot) -> Self {
|
pub fn new(account_paths: Vec<PathBuf>, id: AppendVecId, slot: Slot) -> Self {
|
||||||
let accounts = Accounts::new(Some(account_paths));
|
let accounts = Accounts::new(account_paths);
|
||||||
accounts
|
accounts
|
||||||
.accounts_db
|
.accounts_db
|
||||||
.next_id
|
.next_id
|
||||||
|
@ -86,7 +86,7 @@ impl BankRc {
|
||||||
pub fn accounts_from_stream<R: Read, P: AsRef<Path>>(
|
pub fn accounts_from_stream<R: Read, P: AsRef<Path>>(
|
||||||
&self,
|
&self,
|
||||||
mut stream: &mut BufReader<R>,
|
mut stream: &mut BufReader<R>,
|
||||||
local_paths: String,
|
local_paths: &[PathBuf],
|
||||||
append_vecs_path: P,
|
append_vecs_path: P,
|
||||||
) -> std::result::Result<(), IOError> {
|
) -> std::result::Result<(), IOError> {
|
||||||
let _len: usize =
|
let _len: usize =
|
||||||
|
@ -289,10 +289,10 @@ impl Default for BlockhashQueue {
|
||||||
|
|
||||||
impl Bank {
|
impl Bank {
|
||||||
pub fn new(genesis_config: &GenesisConfig) -> Self {
|
pub fn new(genesis_config: &GenesisConfig) -> Self {
|
||||||
Self::new_with_paths(&genesis_config, None)
|
Self::new_with_paths(&genesis_config, Vec::new())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn new_with_paths(genesis_config: &GenesisConfig, paths: Option<String>) -> Self {
|
pub fn new_with_paths(genesis_config: &GenesisConfig, paths: Vec<PathBuf>) -> Self {
|
||||||
let mut bank = Self::default();
|
let mut bank = Self::default();
|
||||||
bank.ancestors.insert(bank.slot(), 0);
|
bank.ancestors.insert(bank.slot(), 0);
|
||||||
bank.rc.accounts = Arc::new(Accounts::new(paths));
|
bank.rc.accounts = Arc::new(Accounts::new(paths));
|
||||||
|
@ -417,7 +417,7 @@ impl Bank {
|
||||||
|
|
||||||
pub fn create_with_genesis(
|
pub fn create_with_genesis(
|
||||||
genesis_config: &GenesisConfig,
|
genesis_config: &GenesisConfig,
|
||||||
account_paths: String,
|
account_paths: Vec<PathBuf>,
|
||||||
status_cache_rc: &StatusCacheRc,
|
status_cache_rc: &StatusCacheRc,
|
||||||
id: AppendVecId,
|
id: AppendVecId,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
|
@ -3921,7 +3921,7 @@ mod tests {
|
||||||
copy_append_vecs(&bank2.rc.accounts.accounts_db, copied_accounts.path()).unwrap();
|
copy_append_vecs(&bank2.rc.accounts.accounts_db, copied_accounts.path()).unwrap();
|
||||||
dbank
|
dbank
|
||||||
.rc
|
.rc
|
||||||
.accounts_from_stream(&mut reader, dbank_paths, copied_accounts.path())
|
.accounts_from_stream(&mut reader, &dbank_paths, copied_accounts.path())
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(dbank.get_balance(&key1.pubkey()), 0);
|
assert_eq!(dbank.get_balance(&key1.pubkey()), 0);
|
||||||
assert_eq!(dbank.get_balance(&key2.pubkey()), 10);
|
assert_eq!(dbank.get_balance(&key2.pubkey()), 10);
|
||||||
|
|
|
@ -591,12 +591,28 @@ pub fn main() {
|
||||||
solana_net_utils::parse_port_range(matches.value_of("dynamic_port_range").unwrap())
|
solana_net_utils::parse_port_range(matches.value_of("dynamic_port_range").unwrap())
|
||||||
.expect("invalid dynamic_port_range");
|
.expect("invalid dynamic_port_range");
|
||||||
|
|
||||||
if let Some(account_paths) = matches.value_of("account_paths") {
|
let account_paths = if let Some(account_paths) = matches.value_of("account_paths") {
|
||||||
validator_config.account_paths = Some(account_paths.to_string());
|
account_paths.split(',').map(PathBuf::from).collect()
|
||||||
} else {
|
} else {
|
||||||
validator_config.account_paths =
|
vec![ledger_path.join("accounts")]
|
||||||
Some(ledger_path.join("accounts").to_str().unwrap().to_string());
|
};
|
||||||
|
|
||||||
|
// Create and canonicalize account paths to avoid issues with symlink creation
|
||||||
|
validator_config.account_paths = account_paths
|
||||||
|
.into_iter()
|
||||||
|
.map(|account_path| {
|
||||||
|
match fs::create_dir_all(&account_path).and_then(|_| fs::canonicalize(&account_path)) {
|
||||||
|
Ok(account_path) => account_path,
|
||||||
|
Err(err) => {
|
||||||
|
eprintln!(
|
||||||
|
"Unable to access account path: {:?}, err: {:?}",
|
||||||
|
account_path, err
|
||||||
|
);
|
||||||
|
exit(1);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
let snapshot_interval_slots = value_t_or_exit!(matches, "snapshot_interval_slots", usize);
|
let snapshot_interval_slots = value_t_or_exit!(matches, "snapshot_interval_slots", usize);
|
||||||
let snapshot_path = ledger_path.clone().join("snapshot");
|
let snapshot_path = ledger_path.clone().join("snapshot");
|
||||||
|
|
Loading…
Reference in New Issue