make accounts_db own the directory paths (#5230)
* change paths to something accounts_db (the singleton) owns, fixes SIGILL * fail deserialize if paths don't work serialize paths, too * test that paths are populated from a bank snapshot
This commit is contained in:
parent
b41e8333b1
commit
8a12ed029c
|
@ -1,6 +1,6 @@
|
||||||
use crate::accounts_db::{
|
use crate::accounts_db::{
|
||||||
get_paths_vec, AccountInfo, AccountStorage, AccountsDB, AppendVecId, ErrorCounters,
|
AccountInfo, AccountStorage, AccountsDB, AppendVecId, ErrorCounters, InstructionAccounts,
|
||||||
InstructionAccounts, InstructionCredits, InstructionLoaders,
|
InstructionCredits, InstructionLoaders,
|
||||||
};
|
};
|
||||||
use crate::accounts_index::{AccountsIndex, Fork};
|
use crate::accounts_index::{AccountsIndex, Fork};
|
||||||
use crate::append_vec::StoredAccount;
|
use crate::append_vec::StoredAccount;
|
||||||
|
@ -15,22 +15,15 @@ use solana_sdk::hash::{Hash, Hasher};
|
||||||
use solana_sdk::message::Message;
|
use solana_sdk::message::Message;
|
||||||
use solana_sdk::native_loader;
|
use solana_sdk::native_loader;
|
||||||
use solana_sdk::pubkey::Pubkey;
|
use solana_sdk::pubkey::Pubkey;
|
||||||
use solana_sdk::signature::{Keypair, KeypairUtil};
|
|
||||||
use solana_sdk::system_program;
|
use solana_sdk::system_program;
|
||||||
use solana_sdk::sysvar;
|
use solana_sdk::sysvar;
|
||||||
use solana_sdk::transaction::Result;
|
use solana_sdk::transaction::Result;
|
||||||
use solana_sdk::transaction::{Transaction, TransactionError};
|
use solana_sdk::transaction::{Transaction, TransactionError};
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::env;
|
|
||||||
use std::fs::remove_dir_all;
|
|
||||||
use std::io::{BufReader, Read};
|
use std::io::{BufReader, Read};
|
||||||
use std::path::Path;
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
|
||||||
use std::sync::{Arc, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard};
|
use std::sync::{Arc, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard};
|
||||||
|
|
||||||
const ACCOUNTSDB_DIR: &str = "accountsdb";
|
|
||||||
const NUM_ACCOUNT_DIRS: usize = 4;
|
|
||||||
|
|
||||||
#[derive(Default, Debug)]
|
#[derive(Default, Debug)]
|
||||||
struct CreditOnlyLock {
|
struct CreditOnlyLock {
|
||||||
credits: AtomicU64,
|
credits: AtomicU64,
|
||||||
|
@ -50,73 +43,20 @@ pub struct Accounts {
|
||||||
/// and number of locks. On commit_credits(), we do a take() on the option so that the hashmap
|
/// and number of locks. On commit_credits(), we do a take() on the option so that the hashmap
|
||||||
/// is no longer available to be written to.
|
/// is no longer available to be written to.
|
||||||
credit_only_account_locks: Arc<RwLock<Option<HashMap<Pubkey, CreditOnlyLock>>>>,
|
credit_only_account_locks: Arc<RwLock<Option<HashMap<Pubkey, CreditOnlyLock>>>>,
|
||||||
|
|
||||||
/// List of persistent stores
|
|
||||||
pub paths: String,
|
|
||||||
|
|
||||||
/// set to true if object created the directories in paths
|
|
||||||
/// when true, delete parents of 'paths' on drop
|
|
||||||
own_paths: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for Accounts {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
if self.own_paths {
|
|
||||||
let paths = get_paths_vec(&self.paths);
|
|
||||||
paths.iter().for_each(|p| {
|
|
||||||
let _ignored = remove_dir_all(p);
|
|
||||||
|
|
||||||
// it is safe to delete the parent
|
|
||||||
let path = Path::new(p);
|
|
||||||
let _ignored = remove_dir_all(path.parent().unwrap());
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Accounts {
|
impl Accounts {
|
||||||
fn make_new_dir() -> String {
|
|
||||||
static ACCOUNT_DIR: AtomicUsize = AtomicUsize::new(0);
|
|
||||||
let dir = ACCOUNT_DIR.fetch_add(1, Ordering::Relaxed);
|
|
||||||
let out_dir = env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string());
|
|
||||||
let keypair = Keypair::new();
|
|
||||||
format!(
|
|
||||||
"{}/{}/{}/{}",
|
|
||||||
out_dir,
|
|
||||||
ACCOUNTSDB_DIR,
|
|
||||||
keypair.pubkey(),
|
|
||||||
dir.to_string()
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn make_default_paths() -> String {
|
|
||||||
let mut paths = "".to_string();
|
|
||||||
for index in 0..NUM_ACCOUNT_DIRS {
|
|
||||||
if index > 0 {
|
|
||||||
paths.push_str(",");
|
|
||||||
}
|
|
||||||
paths.push_str(&Self::make_new_dir());
|
|
||||||
}
|
|
||||||
paths
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn new(in_paths: Option<String>) -> Self {
|
pub fn new(in_paths: Option<String>) -> Self {
|
||||||
Self::new_with_num_stores(in_paths, 0)
|
Self::new_with_num_stores(in_paths, 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn new_with_num_stores(in_paths: Option<String>, min_num_stores: usize) -> Self {
|
pub fn new_with_num_stores(paths: Option<String>, min_num_stores: usize) -> Self {
|
||||||
let (paths, own_paths) = if in_paths.is_none() {
|
let accounts_db = Arc::new(AccountsDB::new_with_num_stores(paths, min_num_stores));
|
||||||
(Self::make_default_paths(), true)
|
|
||||||
} else {
|
|
||||||
(in_paths.unwrap(), false)
|
|
||||||
};
|
|
||||||
let accounts_db = Arc::new(AccountsDB::new_with_num_stores(&paths, min_num_stores));
|
|
||||||
Accounts {
|
Accounts {
|
||||||
accounts_db,
|
accounts_db,
|
||||||
account_locks: Mutex::new(HashSet::new()),
|
account_locks: Mutex::new(HashSet::new()),
|
||||||
credit_only_account_locks: Arc::new(RwLock::new(Some(HashMap::new()))),
|
credit_only_account_locks: Arc::new(RwLock::new(Some(HashMap::new()))),
|
||||||
paths,
|
|
||||||
own_paths,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub fn new_from_parent(parent: &Accounts) -> Self {
|
pub fn new_from_parent(parent: &Accounts) -> Self {
|
||||||
|
@ -125,8 +65,6 @@ impl Accounts {
|
||||||
accounts_db,
|
accounts_db,
|
||||||
account_locks: Mutex::new(HashSet::new()),
|
account_locks: Mutex::new(HashSet::new()),
|
||||||
credit_only_account_locks: Arc::new(RwLock::new(Some(HashMap::new()))),
|
credit_only_account_locks: Arc::new(RwLock::new(Some(HashMap::new()))),
|
||||||
paths: parent.paths.clone(),
|
|
||||||
own_paths: parent.own_paths,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1207,16 +1145,19 @@ mod tests {
|
||||||
for _ in 1..num {
|
for _ in 1..num {
|
||||||
let idx = thread_rng().gen_range(0, num - 1);
|
let idx = thread_rng().gen_range(0, num - 1);
|
||||||
let ancestors = vec![(0, 0)].into_iter().collect();
|
let ancestors = vec![(0, 0)].into_iter().collect();
|
||||||
let account = accounts.load_slow(&ancestors, &pubkeys[idx]).unwrap();
|
let account = accounts.load_slow(&ancestors, &pubkeys[idx]);
|
||||||
let account1 = Account::new((idx + 1) as u64, 0, &Account::default().owner);
|
let account1 = Some((
|
||||||
assert_eq!(account, (account1, 0));
|
Account::new((idx + 1) as u64, 0, &Account::default().owner),
|
||||||
|
0,
|
||||||
|
));
|
||||||
|
assert_eq!(account, account1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accounts_serialize() {
|
fn test_accounts_serialize() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let accounts = Accounts::new(Some("serialize_accounts".to_string()));
|
let accounts = Accounts::new(None);
|
||||||
|
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_test_accounts(&accounts, &mut pubkeys, 100);
|
create_test_accounts(&accounts, &mut pubkeys, 100);
|
||||||
|
@ -1229,7 +1170,7 @@ mod tests {
|
||||||
serialize_into(&mut writer, &*accounts.accounts_db).unwrap();
|
serialize_into(&mut writer, &*accounts.accounts_db).unwrap();
|
||||||
|
|
||||||
let mut reader = BufReader::new(&buf[..]);
|
let mut reader = BufReader::new(&buf[..]);
|
||||||
let daccounts = Accounts::new(Some("serialize_accounts".to_string()));
|
let daccounts = Accounts::new(Some(accounts.accounts_db.paths()));
|
||||||
assert!(daccounts.update_from_stream(&mut reader).is_ok());
|
assert!(daccounts.update_from_stream(&mut reader).is_ok());
|
||||||
check_accounts(&daccounts, &pubkeys, 100);
|
check_accounts(&daccounts, &pubkeys, 100);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|
|
@ -33,7 +33,7 @@ use solana_sdk::account::{Account, LamportCredit};
|
||||||
use solana_sdk::pubkey::Pubkey;
|
use solana_sdk::pubkey::Pubkey;
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::fmt;
|
use std::fmt;
|
||||||
use std::fs::{create_dir_all, remove_dir_all};
|
use std::fs::remove_dir_all;
|
||||||
use std::io::{BufReader, Cursor, Error, ErrorKind, Read};
|
use std::io::{BufReader, Cursor, Error, ErrorKind, Read};
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
@ -41,7 +41,6 @@ use std::sync::{Arc, RwLock};
|
||||||
use sys_info;
|
use sys_info;
|
||||||
|
|
||||||
const ACCOUNT_DATA_FILE_SIZE: u64 = 4 * 1024 * 1024;
|
const ACCOUNT_DATA_FILE_SIZE: u64 = 4 * 1024 * 1024;
|
||||||
const ACCOUNT_DATA_FILE: &str = "data";
|
|
||||||
pub const NUM_THREADS: u32 = 10;
|
pub const NUM_THREADS: u32 = 10;
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
|
@ -167,11 +166,9 @@ pub struct AccountStorageEntry {
|
||||||
|
|
||||||
impl AccountStorageEntry {
|
impl AccountStorageEntry {
|
||||||
pub fn new(path: &str, fork_id: Fork, id: usize, file_size: u64) -> Self {
|
pub fn new(path: &str, fork_id: Fork, id: usize, file_size: u64) -> Self {
|
||||||
let p = format!("{}/{}.{}", path, fork_id, id);
|
let tail = format!("{}.{}", fork_id, id);
|
||||||
let path = Path::new(&p);
|
let path = Path::new(path).join(&tail);
|
||||||
let _ignored = remove_dir_all(path);
|
let accounts = AppendVec::new(&path, true, file_size as usize);
|
||||||
create_dir_all(path).expect("Create directory failed");
|
|
||||||
let accounts = AppendVec::new(&path.join(ACCOUNT_DATA_FILE), true, file_size as usize);
|
|
||||||
|
|
||||||
AccountStorageEntry {
|
AccountStorageEntry {
|
||||||
id,
|
id,
|
||||||
|
@ -256,6 +253,37 @@ impl AccountStorageEntry {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn get_paths_vec(paths: &str) -> Vec<String> {
|
||||||
|
paths.split(',').map(ToString::to_string).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct TempPaths {
|
||||||
|
pub paths: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for TempPaths {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let paths = get_paths_vec(&self.paths);
|
||||||
|
paths.iter().for_each(|p| {
|
||||||
|
let _ignored = remove_dir_all(p);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_temp_accounts_path(paths: &str) -> TempPaths {
|
||||||
|
let paths = get_paths_vec(paths);
|
||||||
|
let out_dir = std::env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string());
|
||||||
|
let rand = Pubkey::new_rand();
|
||||||
|
let paths: Vec<_> = paths
|
||||||
|
.iter()
|
||||||
|
.map(|path| format!("{}/accounts_db/{}/{}", out_dir, rand, path))
|
||||||
|
.collect();
|
||||||
|
TempPaths {
|
||||||
|
paths: paths.join(","),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// This structure handles the load/store of the accounts
|
// This structure handles the load/store of the accounts
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct AccountsDB {
|
pub struct AccountsDB {
|
||||||
|
@ -272,7 +300,10 @@ pub struct AccountsDB {
|
||||||
write_version: AtomicUsize,
|
write_version: AtomicUsize,
|
||||||
|
|
||||||
/// Set of storage paths to pick from
|
/// Set of storage paths to pick from
|
||||||
paths: Vec<String>,
|
paths: RwLock<Vec<String>>,
|
||||||
|
|
||||||
|
/// Set of paths this accounts_db needs to hold/remove
|
||||||
|
temp_paths: Option<TempPaths>,
|
||||||
|
|
||||||
/// Starting file size of appendvecs
|
/// Starting file size of appendvecs
|
||||||
file_size: u64,
|
file_size: u64,
|
||||||
|
@ -283,10 +314,6 @@ pub struct AccountsDB {
|
||||||
min_num_stores: usize,
|
min_num_stores: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_paths_vec(paths: &str) -> Vec<String> {
|
|
||||||
paths.split(',').map(ToString::to_string).collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for AccountsDB {
|
impl Default for AccountsDB {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
AccountsDB {
|
AccountsDB {
|
||||||
|
@ -294,7 +321,8 @@ impl Default for AccountsDB {
|
||||||
storage: RwLock::new(AccountStorage(HashMap::new())),
|
storage: RwLock::new(AccountStorage(HashMap::new())),
|
||||||
next_id: AtomicUsize::new(0),
|
next_id: AtomicUsize::new(0),
|
||||||
write_version: AtomicUsize::new(0),
|
write_version: AtomicUsize::new(0),
|
||||||
paths: Vec::default(),
|
paths: RwLock::new(Vec::default()),
|
||||||
|
temp_paths: None,
|
||||||
file_size: u64::default(),
|
file_size: u64::default(),
|
||||||
thread_pool: rayon::ThreadPoolBuilder::new()
|
thread_pool: rayon::ThreadPoolBuilder::new()
|
||||||
.num_threads(2)
|
.num_threads(2)
|
||||||
|
@ -306,20 +334,22 @@ impl Default for AccountsDB {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl AccountsDB {
|
impl AccountsDB {
|
||||||
pub fn new_with_num_stores(paths: &str, min_num_stores: usize) -> Self {
|
pub fn new_with_file_size(paths: Option<String>, file_size: u64) -> Self {
|
||||||
let mut new = Self::new(paths);
|
let (paths, temp_paths) = match paths {
|
||||||
new.min_num_stores = min_num_stores;
|
Some(paths) => (get_paths_vec(&paths), None),
|
||||||
new
|
None => {
|
||||||
}
|
let temp_paths = get_temp_accounts_path("0,1,2,3"); // make 4 directories by default
|
||||||
|
(get_paths_vec(&temp_paths.paths), Some(temp_paths))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
pub fn new_with_file_size(paths: &str, file_size: u64) -> Self {
|
|
||||||
let paths = get_paths_vec(&paths);
|
|
||||||
AccountsDB {
|
AccountsDB {
|
||||||
accounts_index: RwLock::new(AccountsIndex::default()),
|
accounts_index: RwLock::new(AccountsIndex::default()),
|
||||||
storage: RwLock::new(AccountStorage(HashMap::new())),
|
storage: RwLock::new(AccountStorage(HashMap::new())),
|
||||||
next_id: AtomicUsize::new(0),
|
next_id: AtomicUsize::new(0),
|
||||||
write_version: AtomicUsize::new(0),
|
write_version: AtomicUsize::new(0),
|
||||||
paths,
|
paths: RwLock::new(paths),
|
||||||
|
temp_paths,
|
||||||
file_size,
|
file_size,
|
||||||
thread_pool: rayon::ThreadPoolBuilder::new()
|
thread_pool: rayon::ThreadPoolBuilder::new()
|
||||||
.num_threads(sys_info::cpu_num().unwrap_or(NUM_THREADS) as usize)
|
.num_threads(sys_info::cpu_num().unwrap_or(NUM_THREADS) as usize)
|
||||||
|
@ -329,18 +359,28 @@ impl AccountsDB {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn new(paths: &str) -> Self {
|
pub fn new_with_num_stores(paths: Option<String>, min_num_stores: usize) -> Self {
|
||||||
|
let mut new = Self::new(paths);
|
||||||
|
new.min_num_stores = min_num_stores;
|
||||||
|
new
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn new(paths: Option<String>) -> Self {
|
||||||
Self::new_with_file_size(paths, ACCOUNT_DATA_FILE_SIZE)
|
Self::new_with_file_size(paths, ACCOUNT_DATA_FILE_SIZE)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn paths(&self) -> String {
|
||||||
|
self.paths.read().unwrap().join(",")
|
||||||
|
}
|
||||||
|
|
||||||
pub fn update_from_stream<R: Read>(
|
pub fn update_from_stream<R: Read>(
|
||||||
&self,
|
&self,
|
||||||
mut stream: &mut BufReader<R>,
|
mut stream: &mut BufReader<R>,
|
||||||
) -> Result<(), std::io::Error> {
|
) -> Result<(), std::io::Error> {
|
||||||
AppendVec::set_account_paths(&self.paths);
|
|
||||||
|
|
||||||
let _len: usize = deserialize_from(&mut stream)
|
let _len: usize = deserialize_from(&mut stream)
|
||||||
.map_err(|_| AccountsDB::get_io_error("len deserialize error"))?;
|
.map_err(|_| AccountsDB::get_io_error("len deserialize error"))?;
|
||||||
|
*self.paths.write().unwrap() = deserialize_from(&mut stream)
|
||||||
|
.map_err(|_| AccountsDB::get_io_error("paths deserialize error"))?;
|
||||||
let mut storage: AccountStorage = deserialize_from(&mut stream)
|
let mut storage: AccountStorage = deserialize_from(&mut stream)
|
||||||
.map_err(|_| AccountsDB::get_io_error("storage deserialize error"))?;
|
.map_err(|_| AccountsDB::get_io_error("storage deserialize error"))?;
|
||||||
let version: u64 = deserialize_from(&mut stream)
|
let version: u64 = deserialize_from(&mut stream)
|
||||||
|
@ -529,8 +569,9 @@ impl AccountsDB {
|
||||||
fork_storage: &mut ForkStores,
|
fork_storage: &mut ForkStores,
|
||||||
size: u64,
|
size: u64,
|
||||||
) -> Arc<AccountStorageEntry> {
|
) -> Arc<AccountStorageEntry> {
|
||||||
let path_index = thread_rng().gen_range(0, self.paths.len());
|
let paths = self.paths.read().unwrap();
|
||||||
let store = Arc::new(self.new_storage_entry(fork_id, &self.paths[path_index], size));
|
let path_index = thread_rng().gen_range(0, paths.len());
|
||||||
|
let store = Arc::new(self.new_storage_entry(fork_id, &paths[path_index], size));
|
||||||
fork_storage.insert(store.id, store.clone());
|
fork_storage.insert(store.id, store.clone());
|
||||||
store
|
store
|
||||||
}
|
}
|
||||||
|
@ -763,10 +804,13 @@ impl Serialize for AccountsDB {
|
||||||
{
|
{
|
||||||
use serde::ser::Error;
|
use serde::ser::Error;
|
||||||
let storage = self.storage.read().unwrap();
|
let storage = self.storage.read().unwrap();
|
||||||
let len = serialized_size(&*storage).unwrap() + std::mem::size_of::<u64>() as u64;
|
let len = serialized_size(&self.paths).unwrap()
|
||||||
|
+ serialized_size(&*storage).unwrap()
|
||||||
|
+ std::mem::size_of::<u64>() as u64;
|
||||||
let mut buf = vec![0u8; len as usize];
|
let mut buf = vec![0u8; len as usize];
|
||||||
let mut wr = Cursor::new(&mut buf[..]);
|
let mut wr = Cursor::new(&mut buf[..]);
|
||||||
let version: u64 = self.write_version.load(Ordering::Relaxed) as u64;
|
let version: u64 = self.write_version.load(Ordering::Relaxed) as u64;
|
||||||
|
serialize_into(&mut wr, &self.paths).map_err(Error::custom)?;
|
||||||
serialize_into(&mut wr, &*storage).map_err(Error::custom)?;
|
serialize_into(&mut wr, &*storage).map_err(Error::custom)?;
|
||||||
serialize_into(&mut wr, &version).map_err(Error::custom)?;
|
serialize_into(&mut wr, &version).map_err(Error::custom)?;
|
||||||
let len = wr.position() as usize;
|
let len = wr.position() as usize;
|
||||||
|
@ -783,54 +827,10 @@ mod tests {
|
||||||
use rand::{thread_rng, Rng};
|
use rand::{thread_rng, Rng};
|
||||||
use solana_sdk::account::Account;
|
use solana_sdk::account::Account;
|
||||||
|
|
||||||
fn cleanup_paths(paths: &str) {
|
|
||||||
let paths = get_paths_vec(&paths);
|
|
||||||
paths.iter().for_each(|p| {
|
|
||||||
let _ignored = remove_dir_all(p);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
struct TempPaths {
|
|
||||||
pub paths: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for TempPaths {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
cleanup_paths(&self.paths);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_tmp_accounts_path(paths: &str) -> TempPaths {
|
|
||||||
let vpaths = get_paths_vec(paths);
|
|
||||||
let out_dir = std::env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string());
|
|
||||||
let vpaths: Vec<_> = vpaths
|
|
||||||
.iter()
|
|
||||||
.map(|path| format!("{}/{}", out_dir, path))
|
|
||||||
.collect();
|
|
||||||
TempPaths {
|
|
||||||
paths: vpaths.join(","),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[macro_export]
|
|
||||||
macro_rules! tmp_accounts_name {
|
|
||||||
() => {
|
|
||||||
&format!("{}-{}", file!(), line!())
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
#[macro_export]
|
|
||||||
macro_rules! get_tmp_accounts_path {
|
|
||||||
() => {
|
|
||||||
get_tmp_accounts_path(tmp_accounts_name!())
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accountsdb_add_root() {
|
fn test_accountsdb_add_root() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let paths = get_tmp_accounts_path!();
|
let db = AccountsDB::new(None);
|
||||||
let db = AccountsDB::new(&paths.paths);
|
|
||||||
let key = Pubkey::default();
|
let key = Pubkey::default();
|
||||||
let account0 = Account::new(1, 0, &key);
|
let account0 = Account::new(1, 0, &key);
|
||||||
|
|
||||||
|
@ -843,8 +843,7 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accountsdb_latest_ancestor() {
|
fn test_accountsdb_latest_ancestor() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let paths = get_tmp_accounts_path!();
|
let db = AccountsDB::new(None);
|
||||||
let db = AccountsDB::new(&paths.paths);
|
|
||||||
let key = Pubkey::default();
|
let key = Pubkey::default();
|
||||||
let account0 = Account::new(1, 0, &key);
|
let account0 = Account::new(1, 0, &key);
|
||||||
|
|
||||||
|
@ -871,8 +870,7 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accountsdb_latest_ancestor_with_root() {
|
fn test_accountsdb_latest_ancestor_with_root() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let paths = get_tmp_accounts_path!();
|
let db = AccountsDB::new(None);
|
||||||
let db = AccountsDB::new(&paths.paths);
|
|
||||||
let key = Pubkey::default();
|
let key = Pubkey::default();
|
||||||
let account0 = Account::new(1, 0, &key);
|
let account0 = Account::new(1, 0, &key);
|
||||||
|
|
||||||
|
@ -892,8 +890,8 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accountsdb_root_one_fork() {
|
fn test_accountsdb_root_one_fork() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let paths = get_tmp_accounts_path!();
|
let db = AccountsDB::new(None);
|
||||||
let db = AccountsDB::new(&paths.paths);
|
|
||||||
let key = Pubkey::default();
|
let key = Pubkey::default();
|
||||||
let account0 = Account::new(1, 0, &key);
|
let account0 = Account::new(1, 0, &key);
|
||||||
|
|
||||||
|
@ -933,8 +931,7 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accountsdb_add_root_many() {
|
fn test_accountsdb_add_root_many() {
|
||||||
let paths = get_tmp_accounts_path!();
|
let db = AccountsDB::new(None);
|
||||||
let db = AccountsDB::new(&paths.paths);
|
|
||||||
|
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_account(&db, &mut pubkeys, 0, 100, 0, 0);
|
create_account(&db, &mut pubkeys, 0, 100, 0, 0);
|
||||||
|
@ -966,8 +963,7 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accountsdb_count_stores() {
|
fn test_accountsdb_count_stores() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let paths = get_tmp_accounts_path!();
|
let db = AccountsDB::new(None);
|
||||||
let db = AccountsDB::new(&paths.paths);
|
|
||||||
|
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_account(
|
create_account(
|
||||||
|
@ -1010,8 +1006,7 @@ mod tests {
|
||||||
let key = Pubkey::default();
|
let key = Pubkey::default();
|
||||||
|
|
||||||
// 1 token in the "root", i.e. db zero
|
// 1 token in the "root", i.e. db zero
|
||||||
let paths = get_tmp_accounts_path!();
|
let db0 = AccountsDB::new(None);
|
||||||
let db0 = AccountsDB::new(&paths.paths);
|
|
||||||
let account0 = Account::new(1, 0, &key);
|
let account0 = Account::new(1, 0, &key);
|
||||||
db0.store(0, &hashmap!(&key => &account0));
|
db0.store(0, &hashmap!(&key => &account0));
|
||||||
|
|
||||||
|
@ -1035,11 +1030,11 @@ mod tests {
|
||||||
space: usize,
|
space: usize,
|
||||||
num_vote: usize,
|
num_vote: usize,
|
||||||
) {
|
) {
|
||||||
|
let ancestors = vec![(fork, 0)].into_iter().collect();
|
||||||
for t in 0..num {
|
for t in 0..num {
|
||||||
let pubkey = Pubkey::new_rand();
|
let pubkey = Pubkey::new_rand();
|
||||||
let account = Account::new((t + 1) as u64, space, &Account::default().owner);
|
let account = Account::new((t + 1) as u64, space, &Account::default().owner);
|
||||||
pubkeys.push(pubkey.clone());
|
pubkeys.push(pubkey.clone());
|
||||||
let ancestors = vec![(fork, 0)].into_iter().collect();
|
|
||||||
assert!(accounts.load_slow(&ancestors, &pubkey).is_none());
|
assert!(accounts.load_slow(&ancestors, &pubkey).is_none());
|
||||||
accounts.store(fork, &hashmap!(&pubkey => &account));
|
accounts.store(fork, &hashmap!(&pubkey => &account));
|
||||||
}
|
}
|
||||||
|
@ -1092,12 +1087,15 @@ mod tests {
|
||||||
num: usize,
|
num: usize,
|
||||||
count: usize,
|
count: usize,
|
||||||
) {
|
) {
|
||||||
for _ in 1..num {
|
let ancestors = vec![(fork, 0)].into_iter().collect();
|
||||||
let idx = thread_rng().gen_range(0, num - 1);
|
for _ in 0..num {
|
||||||
let ancestors = vec![(fork, 0)].into_iter().collect();
|
let idx = thread_rng().gen_range(0, num);
|
||||||
let account = accounts.load_slow(&ancestors, &pubkeys[idx]).unwrap();
|
let account = accounts.load_slow(&ancestors, &pubkeys[idx]);
|
||||||
let account1 = Account::new((idx + count) as u64, 0, &Account::default().owner);
|
let account1 = Some((
|
||||||
assert_eq!(account, (account1, fork));
|
Account::new((idx + count) as u64, 0, &Account::default().owner),
|
||||||
|
fork,
|
||||||
|
));
|
||||||
|
assert_eq!(account, account1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1116,12 +1114,12 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_account_one() {
|
fn test_account_one() {
|
||||||
let paths = get_tmp_accounts_path!();
|
let paths = get_temp_accounts_path("one");
|
||||||
let accounts = AccountsDB::new(&paths.paths);
|
let db = AccountsDB::new(Some(paths.paths.clone()));
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_account(&accounts, &mut pubkeys, 0, 1, 0, 0);
|
create_account(&db, &mut pubkeys, 0, 1, 0, 0);
|
||||||
let ancestors = vec![(0, 0)].into_iter().collect();
|
let ancestors = vec![(0, 0)].into_iter().collect();
|
||||||
let account = accounts.load_slow(&ancestors, &pubkeys[0]).unwrap();
|
let account = db.load_slow(&ancestors, &pubkeys[0]).unwrap();
|
||||||
let mut default_account = Account::default();
|
let mut default_account = Account::default();
|
||||||
default_account.lamports = 1;
|
default_account.lamports = 1;
|
||||||
assert_eq!((default_account, 0), account);
|
assert_eq!((default_account, 0), account);
|
||||||
|
@ -1129,17 +1127,16 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_account_many() {
|
fn test_account_many() {
|
||||||
let paths = get_tmp_accounts_path("many0,many1");
|
let paths = get_temp_accounts_path("many0,many1");
|
||||||
let accounts = AccountsDB::new(&paths.paths);
|
let db = AccountsDB::new(Some(paths.paths.clone()));
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_account(&accounts, &mut pubkeys, 0, 100, 0, 0);
|
create_account(&db, &mut pubkeys, 0, 100, 0, 0);
|
||||||
check_accounts(&accounts, &pubkeys, 0, 100, 1);
|
check_accounts(&db, &pubkeys, 0, 100, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_account_update() {
|
fn test_account_update() {
|
||||||
let paths = get_tmp_accounts_path!();
|
let accounts = AccountsDB::new(None);
|
||||||
let accounts = AccountsDB::new(&paths.paths);
|
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_account(&accounts, &mut pubkeys, 0, 100, 0, 0);
|
create_account(&accounts, &mut pubkeys, 0, 100, 0, 0);
|
||||||
update_accounts(&accounts, &pubkeys, 0, 99);
|
update_accounts(&accounts, &pubkeys, 0, 99);
|
||||||
|
@ -1148,9 +1145,9 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_account_grow_many() {
|
fn test_account_grow_many() {
|
||||||
let paths = get_tmp_accounts_path("many2,many3");
|
let paths = get_temp_accounts_path("many2,many3");
|
||||||
let size = 4096;
|
let size = 4096;
|
||||||
let accounts = AccountsDB::new_with_file_size(&paths.paths, size);
|
let accounts = AccountsDB::new_with_file_size(Some(paths.paths.clone()), size);
|
||||||
let mut keys = vec![];
|
let mut keys = vec![];
|
||||||
for i in 0..9 {
|
for i in 0..9 {
|
||||||
let key = Pubkey::new_rand();
|
let key = Pubkey::new_rand();
|
||||||
|
@ -1184,8 +1181,7 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_account_grow() {
|
fn test_account_grow() {
|
||||||
let paths = get_tmp_accounts_path!();
|
let accounts = AccountsDB::new(None);
|
||||||
let accounts = AccountsDB::new(&paths.paths);
|
|
||||||
let count = [0, 1];
|
let count = [0, 1];
|
||||||
let status = [AccountStorageStatus::Available, AccountStorageStatus::Full];
|
let status = [AccountStorageStatus::Available, AccountStorageStatus::Full];
|
||||||
let pubkey1 = Pubkey::new_rand();
|
let pubkey1 = Pubkey::new_rand();
|
||||||
|
@ -1249,8 +1245,7 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_purge_fork_not_root() {
|
fn test_purge_fork_not_root() {
|
||||||
let paths = get_tmp_accounts_path!();
|
let accounts = AccountsDB::new(None);
|
||||||
let accounts = AccountsDB::new(&paths.paths);
|
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_account(&accounts, &mut pubkeys, 0, 1, 0, 0);
|
create_account(&accounts, &mut pubkeys, 0, 1, 0, 0);
|
||||||
let ancestors = vec![(0, 0)].into_iter().collect();
|
let ancestors = vec![(0, 0)].into_iter().collect();
|
||||||
|
@ -1261,8 +1256,7 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_purge_fork_after_root() {
|
fn test_purge_fork_after_root() {
|
||||||
let paths = get_tmp_accounts_path!();
|
let accounts = AccountsDB::new(None);
|
||||||
let accounts = AccountsDB::new(&paths.paths);
|
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_account(&accounts, &mut pubkeys, 0, 1, 0, 0);
|
create_account(&accounts, &mut pubkeys, 0, 1, 0, 0);
|
||||||
let ancestors = vec![(0, 0)].into_iter().collect();
|
let ancestors = vec![(0, 0)].into_iter().collect();
|
||||||
|
@ -1276,8 +1270,7 @@ mod tests {
|
||||||
//This test is pedantic
|
//This test is pedantic
|
||||||
//A fork is purged when a non root bank is cleaned up. If a fork is behind root but it is
|
//A fork is purged when a non root bank is cleaned up. If a fork is behind root but it is
|
||||||
//not root, it means we are retaining dead banks.
|
//not root, it means we are retaining dead banks.
|
||||||
let paths = get_tmp_accounts_path!();
|
let accounts = AccountsDB::new(None);
|
||||||
let accounts = AccountsDB::new(&paths.paths);
|
|
||||||
let pubkey = Pubkey::new_rand();
|
let pubkey = Pubkey::new_rand();
|
||||||
let account = Account::new(1, 0, &Account::default().owner);
|
let account = Account::new(1, 0, &Account::default().owner);
|
||||||
//store an account
|
//store an account
|
||||||
|
@ -1309,8 +1302,7 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accounts_db_serialize() {
|
fn test_accounts_db_serialize() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let paths = get_tmp_accounts_path!();
|
let accounts = AccountsDB::new(None);
|
||||||
let accounts = AccountsDB::new(&paths.paths);
|
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_account(&accounts, &mut pubkeys, 0, 100, 0, 0);
|
create_account(&accounts, &mut pubkeys, 0, 100, 0, 0);
|
||||||
assert_eq!(check_storage(&accounts, 0, 100), true);
|
assert_eq!(check_storage(&accounts, 0, 100), true);
|
||||||
|
@ -1329,12 +1321,14 @@ mod tests {
|
||||||
assert!(check_storage(&accounts, 1, 10));
|
assert!(check_storage(&accounts, 1, 10));
|
||||||
|
|
||||||
let mut reader = BufReader::new(&buf[..]);
|
let mut reader = BufReader::new(&buf[..]);
|
||||||
let daccounts = AccountsDB::new(&paths.paths);
|
let daccounts = AccountsDB::new(None);
|
||||||
assert!(daccounts.update_from_stream(&mut reader).is_ok());
|
assert!(daccounts.update_from_stream(&mut reader).is_ok());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
daccounts.write_version.load(Ordering::Relaxed),
|
daccounts.write_version.load(Ordering::Relaxed),
|
||||||
accounts.write_version.load(Ordering::Relaxed)
|
accounts.write_version.load(Ordering::Relaxed)
|
||||||
);
|
);
|
||||||
|
assert_eq!(daccounts.paths(), accounts.paths());
|
||||||
|
|
||||||
check_accounts(&daccounts, &pubkeys, 0, 100, 2);
|
check_accounts(&daccounts, &pubkeys, 0, 100, 2);
|
||||||
check_accounts(&daccounts, &pubkeys1, 1, 10, 1);
|
check_accounts(&daccounts, &pubkeys1, 1, 10, 1);
|
||||||
assert!(check_storage(&daccounts, 0, 100));
|
assert!(check_storage(&daccounts, 0, 100));
|
||||||
|
@ -1346,15 +1340,11 @@ mod tests {
|
||||||
fn test_store_account_stress() {
|
fn test_store_account_stress() {
|
||||||
let fork_id = 42;
|
let fork_id = 42;
|
||||||
let num_threads = 2;
|
let num_threads = 2;
|
||||||
let paths = get_tmp_accounts_path!();
|
|
||||||
|
|
||||||
let min_file_bytes = std::mem::size_of::<StorageMeta>()
|
let min_file_bytes = std::mem::size_of::<StorageMeta>()
|
||||||
+ std::mem::size_of::<crate::append_vec::AccountBalance>();
|
+ std::mem::size_of::<crate::append_vec::AccountBalance>();
|
||||||
|
|
||||||
let db = Arc::new(AccountsDB::new_with_file_size(
|
let db = Arc::new(AccountsDB::new_with_file_size(None, min_file_bytes as u64));
|
||||||
&paths.paths,
|
|
||||||
min_file_bytes as u64,
|
|
||||||
));
|
|
||||||
|
|
||||||
db.add_root(fork_id);
|
db.add_root(fork_id);
|
||||||
let thread_hdls: Vec<_> = (0..num_threads)
|
let thread_hdls: Vec<_> = (0..num_threads)
|
||||||
|
@ -1391,8 +1381,7 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accountsdb_scan_accounts() {
|
fn test_accountsdb_scan_accounts() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let paths = get_tmp_accounts_path!();
|
let db = AccountsDB::new(None);
|
||||||
let db = AccountsDB::new(&paths.paths);
|
|
||||||
let key = Pubkey::default();
|
let key = Pubkey::default();
|
||||||
let key0 = Pubkey::new_rand();
|
let key0 = Pubkey::new_rand();
|
||||||
let account0 = Account::new(1, 0, &key);
|
let account0 = Account::new(1, 0, &key);
|
||||||
|
@ -1425,8 +1414,7 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_store_large_account() {
|
fn test_store_large_account() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let paths = get_tmp_accounts_path!();
|
let db = AccountsDB::new(None);
|
||||||
let db = AccountsDB::new(&paths.paths);
|
|
||||||
|
|
||||||
let key = Pubkey::default();
|
let key = Pubkey::default();
|
||||||
let data_len = ACCOUNT_DATA_FILE_SIZE as usize + 7;
|
let data_len = ACCOUNT_DATA_FILE_SIZE as usize + 7;
|
||||||
|
|
|
@ -1,12 +1,10 @@
|
||||||
use bincode::{deserialize_from, serialize_into, serialized_size};
|
use bincode::{deserialize_from, serialize_into, serialized_size};
|
||||||
use lazy_static::lazy_static;
|
|
||||||
use log::warn;
|
|
||||||
use memmap::MmapMut;
|
use memmap::MmapMut;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use solana_sdk::account::Account;
|
use solana_sdk::account::Account;
|
||||||
use solana_sdk::pubkey::Pubkey;
|
use solana_sdk::pubkey::Pubkey;
|
||||||
use std::fmt;
|
use std::fmt;
|
||||||
use std::fs::OpenOptions;
|
use std::fs::{create_dir_all, remove_file, OpenOptions};
|
||||||
use std::io::{Cursor, Seek, SeekFrom, Write};
|
use std::io::{Cursor, Seek, SeekFrom, Write};
|
||||||
use std::mem;
|
use std::mem;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
@ -63,10 +61,6 @@ impl<'a> StoredAccount<'a> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
lazy_static! {
|
|
||||||
static ref ACCOUNT_PATHS: Mutex<Vec<String>> = Mutex::new(vec![]);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
#[allow(clippy::mutex_atomic)]
|
#[allow(clippy::mutex_atomic)]
|
||||||
pub struct AppendVec {
|
pub struct AppendVec {
|
||||||
|
@ -80,13 +74,20 @@ pub struct AppendVec {
|
||||||
|
|
||||||
impl Drop for AppendVec {
|
impl Drop for AppendVec {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
let _ignored = std::fs::remove_dir_all(&self.path.parent().unwrap());
|
let _ignored = remove_file(&self.path);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl AppendVec {
|
impl AppendVec {
|
||||||
#[allow(clippy::mutex_atomic)]
|
#[allow(clippy::mutex_atomic)]
|
||||||
pub fn new(file: &Path, create: bool, size: usize) -> Self {
|
pub fn new(file: &Path, create: bool, size: usize) -> Self {
|
||||||
|
if create {
|
||||||
|
let _ignored = remove_file(file);
|
||||||
|
if let Some(parent) = file.parent() {
|
||||||
|
create_dir_all(parent).expect("Create directory failed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let mut data = OpenOptions::new()
|
let mut data = OpenOptions::new()
|
||||||
.read(true)
|
.read(true)
|
||||||
.write(true)
|
.write(true)
|
||||||
|
@ -152,6 +153,7 @@ impl AppendVec {
|
||||||
|
|
||||||
fn get_slice(&self, offset: usize, size: usize) -> Option<(&[u8], usize)> {
|
fn get_slice(&self, offset: usize, size: usize) -> Option<(&[u8], usize)> {
|
||||||
let len = self.len();
|
let len = self.len();
|
||||||
|
|
||||||
if len < offset + size {
|
if len < offset + size {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
@ -279,10 +281,6 @@ impl AppendVec {
|
||||||
pub fn append_account_test(&self, data: &(StorageMeta, Account)) -> Option<usize> {
|
pub fn append_account_test(&self, data: &(StorageMeta, Account)) -> Option<usize> {
|
||||||
self.append_account(data.0.clone(), &data.1)
|
self.append_account(data.0.clone(), &data.1)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn set_account_paths(paths: &[String]) {
|
|
||||||
ACCOUNT_PATHS.lock().unwrap().extend_from_slice(paths);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub mod test_utils {
|
pub mod test_utils {
|
||||||
|
@ -379,36 +377,16 @@ impl<'a> serde::de::Visitor<'a> for AppendVecVisitor {
|
||||||
let file_size: u64 = deserialize_from(&mut rd).map_err(Error::custom)?;
|
let file_size: u64 = deserialize_from(&mut rd).map_err(Error::custom)?;
|
||||||
let offset: usize = deserialize_from(&mut rd).map_err(Error::custom)?;
|
let offset: usize = deserialize_from(&mut rd).map_err(Error::custom)?;
|
||||||
|
|
||||||
let split_path: Vec<&str> = path.to_str().unwrap().rsplit('/').collect();
|
|
||||||
let account_paths = ACCOUNT_PATHS.lock().unwrap().clone();
|
|
||||||
let mut account_path = path.clone();
|
|
||||||
if split_path.len() >= 2 {
|
|
||||||
for dir_path in account_paths.iter() {
|
|
||||||
let fullpath = format!("{}/{}/{}", dir_path, split_path[1], split_path[0]);
|
|
||||||
let file_path = Path::new(&fullpath);
|
|
||||||
account_path = file_path.to_path_buf();
|
|
||||||
if file_path.exists() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let data = OpenOptions::new()
|
let data = OpenOptions::new()
|
||||||
.read(true)
|
.read(true)
|
||||||
.write(true)
|
.write(true)
|
||||||
.create(false)
|
.create(false)
|
||||||
.open(account_path.as_path());
|
.open(&path)
|
||||||
|
.map_err(|e| Error::custom(e.to_string()))?;
|
||||||
|
|
||||||
if data.is_err() {
|
let map = unsafe { MmapMut::map_mut(&data).map_err(|e| Error::custom(e.to_string()))? };
|
||||||
warn!("account open {:?} failed", account_path);
|
|
||||||
std::fs::create_dir_all(&account_path.parent().unwrap())
|
|
||||||
.expect("Create directory failed");
|
|
||||||
return Ok(AppendVec::new(&account_path, true, file_size as usize));
|
|
||||||
}
|
|
||||||
|
|
||||||
let map = unsafe { MmapMut::map_mut(&data.unwrap()).expect("failed to map the data file") };
|
|
||||||
Ok(AppendVec {
|
Ok(AppendVec {
|
||||||
path: account_path,
|
path,
|
||||||
map,
|
map,
|
||||||
append_offset: Mutex::new(offset),
|
append_offset: Mutex::new(offset),
|
||||||
current_len: AtomicUsize::new(current_len as usize),
|
current_len: AtomicUsize::new(current_len as usize),
|
||||||
|
@ -499,7 +477,7 @@ pub mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_append_vec_serialize() {
|
fn test_append_vec_serialize() {
|
||||||
let path = get_append_vec_path("test_append_serialize");
|
let path = get_append_vec_path("test_append_serialize");
|
||||||
let av: AppendVec = AppendVec::new(Path::new(&path.path), true, 1024 * 1024);
|
let av: AppendVec = AppendVec::new(&Path::new(&path.path).join("0"), true, 1024 * 1024);
|
||||||
let account1 = create_test_account(1);
|
let account1 = create_test_account(1);
|
||||||
let index1 = av.append_account_test(&account1).unwrap();
|
let index1 = av.append_account_test(&account1).unwrap();
|
||||||
assert_eq!(index1, 0);
|
assert_eq!(index1, 0);
|
||||||
|
@ -514,7 +492,6 @@ pub mod tests {
|
||||||
let mut writer = Cursor::new(&mut buf[..]);
|
let mut writer = Cursor::new(&mut buf[..]);
|
||||||
serialize_into(&mut writer, &av).unwrap();
|
serialize_into(&mut writer, &av).unwrap();
|
||||||
|
|
||||||
AppendVec::set_account_paths(&vec![get_append_vec_dir()]);
|
|
||||||
let mut reader = Cursor::new(&mut buf[..]);
|
let mut reader = Cursor::new(&mut buf[..]);
|
||||||
let dav: AppendVec = deserialize_from(&mut reader).unwrap();
|
let dav: AppendVec = deserialize_from(&mut reader).unwrap();
|
||||||
|
|
||||||
|
@ -522,8 +499,10 @@ pub mod tests {
|
||||||
assert_eq!(dav.get_account_test(index1).unwrap(), account1);
|
assert_eq!(dav.get_account_test(index1).unwrap(), account1);
|
||||||
drop(dav);
|
drop(dav);
|
||||||
|
|
||||||
|
// dropping dav above blows away underlying file's directory entry,
|
||||||
|
// which is what we're testing next.
|
||||||
let mut reader = Cursor::new(&mut buf[..]);
|
let mut reader = Cursor::new(&mut buf[..]);
|
||||||
let dav: AppendVec = deserialize_from(&mut reader).unwrap();
|
let dav: Result<AppendVec, Box<bincode::ErrorKind>> = deserialize_from(&mut reader);
|
||||||
assert!(dav.get_account_test(index2).is_none());
|
assert!(dav.is_err());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -2750,10 +2750,7 @@ mod tests {
|
||||||
let mut rdr = Cursor::new(&buf[..]);
|
let mut rdr = Cursor::new(&buf[..]);
|
||||||
let mut dbank: Bank = deserialize_from(&mut rdr).unwrap();
|
let mut dbank: Bank = deserialize_from(&mut rdr).unwrap();
|
||||||
let mut reader = BufReader::new(&buf[rdr.position() as usize..]);
|
let mut reader = BufReader::new(&buf[rdr.position() as usize..]);
|
||||||
dbank.set_bank_rc(
|
dbank.set_bank_rc(&BankRc::new(None, 0), &StatusCacheRc::default());
|
||||||
&BankRc::new(Some(bank0.accounts().paths.clone()), 0),
|
|
||||||
&StatusCacheRc::default(),
|
|
||||||
);
|
|
||||||
assert!(dbank.rc.update_from_stream(&mut reader).is_ok());
|
assert!(dbank.rc.update_from_stream(&mut reader).is_ok());
|
||||||
assert_eq!(dbank.get_balance(&key.pubkey()), 10);
|
assert_eq!(dbank.get_balance(&key.pubkey()), 10);
|
||||||
bank.compare_bank(&dbank);
|
bank.compare_bank(&dbank);
|
||||||
|
|
Loading…
Reference in New Issue