groom accounts_db (#5283)
This commit is contained in:
parent
d1eff5d607
commit
8537da19bb
|
@ -46,12 +46,8 @@ pub struct Accounts {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Accounts {
|
impl Accounts {
|
||||||
pub fn new(in_paths: Option<String>) -> Self {
|
pub fn new(paths: Option<String>) -> Self {
|
||||||
Self::new_with_num_stores(in_paths, 0)
|
let accounts_db = Arc::new(AccountsDB::new(paths));
|
||||||
}
|
|
||||||
|
|
||||||
pub fn new_with_num_stores(paths: Option<String>, min_num_stores: usize) -> Self {
|
|
||||||
let accounts_db = Arc::new(AccountsDB::new_with_num_stores(paths, min_num_stores));
|
|
||||||
|
|
||||||
Accounts {
|
Accounts {
|
||||||
accounts_db,
|
accounts_db,
|
||||||
|
|
|
@ -40,8 +40,9 @@ use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
use std::sync::{Arc, RwLock};
|
use std::sync::{Arc, RwLock};
|
||||||
use sys_info;
|
use sys_info;
|
||||||
|
|
||||||
const ACCOUNT_DATA_FILE_SIZE: u64 = 4 * 1024 * 1024;
|
pub const DEFAULT_FILE_SIZE: u64 = 4 * 1024 * 1024;
|
||||||
pub const NUM_THREADS: u32 = 10;
|
pub const DEFAULT_NUM_THREADS: u32 = 8;
|
||||||
|
pub const DEFAULT_DIRS: &str = "0,1,2,3";
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
pub struct ErrorCounters {
|
pub struct ErrorCounters {
|
||||||
|
@ -316,57 +317,53 @@ pub struct AccountsDB {
|
||||||
|
|
||||||
impl Default for AccountsDB {
|
impl Default for AccountsDB {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
|
let num_threads = sys_info::cpu_num().unwrap_or(DEFAULT_NUM_THREADS) as usize;
|
||||||
|
|
||||||
|
let temp_paths = get_temp_accounts_path(DEFAULT_DIRS); // make 4 directories by default
|
||||||
|
|
||||||
AccountsDB {
|
AccountsDB {
|
||||||
accounts_index: RwLock::new(AccountsIndex::default()),
|
accounts_index: RwLock::new(AccountsIndex::default()),
|
||||||
storage: RwLock::new(AccountStorage(HashMap::new())),
|
storage: RwLock::new(AccountStorage(HashMap::new())),
|
||||||
next_id: AtomicUsize::new(0),
|
next_id: AtomicUsize::new(0),
|
||||||
write_version: AtomicUsize::new(0),
|
write_version: AtomicUsize::new(0),
|
||||||
paths: RwLock::new(Vec::default()),
|
paths: RwLock::new(get_paths_vec(&temp_paths.paths)),
|
||||||
temp_paths: None,
|
temp_paths: Some(temp_paths),
|
||||||
file_size: u64::default(),
|
file_size: DEFAULT_FILE_SIZE,
|
||||||
thread_pool: rayon::ThreadPoolBuilder::new()
|
thread_pool: rayon::ThreadPoolBuilder::new()
|
||||||
.num_threads(2)
|
.num_threads(num_threads)
|
||||||
.build()
|
.build()
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
min_num_stores: 0,
|
min_num_stores: num_threads,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl AccountsDB {
|
impl AccountsDB {
|
||||||
pub fn new_with_file_size(paths: Option<String>, file_size: u64) -> Self {
|
|
||||||
let (paths, temp_paths) = match paths {
|
|
||||||
Some(paths) => (get_paths_vec(&paths), None),
|
|
||||||
None => {
|
|
||||||
let temp_paths = get_temp_accounts_path("0,1,2,3"); // make 4 directories by default
|
|
||||||
(get_paths_vec(&temp_paths.paths), Some(temp_paths))
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
AccountsDB {
|
|
||||||
accounts_index: RwLock::new(AccountsIndex::default()),
|
|
||||||
storage: RwLock::new(AccountStorage(HashMap::new())),
|
|
||||||
next_id: AtomicUsize::new(0),
|
|
||||||
write_version: AtomicUsize::new(0),
|
|
||||||
paths: RwLock::new(paths),
|
|
||||||
temp_paths,
|
|
||||||
file_size,
|
|
||||||
thread_pool: rayon::ThreadPoolBuilder::new()
|
|
||||||
.num_threads(sys_info::cpu_num().unwrap_or(NUM_THREADS) as usize)
|
|
||||||
.build()
|
|
||||||
.unwrap(),
|
|
||||||
min_num_stores: 0,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn new_with_num_stores(paths: Option<String>, min_num_stores: usize) -> Self {
|
|
||||||
let mut new = Self::new(paths);
|
|
||||||
new.min_num_stores = min_num_stores;
|
|
||||||
new
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn new(paths: Option<String>) -> Self {
|
pub fn new(paths: Option<String>) -> Self {
|
||||||
Self::new_with_file_size(paths, ACCOUNT_DATA_FILE_SIZE)
|
if let Some(paths) = paths {
|
||||||
|
AccountsDB {
|
||||||
|
paths: RwLock::new(get_paths_vec(&paths)),
|
||||||
|
temp_paths: None,
|
||||||
|
..AccountsDB::default()
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
AccountsDB::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub fn new_single() -> Self {
|
||||||
|
AccountsDB {
|
||||||
|
min_num_stores: 0,
|
||||||
|
..AccountsDB::new(None)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[cfg(test)]
|
||||||
|
pub fn new_sized(paths: Option<String>, file_size: u64) -> Self {
|
||||||
|
AccountsDB {
|
||||||
|
file_size,
|
||||||
|
..AccountsDB::new(paths)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn paths(&self) -> String {
|
pub fn paths(&self) -> String {
|
||||||
|
@ -556,7 +553,7 @@ impl AccountsDB {
|
||||||
let fork_storage = stores.0.entry(fork_id).or_insert_with(HashMap::new);
|
let fork_storage = stores.0.entry(fork_id).or_insert_with(HashMap::new);
|
||||||
|
|
||||||
// Create more stores so that when scanning the storage all CPUs have work
|
// Create more stores so that when scanning the storage all CPUs have work
|
||||||
while fork_storage.len() < self.min_num_stores {
|
while fork_storage.len() + 1 < self.min_num_stores {
|
||||||
self.create_store(fork_id, fork_storage, self.file_size);
|
self.create_store(fork_id, fork_storage, self.file_size);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -963,21 +960,14 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accountsdb_count_stores() {
|
fn test_accountsdb_count_stores() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let db = AccountsDB::new(None);
|
let db = AccountsDB::new_single();
|
||||||
|
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_account(
|
create_account(&db, &mut pubkeys, 0, 2, DEFAULT_FILE_SIZE as usize / 3, 0);
|
||||||
&db,
|
|
||||||
&mut pubkeys,
|
|
||||||
0,
|
|
||||||
2,
|
|
||||||
ACCOUNT_DATA_FILE_SIZE as usize / 3,
|
|
||||||
0,
|
|
||||||
);
|
|
||||||
assert!(check_storage(&db, 0, 2));
|
assert!(check_storage(&db, 0, 2));
|
||||||
|
|
||||||
let pubkey = Pubkey::new_rand();
|
let pubkey = Pubkey::new_rand();
|
||||||
let account = Account::new(1, ACCOUNT_DATA_FILE_SIZE as usize / 3, &pubkey);
|
let account = Account::new(1, DEFAULT_FILE_SIZE as usize / 3, &pubkey);
|
||||||
db.store(1, &hashmap!(&pubkey => &account));
|
db.store(1, &hashmap!(&pubkey => &account));
|
||||||
db.store(1, &hashmap!(&pubkeys[0] => &account));
|
db.store(1, &hashmap!(&pubkeys[0] => &account));
|
||||||
{
|
{
|
||||||
|
@ -1136,7 +1126,7 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_account_update() {
|
fn test_account_update() {
|
||||||
let accounts = AccountsDB::new(None);
|
let accounts = AccountsDB::new_single();
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_account(&accounts, &mut pubkeys, 0, 100, 0, 0);
|
create_account(&accounts, &mut pubkeys, 0, 100, 0, 0);
|
||||||
update_accounts(&accounts, &pubkeys, 0, 99);
|
update_accounts(&accounts, &pubkeys, 0, 99);
|
||||||
|
@ -1147,7 +1137,7 @@ mod tests {
|
||||||
fn test_account_grow_many() {
|
fn test_account_grow_many() {
|
||||||
let paths = get_temp_accounts_path("many2,many3");
|
let paths = get_temp_accounts_path("many2,many3");
|
||||||
let size = 4096;
|
let size = 4096;
|
||||||
let accounts = AccountsDB::new_with_file_size(Some(paths.paths.clone()), size);
|
let accounts = AccountsDB::new_sized(Some(paths.paths.clone()), size);
|
||||||
let mut keys = vec![];
|
let mut keys = vec![];
|
||||||
for i in 0..9 {
|
for i in 0..9 {
|
||||||
let key = Pubkey::new_rand();
|
let key = Pubkey::new_rand();
|
||||||
|
@ -1181,11 +1171,12 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_account_grow() {
|
fn test_account_grow() {
|
||||||
let accounts = AccountsDB::new(None);
|
let accounts = AccountsDB::new_single();
|
||||||
|
|
||||||
let count = [0, 1];
|
let count = [0, 1];
|
||||||
let status = [AccountStorageStatus::Available, AccountStorageStatus::Full];
|
let status = [AccountStorageStatus::Available, AccountStorageStatus::Full];
|
||||||
let pubkey1 = Pubkey::new_rand();
|
let pubkey1 = Pubkey::new_rand();
|
||||||
let account1 = Account::new(1, ACCOUNT_DATA_FILE_SIZE as usize / 2, &pubkey1);
|
let account1 = Account::new(1, DEFAULT_FILE_SIZE as usize / 2, &pubkey1);
|
||||||
accounts.store(0, &hashmap!(&pubkey1 => &account1));
|
accounts.store(0, &hashmap!(&pubkey1 => &account1));
|
||||||
{
|
{
|
||||||
let stores = accounts.storage.read().unwrap();
|
let stores = accounts.storage.read().unwrap();
|
||||||
|
@ -1195,7 +1186,7 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
let pubkey2 = Pubkey::new_rand();
|
let pubkey2 = Pubkey::new_rand();
|
||||||
let account2 = Account::new(1, ACCOUNT_DATA_FILE_SIZE as usize / 2, &pubkey2);
|
let account2 = Account::new(1, DEFAULT_FILE_SIZE as usize / 2, &pubkey2);
|
||||||
accounts.store(0, &hashmap!(&pubkey2 => &account2));
|
accounts.store(0, &hashmap!(&pubkey2 => &account2));
|
||||||
{
|
{
|
||||||
let stores = accounts.storage.read().unwrap();
|
let stores = accounts.storage.read().unwrap();
|
||||||
|
@ -1302,7 +1293,7 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accounts_db_serialize() {
|
fn test_accounts_db_serialize() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let accounts = AccountsDB::new(None);
|
let accounts = AccountsDB::new_single();
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_account(&accounts, &mut pubkeys, 0, 100, 0, 0);
|
create_account(&accounts, &mut pubkeys, 0, 100, 0, 0);
|
||||||
assert_eq!(check_storage(&accounts, 0, 100), true);
|
assert_eq!(check_storage(&accounts, 0, 100), true);
|
||||||
|
@ -1344,7 +1335,7 @@ mod tests {
|
||||||
let min_file_bytes = std::mem::size_of::<StorageMeta>()
|
let min_file_bytes = std::mem::size_of::<StorageMeta>()
|
||||||
+ std::mem::size_of::<crate::append_vec::AccountBalance>();
|
+ std::mem::size_of::<crate::append_vec::AccountBalance>();
|
||||||
|
|
||||||
let db = Arc::new(AccountsDB::new_with_file_size(None, min_file_bytes as u64));
|
let db = Arc::new(AccountsDB::new_sized(None, min_file_bytes as u64));
|
||||||
|
|
||||||
db.add_root(fork_id);
|
db.add_root(fork_id);
|
||||||
let thread_hdls: Vec<_> = (0..num_threads)
|
let thread_hdls: Vec<_> = (0..num_threads)
|
||||||
|
@ -1417,7 +1408,7 @@ mod tests {
|
||||||
let db = AccountsDB::new(None);
|
let db = AccountsDB::new(None);
|
||||||
|
|
||||||
let key = Pubkey::default();
|
let key = Pubkey::default();
|
||||||
let data_len = ACCOUNT_DATA_FILE_SIZE as usize + 7;
|
let data_len = DEFAULT_FILE_SIZE as usize + 7;
|
||||||
let account = Account::new(1, data_len, &key);
|
let account = Account::new(1, data_len, &key);
|
||||||
|
|
||||||
db.store(0, &hashmap!(&key => &account));
|
db.store(0, &hashmap!(&key => &account));
|
||||||
|
|
|
@ -50,12 +50,6 @@ use std::sync::{Arc, RwLock, RwLockReadGuard};
|
||||||
|
|
||||||
pub const SECONDS_PER_YEAR: f64 = (365.0 * 24.0 * 60.0 * 60.0);
|
pub const SECONDS_PER_YEAR: f64 = (365.0 * 24.0 * 60.0 * 60.0);
|
||||||
|
|
||||||
// Create many append vecs to increase parallelism in scan_ functions.
|
|
||||||
fn default_num_stores() -> usize {
|
|
||||||
const DEFAULT_NUM_STORES: u32 = 8;
|
|
||||||
sys_info::cpu_num().unwrap_or(DEFAULT_NUM_STORES) as usize
|
|
||||||
}
|
|
||||||
|
|
||||||
type BankStatusCache = StatusCache<Result<()>>;
|
type BankStatusCache = StatusCache<Result<()>>;
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
|
@ -69,7 +63,7 @@ pub struct BankRc {
|
||||||
|
|
||||||
impl BankRc {
|
impl BankRc {
|
||||||
pub fn new(account_paths: Option<String>, id: AppendVecId) -> Self {
|
pub fn new(account_paths: Option<String>, id: AppendVecId) -> Self {
|
||||||
let accounts = Accounts::new_with_num_stores(account_paths, default_num_stores());
|
let accounts = Accounts::new(account_paths);
|
||||||
accounts
|
accounts
|
||||||
.accounts_db
|
.accounts_db
|
||||||
.next_id
|
.next_id
|
||||||
|
@ -286,7 +280,7 @@ impl Bank {
|
||||||
pub fn new_with_paths(genesis_block: &GenesisBlock, paths: Option<String>) -> Self {
|
pub fn new_with_paths(genesis_block: &GenesisBlock, paths: Option<String>) -> Self {
|
||||||
let mut bank = Self::default();
|
let mut bank = Self::default();
|
||||||
bank.ancestors.insert(bank.slot(), 0);
|
bank.ancestors.insert(bank.slot(), 0);
|
||||||
bank.rc.accounts = Arc::new(Accounts::new_with_num_stores(paths, default_num_stores()));
|
bank.rc.accounts = Arc::new(Accounts::new(paths));
|
||||||
bank.process_genesis_block(genesis_block);
|
bank.process_genesis_block(genesis_block);
|
||||||
// genesis needs stakes for all epochs up to the epoch implied by
|
// genesis needs stakes for all epochs up to the epoch implied by
|
||||||
// slot = 0 and genesis configuration
|
// slot = 0 and genesis configuration
|
||||||
|
|
Loading…
Reference in New Issue