Set proper count value for account stores (#4797)
* set count values for store accounts * Use AppendVecId type
This commit is contained in:
parent
74a06e4230
commit
9e7f618cff
|
@ -329,8 +329,9 @@ impl BankForks {
|
||||||
names.sort();
|
names.sort();
|
||||||
let mut bank_maps = vec![];
|
let mut bank_maps = vec![];
|
||||||
let status_cache_rc = StatusCacheRc::default();
|
let status_cache_rc = StatusCacheRc::default();
|
||||||
|
let id = (names[names.len() - 1] + 1) as usize;
|
||||||
let mut bank0 =
|
let mut bank0 =
|
||||||
Bank::create_with_genesis(&genesis_block, account_paths.clone(), &status_cache_rc);
|
Bank::create_with_genesis(&genesis_block, account_paths.clone(), &status_cache_rc, id);
|
||||||
bank0.freeze();
|
bank0.freeze();
|
||||||
let bank_root = BankForks::load_snapshots(
|
let bank_root = BankForks::load_snapshots(
|
||||||
&names,
|
&names,
|
||||||
|
|
|
@ -245,6 +245,8 @@ impl AccountStorageEntry {
|
||||||
|
|
||||||
if count > 0 {
|
if count > 0 {
|
||||||
*count_and_status = (count - 1, status);
|
*count_and_status = (count - 1, status);
|
||||||
|
} else {
|
||||||
|
warn!("count value 0 for fork {}", self.fork_id);
|
||||||
}
|
}
|
||||||
count_and_status.0
|
count_and_status.0
|
||||||
}
|
}
|
||||||
|
@ -260,7 +262,7 @@ pub struct AccountsDB {
|
||||||
pub storage: RwLock<AccountStorage>,
|
pub storage: RwLock<AccountStorage>,
|
||||||
|
|
||||||
/// distribute the accounts across storage lists
|
/// distribute the accounts across storage lists
|
||||||
next_id: AtomicUsize,
|
pub next_id: AtomicUsize,
|
||||||
|
|
||||||
/// write version
|
/// write version
|
||||||
write_version: AtomicUsize,
|
write_version: AtomicUsize,
|
||||||
|
@ -325,9 +327,7 @@ impl AccountsDB {
|
||||||
|
|
||||||
let _len: usize = deserialize_from(&mut stream)
|
let _len: usize = deserialize_from(&mut stream)
|
||||||
.map_err(|_| AccountsDB::get_io_error("len deserialize error"))?;
|
.map_err(|_| AccountsDB::get_io_error("len deserialize error"))?;
|
||||||
let accounts_index: AccountsIndex<AccountInfo> = deserialize_from(&mut stream)
|
let mut storage: AccountStorage = deserialize_from(&mut stream)
|
||||||
.map_err(|_| AccountsDB::get_io_error("accounts index deserialize error"))?;
|
|
||||||
let storage: AccountStorage = deserialize_from(&mut stream)
|
|
||||||
.map_err(|_| AccountsDB::get_io_error("storage deserialize error"))?;
|
.map_err(|_| AccountsDB::get_io_error("storage deserialize error"))?;
|
||||||
let version: u64 = deserialize_from(&mut stream)
|
let version: u64 = deserialize_from(&mut stream)
|
||||||
.map_err(|_| AccountsDB::get_io_error("write version deserialize error"))?;
|
.map_err(|_| AccountsDB::get_io_error("write version deserialize error"))?;
|
||||||
|
@ -341,11 +341,13 @@ impl AccountsDB {
|
||||||
ids.sort();
|
ids.sort();
|
||||||
|
|
||||||
{
|
{
|
||||||
let mut index = self.accounts_index.write().unwrap();
|
|
||||||
let union = index.roots.union(&accounts_index.roots);
|
|
||||||
index.roots = union.cloned().collect();
|
|
||||||
index.last_root = accounts_index.last_root;
|
|
||||||
let mut stores = self.storage.write().unwrap();
|
let mut stores = self.storage.write().unwrap();
|
||||||
|
if let Some((_, store0)) = storage.0.remove_entry(&0) {
|
||||||
|
let fork_storage0 = stores.0.entry(0).or_insert_with(HashMap::new);
|
||||||
|
for (id, store) in store0.iter() {
|
||||||
|
fork_storage0.insert(*id, store.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
stores.0.extend(storage.0);
|
stores.0.extend(storage.0);
|
||||||
}
|
}
|
||||||
self.next_id
|
self.next_id
|
||||||
|
@ -627,8 +629,8 @@ impl AccountsDB {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn generate_index(&self) {
|
fn generate_index(&self) {
|
||||||
let mut forks: Vec<Fork> = self.storage.read().unwrap().0.keys().cloned().collect();
|
let storage = self.storage.read().unwrap();
|
||||||
|
let mut forks: Vec<Fork> = storage.0.keys().cloned().collect();
|
||||||
forks.sort();
|
forks.sort();
|
||||||
let mut accounts_index = self.accounts_index.write().unwrap();
|
let mut accounts_index = self.accounts_index.write().unwrap();
|
||||||
accounts_index.roots.insert(0);
|
accounts_index.roots.insert(0);
|
||||||
|
@ -655,8 +657,12 @@ impl AccountsDB {
|
||||||
while let Some(maps) = accumulator.pop() {
|
while let Some(maps) = accumulator.pop() {
|
||||||
AccountsDB::merge(&mut account_maps, &maps);
|
AccountsDB::merge(&mut account_maps, &maps);
|
||||||
}
|
}
|
||||||
for (pubkey, (_, account_info)) in account_maps.iter() {
|
if !account_maps.is_empty() {
|
||||||
accounts_index.add_index(*fork_id, pubkey, account_info.clone());
|
accounts_index.roots.insert(*fork_id);
|
||||||
|
let mut _reclaims: Vec<(u64, AccountInfo)> = vec![];
|
||||||
|
for (pubkey, (_, account_info)) in account_maps.iter() {
|
||||||
|
accounts_index.insert(*fork_id, pubkey, account_info.clone(), &mut _reclaims);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -668,15 +674,11 @@ impl Serialize for AccountsDB {
|
||||||
S: serde::ser::Serializer,
|
S: serde::ser::Serializer,
|
||||||
{
|
{
|
||||||
use serde::ser::Error;
|
use serde::ser::Error;
|
||||||
let accounts_index = self.accounts_index.read().unwrap();
|
|
||||||
let storage = self.storage.read().unwrap();
|
let storage = self.storage.read().unwrap();
|
||||||
let len = serialized_size(&*accounts_index).unwrap()
|
let len = serialized_size(&*storage).unwrap() + std::mem::size_of::<u64>() as u64;
|
||||||
+ serialized_size(&*storage).unwrap()
|
|
||||||
+ std::mem::size_of::<u64>() as u64;
|
|
||||||
let mut buf = vec![0u8; len as usize];
|
let mut buf = vec![0u8; len as usize];
|
||||||
let mut wr = Cursor::new(&mut buf[..]);
|
let mut wr = Cursor::new(&mut buf[..]);
|
||||||
let version: u64 = self.write_version.load(Ordering::Relaxed) as u64;
|
let version: u64 = self.write_version.load(Ordering::Relaxed) as u64;
|
||||||
serialize_into(&mut wr, &*accounts_index).map_err(Error::custom)?;
|
|
||||||
serialize_into(&mut wr, &*storage).map_err(Error::custom)?;
|
serialize_into(&mut wr, &*storage).map_err(Error::custom)?;
|
||||||
serialize_into(&mut wr, &version).map_err(Error::custom)?;
|
serialize_into(&mut wr, &version).map_err(Error::custom)?;
|
||||||
let len = wr.position() as usize;
|
let len = wr.position() as usize;
|
||||||
|
@ -880,7 +882,7 @@ mod tests {
|
||||||
ACCOUNT_DATA_FILE_SIZE as usize / 3,
|
ACCOUNT_DATA_FILE_SIZE as usize / 3,
|
||||||
0,
|
0,
|
||||||
);
|
);
|
||||||
assert!(check_storage(&db, 2));
|
assert!(check_storage(&db, 0, 2));
|
||||||
|
|
||||||
let pubkey = Pubkey::new_rand();
|
let pubkey = Pubkey::new_rand();
|
||||||
let account = Account::new(1, ACCOUNT_DATA_FILE_SIZE as usize / 3, &pubkey);
|
let account = Account::new(1, ACCOUNT_DATA_FILE_SIZE as usize / 3, &pubkey);
|
||||||
|
@ -991,12 +993,17 @@ mod tests {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn check_storage(accounts: &AccountsDB, count: usize) -> bool {
|
fn check_storage(accounts: &AccountsDB, fork: Fork, count: usize) -> bool {
|
||||||
let stores = accounts.storage.read().unwrap();
|
let storage = accounts.storage.read().unwrap();
|
||||||
assert_eq!(stores.0[&0].len(), 1);
|
assert_eq!(storage.0[&fork].len(), 1);
|
||||||
assert_eq!(stores.0[&0][&0].status(), AccountStorageStatus::Available);
|
let fork_storage = storage.0.get(&fork).unwrap();
|
||||||
assert_eq!(stores.0[&0][&0].count(), count);
|
let mut total_count: usize = 0;
|
||||||
stores.0[&0][&0].count() == count
|
for store in fork_storage.values() {
|
||||||
|
assert_eq!(store.status(), AccountStorageStatus::Available);
|
||||||
|
total_count += store.count();
|
||||||
|
}
|
||||||
|
assert_eq!(total_count, count);
|
||||||
|
total_count == count
|
||||||
}
|
}
|
||||||
|
|
||||||
fn check_accounts(
|
fn check_accounts(
|
||||||
|
@ -1057,7 +1064,7 @@ mod tests {
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_account(&accounts, &mut pubkeys, 0, 100, 0, 0);
|
create_account(&accounts, &mut pubkeys, 0, 100, 0, 0);
|
||||||
update_accounts(&accounts, &pubkeys, 0, 99);
|
update_accounts(&accounts, &pubkeys, 0, 99);
|
||||||
assert_eq!(check_storage(&accounts, 100), true);
|
assert_eq!(check_storage(&accounts, 0, 100), true);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
@ -1232,7 +1239,7 @@ mod tests {
|
||||||
let accounts = AccountsDB::new(&paths.paths);
|
let accounts = AccountsDB::new(&paths.paths);
|
||||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||||
create_account(&accounts, &mut pubkeys, 0, 100, 0, 0);
|
create_account(&accounts, &mut pubkeys, 0, 100, 0, 0);
|
||||||
assert_eq!(check_storage(&accounts, 100), true);
|
assert_eq!(check_storage(&accounts, 0, 100), true);
|
||||||
check_accounts(&accounts, &pubkeys, 0, 100, 1);
|
check_accounts(&accounts, &pubkeys, 0, 100, 1);
|
||||||
modify_accounts(&accounts, &pubkeys, 0, 100, 2);
|
modify_accounts(&accounts, &pubkeys, 0, 100, 2);
|
||||||
check_accounts(&accounts, &pubkeys, 0, 100, 2);
|
check_accounts(&accounts, &pubkeys, 0, 100, 2);
|
||||||
|
@ -1244,6 +1251,8 @@ mod tests {
|
||||||
let mut buf = vec![0u8; serialized_size(&accounts).unwrap() as usize];
|
let mut buf = vec![0u8; serialized_size(&accounts).unwrap() as usize];
|
||||||
let mut writer = Cursor::new(&mut buf[..]);
|
let mut writer = Cursor::new(&mut buf[..]);
|
||||||
serialize_into(&mut writer, &accounts).unwrap();
|
serialize_into(&mut writer, &accounts).unwrap();
|
||||||
|
assert!(check_storage(&accounts, 0, 100));
|
||||||
|
assert!(check_storage(&accounts, 1, 10));
|
||||||
|
|
||||||
let mut reader = BufReader::new(&buf[..]);
|
let mut reader = BufReader::new(&buf[..]);
|
||||||
let daccounts = AccountsDB::new(&paths.paths);
|
let daccounts = AccountsDB::new(&paths.paths);
|
||||||
|
@ -1254,6 +1263,8 @@ mod tests {
|
||||||
);
|
);
|
||||||
check_accounts(&daccounts, &pubkeys, 0, 100, 2);
|
check_accounts(&daccounts, &pubkeys, 0, 100, 2);
|
||||||
check_accounts(&daccounts, &pubkeys1, 1, 10, 1);
|
check_accounts(&daccounts, &pubkeys1, 1, 10, 1);
|
||||||
|
assert!(check_storage(&daccounts, 0, 100));
|
||||||
|
assert!(check_storage(&daccounts, 1, 10));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
|
@ -1,21 +1,18 @@
|
||||||
use hashbrown::HashMap;
|
use hashbrown::HashMap;
|
||||||
use log::*;
|
use log::*;
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
use solana_sdk::pubkey::Pubkey;
|
use solana_sdk::pubkey::Pubkey;
|
||||||
use std::collections;
|
use std::collections;
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
|
|
||||||
pub type Fork = u64;
|
pub type Fork = u64;
|
||||||
|
|
||||||
#[derive(Debug, Default, Deserialize, Serialize)]
|
#[derive(Debug, Default)]
|
||||||
pub struct AccountsIndex<T> {
|
pub struct AccountsIndex<T> {
|
||||||
#[serde(skip)]
|
|
||||||
pub account_maps: HashMap<Pubkey, Vec<(Fork, T)>>,
|
pub account_maps: HashMap<Pubkey, Vec<(Fork, T)>>,
|
||||||
|
|
||||||
pub roots: HashSet<Fork>,
|
pub roots: HashSet<Fork>,
|
||||||
|
|
||||||
//This value that needs to be stored to recover the index from AppendVec
|
//This value that needs to be stored to recover the index from AppendVec
|
||||||
#[serde(skip)]
|
|
||||||
pub last_root: Fork,
|
pub last_root: Fork,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -325,6 +325,7 @@ impl Serialize for AppendVec {
|
||||||
+ std::mem::size_of::<usize>() as u64;
|
+ std::mem::size_of::<usize>() as u64;
|
||||||
let mut buf = vec![0u8; len as usize];
|
let mut buf = vec![0u8; len as usize];
|
||||||
let mut wr = Cursor::new(&mut buf[..]);
|
let mut wr = Cursor::new(&mut buf[..]);
|
||||||
|
self.map.flush().map_err(Error::custom)?;
|
||||||
serialize_into(&mut wr, &self.path).map_err(Error::custom)?;
|
serialize_into(&mut wr, &self.path).map_err(Error::custom)?;
|
||||||
serialize_into(&mut wr, &(self.current_len.load(Ordering::Relaxed) as u64))
|
serialize_into(&mut wr, &(self.current_len.load(Ordering::Relaxed) as u64))
|
||||||
.map_err(Error::custom)?;
|
.map_err(Error::custom)?;
|
||||||
|
|
|
@ -4,7 +4,7 @@
|
||||||
//! already been signed and verified.
|
//! already been signed and verified.
|
||||||
use crate::accounts::Accounts;
|
use crate::accounts::Accounts;
|
||||||
use crate::accounts_db::{
|
use crate::accounts_db::{
|
||||||
ErrorCounters, InstructionAccounts, InstructionCredits, InstructionLoaders,
|
AppendVecId, ErrorCounters, InstructionAccounts, InstructionCredits, InstructionLoaders,
|
||||||
};
|
};
|
||||||
use crate::accounts_index::Fork;
|
use crate::accounts_index::Fork;
|
||||||
use crate::blockhash_queue::BlockhashQueue;
|
use crate::blockhash_queue::BlockhashQueue;
|
||||||
|
@ -62,8 +62,12 @@ pub struct BankRc {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl BankRc {
|
impl BankRc {
|
||||||
pub fn new(account_paths: Option<String>) -> Self {
|
pub fn new(account_paths: Option<String>, id: AppendVecId) -> Self {
|
||||||
let accounts = Accounts::new(account_paths);
|
let accounts = Accounts::new(account_paths);
|
||||||
|
accounts
|
||||||
|
.accounts_db
|
||||||
|
.next_id
|
||||||
|
.store(id as usize, Ordering::Relaxed);
|
||||||
BankRc {
|
BankRc {
|
||||||
accounts: Arc::new(accounts),
|
accounts: Arc::new(accounts),
|
||||||
parent: RwLock::new(None),
|
parent: RwLock::new(None),
|
||||||
|
@ -374,9 +378,10 @@ impl Bank {
|
||||||
genesis_block: &GenesisBlock,
|
genesis_block: &GenesisBlock,
|
||||||
account_paths: Option<String>,
|
account_paths: Option<String>,
|
||||||
status_cache_rc: &StatusCacheRc,
|
status_cache_rc: &StatusCacheRc,
|
||||||
|
id: AppendVecId,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let mut bank = Self::default();
|
let mut bank = Self::default();
|
||||||
bank.set_bank_rc(&BankRc::new(account_paths), &status_cache_rc);
|
bank.set_bank_rc(&BankRc::new(account_paths, id), &status_cache_rc);
|
||||||
bank.process_genesis_block(genesis_block);
|
bank.process_genesis_block(genesis_block);
|
||||||
bank.ancestors.insert(0, 0);
|
bank.ancestors.insert(0, 0);
|
||||||
bank
|
bank
|
||||||
|
@ -2583,7 +2588,7 @@ mod tests {
|
||||||
let mut dbank: Bank = deserialize_from(&mut rdr).unwrap();
|
let mut dbank: Bank = deserialize_from(&mut rdr).unwrap();
|
||||||
let mut reader = BufReader::new(&buf[rdr.position() as usize..]);
|
let mut reader = BufReader::new(&buf[rdr.position() as usize..]);
|
||||||
dbank.set_bank_rc(
|
dbank.set_bank_rc(
|
||||||
&BankRc::new(Some(bank0.accounts().paths.clone())),
|
&BankRc::new(Some(bank0.accounts().paths.clone()), 0),
|
||||||
&StatusCacheRc::default(),
|
&StatusCacheRc::default(),
|
||||||
);
|
);
|
||||||
assert!(dbank.rc.update_from_stream(&mut reader).is_ok());
|
assert!(dbank.rc.update_from_stream(&mut reader).is_ok());
|
||||||
|
|
Loading…
Reference in New Issue