Abstract out AppendVec into AccountsFile enum (#29815)
Abstracts out AppendVec into AccountsFile enum. This will allow different implementations of AccountsFile that support the AccountsDB.
This commit is contained in:
parent
4194661bcf
commit
aeb6df3507
|
@ -24,6 +24,7 @@ use {
|
||||||
account_storage::{AccountStorage, AccountStorageStatus, ShrinkInProgress},
|
account_storage::{AccountStorage, AccountStorageStatus, ShrinkInProgress},
|
||||||
accounts_background_service::{DroppedSlotsSender, SendDroppedBankCallback},
|
accounts_background_service::{DroppedSlotsSender, SendDroppedBankCallback},
|
||||||
accounts_cache::{AccountsCache, CachedAccount, SlotCache},
|
accounts_cache::{AccountsCache, CachedAccount, SlotCache},
|
||||||
|
accounts_file::AccountsFile,
|
||||||
accounts_hash::{
|
accounts_hash::{
|
||||||
AccountsDeltaHash, AccountsHash, AccountsHasher, CalcAccountsHashConfig,
|
AccountsDeltaHash, AccountsHash, AccountsHasher, CalcAccountsHashConfig,
|
||||||
CalculateHashIntermediate, HashStats, ZeroLamportAccounts,
|
CalculateHashIntermediate, HashStats, ZeroLamportAccounts,
|
||||||
|
@ -1008,7 +1009,7 @@ pub struct AccountStorageEntry {
|
||||||
pub(crate) slot: AtomicU64,
|
pub(crate) slot: AtomicU64,
|
||||||
|
|
||||||
/// storage holding the accounts
|
/// storage holding the accounts
|
||||||
pub(crate) accounts: AppendVec,
|
pub(crate) accounts: AccountsFile,
|
||||||
|
|
||||||
/// Keeps track of the number of accounts stored in a specific AppendVec.
|
/// Keeps track of the number of accounts stored in a specific AppendVec.
|
||||||
/// This is periodically checked to reuse the stores that do not have
|
/// This is periodically checked to reuse the stores that do not have
|
||||||
|
@ -1030,9 +1031,9 @@ pub struct AccountStorageEntry {
|
||||||
|
|
||||||
impl AccountStorageEntry {
|
impl AccountStorageEntry {
|
||||||
pub fn new(path: &Path, slot: Slot, id: AppendVecId, file_size: u64) -> Self {
|
pub fn new(path: &Path, slot: Slot, id: AppendVecId, file_size: u64) -> Self {
|
||||||
let tail = AppendVec::file_name(slot, id);
|
let tail = AccountsFile::file_name(slot, id);
|
||||||
let path = Path::new(path).join(tail);
|
let path = Path::new(path).join(tail);
|
||||||
let accounts = AppendVec::new(&path, true, file_size as usize);
|
let accounts = AccountsFile::AppendVec(AppendVec::new(&path, true, file_size as usize));
|
||||||
|
|
||||||
Self {
|
Self {
|
||||||
id: AtomicAppendVecId::new(id),
|
id: AtomicAppendVecId::new(id),
|
||||||
|
@ -1047,7 +1048,7 @@ impl AccountStorageEntry {
|
||||||
pub(crate) fn new_existing(
|
pub(crate) fn new_existing(
|
||||||
slot: Slot,
|
slot: Slot,
|
||||||
id: AppendVecId,
|
id: AppendVecId,
|
||||||
accounts: AppendVec,
|
accounts: AccountsFile,
|
||||||
num_accounts: usize,
|
num_accounts: usize,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
|
@ -10336,7 +10337,7 @@ pub mod tests {
|
||||||
let slot_expected: Slot = 0;
|
let slot_expected: Slot = 0;
|
||||||
let size: usize = 123;
|
let size: usize = 123;
|
||||||
let mut data = AccountStorageEntry::new(&paths[0], slot_expected, 0, size as u64);
|
let mut data = AccountStorageEntry::new(&paths[0], slot_expected, 0, size as u64);
|
||||||
let av = AppendVec::new(&tf.path, true, 1024 * 1024);
|
let av = AccountsFile::AppendVec(AppendVec::new(&tf.path, true, 1024 * 1024));
|
||||||
data.accounts = av;
|
data.accounts = av;
|
||||||
|
|
||||||
let storage = Arc::new(data);
|
let storage = Arc::new(data);
|
||||||
|
@ -10450,7 +10451,7 @@ pub mod tests {
|
||||||
let slot_expected: Slot = 0;
|
let slot_expected: Slot = 0;
|
||||||
let size: usize = 123;
|
let size: usize = 123;
|
||||||
let mut data = AccountStorageEntry::new(&paths[0], slot_expected, 0, size as u64);
|
let mut data = AccountStorageEntry::new(&paths[0], slot_expected, 0, size as u64);
|
||||||
let av = AppendVec::new(&tf.path, true, 1024 * 1024);
|
let av = AccountsFile::AppendVec(AppendVec::new(&tf.path, true, 1024 * 1024));
|
||||||
data.accounts = av;
|
data.accounts = av;
|
||||||
|
|
||||||
let storage = Arc::new(data);
|
let storage = Arc::new(data);
|
||||||
|
@ -10527,7 +10528,7 @@ pub mod tests {
|
||||||
let (_temp_dirs, paths) = get_temp_accounts_paths(1).unwrap();
|
let (_temp_dirs, paths) = get_temp_accounts_paths(1).unwrap();
|
||||||
let size: usize = aligned_stored_size(account_data_size.unwrap_or(123) as usize);
|
let size: usize = aligned_stored_size(account_data_size.unwrap_or(123) as usize);
|
||||||
let mut data = AccountStorageEntry::new(&paths[0], slot, id, size as u64);
|
let mut data = AccountStorageEntry::new(&paths[0], slot, id, size as u64);
|
||||||
let av = AppendVec::new(&tf.path, true, (1024 * 1024).max(size));
|
let av = AccountsFile::AppendVec(AppendVec::new(&tf.path, true, (1024 * 1024).max(size)));
|
||||||
data.accounts = av;
|
data.accounts = av;
|
||||||
|
|
||||||
let arc = Arc::new(data);
|
let arc = Arc::new(data);
|
||||||
|
|
|
@ -0,0 +1,163 @@
|
||||||
|
use {
|
||||||
|
crate::{append_vec::*, storable_accounts::StorableAccounts},
|
||||||
|
solana_sdk::{account::ReadableAccount, clock::Slot, hash::Hash, pubkey::Pubkey},
|
||||||
|
std::{borrow::Borrow, io, path::PathBuf},
|
||||||
|
};
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
/// An enum for accessing an accounts file which can be implemented
|
||||||
|
/// under different formats.
|
||||||
|
pub enum AccountsFile {
|
||||||
|
AppendVec(AppendVec),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AccountsFile {
|
||||||
|
/// By default, all AccountsFile will remove its underlying file on
|
||||||
|
/// drop. Calling this function to disable such behavior for this
|
||||||
|
/// instance.
|
||||||
|
pub fn set_no_remove_on_drop(&mut self) {
|
||||||
|
match self {
|
||||||
|
Self::AppendVec(av) => av.set_no_remove_on_drop(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn flush(&self) -> io::Result<()> {
|
||||||
|
match self {
|
||||||
|
Self::AppendVec(av) => av.flush(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn reset(&self) {
|
||||||
|
match self {
|
||||||
|
Self::AppendVec(av) => av.reset(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn remaining_bytes(&self) -> u64 {
|
||||||
|
match self {
|
||||||
|
Self::AppendVec(av) => av.remaining_bytes(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn len(&self) -> usize {
|
||||||
|
match self {
|
||||||
|
Self::AppendVec(av) => av.len(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_empty(&self) -> bool {
|
||||||
|
match self {
|
||||||
|
Self::AppendVec(av) => av.is_empty(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn capacity(&self) -> u64 {
|
||||||
|
match self {
|
||||||
|
Self::AppendVec(av) => av.capacity(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn file_name(slot: Slot, id: impl std::fmt::Display) -> String {
|
||||||
|
format!("{slot}.{id}")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return (account metadata, next_index) pair for the account at the
|
||||||
|
/// specified `index` if any. Otherwise return None. Also return the
|
||||||
|
/// index of the next entry.
|
||||||
|
pub fn get_account(&self, index: usize) -> Option<(StoredAccountMeta<'_>, usize)> {
|
||||||
|
match self {
|
||||||
|
Self::AppendVec(av) => av.get_account(index),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn account_matches_owners(
|
||||||
|
&self,
|
||||||
|
offset: usize,
|
||||||
|
owners: &[&Pubkey],
|
||||||
|
) -> Result<(), MatchAccountOwnerError> {
|
||||||
|
match self {
|
||||||
|
Self::AppendVec(av) => av.account_matches_owners(offset, owners),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return the path of the underlying account file.
|
||||||
|
pub fn get_path(&self) -> PathBuf {
|
||||||
|
match self {
|
||||||
|
Self::AppendVec(av) => av.get_path(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return iterator for account metadata
|
||||||
|
pub fn account_iter(&self) -> AccountsFileIter {
|
||||||
|
AccountsFileIter::new(self)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return a vector of account metadata for each account, starting from `offset`.
|
||||||
|
pub fn accounts(&self, offset: usize) -> Vec<StoredAccountMeta> {
|
||||||
|
match self {
|
||||||
|
Self::AppendVec(av) => av.accounts(offset),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Copy each account metadata, account and hash to the internal buffer.
|
||||||
|
/// If there is no room to write the first entry, None is returned.
|
||||||
|
/// Otherwise, returns the starting offset of each account metadata.
|
||||||
|
/// Plus, the final return value is the offset where the next entry would be appended.
|
||||||
|
/// So, return.len() is 1 + (number of accounts written)
|
||||||
|
/// After each account is appended, the internal `current_len` is updated
|
||||||
|
/// and will be available to other threads.
|
||||||
|
pub fn append_accounts<
|
||||||
|
'a,
|
||||||
|
'b,
|
||||||
|
T: ReadableAccount + Sync,
|
||||||
|
U: StorableAccounts<'a, T>,
|
||||||
|
V: Borrow<Hash>,
|
||||||
|
>(
|
||||||
|
&self,
|
||||||
|
accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>,
|
||||||
|
skip: usize,
|
||||||
|
) -> Option<Vec<usize>> {
|
||||||
|
match self {
|
||||||
|
Self::AppendVec(av) => av.append_accounts(accounts, skip),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct AccountsFileIter<'a> {
|
||||||
|
file_entry: &'a AccountsFile,
|
||||||
|
offset: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> AccountsFileIter<'a> {
|
||||||
|
pub fn new(file_entry: &'a AccountsFile) -> Self {
|
||||||
|
Self {
|
||||||
|
file_entry,
|
||||||
|
offset: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> Iterator for AccountsFileIter<'a> {
|
||||||
|
type Item = StoredAccountMeta<'a>;
|
||||||
|
|
||||||
|
fn next(&mut self) -> Option<Self::Item> {
|
||||||
|
if let Some((account, next_offset)) = self.file_entry.get_account(self.offset) {
|
||||||
|
self.offset = next_offset;
|
||||||
|
Some(account)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub mod tests {
|
||||||
|
use crate::accounts_file::AccountsFile;
|
||||||
|
impl AccountsFile {
|
||||||
|
pub(crate) fn set_current_len_for_tests(&self, len: usize) {
|
||||||
|
match self {
|
||||||
|
Self::AppendVec(av) => av.set_current_len_for_tests(len),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -11,9 +11,10 @@ use {
|
||||||
ShrinkCollectAliveSeparatedByRefs, ShrinkStatsSub, StoreReclaims,
|
ShrinkCollectAliveSeparatedByRefs, ShrinkStatsSub, StoreReclaims,
|
||||||
INCLUDE_SLOT_IN_HASH_IRRELEVANT_APPEND_VEC_OPERATION,
|
INCLUDE_SLOT_IN_HASH_IRRELEVANT_APPEND_VEC_OPERATION,
|
||||||
},
|
},
|
||||||
|
accounts_file::AccountsFile,
|
||||||
accounts_index::ZeroLamport,
|
accounts_index::ZeroLamport,
|
||||||
active_stats::ActiveStatItem,
|
active_stats::ActiveStatItem,
|
||||||
append_vec::{aligned_stored_size, AppendVec, StoredAccountMeta},
|
append_vec::{aligned_stored_size, StoredAccountMeta},
|
||||||
storable_accounts::{StorableAccounts, StorableAccountsBySlot},
|
storable_accounts::{StorableAccounts, StorableAccountsBySlot},
|
||||||
},
|
},
|
||||||
rand::{thread_rng, Rng},
|
rand::{thread_rng, Rng},
|
||||||
|
@ -748,8 +749,10 @@ pub fn get_ancient_append_vec_capacity() -> u64 {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// is this a max-size append vec designed to be used as an ancient append vec?
|
/// is this a max-size append vec designed to be used as an ancient append vec?
|
||||||
pub fn is_ancient(storage: &AppendVec) -> bool {
|
pub fn is_ancient(storage: &AccountsFile) -> bool {
|
||||||
storage.capacity() >= get_ancient_append_vec_capacity()
|
match storage {
|
||||||
|
AccountsFile::AppendVec(storage) => storage.capacity() >= get_ancient_append_vec_capacity(),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
@ -766,7 +769,9 @@ pub mod tests {
|
||||||
},
|
},
|
||||||
INCLUDE_SLOT_IN_HASH_TESTS,
|
INCLUDE_SLOT_IN_HASH_TESTS,
|
||||||
},
|
},
|
||||||
append_vec::{aligned_stored_size, AccountMeta, StoredAccountMeta, StoredMeta},
|
append_vec::{
|
||||||
|
aligned_stored_size, AccountMeta, AppendVec, StoredAccountMeta, StoredMeta,
|
||||||
|
},
|
||||||
storable_accounts::StorableAccountsBySlot,
|
storable_accounts::StorableAccountsBySlot,
|
||||||
},
|
},
|
||||||
solana_sdk::{
|
solana_sdk::{
|
||||||
|
@ -1670,7 +1675,7 @@ pub mod tests {
|
||||||
] {
|
] {
|
||||||
let tf = crate::append_vec::test_utils::get_append_vec_path("test_is_ancient");
|
let tf = crate::append_vec::test_utils::get_append_vec_path("test_is_ancient");
|
||||||
let (_temp_dirs, _paths) = get_temp_accounts_paths(1).unwrap();
|
let (_temp_dirs, _paths) = get_temp_accounts_paths(1).unwrap();
|
||||||
let av = AppendVec::new(&tf.path, true, size as usize);
|
let av = AccountsFile::AppendVec(AppendVec::new(&tf.path, true, size as usize));
|
||||||
|
|
||||||
assert_eq!(expected_ancient, is_ancient(&av));
|
assert_eq!(expected_ancient, is_ancient(&av));
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,6 +12,7 @@ pub mod accounts;
|
||||||
pub mod accounts_background_service;
|
pub mod accounts_background_service;
|
||||||
pub mod accounts_cache;
|
pub mod accounts_cache;
|
||||||
pub mod accounts_db;
|
pub mod accounts_db;
|
||||||
|
pub mod accounts_file;
|
||||||
pub mod accounts_hash;
|
pub mod accounts_hash;
|
||||||
pub mod accounts_index;
|
pub mod accounts_index;
|
||||||
pub mod accounts_index_storage;
|
pub mod accounts_index_storage;
|
||||||
|
|
|
@ -5,6 +5,7 @@ use {
|
||||||
AccountShrinkThreshold, AccountStorageEntry, AccountsDb, AccountsDbConfig, AppendVecId,
|
AccountShrinkThreshold, AccountStorageEntry, AccountsDb, AccountsDbConfig, AppendVecId,
|
||||||
AtomicAppendVecId, BankHashStats, IndexGenerationInfo,
|
AtomicAppendVecId, BankHashStats, IndexGenerationInfo,
|
||||||
},
|
},
|
||||||
|
accounts_file::AccountsFile,
|
||||||
accounts_hash::{AccountsDeltaHash, AccountsHash},
|
accounts_hash::{AccountsDeltaHash, AccountsHash},
|
||||||
accounts_index::AccountSecondaryIndexes,
|
accounts_index::AccountSecondaryIndexes,
|
||||||
accounts_update_notifier_interface::AccountsUpdateNotifier,
|
accounts_update_notifier_interface::AccountsUpdateNotifier,
|
||||||
|
@ -573,11 +574,12 @@ fn reconstruct_single_storage(
|
||||||
current_len: usize,
|
current_len: usize,
|
||||||
append_vec_id: AppendVecId,
|
append_vec_id: AppendVecId,
|
||||||
) -> io::Result<Arc<AccountStorageEntry>> {
|
) -> io::Result<Arc<AccountStorageEntry>> {
|
||||||
let (accounts, num_accounts) = AppendVec::new_from_file(append_vec_path, current_len)?;
|
let (append_vec, num_accounts) = AppendVec::new_from_file(append_vec_path, current_len)?;
|
||||||
|
let accounts_file = AccountsFile::AppendVec(append_vec);
|
||||||
Ok(Arc::new(AccountStorageEntry::new_existing(
|
Ok(Arc::new(AccountStorageEntry::new_existing(
|
||||||
*slot,
|
*slot,
|
||||||
append_vec_id,
|
append_vec_id,
|
||||||
accounts,
|
accounts_file,
|
||||||
num_accounts,
|
num_accounts,
|
||||||
)))
|
)))
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,6 +8,7 @@ use {
|
||||||
accounts_db::{
|
accounts_db::{
|
||||||
get_temp_accounts_paths, test_utils::create_test_accounts, AccountShrinkThreshold,
|
get_temp_accounts_paths, test_utils::create_test_accounts, AccountShrinkThreshold,
|
||||||
},
|
},
|
||||||
|
accounts_file::AccountsFile,
|
||||||
accounts_hash::{AccountsDeltaHash, AccountsHash},
|
accounts_hash::{AccountsDeltaHash, AccountsHash},
|
||||||
append_vec::AppendVec,
|
append_vec::AppendVec,
|
||||||
bank::{Bank, BankTestConfig},
|
bank::{Bank, BankTestConfig},
|
||||||
|
@ -56,10 +57,11 @@ fn copy_append_vecs<P: AsRef<Path>>(
|
||||||
// Read new file into append-vec and build new entry
|
// Read new file into append-vec and build new entry
|
||||||
let (append_vec, num_accounts) =
|
let (append_vec, num_accounts) =
|
||||||
AppendVec::new_from_file(output_path, storage_entry.accounts.len())?;
|
AppendVec::new_from_file(output_path, storage_entry.accounts.len())?;
|
||||||
|
let accounts_file = AccountsFile::AppendVec(append_vec);
|
||||||
let new_storage_entry = AccountStorageEntry::new_existing(
|
let new_storage_entry = AccountStorageEntry::new_existing(
|
||||||
storage_entry.slot(),
|
storage_entry.slot(),
|
||||||
storage_entry.append_vec_id(),
|
storage_entry.append_vec_id(),
|
||||||
append_vec,
|
accounts_file,
|
||||||
num_accounts,
|
num_accounts,
|
||||||
);
|
);
|
||||||
next_append_vec_id = next_append_vec_id.max(new_storage_entry.append_vec_id());
|
next_append_vec_id = next_append_vec_id.max(new_storage_entry.append_vec_id());
|
||||||
|
|
|
@ -196,6 +196,7 @@ pub mod tests {
|
||||||
super::*,
|
super::*,
|
||||||
crate::{
|
crate::{
|
||||||
accounts_db::{AccountStorageEntry, AppendVecId},
|
accounts_db::{AccountStorageEntry, AppendVecId},
|
||||||
|
accounts_file::AccountsFile,
|
||||||
append_vec::AppendVec,
|
append_vec::AppendVec,
|
||||||
},
|
},
|
||||||
std::sync::Arc,
|
std::sync::Arc,
|
||||||
|
@ -369,7 +370,7 @@ pub mod tests {
|
||||||
let size: usize = 123;
|
let size: usize = 123;
|
||||||
let slot = 0;
|
let slot = 0;
|
||||||
let mut data = AccountStorageEntry::new(&paths[0], slot, id, size as u64);
|
let mut data = AccountStorageEntry::new(&paths[0], slot, id, size as u64);
|
||||||
let av = AppendVec::new(&tf.path, true, 1024 * 1024);
|
let av = AccountsFile::AppendVec(AppendVec::new(&tf.path, true, 1024 * 1024));
|
||||||
data.accounts = av;
|
data.accounts = av;
|
||||||
|
|
||||||
Arc::new(data)
|
Arc::new(data)
|
||||||
|
|
Loading…
Reference in New Issue