Abstract out AppendVec into AccountsFile enum (#29815)
Abstracts out AppendVec into AccountsFile enum. This will allow different implementations of AccountsFile that support the AccountsDB.
This commit is contained in:
parent
4194661bcf
commit
aeb6df3507
|
@ -24,6 +24,7 @@ use {
|
|||
account_storage::{AccountStorage, AccountStorageStatus, ShrinkInProgress},
|
||||
accounts_background_service::{DroppedSlotsSender, SendDroppedBankCallback},
|
||||
accounts_cache::{AccountsCache, CachedAccount, SlotCache},
|
||||
accounts_file::AccountsFile,
|
||||
accounts_hash::{
|
||||
AccountsDeltaHash, AccountsHash, AccountsHasher, CalcAccountsHashConfig,
|
||||
CalculateHashIntermediate, HashStats, ZeroLamportAccounts,
|
||||
|
@ -1008,7 +1009,7 @@ pub struct AccountStorageEntry {
|
|||
pub(crate) slot: AtomicU64,
|
||||
|
||||
/// storage holding the accounts
|
||||
pub(crate) accounts: AppendVec,
|
||||
pub(crate) accounts: AccountsFile,
|
||||
|
||||
/// Keeps track of the number of accounts stored in a specific AppendVec.
|
||||
/// This is periodically checked to reuse the stores that do not have
|
||||
|
@ -1030,9 +1031,9 @@ pub struct AccountStorageEntry {
|
|||
|
||||
impl AccountStorageEntry {
|
||||
pub fn new(path: &Path, slot: Slot, id: AppendVecId, file_size: u64) -> Self {
|
||||
let tail = AppendVec::file_name(slot, id);
|
||||
let tail = AccountsFile::file_name(slot, id);
|
||||
let path = Path::new(path).join(tail);
|
||||
let accounts = AppendVec::new(&path, true, file_size as usize);
|
||||
let accounts = AccountsFile::AppendVec(AppendVec::new(&path, true, file_size as usize));
|
||||
|
||||
Self {
|
||||
id: AtomicAppendVecId::new(id),
|
||||
|
@ -1047,7 +1048,7 @@ impl AccountStorageEntry {
|
|||
pub(crate) fn new_existing(
|
||||
slot: Slot,
|
||||
id: AppendVecId,
|
||||
accounts: AppendVec,
|
||||
accounts: AccountsFile,
|
||||
num_accounts: usize,
|
||||
) -> Self {
|
||||
Self {
|
||||
|
@ -10336,7 +10337,7 @@ pub mod tests {
|
|||
let slot_expected: Slot = 0;
|
||||
let size: usize = 123;
|
||||
let mut data = AccountStorageEntry::new(&paths[0], slot_expected, 0, size as u64);
|
||||
let av = AppendVec::new(&tf.path, true, 1024 * 1024);
|
||||
let av = AccountsFile::AppendVec(AppendVec::new(&tf.path, true, 1024 * 1024));
|
||||
data.accounts = av;
|
||||
|
||||
let storage = Arc::new(data);
|
||||
|
@ -10450,7 +10451,7 @@ pub mod tests {
|
|||
let slot_expected: Slot = 0;
|
||||
let size: usize = 123;
|
||||
let mut data = AccountStorageEntry::new(&paths[0], slot_expected, 0, size as u64);
|
||||
let av = AppendVec::new(&tf.path, true, 1024 * 1024);
|
||||
let av = AccountsFile::AppendVec(AppendVec::new(&tf.path, true, 1024 * 1024));
|
||||
data.accounts = av;
|
||||
|
||||
let storage = Arc::new(data);
|
||||
|
@ -10527,7 +10528,7 @@ pub mod tests {
|
|||
let (_temp_dirs, paths) = get_temp_accounts_paths(1).unwrap();
|
||||
let size: usize = aligned_stored_size(account_data_size.unwrap_or(123) as usize);
|
||||
let mut data = AccountStorageEntry::new(&paths[0], slot, id, size as u64);
|
||||
let av = AppendVec::new(&tf.path, true, (1024 * 1024).max(size));
|
||||
let av = AccountsFile::AppendVec(AppendVec::new(&tf.path, true, (1024 * 1024).max(size)));
|
||||
data.accounts = av;
|
||||
|
||||
let arc = Arc::new(data);
|
||||
|
|
|
@ -0,0 +1,163 @@
|
|||
use {
|
||||
crate::{append_vec::*, storable_accounts::StorableAccounts},
|
||||
solana_sdk::{account::ReadableAccount, clock::Slot, hash::Hash, pubkey::Pubkey},
|
||||
std::{borrow::Borrow, io, path::PathBuf},
|
||||
};
|
||||
|
||||
#[derive(Debug)]
|
||||
/// An enum for accessing an accounts file which can be implemented
|
||||
/// under different formats.
|
||||
pub enum AccountsFile {
|
||||
AppendVec(AppendVec),
|
||||
}
|
||||
|
||||
impl AccountsFile {
|
||||
/// By default, all AccountsFile will remove its underlying file on
|
||||
/// drop. Calling this function to disable such behavior for this
|
||||
/// instance.
|
||||
pub fn set_no_remove_on_drop(&mut self) {
|
||||
match self {
|
||||
Self::AppendVec(av) => av.set_no_remove_on_drop(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn flush(&self) -> io::Result<()> {
|
||||
match self {
|
||||
Self::AppendVec(av) => av.flush(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn reset(&self) {
|
||||
match self {
|
||||
Self::AppendVec(av) => av.reset(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn remaining_bytes(&self) -> u64 {
|
||||
match self {
|
||||
Self::AppendVec(av) => av.remaining_bytes(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
match self {
|
||||
Self::AppendVec(av) => av.len(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
match self {
|
||||
Self::AppendVec(av) => av.is_empty(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn capacity(&self) -> u64 {
|
||||
match self {
|
||||
Self::AppendVec(av) => av.capacity(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn file_name(slot: Slot, id: impl std::fmt::Display) -> String {
|
||||
format!("{slot}.{id}")
|
||||
}
|
||||
|
||||
/// Return (account metadata, next_index) pair for the account at the
|
||||
/// specified `index` if any. Otherwise return None. Also return the
|
||||
/// index of the next entry.
|
||||
pub fn get_account(&self, index: usize) -> Option<(StoredAccountMeta<'_>, usize)> {
|
||||
match self {
|
||||
Self::AppendVec(av) => av.get_account(index),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn account_matches_owners(
|
||||
&self,
|
||||
offset: usize,
|
||||
owners: &[&Pubkey],
|
||||
) -> Result<(), MatchAccountOwnerError> {
|
||||
match self {
|
||||
Self::AppendVec(av) => av.account_matches_owners(offset, owners),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the path of the underlying account file.
|
||||
pub fn get_path(&self) -> PathBuf {
|
||||
match self {
|
||||
Self::AppendVec(av) => av.get_path(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return iterator for account metadata
|
||||
pub fn account_iter(&self) -> AccountsFileIter {
|
||||
AccountsFileIter::new(self)
|
||||
}
|
||||
|
||||
/// Return a vector of account metadata for each account, starting from `offset`.
|
||||
pub fn accounts(&self, offset: usize) -> Vec<StoredAccountMeta> {
|
||||
match self {
|
||||
Self::AppendVec(av) => av.accounts(offset),
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy each account metadata, account and hash to the internal buffer.
|
||||
/// If there is no room to write the first entry, None is returned.
|
||||
/// Otherwise, returns the starting offset of each account metadata.
|
||||
/// Plus, the final return value is the offset where the next entry would be appended.
|
||||
/// So, return.len() is 1 + (number of accounts written)
|
||||
/// After each account is appended, the internal `current_len` is updated
|
||||
/// and will be available to other threads.
|
||||
pub fn append_accounts<
|
||||
'a,
|
||||
'b,
|
||||
T: ReadableAccount + Sync,
|
||||
U: StorableAccounts<'a, T>,
|
||||
V: Borrow<Hash>,
|
||||
>(
|
||||
&self,
|
||||
accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>,
|
||||
skip: usize,
|
||||
) -> Option<Vec<usize>> {
|
||||
match self {
|
||||
Self::AppendVec(av) => av.append_accounts(accounts, skip),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct AccountsFileIter<'a> {
|
||||
file_entry: &'a AccountsFile,
|
||||
offset: usize,
|
||||
}
|
||||
|
||||
impl<'a> AccountsFileIter<'a> {
|
||||
pub fn new(file_entry: &'a AccountsFile) -> Self {
|
||||
Self {
|
||||
file_entry,
|
||||
offset: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Iterator for AccountsFileIter<'a> {
|
||||
type Item = StoredAccountMeta<'a>;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
if let Some((account, next_offset)) = self.file_entry.get_account(self.offset) {
|
||||
self.offset = next_offset;
|
||||
Some(account)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub mod tests {
|
||||
use crate::accounts_file::AccountsFile;
|
||||
impl AccountsFile {
|
||||
pub(crate) fn set_current_len_for_tests(&self, len: usize) {
|
||||
match self {
|
||||
Self::AppendVec(av) => av.set_current_len_for_tests(len),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -11,9 +11,10 @@ use {
|
|||
ShrinkCollectAliveSeparatedByRefs, ShrinkStatsSub, StoreReclaims,
|
||||
INCLUDE_SLOT_IN_HASH_IRRELEVANT_APPEND_VEC_OPERATION,
|
||||
},
|
||||
accounts_file::AccountsFile,
|
||||
accounts_index::ZeroLamport,
|
||||
active_stats::ActiveStatItem,
|
||||
append_vec::{aligned_stored_size, AppendVec, StoredAccountMeta},
|
||||
append_vec::{aligned_stored_size, StoredAccountMeta},
|
||||
storable_accounts::{StorableAccounts, StorableAccountsBySlot},
|
||||
},
|
||||
rand::{thread_rng, Rng},
|
||||
|
@ -748,8 +749,10 @@ pub fn get_ancient_append_vec_capacity() -> u64 {
|
|||
}
|
||||
|
||||
/// is this a max-size append vec designed to be used as an ancient append vec?
|
||||
pub fn is_ancient(storage: &AppendVec) -> bool {
|
||||
storage.capacity() >= get_ancient_append_vec_capacity()
|
||||
pub fn is_ancient(storage: &AccountsFile) -> bool {
|
||||
match storage {
|
||||
AccountsFile::AppendVec(storage) => storage.capacity() >= get_ancient_append_vec_capacity(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
@ -766,7 +769,9 @@ pub mod tests {
|
|||
},
|
||||
INCLUDE_SLOT_IN_HASH_TESTS,
|
||||
},
|
||||
append_vec::{aligned_stored_size, AccountMeta, StoredAccountMeta, StoredMeta},
|
||||
append_vec::{
|
||||
aligned_stored_size, AccountMeta, AppendVec, StoredAccountMeta, StoredMeta,
|
||||
},
|
||||
storable_accounts::StorableAccountsBySlot,
|
||||
},
|
||||
solana_sdk::{
|
||||
|
@ -1670,7 +1675,7 @@ pub mod tests {
|
|||
] {
|
||||
let tf = crate::append_vec::test_utils::get_append_vec_path("test_is_ancient");
|
||||
let (_temp_dirs, _paths) = get_temp_accounts_paths(1).unwrap();
|
||||
let av = AppendVec::new(&tf.path, true, size as usize);
|
||||
let av = AccountsFile::AppendVec(AppendVec::new(&tf.path, true, size as usize));
|
||||
|
||||
assert_eq!(expected_ancient, is_ancient(&av));
|
||||
}
|
||||
|
|
|
@ -12,6 +12,7 @@ pub mod accounts;
|
|||
pub mod accounts_background_service;
|
||||
pub mod accounts_cache;
|
||||
pub mod accounts_db;
|
||||
pub mod accounts_file;
|
||||
pub mod accounts_hash;
|
||||
pub mod accounts_index;
|
||||
pub mod accounts_index_storage;
|
||||
|
|
|
@ -5,6 +5,7 @@ use {
|
|||
AccountShrinkThreshold, AccountStorageEntry, AccountsDb, AccountsDbConfig, AppendVecId,
|
||||
AtomicAppendVecId, BankHashStats, IndexGenerationInfo,
|
||||
},
|
||||
accounts_file::AccountsFile,
|
||||
accounts_hash::{AccountsDeltaHash, AccountsHash},
|
||||
accounts_index::AccountSecondaryIndexes,
|
||||
accounts_update_notifier_interface::AccountsUpdateNotifier,
|
||||
|
@ -573,11 +574,12 @@ fn reconstruct_single_storage(
|
|||
current_len: usize,
|
||||
append_vec_id: AppendVecId,
|
||||
) -> io::Result<Arc<AccountStorageEntry>> {
|
||||
let (accounts, num_accounts) = AppendVec::new_from_file(append_vec_path, current_len)?;
|
||||
let (append_vec, num_accounts) = AppendVec::new_from_file(append_vec_path, current_len)?;
|
||||
let accounts_file = AccountsFile::AppendVec(append_vec);
|
||||
Ok(Arc::new(AccountStorageEntry::new_existing(
|
||||
*slot,
|
||||
append_vec_id,
|
||||
accounts,
|
||||
accounts_file,
|
||||
num_accounts,
|
||||
)))
|
||||
}
|
||||
|
|
|
@ -8,6 +8,7 @@ use {
|
|||
accounts_db::{
|
||||
get_temp_accounts_paths, test_utils::create_test_accounts, AccountShrinkThreshold,
|
||||
},
|
||||
accounts_file::AccountsFile,
|
||||
accounts_hash::{AccountsDeltaHash, AccountsHash},
|
||||
append_vec::AppendVec,
|
||||
bank::{Bank, BankTestConfig},
|
||||
|
@ -56,10 +57,11 @@ fn copy_append_vecs<P: AsRef<Path>>(
|
|||
// Read new file into append-vec and build new entry
|
||||
let (append_vec, num_accounts) =
|
||||
AppendVec::new_from_file(output_path, storage_entry.accounts.len())?;
|
||||
let accounts_file = AccountsFile::AppendVec(append_vec);
|
||||
let new_storage_entry = AccountStorageEntry::new_existing(
|
||||
storage_entry.slot(),
|
||||
storage_entry.append_vec_id(),
|
||||
append_vec,
|
||||
accounts_file,
|
||||
num_accounts,
|
||||
);
|
||||
next_append_vec_id = next_append_vec_id.max(new_storage_entry.append_vec_id());
|
||||
|
|
|
@ -196,6 +196,7 @@ pub mod tests {
|
|||
super::*,
|
||||
crate::{
|
||||
accounts_db::{AccountStorageEntry, AppendVecId},
|
||||
accounts_file::AccountsFile,
|
||||
append_vec::AppendVec,
|
||||
},
|
||||
std::sync::Arc,
|
||||
|
@ -369,7 +370,7 @@ pub mod tests {
|
|||
let size: usize = 123;
|
||||
let slot = 0;
|
||||
let mut data = AccountStorageEntry::new(&paths[0], slot, id, size as u64);
|
||||
let av = AppendVec::new(&tf.path, true, 1024 * 1024);
|
||||
let av = AccountsFile::AppendVec(AppendVec::new(&tf.path, true, 1024 * 1024));
|
||||
data.accounts = av;
|
||||
|
||||
Arc::new(data)
|
||||
|
|
Loading…
Reference in New Issue