Move account-meta structs from append_vec.rs to account_meta.rs (#30443)

#### Problem
As we start supporting new storage formats, those account-meta
structs will be shared by different storage formats and thus need
a new home.

#### Summary of Changes
This PR creates meta.rs under account_storage and moves all the account-meta
structs out from append_vec.rs.
This commit is contained in:
Yueh-Hsuan Chiang 2023-02-22 16:10:34 -08:00 committed by GitHub
parent 69ea295b07
commit ac7e7aa8f0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 242 additions and 225 deletions

View File

@ -8,8 +8,8 @@ use {
solana_measure::measure::Measure,
solana_metrics::*,
solana_runtime::{
account_storage::meta::StoredAccountMeta,
accounts_update_notifier_interface::AccountsUpdateNotifierInterface,
append_vec::StoredAccountMeta,
},
solana_sdk::{
account::{AccountSharedData, ReadableAccount},

View File

@ -4,10 +4,11 @@ extern crate test;
use {
rand::{thread_rng, Rng},
solana_runtime::{
account_storage::meta::{StorableAccountsWithHashesAndWriteVersions, StoredMeta},
accounts_db::INCLUDE_SLOT_IN_HASH_TESTS,
append_vec::{
test_utils::{create_test_account, get_append_vec_path},
AppendVec, StorableAccountsWithHashesAndWriteVersions, StoredMeta,
AppendVec,
},
},
solana_sdk::{

View File

@ -7,6 +7,8 @@ use {
std::sync::Arc,
};
pub mod meta;
#[derive(Clone, Debug)]
pub struct AccountStorageReference {
/// the single storage for a given slot

View File

@ -0,0 +1,199 @@
use {
crate::storable_accounts::StorableAccounts,
solana_sdk::{
account::{Account, AccountSharedData, ReadableAccount},
hash::Hash,
pubkey::Pubkey,
stake_history::Epoch,
},
std::{borrow::Borrow, marker::PhantomData},
};
pub type StoredMetaWriteVersion = u64;
/// Goal is to eliminate copies and data reshaping given various code paths that store accounts.
/// This struct contains what is needed to store accounts to a storage
/// 1. account & pubkey (StorableAccounts)
/// 2. hash per account (Maybe in StorableAccounts, otherwise has to be passed in separately)
/// 3. write version per account (Maybe in StorableAccounts, otherwise has to be passed in separately)
pub struct StorableAccountsWithHashesAndWriteVersions<
'a: 'b,
'b,
T: ReadableAccount + Sync + 'b,
U: StorableAccounts<'a, T>,
V: Borrow<Hash>,
> {
/// accounts to store
/// always has pubkey and account
/// may also have hash and write_version per account
pub(crate) accounts: &'b U,
/// if accounts does not have hash and write version, this has a hash and write version per account
hashes_and_write_versions: Option<(Vec<V>, Vec<StoredMetaWriteVersion>)>,
_phantom: PhantomData<&'a T>,
}
impl<'a: 'b, 'b, T: ReadableAccount + Sync + 'b, U: StorableAccounts<'a, T>, V: Borrow<Hash>>
StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>
{
/// used when accounts contains hash and write version already
pub fn new(accounts: &'b U) -> Self {
assert!(accounts.has_hash_and_write_version());
Self {
accounts,
hashes_and_write_versions: None,
_phantom: PhantomData,
}
}
/// used when accounts does NOT contains hash or write version
/// In this case, hashes and write_versions have to be passed in separately and zipped together.
pub fn new_with_hashes_and_write_versions(
accounts: &'b U,
hashes: Vec<V>,
write_versions: Vec<StoredMetaWriteVersion>,
) -> Self {
assert!(!accounts.has_hash_and_write_version());
assert_eq!(accounts.len(), hashes.len());
assert_eq!(write_versions.len(), hashes.len());
Self {
accounts,
hashes_and_write_versions: Some((hashes, write_versions)),
_phantom: PhantomData,
}
}
/// get all account fields at 'index'
pub fn get(&self, index: usize) -> (Option<&T>, &Pubkey, &Hash, StoredMetaWriteVersion) {
let account = self.accounts.account_default_if_zero_lamport(index);
let pubkey = self.accounts.pubkey(index);
let (hash, write_version) = if self.accounts.has_hash_and_write_version() {
(
self.accounts.hash(index),
self.accounts.write_version(index),
)
} else {
let item = self.hashes_and_write_versions.as_ref().unwrap();
(item.0[index].borrow(), item.1[index])
};
(account, pubkey, hash, write_version)
}
/// None if account at index has lamports == 0
/// Otherwise, Some(account)
/// This is the only way to access the account.
pub fn account(&self, index: usize) -> Option<&T> {
self.accounts.account_default_if_zero_lamport(index)
}
/// # accounts to write
pub fn len(&self) -> usize {
self.accounts.len()
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
/// References to account data stored elsewhere. Getting an `Account` requires cloning
/// (see `StoredAccountMeta::clone_account()`).
#[derive(PartialEq, Eq, Debug)]
pub struct StoredAccountMeta<'a> {
pub meta: &'a StoredMeta,
/// account data
pub account_meta: &'a AccountMeta,
pub data: &'a [u8],
pub offset: usize,
pub stored_size: usize,
pub hash: &'a Hash,
}
impl<'a> StoredAccountMeta<'a> {
/// Return a new Account by copying all the data referenced by the `StoredAccountMeta`.
pub fn clone_account(&self) -> AccountSharedData {
AccountSharedData::from(Account {
lamports: self.account_meta.lamports,
owner: self.account_meta.owner,
executable: self.account_meta.executable,
rent_epoch: self.account_meta.rent_epoch,
data: self.data.to_vec(),
})
}
pub fn pubkey(&self) -> &Pubkey {
&self.meta.pubkey
}
pub(crate) fn sanitize(&self) -> bool {
self.sanitize_executable() && self.sanitize_lamports()
}
pub(crate) fn sanitize_executable(&self) -> bool {
// Sanitize executable to ensure higher 7-bits are cleared correctly.
self.ref_executable_byte() & !1 == 0
}
pub(crate) fn sanitize_lamports(&self) -> bool {
// Sanitize 0 lamports to ensure to be same as AccountSharedData::default()
self.account_meta.lamports != 0 || self.clone_account() == AccountSharedData::default()
}
pub(crate) fn ref_executable_byte(&self) -> &u8 {
// Use extra references to avoid value silently clamped to 1 (=true) and 0 (=false)
// Yes, this really happens; see test_new_from_file_crafted_executable
let executable_bool: &bool = &self.account_meta.executable;
// UNSAFE: Force to interpret mmap-backed bool as u8 to really read the actual memory content
let executable_byte: &u8 = unsafe { &*(executable_bool as *const bool as *const u8) };
executable_byte
}
}
/// Meta contains enough context to recover the index from storage itself
/// This struct will be backed by mmaped and snapshotted data files.
/// So the data layout must be stable and consistent across the entire cluster!
#[derive(Clone, PartialEq, Eq, Debug)]
#[repr(C)]
pub struct StoredMeta {
/// global write version
/// This will be made completely obsolete such that we stop storing it.
/// We will not support multiple append vecs per slot anymore, so this concept is no longer necessary.
/// Order of stores of an account to an append vec will determine 'latest' account data per pubkey.
pub write_version_obsolete: StoredMetaWriteVersion,
pub data_len: u64,
/// key for the account
pub pubkey: Pubkey,
}
/// This struct will be backed by mmaped and snapshotted data files.
/// So the data layout must be stable and consistent across the entire cluster!
#[derive(Serialize, Deserialize, Clone, Debug, Default, Eq, PartialEq)]
#[repr(C)]
pub struct AccountMeta {
/// lamports in the account
pub lamports: u64,
/// the epoch at which this account will next owe rent
pub rent_epoch: Epoch,
/// the program that owns this account. If executable, the program that loads this account.
pub owner: Pubkey,
/// this account's data contains a loaded program (and is now read-only)
pub executable: bool,
}
impl<'a, T: ReadableAccount> From<&'a T> for AccountMeta {
fn from(account: &'a T) -> Self {
Self {
lamports: account.lamports(),
owner: *account.owner(),
executable: account.executable(),
rent_epoch: account.rent_epoch(),
}
}
}
impl<'a, T: ReadableAccount> From<Option<&'a T>> for AccountMeta {
fn from(account: Option<&'a T>) -> Self {
match account {
Some(account) => AccountMeta::from(account),
None => AccountMeta::default(),
}
}
}

View File

@ -21,7 +21,13 @@
use {
crate::{
account_info::{AccountInfo, StorageLocation},
account_storage::{AccountStorage, AccountStorageStatus, ShrinkInProgress},
account_storage::{
meta::{
StorableAccountsWithHashesAndWriteVersions, StoredAccountMeta,
StoredMetaWriteVersion,
},
AccountStorage, AccountStorageStatus, ShrinkInProgress,
},
accounts_background_service::{DroppedSlotsSender, SendDroppedBankCallback},
accounts_cache::{AccountsCache, CachedAccount, SlotCache},
accounts_file::AccountsFile,
@ -44,9 +50,8 @@ use {
get_ancient_append_vec_capacity, is_ancient, AccountsToStore, StorageSelector,
},
append_vec::{
aligned_stored_size, AppendVec, MatchAccountOwnerError,
StorableAccountsWithHashesAndWriteVersions, StoredAccountMeta, StoredMetaWriteVersion,
APPEND_VEC_MMAPPED_FILES_OPEN, STORE_META_OVERHEAD,
aligned_stored_size, AppendVec, MatchAccountOwnerError, APPEND_VEC_MMAPPED_FILES_OPEN,
STORE_META_OVERHEAD,
},
cache_hash_data::{CacheHashData, CacheHashDataFile},
contains::Contains,
@ -9401,13 +9406,14 @@ pub mod tests {
super::*,
crate::{
account_info::StoredSize,
account_storage::meta::{AccountMeta, StoredMeta},
accounts::Accounts,
accounts_hash::MERKLE_FANOUT,
accounts_index::{
tests::*, AccountIndex, AccountSecondaryIndexes,
AccountSecondaryIndexesIncludeExclude, ReadAccountMapEntry, RefCount,
},
append_vec::{test_utils::TempFile, AccountMeta, StoredMeta},
append_vec::test_utils::TempFile,
cache_hash_data_stats::CacheHashDataStats,
inline_spl_token,
secondary_index::MAX_NUM_LARGEST_INDEX_KEYS_RETURNED,
@ -12153,8 +12159,7 @@ pub mod tests {
let slot = 42;
let num_threads = 2;
let min_file_bytes = std::mem::size_of::<StoredMeta>()
+ std::mem::size_of::<crate::append_vec::AccountMeta>();
let min_file_bytes = std::mem::size_of::<StoredMeta>() + std::mem::size_of::<AccountMeta>();
let db = Arc::new(AccountsDb::new_sized(Vec::new(), min_file_bytes as u64));

View File

@ -1,7 +1,7 @@
use {
crate::{
account_storage::meta::{StoredAccountMeta, StoredMeta},
accounts_db::AccountsDb,
append_vec::{StoredAccountMeta, StoredMeta},
},
solana_measure::measure::Measure,
solana_metrics::*,
@ -162,11 +162,11 @@ impl AccountsDb {
pub mod tests {
use {
crate::{
account_storage::meta::StoredAccountMeta,
accounts_db::AccountsDb,
accounts_update_notifier_interface::{
AccountsUpdateNotifier, AccountsUpdateNotifierInterface,
},
append_vec::StoredAccountMeta,
},
dashmap::DashMap,
solana_sdk::{

View File

@ -1,9 +1,7 @@
use {
crate::{
append_vec::{
AppendVec, MatchAccountOwnerError, StorableAccountsWithHashesAndWriteVersions,
StoredAccountMeta,
},
account_storage::meta::{StorableAccountsWithHashesAndWriteVersions, StoredAccountMeta},
append_vec::{AppendVec, MatchAccountOwnerError},
storable_accounts::StorableAccounts,
},
solana_sdk::{account::ReadableAccount, clock::Slot, hash::Hash, pubkey::Pubkey},

View File

@ -1,5 +1,5 @@
use {
crate::append_vec::StoredAccountMeta,
crate::account_storage::meta::StoredAccountMeta,
solana_sdk::{
account::AccountSharedData, clock::Slot, pubkey::Pubkey, transaction::SanitizedTransaction,
},

View File

@ -5,7 +5,7 @@
//! Otherwise, an ancient append vec is the same as any other append vec
use {
crate::{
account_storage::ShrinkInProgress,
account_storage::{meta::StoredAccountMeta, ShrinkInProgress},
accounts_db::{
AccountStorageEntry, AccountsDb, AliveAccounts, GetUniqueAccountsResult, ShrinkCollect,
ShrinkCollectAliveSeparatedByRefs, ShrinkStatsSub, StoreReclaims,
@ -14,7 +14,7 @@ use {
accounts_file::AccountsFile,
accounts_index::ZeroLamport,
active_stats::ActiveStatItem,
append_vec::{aligned_stored_size, StoredAccountMeta},
append_vec::aligned_stored_size,
storable_accounts::{StorableAccounts, StorableAccountsBySlot},
},
rand::{thread_rng, Rng},
@ -760,6 +760,7 @@ pub mod tests {
use {
super::*,
crate::{
account_storage::meta::{AccountMeta, StoredAccountMeta, StoredMeta},
accounts_db::{
get_temp_accounts_paths,
tests::{
@ -769,9 +770,7 @@ pub mod tests {
},
INCLUDE_SLOT_IN_HASH_TESTS,
},
append_vec::{
aligned_stored_size, AccountMeta, AppendVec, StoredAccountMeta, StoredMeta,
},
append_vec::{aligned_stored_size, AppendVec},
storable_accounts::StorableAccountsBySlot,
},
solana_sdk::{

View File

@ -5,22 +5,20 @@
//! <https://docs.solana.com/implemented-proposals/persistent-account-storage>
use {
crate::storable_accounts::StorableAccounts,
crate::{
account_storage::meta::{
AccountMeta, StorableAccountsWithHashesAndWriteVersions, StoredAccountMeta, StoredMeta,
},
storable_accounts::StorableAccounts,
},
log::*,
memmap2::MmapMut,
serde::{Deserialize, Serialize},
solana_sdk::{
account::{Account, AccountSharedData, ReadableAccount},
clock::{Epoch, Slot},
hash::Hash,
pubkey::Pubkey,
},
solana_sdk::{account::ReadableAccount, clock::Slot, hash::Hash, pubkey::Pubkey},
std::{
borrow::Borrow,
convert::TryFrom,
fs::{remove_file, OpenOptions},
io::{self, Seek, SeekFrom, Write},
marker::PhantomData,
mem,
path::{Path, PathBuf},
sync::{
@ -55,195 +53,6 @@ pub fn aligned_stored_size(data_len: usize) -> usize {
pub const MAXIMUM_APPEND_VEC_FILE_SIZE: u64 = 16 * 1024 * 1024 * 1024; // 16 GiB
pub type StoredMetaWriteVersion = u64;
/// Goal is to eliminate copies and data reshaping given various code paths that store accounts.
/// This struct contains what is needed to store accounts to a storage
/// 1. account & pubkey (StorableAccounts)
/// 2. hash per account (Maybe in StorableAccounts, otherwise has to be passed in separately)
/// 3. write version per account (Maybe in StorableAccounts, otherwise has to be passed in separately)
pub struct StorableAccountsWithHashesAndWriteVersions<
'a: 'b,
'b,
T: ReadableAccount + Sync + 'b,
U: StorableAccounts<'a, T>,
V: Borrow<Hash>,
> {
/// accounts to store
/// always has pubkey and account
/// may also have hash and write_version per account
accounts: &'b U,
/// if accounts does not have hash and write version, this has a hash and write version per account
hashes_and_write_versions: Option<(Vec<V>, Vec<StoredMetaWriteVersion>)>,
_phantom: PhantomData<&'a T>,
}
impl<'a: 'b, 'b, T: ReadableAccount + Sync + 'b, U: StorableAccounts<'a, T>, V: Borrow<Hash>>
StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>
{
/// used when accounts contains hash and write version already
pub fn new(accounts: &'b U) -> Self {
assert!(accounts.has_hash_and_write_version());
Self {
accounts,
hashes_and_write_versions: None,
_phantom: PhantomData,
}
}
/// used when accounts does NOT contains hash or write version
/// In this case, hashes and write_versions have to be passed in separately and zipped together.
pub fn new_with_hashes_and_write_versions(
accounts: &'b U,
hashes: Vec<V>,
write_versions: Vec<StoredMetaWriteVersion>,
) -> Self {
assert!(!accounts.has_hash_and_write_version());
assert_eq!(accounts.len(), hashes.len());
assert_eq!(write_versions.len(), hashes.len());
Self {
accounts,
hashes_and_write_versions: Some((hashes, write_versions)),
_phantom: PhantomData,
}
}
/// get all account fields at 'index'
pub fn get(&self, index: usize) -> (Option<&T>, &Pubkey, &Hash, StoredMetaWriteVersion) {
let account = self.accounts.account_default_if_zero_lamport(index);
let pubkey = self.accounts.pubkey(index);
let (hash, write_version) = if self.accounts.has_hash_and_write_version() {
(
self.accounts.hash(index),
self.accounts.write_version(index),
)
} else {
let item = self.hashes_and_write_versions.as_ref().unwrap();
(item.0[index].borrow(), item.1[index])
};
(account, pubkey, hash, write_version)
}
/// None if account at index has lamports == 0
/// Otherwise, Some(account)
/// This is the only way to access the account.
pub fn account(&self, index: usize) -> Option<&T> {
self.accounts.account_default_if_zero_lamport(index)
}
/// # accounts to write
pub fn len(&self) -> usize {
self.accounts.len()
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
/// Meta contains enough context to recover the index from storage itself
/// This struct will be backed by mmaped and snapshotted data files.
/// So the data layout must be stable and consistent across the entire cluster!
#[derive(Clone, PartialEq, Eq, Debug)]
#[repr(C)]
pub struct StoredMeta {
/// global write version
/// This will be made completely obsolete such that we stop storing it.
/// We will not support multiple append vecs per slot anymore, so this concept is no longer necessary.
/// Order of stores of an account to an append vec will determine 'latest' account data per pubkey.
pub write_version_obsolete: StoredMetaWriteVersion,
pub data_len: u64,
/// key for the account
pub pubkey: Pubkey,
}
/// This struct will be backed by mmaped and snapshotted data files.
/// So the data layout must be stable and consistent across the entire cluster!
#[derive(Serialize, Deserialize, Clone, Debug, Default, Eq, PartialEq)]
#[repr(C)]
pub struct AccountMeta {
/// lamports in the account
pub lamports: u64,
/// the epoch at which this account will next owe rent
pub rent_epoch: Epoch,
/// the program that owns this account. If executable, the program that loads this account.
pub owner: Pubkey,
/// this account's data contains a loaded program (and is now read-only)
pub executable: bool,
}
impl<'a, T: ReadableAccount> From<&'a T> for AccountMeta {
fn from(account: &'a T) -> Self {
Self {
lamports: account.lamports(),
owner: *account.owner(),
executable: account.executable(),
rent_epoch: account.rent_epoch(),
}
}
}
impl<'a, T: ReadableAccount> From<Option<&'a T>> for AccountMeta {
fn from(account: Option<&'a T>) -> Self {
match account {
Some(account) => AccountMeta::from(account),
None => AccountMeta::default(),
}
}
}
/// References to account data stored elsewhere. Getting an `Account` requires cloning
/// (see `StoredAccountMeta::clone_account()`).
#[derive(PartialEq, Eq, Debug)]
pub struct StoredAccountMeta<'a> {
pub meta: &'a StoredMeta,
/// account data
pub account_meta: &'a AccountMeta,
pub data: &'a [u8],
pub offset: usize,
pub stored_size: usize,
pub hash: &'a Hash,
}
impl<'a> StoredAccountMeta<'a> {
/// Return a new Account by copying all the data referenced by the `StoredAccountMeta`.
pub fn clone_account(&self) -> AccountSharedData {
AccountSharedData::from(Account {
lamports: self.account_meta.lamports,
owner: self.account_meta.owner,
executable: self.account_meta.executable,
rent_epoch: self.account_meta.rent_epoch,
data: self.data.to_vec(),
})
}
pub fn pubkey(&self) -> &Pubkey {
&self.meta.pubkey
}
fn sanitize(&self) -> bool {
self.sanitize_executable() && self.sanitize_lamports()
}
fn sanitize_executable(&self) -> bool {
// Sanitize executable to ensure higher 7-bits are cleared correctly.
self.ref_executable_byte() & !1 == 0
}
fn sanitize_lamports(&self) -> bool {
// Sanitize 0 lamports to ensure to be same as AccountSharedData::default()
self.account_meta.lamports != 0 || self.clone_account() == AccountSharedData::default()
}
fn ref_executable_byte(&self) -> &u8 {
// Use extra references to avoid value silently clamped to 1 (=true) and 0 (=false)
// Yes, this really happens; see test_new_from_file_crafted_executable
let executable_bool: &bool = &self.account_meta.executable;
// UNSAFE: Force to interpret mmap-backed bool as u8 to really read the actual memory content
let executable_byte: &u8 = unsafe { &*(executable_bool as *const bool as *const u8) };
executable_byte
}
}
pub struct AppendVecAccountsIter<'a> {
append_vec: &'a AppendVec,
offset: usize,
@ -635,7 +444,10 @@ impl AppendVec {
}
#[cfg(test)]
pub fn get_account_test(&self, offset: usize) -> Option<(StoredMeta, AccountSharedData)> {
pub fn get_account_test(
&self,
offset: usize,
) -> Option<(StoredMeta, solana_sdk::account::AccountSharedData)> {
let (stored_account, _) = self.get_account(offset)?;
let meta = stored_account.meta.clone();
Some((meta, stored_account.clone_account()))
@ -743,7 +555,7 @@ pub mod tests {
memoffset::offset_of,
rand::{thread_rng, Rng},
solana_sdk::{
account::{accounts_equal, WritableAccount},
account::{accounts_equal, Account, AccountSharedData, WritableAccount},
timing::duration_as_ms,
},
std::time::Instant,

View File

@ -1,5 +1,6 @@
use {
crate::{
account_storage::meta::StoredMetaWriteVersion,
accounts::Accounts,
accounts_db::{
AccountShrinkThreshold, AccountStorageEntry, AccountsDb, AccountsDbConfig, AppendVecId,
@ -9,7 +10,7 @@ use {
accounts_hash::{AccountsDeltaHash, AccountsHash},
accounts_index::AccountSecondaryIndexes,
accounts_update_notifier_interface::AccountsUpdateNotifier,
append_vec::{AppendVec, StoredMetaWriteVersion},
append_vec::AppendVec,
bank::{Bank, BankFieldsToDeserialize, BankIncrementalSnapshotPersistence, BankRc},
blockhash_queue::BlockhashQueue,
builtins::Builtins,

View File

@ -1,6 +1,6 @@
//! trait for abstracting underlying storage of pubkey and account pairs to be written
use {
crate::{accounts_db::IncludeSlotInHash, append_vec::StoredAccountMeta},
crate::{account_storage::meta::StoredAccountMeta, accounts_db::IncludeSlotInHash},
solana_sdk::{account::ReadableAccount, clock::Slot, hash::Hash, pubkey::Pubkey},
};
@ -304,8 +304,8 @@ pub mod tests {
use {
super::*,
crate::{
account_storage::meta::{AccountMeta, StoredAccountMeta, StoredMeta},
accounts_db::INCLUDE_SLOT_IN_HASH_TESTS,
append_vec::{AccountMeta, StoredAccountMeta, StoredMeta},
},
solana_sdk::{
account::{accounts_equal, AccountSharedData, WritableAccount},

View File

@ -1,7 +1,7 @@
use {
clap::{crate_description, crate_name, value_t, value_t_or_exit, App, Arg},
log::*,
solana_runtime::append_vec::{AppendVec, StoredAccountMeta},
solana_runtime::{account_storage::meta::StoredAccountMeta, append_vec::AppendVec},
solana_sdk::{account::AccountSharedData, hash::Hash, pubkey::Pubkey},
};