Move account-meta structs from append_vec.rs to account_meta.rs (#30443)
#### Problem As we start supporting new storage formats, those account-meta structs will be shared by different storage formats and thus need a new home. #### Summary of Changes This PR creates meta.rs under account_storage and moves all the account-meta structs out from append_vec.rs.
This commit is contained in:
parent
69ea295b07
commit
ac7e7aa8f0
|
@ -8,8 +8,8 @@ use {
|
||||||
solana_measure::measure::Measure,
|
solana_measure::measure::Measure,
|
||||||
solana_metrics::*,
|
solana_metrics::*,
|
||||||
solana_runtime::{
|
solana_runtime::{
|
||||||
|
account_storage::meta::StoredAccountMeta,
|
||||||
accounts_update_notifier_interface::AccountsUpdateNotifierInterface,
|
accounts_update_notifier_interface::AccountsUpdateNotifierInterface,
|
||||||
append_vec::StoredAccountMeta,
|
|
||||||
},
|
},
|
||||||
solana_sdk::{
|
solana_sdk::{
|
||||||
account::{AccountSharedData, ReadableAccount},
|
account::{AccountSharedData, ReadableAccount},
|
||||||
|
|
|
@ -4,10 +4,11 @@ extern crate test;
|
||||||
use {
|
use {
|
||||||
rand::{thread_rng, Rng},
|
rand::{thread_rng, Rng},
|
||||||
solana_runtime::{
|
solana_runtime::{
|
||||||
|
account_storage::meta::{StorableAccountsWithHashesAndWriteVersions, StoredMeta},
|
||||||
accounts_db::INCLUDE_SLOT_IN_HASH_TESTS,
|
accounts_db::INCLUDE_SLOT_IN_HASH_TESTS,
|
||||||
append_vec::{
|
append_vec::{
|
||||||
test_utils::{create_test_account, get_append_vec_path},
|
test_utils::{create_test_account, get_append_vec_path},
|
||||||
AppendVec, StorableAccountsWithHashesAndWriteVersions, StoredMeta,
|
AppendVec,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
solana_sdk::{
|
solana_sdk::{
|
||||||
|
|
|
@ -7,6 +7,8 @@ use {
|
||||||
std::sync::Arc,
|
std::sync::Arc,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
pub mod meta;
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub struct AccountStorageReference {
|
pub struct AccountStorageReference {
|
||||||
/// the single storage for a given slot
|
/// the single storage for a given slot
|
||||||
|
|
|
@ -0,0 +1,199 @@
|
||||||
|
use {
|
||||||
|
crate::storable_accounts::StorableAccounts,
|
||||||
|
solana_sdk::{
|
||||||
|
account::{Account, AccountSharedData, ReadableAccount},
|
||||||
|
hash::Hash,
|
||||||
|
pubkey::Pubkey,
|
||||||
|
stake_history::Epoch,
|
||||||
|
},
|
||||||
|
std::{borrow::Borrow, marker::PhantomData},
|
||||||
|
};
|
||||||
|
|
||||||
|
pub type StoredMetaWriteVersion = u64;
|
||||||
|
|
||||||
|
/// Goal is to eliminate copies and data reshaping given various code paths that store accounts.
|
||||||
|
/// This struct contains what is needed to store accounts to a storage
|
||||||
|
/// 1. account & pubkey (StorableAccounts)
|
||||||
|
/// 2. hash per account (Maybe in StorableAccounts, otherwise has to be passed in separately)
|
||||||
|
/// 3. write version per account (Maybe in StorableAccounts, otherwise has to be passed in separately)
|
||||||
|
pub struct StorableAccountsWithHashesAndWriteVersions<
|
||||||
|
'a: 'b,
|
||||||
|
'b,
|
||||||
|
T: ReadableAccount + Sync + 'b,
|
||||||
|
U: StorableAccounts<'a, T>,
|
||||||
|
V: Borrow<Hash>,
|
||||||
|
> {
|
||||||
|
/// accounts to store
|
||||||
|
/// always has pubkey and account
|
||||||
|
/// may also have hash and write_version per account
|
||||||
|
pub(crate) accounts: &'b U,
|
||||||
|
/// if accounts does not have hash and write version, this has a hash and write version per account
|
||||||
|
hashes_and_write_versions: Option<(Vec<V>, Vec<StoredMetaWriteVersion>)>,
|
||||||
|
_phantom: PhantomData<&'a T>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a: 'b, 'b, T: ReadableAccount + Sync + 'b, U: StorableAccounts<'a, T>, V: Borrow<Hash>>
|
||||||
|
StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>
|
||||||
|
{
|
||||||
|
/// used when accounts contains hash and write version already
|
||||||
|
pub fn new(accounts: &'b U) -> Self {
|
||||||
|
assert!(accounts.has_hash_and_write_version());
|
||||||
|
Self {
|
||||||
|
accounts,
|
||||||
|
hashes_and_write_versions: None,
|
||||||
|
_phantom: PhantomData,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/// used when accounts does NOT contains hash or write version
|
||||||
|
/// In this case, hashes and write_versions have to be passed in separately and zipped together.
|
||||||
|
pub fn new_with_hashes_and_write_versions(
|
||||||
|
accounts: &'b U,
|
||||||
|
hashes: Vec<V>,
|
||||||
|
write_versions: Vec<StoredMetaWriteVersion>,
|
||||||
|
) -> Self {
|
||||||
|
assert!(!accounts.has_hash_and_write_version());
|
||||||
|
assert_eq!(accounts.len(), hashes.len());
|
||||||
|
assert_eq!(write_versions.len(), hashes.len());
|
||||||
|
Self {
|
||||||
|
accounts,
|
||||||
|
hashes_and_write_versions: Some((hashes, write_versions)),
|
||||||
|
_phantom: PhantomData,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// get all account fields at 'index'
|
||||||
|
pub fn get(&self, index: usize) -> (Option<&T>, &Pubkey, &Hash, StoredMetaWriteVersion) {
|
||||||
|
let account = self.accounts.account_default_if_zero_lamport(index);
|
||||||
|
let pubkey = self.accounts.pubkey(index);
|
||||||
|
let (hash, write_version) = if self.accounts.has_hash_and_write_version() {
|
||||||
|
(
|
||||||
|
self.accounts.hash(index),
|
||||||
|
self.accounts.write_version(index),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
let item = self.hashes_and_write_versions.as_ref().unwrap();
|
||||||
|
(item.0[index].borrow(), item.1[index])
|
||||||
|
};
|
||||||
|
(account, pubkey, hash, write_version)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// None if account at index has lamports == 0
|
||||||
|
/// Otherwise, Some(account)
|
||||||
|
/// This is the only way to access the account.
|
||||||
|
pub fn account(&self, index: usize) -> Option<&T> {
|
||||||
|
self.accounts.account_default_if_zero_lamport(index)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// # accounts to write
|
||||||
|
pub fn len(&self) -> usize {
|
||||||
|
self.accounts.len()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_empty(&self) -> bool {
|
||||||
|
self.len() == 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// References to account data stored elsewhere. Getting an `Account` requires cloning
|
||||||
|
/// (see `StoredAccountMeta::clone_account()`).
|
||||||
|
#[derive(PartialEq, Eq, Debug)]
|
||||||
|
pub struct StoredAccountMeta<'a> {
|
||||||
|
pub meta: &'a StoredMeta,
|
||||||
|
/// account data
|
||||||
|
pub account_meta: &'a AccountMeta,
|
||||||
|
pub data: &'a [u8],
|
||||||
|
pub offset: usize,
|
||||||
|
pub stored_size: usize,
|
||||||
|
pub hash: &'a Hash,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> StoredAccountMeta<'a> {
|
||||||
|
/// Return a new Account by copying all the data referenced by the `StoredAccountMeta`.
|
||||||
|
pub fn clone_account(&self) -> AccountSharedData {
|
||||||
|
AccountSharedData::from(Account {
|
||||||
|
lamports: self.account_meta.lamports,
|
||||||
|
owner: self.account_meta.owner,
|
||||||
|
executable: self.account_meta.executable,
|
||||||
|
rent_epoch: self.account_meta.rent_epoch,
|
||||||
|
data: self.data.to_vec(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn pubkey(&self) -> &Pubkey {
|
||||||
|
&self.meta.pubkey
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn sanitize(&self) -> bool {
|
||||||
|
self.sanitize_executable() && self.sanitize_lamports()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn sanitize_executable(&self) -> bool {
|
||||||
|
// Sanitize executable to ensure higher 7-bits are cleared correctly.
|
||||||
|
self.ref_executable_byte() & !1 == 0
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn sanitize_lamports(&self) -> bool {
|
||||||
|
// Sanitize 0 lamports to ensure to be same as AccountSharedData::default()
|
||||||
|
self.account_meta.lamports != 0 || self.clone_account() == AccountSharedData::default()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn ref_executable_byte(&self) -> &u8 {
|
||||||
|
// Use extra references to avoid value silently clamped to 1 (=true) and 0 (=false)
|
||||||
|
// Yes, this really happens; see test_new_from_file_crafted_executable
|
||||||
|
let executable_bool: &bool = &self.account_meta.executable;
|
||||||
|
// UNSAFE: Force to interpret mmap-backed bool as u8 to really read the actual memory content
|
||||||
|
let executable_byte: &u8 = unsafe { &*(executable_bool as *const bool as *const u8) };
|
||||||
|
executable_byte
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Meta contains enough context to recover the index from storage itself
|
||||||
|
/// This struct will be backed by mmaped and snapshotted data files.
|
||||||
|
/// So the data layout must be stable and consistent across the entire cluster!
|
||||||
|
#[derive(Clone, PartialEq, Eq, Debug)]
|
||||||
|
#[repr(C)]
|
||||||
|
pub struct StoredMeta {
|
||||||
|
/// global write version
|
||||||
|
/// This will be made completely obsolete such that we stop storing it.
|
||||||
|
/// We will not support multiple append vecs per slot anymore, so this concept is no longer necessary.
|
||||||
|
/// Order of stores of an account to an append vec will determine 'latest' account data per pubkey.
|
||||||
|
pub write_version_obsolete: StoredMetaWriteVersion,
|
||||||
|
pub data_len: u64,
|
||||||
|
/// key for the account
|
||||||
|
pub pubkey: Pubkey,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// This struct will be backed by mmaped and snapshotted data files.
|
||||||
|
/// So the data layout must be stable and consistent across the entire cluster!
|
||||||
|
#[derive(Serialize, Deserialize, Clone, Debug, Default, Eq, PartialEq)]
|
||||||
|
#[repr(C)]
|
||||||
|
pub struct AccountMeta {
|
||||||
|
/// lamports in the account
|
||||||
|
pub lamports: u64,
|
||||||
|
/// the epoch at which this account will next owe rent
|
||||||
|
pub rent_epoch: Epoch,
|
||||||
|
/// the program that owns this account. If executable, the program that loads this account.
|
||||||
|
pub owner: Pubkey,
|
||||||
|
/// this account's data contains a loaded program (and is now read-only)
|
||||||
|
pub executable: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a, T: ReadableAccount> From<&'a T> for AccountMeta {
|
||||||
|
fn from(account: &'a T) -> Self {
|
||||||
|
Self {
|
||||||
|
lamports: account.lamports(),
|
||||||
|
owner: *account.owner(),
|
||||||
|
executable: account.executable(),
|
||||||
|
rent_epoch: account.rent_epoch(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a, T: ReadableAccount> From<Option<&'a T>> for AccountMeta {
|
||||||
|
fn from(account: Option<&'a T>) -> Self {
|
||||||
|
match account {
|
||||||
|
Some(account) => AccountMeta::from(account),
|
||||||
|
None => AccountMeta::default(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -21,7 +21,13 @@
|
||||||
use {
|
use {
|
||||||
crate::{
|
crate::{
|
||||||
account_info::{AccountInfo, StorageLocation},
|
account_info::{AccountInfo, StorageLocation},
|
||||||
account_storage::{AccountStorage, AccountStorageStatus, ShrinkInProgress},
|
account_storage::{
|
||||||
|
meta::{
|
||||||
|
StorableAccountsWithHashesAndWriteVersions, StoredAccountMeta,
|
||||||
|
StoredMetaWriteVersion,
|
||||||
|
},
|
||||||
|
AccountStorage, AccountStorageStatus, ShrinkInProgress,
|
||||||
|
},
|
||||||
accounts_background_service::{DroppedSlotsSender, SendDroppedBankCallback},
|
accounts_background_service::{DroppedSlotsSender, SendDroppedBankCallback},
|
||||||
accounts_cache::{AccountsCache, CachedAccount, SlotCache},
|
accounts_cache::{AccountsCache, CachedAccount, SlotCache},
|
||||||
accounts_file::AccountsFile,
|
accounts_file::AccountsFile,
|
||||||
|
@ -44,9 +50,8 @@ use {
|
||||||
get_ancient_append_vec_capacity, is_ancient, AccountsToStore, StorageSelector,
|
get_ancient_append_vec_capacity, is_ancient, AccountsToStore, StorageSelector,
|
||||||
},
|
},
|
||||||
append_vec::{
|
append_vec::{
|
||||||
aligned_stored_size, AppendVec, MatchAccountOwnerError,
|
aligned_stored_size, AppendVec, MatchAccountOwnerError, APPEND_VEC_MMAPPED_FILES_OPEN,
|
||||||
StorableAccountsWithHashesAndWriteVersions, StoredAccountMeta, StoredMetaWriteVersion,
|
STORE_META_OVERHEAD,
|
||||||
APPEND_VEC_MMAPPED_FILES_OPEN, STORE_META_OVERHEAD,
|
|
||||||
},
|
},
|
||||||
cache_hash_data::{CacheHashData, CacheHashDataFile},
|
cache_hash_data::{CacheHashData, CacheHashDataFile},
|
||||||
contains::Contains,
|
contains::Contains,
|
||||||
|
@ -9401,13 +9406,14 @@ pub mod tests {
|
||||||
super::*,
|
super::*,
|
||||||
crate::{
|
crate::{
|
||||||
account_info::StoredSize,
|
account_info::StoredSize,
|
||||||
|
account_storage::meta::{AccountMeta, StoredMeta},
|
||||||
accounts::Accounts,
|
accounts::Accounts,
|
||||||
accounts_hash::MERKLE_FANOUT,
|
accounts_hash::MERKLE_FANOUT,
|
||||||
accounts_index::{
|
accounts_index::{
|
||||||
tests::*, AccountIndex, AccountSecondaryIndexes,
|
tests::*, AccountIndex, AccountSecondaryIndexes,
|
||||||
AccountSecondaryIndexesIncludeExclude, ReadAccountMapEntry, RefCount,
|
AccountSecondaryIndexesIncludeExclude, ReadAccountMapEntry, RefCount,
|
||||||
},
|
},
|
||||||
append_vec::{test_utils::TempFile, AccountMeta, StoredMeta},
|
append_vec::test_utils::TempFile,
|
||||||
cache_hash_data_stats::CacheHashDataStats,
|
cache_hash_data_stats::CacheHashDataStats,
|
||||||
inline_spl_token,
|
inline_spl_token,
|
||||||
secondary_index::MAX_NUM_LARGEST_INDEX_KEYS_RETURNED,
|
secondary_index::MAX_NUM_LARGEST_INDEX_KEYS_RETURNED,
|
||||||
|
@ -12153,8 +12159,7 @@ pub mod tests {
|
||||||
let slot = 42;
|
let slot = 42;
|
||||||
let num_threads = 2;
|
let num_threads = 2;
|
||||||
|
|
||||||
let min_file_bytes = std::mem::size_of::<StoredMeta>()
|
let min_file_bytes = std::mem::size_of::<StoredMeta>() + std::mem::size_of::<AccountMeta>();
|
||||||
+ std::mem::size_of::<crate::append_vec::AccountMeta>();
|
|
||||||
|
|
||||||
let db = Arc::new(AccountsDb::new_sized(Vec::new(), min_file_bytes as u64));
|
let db = Arc::new(AccountsDb::new_sized(Vec::new(), min_file_bytes as u64));
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
use {
|
use {
|
||||||
crate::{
|
crate::{
|
||||||
|
account_storage::meta::{StoredAccountMeta, StoredMeta},
|
||||||
accounts_db::AccountsDb,
|
accounts_db::AccountsDb,
|
||||||
append_vec::{StoredAccountMeta, StoredMeta},
|
|
||||||
},
|
},
|
||||||
solana_measure::measure::Measure,
|
solana_measure::measure::Measure,
|
||||||
solana_metrics::*,
|
solana_metrics::*,
|
||||||
|
@ -162,11 +162,11 @@ impl AccountsDb {
|
||||||
pub mod tests {
|
pub mod tests {
|
||||||
use {
|
use {
|
||||||
crate::{
|
crate::{
|
||||||
|
account_storage::meta::StoredAccountMeta,
|
||||||
accounts_db::AccountsDb,
|
accounts_db::AccountsDb,
|
||||||
accounts_update_notifier_interface::{
|
accounts_update_notifier_interface::{
|
||||||
AccountsUpdateNotifier, AccountsUpdateNotifierInterface,
|
AccountsUpdateNotifier, AccountsUpdateNotifierInterface,
|
||||||
},
|
},
|
||||||
append_vec::StoredAccountMeta,
|
|
||||||
},
|
},
|
||||||
dashmap::DashMap,
|
dashmap::DashMap,
|
||||||
solana_sdk::{
|
solana_sdk::{
|
||||||
|
|
|
@ -1,9 +1,7 @@
|
||||||
use {
|
use {
|
||||||
crate::{
|
crate::{
|
||||||
append_vec::{
|
account_storage::meta::{StorableAccountsWithHashesAndWriteVersions, StoredAccountMeta},
|
||||||
AppendVec, MatchAccountOwnerError, StorableAccountsWithHashesAndWriteVersions,
|
append_vec::{AppendVec, MatchAccountOwnerError},
|
||||||
StoredAccountMeta,
|
|
||||||
},
|
|
||||||
storable_accounts::StorableAccounts,
|
storable_accounts::StorableAccounts,
|
||||||
},
|
},
|
||||||
solana_sdk::{account::ReadableAccount, clock::Slot, hash::Hash, pubkey::Pubkey},
|
solana_sdk::{account::ReadableAccount, clock::Slot, hash::Hash, pubkey::Pubkey},
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
use {
|
use {
|
||||||
crate::append_vec::StoredAccountMeta,
|
crate::account_storage::meta::StoredAccountMeta,
|
||||||
solana_sdk::{
|
solana_sdk::{
|
||||||
account::AccountSharedData, clock::Slot, pubkey::Pubkey, transaction::SanitizedTransaction,
|
account::AccountSharedData, clock::Slot, pubkey::Pubkey, transaction::SanitizedTransaction,
|
||||||
},
|
},
|
||||||
|
|
|
@ -5,7 +5,7 @@
|
||||||
//! Otherwise, an ancient append vec is the same as any other append vec
|
//! Otherwise, an ancient append vec is the same as any other append vec
|
||||||
use {
|
use {
|
||||||
crate::{
|
crate::{
|
||||||
account_storage::ShrinkInProgress,
|
account_storage::{meta::StoredAccountMeta, ShrinkInProgress},
|
||||||
accounts_db::{
|
accounts_db::{
|
||||||
AccountStorageEntry, AccountsDb, AliveAccounts, GetUniqueAccountsResult, ShrinkCollect,
|
AccountStorageEntry, AccountsDb, AliveAccounts, GetUniqueAccountsResult, ShrinkCollect,
|
||||||
ShrinkCollectAliveSeparatedByRefs, ShrinkStatsSub, StoreReclaims,
|
ShrinkCollectAliveSeparatedByRefs, ShrinkStatsSub, StoreReclaims,
|
||||||
|
@ -14,7 +14,7 @@ use {
|
||||||
accounts_file::AccountsFile,
|
accounts_file::AccountsFile,
|
||||||
accounts_index::ZeroLamport,
|
accounts_index::ZeroLamport,
|
||||||
active_stats::ActiveStatItem,
|
active_stats::ActiveStatItem,
|
||||||
append_vec::{aligned_stored_size, StoredAccountMeta},
|
append_vec::aligned_stored_size,
|
||||||
storable_accounts::{StorableAccounts, StorableAccountsBySlot},
|
storable_accounts::{StorableAccounts, StorableAccountsBySlot},
|
||||||
},
|
},
|
||||||
rand::{thread_rng, Rng},
|
rand::{thread_rng, Rng},
|
||||||
|
@ -760,6 +760,7 @@ pub mod tests {
|
||||||
use {
|
use {
|
||||||
super::*,
|
super::*,
|
||||||
crate::{
|
crate::{
|
||||||
|
account_storage::meta::{AccountMeta, StoredAccountMeta, StoredMeta},
|
||||||
accounts_db::{
|
accounts_db::{
|
||||||
get_temp_accounts_paths,
|
get_temp_accounts_paths,
|
||||||
tests::{
|
tests::{
|
||||||
|
@ -769,9 +770,7 @@ pub mod tests {
|
||||||
},
|
},
|
||||||
INCLUDE_SLOT_IN_HASH_TESTS,
|
INCLUDE_SLOT_IN_HASH_TESTS,
|
||||||
},
|
},
|
||||||
append_vec::{
|
append_vec::{aligned_stored_size, AppendVec},
|
||||||
aligned_stored_size, AccountMeta, AppendVec, StoredAccountMeta, StoredMeta,
|
|
||||||
},
|
|
||||||
storable_accounts::StorableAccountsBySlot,
|
storable_accounts::StorableAccountsBySlot,
|
||||||
},
|
},
|
||||||
solana_sdk::{
|
solana_sdk::{
|
||||||
|
|
|
@ -5,22 +5,20 @@
|
||||||
//! <https://docs.solana.com/implemented-proposals/persistent-account-storage>
|
//! <https://docs.solana.com/implemented-proposals/persistent-account-storage>
|
||||||
|
|
||||||
use {
|
use {
|
||||||
crate::storable_accounts::StorableAccounts,
|
crate::{
|
||||||
|
account_storage::meta::{
|
||||||
|
AccountMeta, StorableAccountsWithHashesAndWriteVersions, StoredAccountMeta, StoredMeta,
|
||||||
|
},
|
||||||
|
storable_accounts::StorableAccounts,
|
||||||
|
},
|
||||||
log::*,
|
log::*,
|
||||||
memmap2::MmapMut,
|
memmap2::MmapMut,
|
||||||
serde::{Deserialize, Serialize},
|
solana_sdk::{account::ReadableAccount, clock::Slot, hash::Hash, pubkey::Pubkey},
|
||||||
solana_sdk::{
|
|
||||||
account::{Account, AccountSharedData, ReadableAccount},
|
|
||||||
clock::{Epoch, Slot},
|
|
||||||
hash::Hash,
|
|
||||||
pubkey::Pubkey,
|
|
||||||
},
|
|
||||||
std::{
|
std::{
|
||||||
borrow::Borrow,
|
borrow::Borrow,
|
||||||
convert::TryFrom,
|
convert::TryFrom,
|
||||||
fs::{remove_file, OpenOptions},
|
fs::{remove_file, OpenOptions},
|
||||||
io::{self, Seek, SeekFrom, Write},
|
io::{self, Seek, SeekFrom, Write},
|
||||||
marker::PhantomData,
|
|
||||||
mem,
|
mem,
|
||||||
path::{Path, PathBuf},
|
path::{Path, PathBuf},
|
||||||
sync::{
|
sync::{
|
||||||
|
@ -55,195 +53,6 @@ pub fn aligned_stored_size(data_len: usize) -> usize {
|
||||||
|
|
||||||
pub const MAXIMUM_APPEND_VEC_FILE_SIZE: u64 = 16 * 1024 * 1024 * 1024; // 16 GiB
|
pub const MAXIMUM_APPEND_VEC_FILE_SIZE: u64 = 16 * 1024 * 1024 * 1024; // 16 GiB
|
||||||
|
|
||||||
pub type StoredMetaWriteVersion = u64;
|
|
||||||
|
|
||||||
/// Goal is to eliminate copies and data reshaping given various code paths that store accounts.
|
|
||||||
/// This struct contains what is needed to store accounts to a storage
|
|
||||||
/// 1. account & pubkey (StorableAccounts)
|
|
||||||
/// 2. hash per account (Maybe in StorableAccounts, otherwise has to be passed in separately)
|
|
||||||
/// 3. write version per account (Maybe in StorableAccounts, otherwise has to be passed in separately)
|
|
||||||
pub struct StorableAccountsWithHashesAndWriteVersions<
|
|
||||||
'a: 'b,
|
|
||||||
'b,
|
|
||||||
T: ReadableAccount + Sync + 'b,
|
|
||||||
U: StorableAccounts<'a, T>,
|
|
||||||
V: Borrow<Hash>,
|
|
||||||
> {
|
|
||||||
/// accounts to store
|
|
||||||
/// always has pubkey and account
|
|
||||||
/// may also have hash and write_version per account
|
|
||||||
accounts: &'b U,
|
|
||||||
/// if accounts does not have hash and write version, this has a hash and write version per account
|
|
||||||
hashes_and_write_versions: Option<(Vec<V>, Vec<StoredMetaWriteVersion>)>,
|
|
||||||
_phantom: PhantomData<&'a T>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<'a: 'b, 'b, T: ReadableAccount + Sync + 'b, U: StorableAccounts<'a, T>, V: Borrow<Hash>>
|
|
||||||
StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>
|
|
||||||
{
|
|
||||||
/// used when accounts contains hash and write version already
|
|
||||||
pub fn new(accounts: &'b U) -> Self {
|
|
||||||
assert!(accounts.has_hash_and_write_version());
|
|
||||||
Self {
|
|
||||||
accounts,
|
|
||||||
hashes_and_write_versions: None,
|
|
||||||
_phantom: PhantomData,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
/// used when accounts does NOT contains hash or write version
|
|
||||||
/// In this case, hashes and write_versions have to be passed in separately and zipped together.
|
|
||||||
pub fn new_with_hashes_and_write_versions(
|
|
||||||
accounts: &'b U,
|
|
||||||
hashes: Vec<V>,
|
|
||||||
write_versions: Vec<StoredMetaWriteVersion>,
|
|
||||||
) -> Self {
|
|
||||||
assert!(!accounts.has_hash_and_write_version());
|
|
||||||
assert_eq!(accounts.len(), hashes.len());
|
|
||||||
assert_eq!(write_versions.len(), hashes.len());
|
|
||||||
Self {
|
|
||||||
accounts,
|
|
||||||
hashes_and_write_versions: Some((hashes, write_versions)),
|
|
||||||
_phantom: PhantomData,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// get all account fields at 'index'
|
|
||||||
pub fn get(&self, index: usize) -> (Option<&T>, &Pubkey, &Hash, StoredMetaWriteVersion) {
|
|
||||||
let account = self.accounts.account_default_if_zero_lamport(index);
|
|
||||||
let pubkey = self.accounts.pubkey(index);
|
|
||||||
let (hash, write_version) = if self.accounts.has_hash_and_write_version() {
|
|
||||||
(
|
|
||||||
self.accounts.hash(index),
|
|
||||||
self.accounts.write_version(index),
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
let item = self.hashes_and_write_versions.as_ref().unwrap();
|
|
||||||
(item.0[index].borrow(), item.1[index])
|
|
||||||
};
|
|
||||||
(account, pubkey, hash, write_version)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// None if account at index has lamports == 0
|
|
||||||
/// Otherwise, Some(account)
|
|
||||||
/// This is the only way to access the account.
|
|
||||||
pub fn account(&self, index: usize) -> Option<&T> {
|
|
||||||
self.accounts.account_default_if_zero_lamport(index)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// # accounts to write
|
|
||||||
pub fn len(&self) -> usize {
|
|
||||||
self.accounts.len()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn is_empty(&self) -> bool {
|
|
||||||
self.len() == 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Meta contains enough context to recover the index from storage itself
|
|
||||||
/// This struct will be backed by mmaped and snapshotted data files.
|
|
||||||
/// So the data layout must be stable and consistent across the entire cluster!
|
|
||||||
#[derive(Clone, PartialEq, Eq, Debug)]
|
|
||||||
#[repr(C)]
|
|
||||||
pub struct StoredMeta {
|
|
||||||
/// global write version
|
|
||||||
/// This will be made completely obsolete such that we stop storing it.
|
|
||||||
/// We will not support multiple append vecs per slot anymore, so this concept is no longer necessary.
|
|
||||||
/// Order of stores of an account to an append vec will determine 'latest' account data per pubkey.
|
|
||||||
pub write_version_obsolete: StoredMetaWriteVersion,
|
|
||||||
pub data_len: u64,
|
|
||||||
/// key for the account
|
|
||||||
pub pubkey: Pubkey,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// This struct will be backed by mmaped and snapshotted data files.
|
|
||||||
/// So the data layout must be stable and consistent across the entire cluster!
|
|
||||||
#[derive(Serialize, Deserialize, Clone, Debug, Default, Eq, PartialEq)]
|
|
||||||
#[repr(C)]
|
|
||||||
pub struct AccountMeta {
|
|
||||||
/// lamports in the account
|
|
||||||
pub lamports: u64,
|
|
||||||
/// the epoch at which this account will next owe rent
|
|
||||||
pub rent_epoch: Epoch,
|
|
||||||
/// the program that owns this account. If executable, the program that loads this account.
|
|
||||||
pub owner: Pubkey,
|
|
||||||
/// this account's data contains a loaded program (and is now read-only)
|
|
||||||
pub executable: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<'a, T: ReadableAccount> From<&'a T> for AccountMeta {
|
|
||||||
fn from(account: &'a T) -> Self {
|
|
||||||
Self {
|
|
||||||
lamports: account.lamports(),
|
|
||||||
owner: *account.owner(),
|
|
||||||
executable: account.executable(),
|
|
||||||
rent_epoch: account.rent_epoch(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<'a, T: ReadableAccount> From<Option<&'a T>> for AccountMeta {
|
|
||||||
fn from(account: Option<&'a T>) -> Self {
|
|
||||||
match account {
|
|
||||||
Some(account) => AccountMeta::from(account),
|
|
||||||
None => AccountMeta::default(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// References to account data stored elsewhere. Getting an `Account` requires cloning
|
|
||||||
/// (see `StoredAccountMeta::clone_account()`).
|
|
||||||
#[derive(PartialEq, Eq, Debug)]
|
|
||||||
pub struct StoredAccountMeta<'a> {
|
|
||||||
pub meta: &'a StoredMeta,
|
|
||||||
/// account data
|
|
||||||
pub account_meta: &'a AccountMeta,
|
|
||||||
pub data: &'a [u8],
|
|
||||||
pub offset: usize,
|
|
||||||
pub stored_size: usize,
|
|
||||||
pub hash: &'a Hash,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<'a> StoredAccountMeta<'a> {
|
|
||||||
/// Return a new Account by copying all the data referenced by the `StoredAccountMeta`.
|
|
||||||
pub fn clone_account(&self) -> AccountSharedData {
|
|
||||||
AccountSharedData::from(Account {
|
|
||||||
lamports: self.account_meta.lamports,
|
|
||||||
owner: self.account_meta.owner,
|
|
||||||
executable: self.account_meta.executable,
|
|
||||||
rent_epoch: self.account_meta.rent_epoch,
|
|
||||||
data: self.data.to_vec(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn pubkey(&self) -> &Pubkey {
|
|
||||||
&self.meta.pubkey
|
|
||||||
}
|
|
||||||
|
|
||||||
fn sanitize(&self) -> bool {
|
|
||||||
self.sanitize_executable() && self.sanitize_lamports()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn sanitize_executable(&self) -> bool {
|
|
||||||
// Sanitize executable to ensure higher 7-bits are cleared correctly.
|
|
||||||
self.ref_executable_byte() & !1 == 0
|
|
||||||
}
|
|
||||||
|
|
||||||
fn sanitize_lamports(&self) -> bool {
|
|
||||||
// Sanitize 0 lamports to ensure to be same as AccountSharedData::default()
|
|
||||||
self.account_meta.lamports != 0 || self.clone_account() == AccountSharedData::default()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn ref_executable_byte(&self) -> &u8 {
|
|
||||||
// Use extra references to avoid value silently clamped to 1 (=true) and 0 (=false)
|
|
||||||
// Yes, this really happens; see test_new_from_file_crafted_executable
|
|
||||||
let executable_bool: &bool = &self.account_meta.executable;
|
|
||||||
// UNSAFE: Force to interpret mmap-backed bool as u8 to really read the actual memory content
|
|
||||||
let executable_byte: &u8 = unsafe { &*(executable_bool as *const bool as *const u8) };
|
|
||||||
executable_byte
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct AppendVecAccountsIter<'a> {
|
pub struct AppendVecAccountsIter<'a> {
|
||||||
append_vec: &'a AppendVec,
|
append_vec: &'a AppendVec,
|
||||||
offset: usize,
|
offset: usize,
|
||||||
|
@ -635,7 +444,10 @@ impl AppendVec {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub fn get_account_test(&self, offset: usize) -> Option<(StoredMeta, AccountSharedData)> {
|
pub fn get_account_test(
|
||||||
|
&self,
|
||||||
|
offset: usize,
|
||||||
|
) -> Option<(StoredMeta, solana_sdk::account::AccountSharedData)> {
|
||||||
let (stored_account, _) = self.get_account(offset)?;
|
let (stored_account, _) = self.get_account(offset)?;
|
||||||
let meta = stored_account.meta.clone();
|
let meta = stored_account.meta.clone();
|
||||||
Some((meta, stored_account.clone_account()))
|
Some((meta, stored_account.clone_account()))
|
||||||
|
@ -743,7 +555,7 @@ pub mod tests {
|
||||||
memoffset::offset_of,
|
memoffset::offset_of,
|
||||||
rand::{thread_rng, Rng},
|
rand::{thread_rng, Rng},
|
||||||
solana_sdk::{
|
solana_sdk::{
|
||||||
account::{accounts_equal, WritableAccount},
|
account::{accounts_equal, Account, AccountSharedData, WritableAccount},
|
||||||
timing::duration_as_ms,
|
timing::duration_as_ms,
|
||||||
},
|
},
|
||||||
std::time::Instant,
|
std::time::Instant,
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
use {
|
use {
|
||||||
crate::{
|
crate::{
|
||||||
|
account_storage::meta::StoredMetaWriteVersion,
|
||||||
accounts::Accounts,
|
accounts::Accounts,
|
||||||
accounts_db::{
|
accounts_db::{
|
||||||
AccountShrinkThreshold, AccountStorageEntry, AccountsDb, AccountsDbConfig, AppendVecId,
|
AccountShrinkThreshold, AccountStorageEntry, AccountsDb, AccountsDbConfig, AppendVecId,
|
||||||
|
@ -9,7 +10,7 @@ use {
|
||||||
accounts_hash::{AccountsDeltaHash, AccountsHash},
|
accounts_hash::{AccountsDeltaHash, AccountsHash},
|
||||||
accounts_index::AccountSecondaryIndexes,
|
accounts_index::AccountSecondaryIndexes,
|
||||||
accounts_update_notifier_interface::AccountsUpdateNotifier,
|
accounts_update_notifier_interface::AccountsUpdateNotifier,
|
||||||
append_vec::{AppendVec, StoredMetaWriteVersion},
|
append_vec::AppendVec,
|
||||||
bank::{Bank, BankFieldsToDeserialize, BankIncrementalSnapshotPersistence, BankRc},
|
bank::{Bank, BankFieldsToDeserialize, BankIncrementalSnapshotPersistence, BankRc},
|
||||||
blockhash_queue::BlockhashQueue,
|
blockhash_queue::BlockhashQueue,
|
||||||
builtins::Builtins,
|
builtins::Builtins,
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
//! trait for abstracting underlying storage of pubkey and account pairs to be written
|
//! trait for abstracting underlying storage of pubkey and account pairs to be written
|
||||||
use {
|
use {
|
||||||
crate::{accounts_db::IncludeSlotInHash, append_vec::StoredAccountMeta},
|
crate::{account_storage::meta::StoredAccountMeta, accounts_db::IncludeSlotInHash},
|
||||||
solana_sdk::{account::ReadableAccount, clock::Slot, hash::Hash, pubkey::Pubkey},
|
solana_sdk::{account::ReadableAccount, clock::Slot, hash::Hash, pubkey::Pubkey},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -304,8 +304,8 @@ pub mod tests {
|
||||||
use {
|
use {
|
||||||
super::*,
|
super::*,
|
||||||
crate::{
|
crate::{
|
||||||
|
account_storage::meta::{AccountMeta, StoredAccountMeta, StoredMeta},
|
||||||
accounts_db::INCLUDE_SLOT_IN_HASH_TESTS,
|
accounts_db::INCLUDE_SLOT_IN_HASH_TESTS,
|
||||||
append_vec::{AccountMeta, StoredAccountMeta, StoredMeta},
|
|
||||||
},
|
},
|
||||||
solana_sdk::{
|
solana_sdk::{
|
||||||
account::{accounts_equal, AccountSharedData, WritableAccount},
|
account::{accounts_equal, AccountSharedData, WritableAccount},
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
use {
|
use {
|
||||||
clap::{crate_description, crate_name, value_t, value_t_or_exit, App, Arg},
|
clap::{crate_description, crate_name, value_t, value_t_or_exit, App, Arg},
|
||||||
log::*,
|
log::*,
|
||||||
solana_runtime::append_vec::{AppendVec, StoredAccountMeta},
|
solana_runtime::{account_storage::meta::StoredAccountMeta, append_vec::AppendVec},
|
||||||
solana_sdk::{account::AccountSharedData, hash::Hash, pubkey::Pubkey},
|
solana_sdk::{account::AccountSharedData, hash::Hash, pubkey::Pubkey},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue