blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Uses AccountHash in StorableAccountsWithHashesAndWriteVersions (#33751)

This commit is contained in:
Brooks 2023-10-18 18:58:19 -04:00 committed by GitHub
parent 93d882f158
commit 1045548606
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 85 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
accounts-db/benches/append_vec.rs
View File

@ -8,6 +8,7 @@ use {
StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo, StoredMeta, StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo, StoredMeta,
}, },
accounts_db::INCLUDE_SLOT_IN_HASH_TESTS, accounts_db::INCLUDE_SLOT_IN_HASH_TESTS,
accounts_hash::AccountHash,
append_vec::{ append_vec::{
test_utils::{create_test_account, get_append_vec_path}, test_utils::{create_test_account, get_append_vec_path},
AppendVec, AppendVec,
@ -33,7 +34,7 @@ fn append_account(
vec: &AppendVec, vec: &AppendVec,
storage_meta: StoredMeta, storage_meta: StoredMeta,
account: &AccountSharedData, account: &AccountSharedData,
hash: Hash, hash: AccountHash,
) -> Option<StoredAccountInfo> { ) -> Option<StoredAccountInfo> {
let slot_ignored = Slot::MAX; let slot_ignored = Slot::MAX;
let accounts = [(&storage_meta.pubkey, account)]; let accounts = [(&storage_meta.pubkey, account)];
@ -55,7 +56,7 @@ fn append_vec_append(bencher: &mut Bencher) {
let vec = AppendVec::new(&path.path, true, 64 * 1024); let vec = AppendVec::new(&path.path, true, 64 * 1024);
bencher.iter(|| { bencher.iter(|| {
let (meta, account) = create_test_account(0); let (meta, account) = create_test_account(0);
if append_account(&vec, meta, &account, Hash::default()).is_none() { if append_account(&vec, meta, &account, AccountHash(Hash::default())).is_none() {
vec.reset(); vec.reset();
} }
}); });
@ -65,7 +66,8 @@ fn add_test_accounts(vec: &AppendVec, size: usize) -> Vec<(usize, usize)> {
(0..size) (0..size)
.filter_map(|sample| { .filter_map(|sample| {
let (meta, account) = create_test_account(sample); let (meta, account) = create_test_account(sample);
append_account(vec, meta, &account, Hash::default()).map(|info| (sample, info.offset)) append_account(vec, meta, &account, AccountHash(Hash::default()))
.map(|info| (sample, info.offset))
}) })
.collect() .collect()
} }
@ -110,7 +112,7 @@ fn append_vec_concurrent_append_read(bencher: &mut Bencher) {
spawn(move || loop { spawn(move || loop {
let sample = indexes1.lock().unwrap().len(); let sample = indexes1.lock().unwrap().len();
let (meta, account) = create_test_account(sample); let (meta, account) = create_test_account(sample);
if let Some(info) = append_account(&vec1, meta, &account, Hash::default()) { if let Some(info) = append_account(&vec1, meta, &account, AccountHash(Hash::default())) {
indexes1.lock().unwrap().push((sample, info.offset)) indexes1.lock().unwrap().push((sample, info.offset))
} else { } else {
break; break;
@ -150,7 +152,7 @@ fn append_vec_concurrent_read_append(bencher: &mut Bencher) {
bencher.iter(|| { bencher.iter(|| {
let sample: usize = thread_rng().gen_range(0..256); let sample: usize = thread_rng().gen_range(0..256);
let (meta, account) = create_test_account(sample); let (meta, account) = create_test_account(sample);
if let Some(info) = append_account(&vec, meta, &account, Hash::default()) { if let Some(info) = append_account(&vec, meta, &account, AccountHash(Hash::default())) {
indexes.lock().unwrap().push((sample, info.offset)) indexes.lock().unwrap().push((sample, info.offset))
} }
}); });

27
accounts-db/src/account_storage/meta.rs
View File

@ -1,5 +1,6 @@
use { use {
crate::{ crate::{
accounts_hash::AccountHash,
append_vec::AppendVecStoredAccountMeta, append_vec::AppendVecStoredAccountMeta,
storable_accounts::StorableAccounts, storable_accounts::StorableAccounts,
tiered_storage::{hot::HotAccountMeta, readable::TieredReadableAccount}, tiered_storage::{hot::HotAccountMeta, readable::TieredReadableAccount},
@ -17,7 +18,7 @@ pub struct StoredAccountInfo {
} }
lazy_static! { lazy_static! {
static ref DEFAULT_ACCOUNT_HASH: Hash = Hash::default(); static ref DEFAULT_ACCOUNT_HASH: AccountHash = AccountHash(Hash::default());
} }
/// Goal is to eliminate copies and data reshaping given various code paths that store accounts. /// Goal is to eliminate copies and data reshaping given various code paths that store accounts.
@ -30,7 +31,7 @@ pub struct StorableAccountsWithHashesAndWriteVersions<
'b, 'b,
T: ReadableAccount + Sync + 'b, T: ReadableAccount + Sync + 'b,
U: StorableAccounts<'a, T>, U: StorableAccounts<'a, T>,
V: Borrow<Hash>, V: Borrow<AccountHash>,
> { > {
/// accounts to store /// accounts to store
/// always has pubkey and account /// always has pubkey and account
@ -41,8 +42,13 @@ pub struct StorableAccountsWithHashesAndWriteVersions<
_phantom: PhantomData<&'a T>, _phantom: PhantomData<&'a T>,
} }
impl<'a: 'b, 'b, T: ReadableAccount + Sync + 'b, U: StorableAccounts<'a, T>, V: Borrow<Hash>> impl<
StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V> 'a: 'b,
'b,
T: ReadableAccount + Sync + 'b,
U: StorableAccounts<'a, T>,
V: Borrow<AccountHash>,
> StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>
{ {
/// used when accounts contains hash and write version already /// used when accounts contains hash and write version already
pub fn new(accounts: &'b U) -> Self { pub fn new(accounts: &'b U) -> Self {
@ -71,12 +77,12 @@ impl<'a: 'b, 'b, T: ReadableAccount + Sync + 'b, U: StorableAccounts<'a, T>, V:
} }
/// get all account fields at 'index' /// get all account fields at 'index'
pub fn get(&self, index: usize) -> (Option<&T>, &Pubkey, &Hash, StoredMetaWriteVersion) { pub fn get(&self, index: usize) -> (Option<&T>, &Pubkey, &AccountHash, StoredMetaWriteVersion) {
let account = self.accounts.account_default_if_zero_lamport(index); let account = self.accounts.account_default_if_zero_lamport(index);
let pubkey = self.accounts.pubkey(index); let pubkey = self.accounts.pubkey(index);
let (hash, write_version) = if self.accounts.has_hash_and_write_version() { let (hash, write_version) = if self.accounts.has_hash_and_write_version() {
( (
&self.accounts.hash(index).0, self.accounts.hash(index),
self.accounts.write_version(index), self.accounts.write_version(index),
) )
} else { } else {
@ -119,11 +125,12 @@ impl<'storage> StoredAccountMeta<'storage> {
} }
} }
pub fn hash(&self) -> &'storage Hash { pub fn hash(&self) -> &'storage AccountHash {
match self { let hash = match self {
Self::AppendVec(av) => av.hash(), Self::AppendVec(av) => av.hash(),
Self::Hot(hot) => hot.hash().unwrap_or(&DEFAULT_ACCOUNT_HASH), Self::Hot(hot) => hot.hash().unwrap_or(&DEFAULT_ACCOUNT_HASH.0),
} };
bytemuck::cast_ref(hash)
} }
pub fn stored_size(&self) -> usize { pub fn stored_size(&self) -> usize {

46
accounts-db/src/accounts_db.rs
View File

@ -384,7 +384,7 @@ impl CurrentAncientAppendVec {
INCLUDE_SLOT_IN_HASH_IRRELEVANT_APPEND_VEC_OPERATION, INCLUDE_SLOT_IN_HASH_IRRELEVANT_APPEND_VEC_OPERATION,
accounts_to_store.slot, accounts_to_store.slot,
), ),
None::<Vec<Hash>>, None::<Vec<AccountHash>>,
self.append_vec(), self.append_vec(),
None, None,
StoreReclaims::Ignore, StoreReclaims::Ignore,
@ -896,7 +896,7 @@ pub enum LoadedAccount<'a> {
impl<'a> LoadedAccount<'a> { impl<'a> LoadedAccount<'a> {
pub fn loaded_hash(&self) -> AccountHash { pub fn loaded_hash(&self) -> AccountHash {
match self { match self {
LoadedAccount::Stored(stored_account_meta) => AccountHash(*stored_account_meta.hash()), LoadedAccount::Stored(stored_account_meta) => *stored_account_meta.hash(),
LoadedAccount::Cached(cached_account) => cached_account.hash(), LoadedAccount::Cached(cached_account) => cached_account.hash(),
} }
} }
@ -4133,7 +4133,7 @@ impl AccountsDb {
&shrink_collect.alive_accounts.alive_accounts()[..], &shrink_collect.alive_accounts.alive_accounts()[..],
INCLUDE_SLOT_IN_HASH_IRRELEVANT_APPEND_VEC_OPERATION, INCLUDE_SLOT_IN_HASH_IRRELEVANT_APPEND_VEC_OPERATION,
), ),
None::<Vec<&Hash>>, None::<Vec<AccountHash>>,
shrink_in_progress.new_storage(), shrink_in_progress.new_storage(),
None, None,
StoreReclaims::Ignore, StoreReclaims::Ignore,
@ -6256,7 +6256,7 @@ impl AccountsDb {
'b, 'b,
T: ReadableAccount + Sync, T: ReadableAccount + Sync,
U: StorableAccounts<'a, T>, U: StorableAccounts<'a, T>,
V: Borrow<Hash>, V: Borrow<AccountHash>,
>( >(
&self, &self,
slot: Slot, slot: Slot,
@ -6623,7 +6623,7 @@ impl AccountsDb {
let pubkeys = self.get_filler_account_pubkeys(filler_accounts as usize); let pubkeys = self.get_filler_account_pubkeys(filler_accounts as usize);
pubkeys.iter().for_each(|key| { pubkeys.iter().for_each(|key| {
accounts.push((key, &account)); accounts.push((key, &account));
hashes.push(hash); hashes.push(AccountHash(hash));
}); });
self.store_accounts_frozen( self.store_accounts_frozen(
(slot, &accounts[..], include_slot_in_hash), (slot, &accounts[..], include_slot_in_hash),
@ -6695,7 +6695,9 @@ impl AccountsDb {
INCLUDE_SLOT_IN_HASH_IRRELEVANT_APPEND_VEC_OPERATION, INCLUDE_SLOT_IN_HASH_IRRELEVANT_APPEND_VEC_OPERATION,
); );
let storable = let storable =
StorableAccountsWithHashesAndWriteVersions::<'_, '_, _, _, &Hash>::new(&to_store); StorableAccountsWithHashesAndWriteVersions::<'_, '_, _, _, &AccountHash>::new(
&to_store,
);
storage.accounts.append_accounts(&storable, 0); storage.accounts.append_accounts(&storable, 0);
Arc::new(storage) Arc::new(storage)
@ -6807,7 +6809,7 @@ impl AccountsDb {
>( >(
&self, &self,
accounts: &'c impl StorableAccounts<'b, T>, accounts: &'c impl StorableAccounts<'b, T>,
hashes: Option<Vec<impl Borrow<Hash>>>, hashes: Option<Vec<impl Borrow<AccountHash>>>,
mut write_version_producer: P, mut write_version_producer: P,
store_to: &StoreTo, store_to: &StoreTo,
transactions: Option<&[Option<&'a SanitizedTransaction>]>, transactions: Option<&[Option<&'a SanitizedTransaction>]>,
@ -6847,7 +6849,7 @@ impl AccountsDb {
self.write_accounts_to_storage( self.write_accounts_to_storage(
slot, slot,
storage, storage,
&StorableAccountsWithHashesAndWriteVersions::<'_, '_, _, _, &Hash>::new( &StorableAccountsWithHashesAndWriteVersions::<'_, '_, _, _, &AccountHash>::new(
accounts, accounts,
), ),
) )
@ -8555,7 +8557,7 @@ impl AccountsDb {
// we use default hashes for now since the same account may be stored to the cache multiple times // we use default hashes for now since the same account may be stored to the cache multiple times
self.store_accounts_unfrozen( self.store_accounts_unfrozen(
accounts, accounts,
None::<Vec<Hash>>, None::<Vec<AccountHash>>,
store_to, store_to,
transactions, transactions,
reclaim, reclaim,
@ -8709,7 +8711,7 @@ impl AccountsDb {
fn store_accounts_unfrozen<'a, T: ReadableAccount + Sync + ZeroLamport + 'a>( fn store_accounts_unfrozen<'a, T: ReadableAccount + Sync + ZeroLamport + 'a>(
&self, &self,
accounts: impl StorableAccounts<'a, T>, accounts: impl StorableAccounts<'a, T>,
hashes: Option<Vec<impl Borrow<Hash>>>, hashes: Option<Vec<impl Borrow<AccountHash>>>,
store_to: &StoreTo, store_to: &StoreTo,
transactions: Option<&'a [Option<&'a SanitizedTransaction>]>, transactions: Option<&'a [Option<&'a SanitizedTransaction>]>,
reclaim: StoreReclaims, reclaim: StoreReclaims,
@ -8738,7 +8740,7 @@ impl AccountsDb {
pub fn store_accounts_frozen<'a, T: ReadableAccount + Sync + ZeroLamport + 'a>( pub fn store_accounts_frozen<'a, T: ReadableAccount + Sync + ZeroLamport + 'a>(
&self, &self,
accounts: impl StorableAccounts<'a, T>, accounts: impl StorableAccounts<'a, T>,
hashes: Option<Vec<impl Borrow<Hash>>>, hashes: Option<Vec<impl Borrow<AccountHash>>>,
storage: &Arc<AccountStorageEntry>, storage: &Arc<AccountStorageEntry>,
write_version_producer: Option<Box<dyn Iterator<Item = StoredMetaWriteVersion>>>, write_version_producer: Option<Box<dyn Iterator<Item = StoredMetaWriteVersion>>>,
reclaim: StoreReclaims, reclaim: StoreReclaims,
@ -8762,7 +8764,7 @@ impl AccountsDb {
fn store_accounts_custom<'a, T: ReadableAccount + Sync + ZeroLamport + 'a>( fn store_accounts_custom<'a, T: ReadableAccount + Sync + ZeroLamport + 'a>(
&self, &self,
accounts: impl StorableAccounts<'a, T>, accounts: impl StorableAccounts<'a, T>,
hashes: Option<Vec<impl Borrow<Hash>>>, hashes: Option<Vec<impl Borrow<AccountHash>>>,
write_version_producer: Option<Box<dyn Iterator<Item = u64>>>, write_version_producer: Option<Box<dyn Iterator<Item = u64>>>,
store_to: &StoreTo, store_to: &StoreTo,
reset_accounts: bool, reset_accounts: bool,
@ -10125,7 +10127,10 @@ pub mod tests {
let expected_accounts_data_len = data.last().unwrap().1.data().len(); let expected_accounts_data_len = data.last().unwrap().1.data().len();
let expected_alive_bytes = aligned_stored_size(expected_accounts_data_len); let expected_alive_bytes = aligned_stored_size(expected_accounts_data_len);
let storable = (slot0, &data[..], INCLUDE_SLOT_IN_HASH_TESTS); let storable = (slot0, &data[..], INCLUDE_SLOT_IN_HASH_TESTS);
let hashes = data.iter().map(|_| Hash::default()).collect::<Vec<_>>(); let hashes = data
.iter()
.map(|_| AccountHash(Hash::default()))
.collect::<Vec<_>>();
let write_versions = data.iter().map(|_| 0).collect::<Vec<_>>(); let write_versions = data.iter().map(|_| 0).collect::<Vec<_>>();
let append = let append =
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions( StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
@ -10619,7 +10624,7 @@ pub mod tests {
accounts_db.storage.remove(&storage.slot(), false); accounts_db.storage.remove(&storage.slot(), false);
}); });
let hash = Hash::default(); let hash = AccountHash(Hash::default());
// replace the sample storages, storing default hash values so that we rehash during scan // replace the sample storages, storing default hash values so that we rehash during scan
let storages = storages let storages = storages
@ -10666,7 +10671,8 @@ pub mod tests {
sample_storages_and_accounts(&accounts_db, INCLUDE_SLOT_IN_HASH_TESTS); sample_storages_and_accounts(&accounts_db, INCLUDE_SLOT_IN_HASH_TESTS);
let max_slot = storages.iter().map(|storage| storage.slot()).max().unwrap(); let max_slot = storages.iter().map(|storage| storage.slot()).max().unwrap();
let hash = Hash::from_str("7JcmM6TFZMkcDkZe6RKVkGaWwN5dXciGC4fa3RxvqQc9").unwrap(); let hash =
AccountHash(Hash::from_str("7JcmM6TFZMkcDkZe6RKVkGaWwN5dXciGC4fa3RxvqQc9").unwrap());
// replace the sample storages, storing bogus hash values so that we trigger the hash mismatch // replace the sample storages, storing bogus hash values so that we trigger the hash mismatch
let storages = storages let storages = storages
@ -11169,7 +11175,7 @@ pub mod tests {
let accounts = [(pubkey, account)]; let accounts = [(pubkey, account)];
let slice = &accounts[..]; let slice = &accounts[..];
let account_data = (slot, slice); let account_data = (slot, slice);
let hash = Hash::default(); let hash = AccountHash(Hash::default());
let storable_accounts = let storable_accounts =
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions( StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&account_data, &account_data,
@ -12769,7 +12775,7 @@ pub mod tests {
// put wrong hash value in store so we get a mismatch // put wrong hash value in store so we get a mismatch
db.store_accounts_unfrozen( db.store_accounts_unfrozen(
(some_slot, &[(&key, &account)][..]), (some_slot, &[(&key, &account)][..]),
Some(vec![&Hash::default()]), Some(vec![&AccountHash(Hash::default())]),
&StoreTo::Storage(&db.find_storage_candidate(some_slot, 1)), &StoreTo::Storage(&db.find_storage_candidate(some_slot, 1)),
None, None,
StoreReclaims::Default, StoreReclaims::Default,
@ -13003,7 +13009,7 @@ pub mod tests {
db.update_accounts_hash_for_tests(some_slot, &ancestors, false, false); db.update_accounts_hash_for_tests(some_slot, &ancestors, false, false);
// provide bogus account hashes // provide bogus account hashes
let some_hash = Hash::new(&[0xca; HASH_BYTES]); let some_hash = AccountHash(Hash::new(&[0xca; HASH_BYTES]));
db.store_accounts_unfrozen( db.store_accounts_unfrozen(
(some_slot, accounts), (some_slot, accounts),
Some(vec![&some_hash]), Some(vec![&some_hash]),
@ -15841,7 +15847,7 @@ pub mod tests {
accounts.accounts_index.set_startup(Startup::Startup); accounts.accounts_index.set_startup(Startup::Startup);
let storage = accounts.create_and_insert_store(slot0, 4_000, "flush_slot_cache"); let storage = accounts.create_and_insert_store(slot0, 4_000, "flush_slot_cache");
let hashes = vec![Hash::default(); 1]; let hashes = vec![AccountHash(Hash::default()); 1];
let write_version = vec![0; 1]; let write_version = vec![0; 1];
storage.accounts.append_accounts( storage.accounts.append_accounts(
&StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions( &StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
@ -15906,7 +15912,7 @@ pub mod tests {
let account_big = AccountSharedData::new(1, 1000, AccountSharedData::default().owner()); let account_big = AccountSharedData::new(1, 1000, AccountSharedData::default().owner());
let slot0 = 0; let slot0 = 0;
let storage = accounts.create_and_insert_store(slot0, 4_000, "flush_slot_cache"); let storage = accounts.create_and_insert_store(slot0, 4_000, "flush_slot_cache");
let hashes = vec![Hash::default(); 2]; let hashes = vec![AccountHash(Hash::default()); 2];
let write_version = vec![0; 2]; let write_version = vec![0; 2];
storage.accounts.append_accounts( storage.accounts.append_accounts(
&StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions( &StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(

5
accounts-db/src/accounts_file.rs
View File

@ -3,11 +3,12 @@ use {
account_storage::meta::{ account_storage::meta::{
StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo, StoredAccountMeta, StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo, StoredAccountMeta,
}, },
accounts_hash::AccountHash,
append_vec::{AppendVec, AppendVecError, MatchAccountOwnerError}, append_vec::{AppendVec, AppendVecError, MatchAccountOwnerError},
storable_accounts::StorableAccounts, storable_accounts::StorableAccounts,
tiered_storage::error::TieredStorageError, tiered_storage::error::TieredStorageError,
}, },
solana_sdk::{account::ReadableAccount, clock::Slot, hash::Hash, pubkey::Pubkey}, solana_sdk::{account::ReadableAccount, clock::Slot, pubkey::Pubkey},
std::{ std::{
borrow::Borrow, borrow::Borrow,
mem, mem,
@ -154,7 +155,7 @@ impl AccountsFile {
'b, 'b,
T: ReadableAccount + Sync, T: ReadableAccount + Sync,
U: StorableAccounts<'a, T>, U: StorableAccounts<'a, T>,
V: Borrow<Hash>, V: Borrow<AccountHash>,
>( >(
&self, &self,
accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>, accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>,

6
accounts-db/src/accounts_hash.rs
View File

@ -1248,12 +1248,6 @@ pub struct AccountHash(pub Hash);
// This also ensures there are no padding bytes, which is requried to safely implement Pod // This also ensures there are no padding bytes, which is requried to safely implement Pod
const _: () = assert!(std::mem::size_of::<AccountHash>() == std::mem::size_of::<Hash>()); const _: () = assert!(std::mem::size_of::<AccountHash>() == std::mem::size_of::<Hash>());
impl Borrow<Hash> for AccountHash {
fn borrow(&self) -> &Hash {
&self.0
}
}
/// Hash of accounts /// Hash of accounts
#[derive(Debug, Copy, Clone, Eq, PartialEq)] #[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum AccountsHashKind { pub enum AccountsHashKind {

7
accounts-db/src/ancient_append_vecs.rs
View File

@ -12,6 +12,7 @@ use {
INCLUDE_SLOT_IN_HASH_IRRELEVANT_APPEND_VEC_OPERATION, INCLUDE_SLOT_IN_HASH_IRRELEVANT_APPEND_VEC_OPERATION,
}, },
accounts_file::AccountsFile, accounts_file::AccountsFile,
accounts_hash::AccountHash,
accounts_index::{AccountsIndexScanResult, ZeroLamport}, accounts_index::{AccountsIndexScanResult, ZeroLamport},
active_stats::ActiveStatItem, active_stats::ActiveStatItem,
append_vec::aligned_stored_size, append_vec::aligned_stored_size,
@ -20,9 +21,7 @@ use {
rand::{thread_rng, Rng}, rand::{thread_rng, Rng},
rayon::prelude::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}, rayon::prelude::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator},
solana_measure::measure_us, solana_measure::measure_us,
solana_sdk::{ solana_sdk::{account::ReadableAccount, clock::Slot, pubkey::Pubkey, saturating_add_assign},
account::ReadableAccount, clock::Slot, hash::Hash, pubkey::Pubkey, saturating_add_assign,
},
std::{ std::{
collections::HashMap, collections::HashMap,
num::NonZeroU64, num::NonZeroU64,
@ -372,7 +371,7 @@ impl AccountsDb {
measure_us!(self.get_store_for_shrink(target_slot, bytes)); measure_us!(self.get_store_for_shrink(target_slot, bytes));
let (store_accounts_timing, rewrite_elapsed_us) = measure_us!(self.store_accounts_frozen( let (store_accounts_timing, rewrite_elapsed_us) = measure_us!(self.store_accounts_frozen(
accounts_to_write, accounts_to_write,
None::<Vec<Hash>>, None::<Vec<AccountHash>>,
shrink_in_progress.new_storage(), shrink_in_progress.new_storage(),
None, None,
StoreReclaims::Ignore, StoreReclaims::Ignore,

20
accounts-db/src/append_vec.rs
View File

@ -11,6 +11,7 @@ use {
StoredAccountMeta, StoredMeta, StoredMetaWriteVersion, StoredAccountMeta, StoredMeta, StoredMetaWriteVersion,
}, },
accounts_file::{AccountsFileError, Result, ALIGN_BOUNDARY_OFFSET}, accounts_file::{AccountsFileError, Result, ALIGN_BOUNDARY_OFFSET},
accounts_hash::AccountHash,
storable_accounts::StorableAccounts, storable_accounts::StorableAccounts,
u64_align, u64_align,
}, },
@ -575,7 +576,7 @@ impl AppendVec {
'b, 'b,
T: ReadableAccount + Sync, T: ReadableAccount + Sync,
U: StorableAccounts<'a, T>, U: StorableAccounts<'a, T>,
V: Borrow<Hash>, V: Borrow<AccountHash>,
>( >(
&self, &self,
accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>, accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>,
@ -611,7 +612,7 @@ impl AppendVec {
.map(|account| account.data()) .map(|account| account.data())
.unwrap_or_default() .unwrap_or_default()
.as_ptr(); .as_ptr();
let hash_ptr = hash.as_ref().as_ptr(); let hash_ptr = hash.0.as_ref().as_ptr();
let ptrs = [ let ptrs = [
(meta_ptr as *const u8, mem::size_of::<StoredMeta>()), (meta_ptr as *const u8, mem::size_of::<StoredMeta>()),
(account_meta_ptr as *const u8, mem::size_of::<AccountMeta>()), (account_meta_ptr as *const u8, mem::size_of::<AccountMeta>()),
@ -669,7 +670,7 @@ pub mod tests {
let accounts = [(&data.0.pubkey, &data.1)]; let accounts = [(&data.0.pubkey, &data.1)];
let slice = &accounts[..]; let slice = &accounts[..];
let account_data = (slot_ignored, slice); let account_data = (slot_ignored, slice);
let hash = Hash::default(); let hash = AccountHash(Hash::default());
let storable_accounts = let storable_accounts =
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions( StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&account_data, &account_data,
@ -740,7 +741,7 @@ pub mod tests {
// for (Slot, &'a [(&'a Pubkey, &'a T)], IncludeSlotInHash) // for (Slot, &'a [(&'a Pubkey, &'a T)], IncludeSlotInHash)
let slot = 0 as Slot; let slot = 0 as Slot;
let pubkey = Pubkey::default(); let pubkey = Pubkey::default();
StorableAccountsWithHashesAndWriteVersions::<'_, '_, _, _, &Hash>::new(&( StorableAccountsWithHashesAndWriteVersions::<'_, '_, _, _, &AccountHash>::new(&(
slot, slot,
&[(&pubkey, &account)][..], &[(&pubkey, &account)][..],
INCLUDE_SLOT_IN_HASH_TESTS, INCLUDE_SLOT_IN_HASH_TESTS,
@ -755,7 +756,7 @@ pub mod tests {
// mismatch between lens of accounts, hashes, write_versions // mismatch between lens of accounts, hashes, write_versions
let mut hashes = Vec::default(); let mut hashes = Vec::default();
if correct_hashes { if correct_hashes {
hashes.push(Hash::default()); hashes.push(AccountHash(Hash::default()));
} }
let mut write_versions = Vec::default(); let mut write_versions = Vec::default();
if correct_write_versions { if correct_write_versions {
@ -798,7 +799,7 @@ pub mod tests {
let account = AccountSharedData::default(); let account = AccountSharedData::default();
let slot = 0 as Slot; let slot = 0 as Slot;
let pubkeys = [Pubkey::default()]; let pubkeys = [Pubkey::default()];
let hashes = Vec::<Hash>::default(); let hashes = Vec::<AccountHash>::default();
let write_versions = Vec::default(); let write_versions = Vec::default();
let mut accounts = vec![(&pubkeys[0], &account)]; let mut accounts = vec![(&pubkeys[0], &account)];
accounts.clear(); accounts.clear();
@ -819,7 +820,10 @@ pub mod tests {
let account = AccountSharedData::default(); let account = AccountSharedData::default();
let slot = 0 as Slot; let slot = 0 as Slot;
let pubkeys = [Pubkey::from([5; 32]), Pubkey::from([6; 32])]; let pubkeys = [Pubkey::from([5; 32]), Pubkey::from([6; 32])];
let hashes = vec![Hash::new(&[3; 32]), Hash::new(&[4; 32])]; let hashes = vec![
AccountHash(Hash::new(&[3; 32])),
AccountHash(Hash::new(&[4; 32])),
];
let write_versions = vec![42, 43]; let write_versions = vec![42, 43];
let accounts = [(&pubkeys[0], &account), (&pubkeys[1], &account)]; let accounts = [(&pubkeys[0], &account), (&pubkeys[1], &account)];
let accounts2 = (slot, &accounts[..], INCLUDE_SLOT_IN_HASH_TESTS); let accounts2 = (slot, &accounts[..], INCLUDE_SLOT_IN_HASH_TESTS);
@ -850,7 +854,7 @@ pub mod tests {
// for (Slot, &'a [(&'a Pubkey, &'a T)], IncludeSlotInHash) // for (Slot, &'a [(&'a Pubkey, &'a T)], IncludeSlotInHash)
let slot = 0 as Slot; let slot = 0 as Slot;
let pubkey = Pubkey::default(); let pubkey = Pubkey::default();
let hashes = vec![Hash::default()]; let hashes = vec![AccountHash(Hash::default())];
let write_versions = vec![0]; let write_versions = vec![0];
let accounts = [(&pubkey, &account)]; let accounts = [(&pubkey, &account)];
let accounts2 = (slot, &accounts[..], INCLUDE_SLOT_IN_HASH_TESTS); let accounts2 = (slot, &accounts[..], INCLUDE_SLOT_IN_HASH_TESTS);

8
accounts-db/src/storable_accounts.rs
View File

@ -178,7 +178,7 @@ impl<'a> StorableAccounts<'a, StoredAccountMeta<'a>>
true true
} }
fn hash(&self, index: usize) -> &AccountHash { fn hash(&self, index: usize) -> &AccountHash {
bytemuck::cast_ref(self.account(index).hash()) self.account(index).hash()
} }
fn write_version(&self, index: usize) -> u64 { fn write_version(&self, index: usize) -> u64 {
self.account(index).write_version() self.account(index).write_version()
@ -282,7 +282,7 @@ impl<'a> StorableAccounts<'a, StoredAccountMeta<'a>> for StorableAccountsBySlot<
true true
} }
fn hash(&self, index: usize) -> &AccountHash { fn hash(&self, index: usize) -> &AccountHash {
bytemuck::cast_ref(self.account(index).hash()) self.account(index).hash()
} }
fn write_version(&self, index: usize) -> u64 { fn write_version(&self, index: usize) -> u64 {
self.account(index).write_version() self.account(index).write_version()
@ -322,7 +322,7 @@ impl<'a> StorableAccounts<'a, StoredAccountMeta<'a>>
true true
} }
fn hash(&self, index: usize) -> &AccountHash { fn hash(&self, index: usize) -> &AccountHash {
bytemuck::cast_ref(self.account(index).hash()) self.account(index).hash()
} }
fn write_version(&self, index: usize) -> u64 { fn write_version(&self, index: usize) -> u64 {
self.account(index).write_version() self.account(index).write_version()
@ -606,7 +606,7 @@ pub mod tests {
let index = index as usize; let index = index as usize;
assert_eq!(storable.account(index), &raw2[index]); assert_eq!(storable.account(index), &raw2[index]);
assert_eq!(storable.pubkey(index), raw2[index].pubkey()); assert_eq!(storable.pubkey(index), raw2[index].pubkey());
assert_eq!(&storable.hash(index).0, raw2[index].hash()); assert_eq!(storable.hash(index), raw2[index].hash());
assert_eq!(storable.slot(index), expected_slots[index]); assert_eq!(storable.slot(index), expected_slots[index]);
assert_eq!(storable.write_version(index), raw2[index].write_version()); assert_eq!(storable.write_version(index), raw2[index].write_version());
}) })

10
accounts-db/src/tiered_storage.rs
View File

@ -14,13 +14,14 @@ pub mod writer;
use { use {
crate::{ crate::{
account_storage::meta::{StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo}, account_storage::meta::{StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo},
accounts_hash::AccountHash,
storable_accounts::StorableAccounts, storable_accounts::StorableAccounts,
}, },
error::TieredStorageError, error::TieredStorageError,
footer::{AccountBlockFormat, AccountMetaFormat, OwnersBlockFormat}, footer::{AccountBlockFormat, AccountMetaFormat, OwnersBlockFormat},
index::AccountIndexFormat, index::AccountIndexFormat,
readable::TieredStorageReader, readable::TieredStorageReader,
solana_sdk::{account::ReadableAccount, hash::Hash}, solana_sdk::account::ReadableAccount,
std::{ std::{
borrow::Borrow, borrow::Borrow,
fs::OpenOptions, fs::OpenOptions,
@ -96,7 +97,7 @@ impl TieredStorage {
'b, 'b,
T: ReadableAccount + Sync, T: ReadableAccount + Sync,
U: StorableAccounts<'a, T>, U: StorableAccounts<'a, T>,
V: Borrow<Hash>, V: Borrow<AccountHash>,
>( >(
&self, &self,
accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>, accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>,
@ -157,6 +158,7 @@ mod tests {
solana_sdk::{ solana_sdk::{
account::{Account, AccountSharedData}, account::{Account, AccountSharedData},
clock::Slot, clock::Slot,
hash::Hash,
pubkey::Pubkey, pubkey::Pubkey,
system_instruction::MAX_PERMITTED_DATA_LENGTH, system_instruction::MAX_PERMITTED_DATA_LENGTH,
}, },
@ -182,7 +184,7 @@ mod tests {
let storable_accounts = let storable_accounts =
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions( StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&account_data, &account_data,
Vec::<&Hash>::new(), Vec::<AccountHash>::new(),
Vec::<StoredMetaWriteVersion>::new(), Vec::<StoredMetaWriteVersion>::new(),
); );
@ -341,7 +343,7 @@ mod tests {
// Slot information is not used here // Slot information is not used here
let account_data = (Slot::MAX, &account_refs[..]); let account_data = (Slot::MAX, &account_refs[..]);
let hashes: Vec<_> = std::iter::repeat_with(Hash::new_unique) let hashes: Vec<_> = std::iter::repeat_with(|| AccountHash(Hash::new_unique()))
.take(account_data_sizes.len()) .take(account_data_sizes.len())
.collect(); .collect();
let write_versions: Vec<_> = accounts let write_versions: Vec<_> = accounts

5
accounts-db/src/tiered_storage/writer.rs
View File

@ -3,13 +3,14 @@
use { use {
crate::{ crate::{
account_storage::meta::{StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo}, account_storage::meta::{StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo},
accounts_hash::AccountHash,
storable_accounts::StorableAccounts, storable_accounts::StorableAccounts,
tiered_storage::{ tiered_storage::{
error::TieredStorageError, file::TieredStorageFile, footer::TieredStorageFooter, error::TieredStorageError, file::TieredStorageFile, footer::TieredStorageFooter,
TieredStorageFormat, TieredStorageResult, TieredStorageFormat, TieredStorageResult,
}, },
}, },
solana_sdk::{account::ReadableAccount, hash::Hash}, solana_sdk::account::ReadableAccount,
std::{borrow::Borrow, path::Path}, std::{borrow::Borrow, path::Path},
}; };
@ -35,7 +36,7 @@ impl<'format> TieredStorageWriter<'format> {
'b, 'b,
T: ReadableAccount + Sync, T: ReadableAccount + Sync,
U: StorableAccounts<'a, T>, U: StorableAccounts<'a, T>,
V: Borrow<Hash>, V: Borrow<AccountHash>,
>( >(
&self, &self,
accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>, accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>,

6
accounts-db/store-tool/src/main.rs
View File

@ -1,7 +1,9 @@
use { use {
clap::{crate_description, crate_name, value_t, value_t_or_exit, App, Arg}, clap::{crate_description, crate_name, value_t, value_t_or_exit, App, Arg},
log::*, log::*,
solana_accounts_db::{account_storage::meta::StoredAccountMeta, append_vec::AppendVec}, solana_accounts_db::{
account_storage::meta::StoredAccountMeta, accounts_hash::AccountHash, append_vec::AppendVec,
},
solana_sdk::{ solana_sdk::{
account::{AccountSharedData, ReadableAccount}, account::{AccountSharedData, ReadableAccount},
hash::Hash, hash::Hash,
@ -65,7 +67,7 @@ fn main() {
} }
fn is_account_zeroed(account: &StoredAccountMeta) -> bool { fn is_account_zeroed(account: &StoredAccountMeta) -> bool {
account.hash() == &Hash::default() account.hash() == &AccountHash(Hash::default())
&& account.data_len() == 0 && account.data_len() == 0
&& account.write_version() == 0 && account.write_version() == 0
&& account.pubkey() == &Pubkey::default() && account.pubkey() == &Pubkey::default()