Removes write version from StorableAccountsWithHashesAndWriteVersions (#561)

This commit is contained in:
Brooks 2024-04-03 12:41:20 -04:00 committed by GitHub
parent 7b204e7d01
commit afa65c6690
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 88 additions and 249 deletions

View File

@ -4,9 +4,7 @@ extern crate test;
use { use {
rand::{thread_rng, Rng}, rand::{thread_rng, Rng},
solana_accounts_db::{ solana_accounts_db::{
account_storage::meta::{ account_storage::meta::{StorableAccountsWithHashes, StoredAccountInfo, StoredMeta},
StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo, StoredMeta,
},
accounts_hash::AccountHash, accounts_hash::AccountHash,
append_vec::{ append_vec::{
test_utils::{create_test_account, get_append_vec_path}, test_utils::{create_test_account, get_append_vec_path},
@ -39,12 +37,7 @@ fn append_account(
let accounts = [(&storage_meta.pubkey, account)]; let accounts = [(&storage_meta.pubkey, account)];
let slice = &accounts[..]; let slice = &accounts[..];
let accounts = (slot_ignored, slice); let accounts = (slot_ignored, slice);
let storable_accounts = let storable_accounts = StorableAccountsWithHashes::new_with_hashes(&accounts, vec![&hash]);
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&accounts,
vec![&hash],
vec![storage_meta.write_version_obsolete],
);
let res = vec.append_accounts(&storable_accounts, 0); let res = vec.append_accounts(&storable_accounts, 0);
res.and_then(|res| res.first().cloned()) res.and_then(|res| res.first().cloned())
} }

View File

@ -2,7 +2,7 @@
use { use {
criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion, Throughput}, criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion, Throughput},
solana_accounts_db::{ solana_accounts_db::{
account_storage::meta::StorableAccountsWithHashesAndWriteVersions, account_storage::meta::StorableAccountsWithHashes,
accounts_hash::AccountHash, accounts_hash::AccountHash,
append_vec::{self, AppendVec}, append_vec::{self, AppendVec},
tiered_storage::hot::HotStorageWriter, tiered_storage::hot::HotStorageWriter,
@ -46,11 +46,9 @@ fn bench_write_accounts_file(c: &mut Criterion) {
.collect(); .collect();
let accounts_refs: Vec<_> = accounts.iter().collect(); let accounts_refs: Vec<_> = accounts.iter().collect();
let accounts_data = (Slot::MAX, accounts_refs.as_slice()); let accounts_data = (Slot::MAX, accounts_refs.as_slice());
let storable_accounts = let storable_accounts = StorableAccountsWithHashes::new_with_hashes(
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&accounts_data, &accounts_data,
vec![AccountHash(Hash::default()); accounts_count], vec![AccountHash(Hash::default()); accounts_count],
vec![0; accounts_count],
); );
group.bench_function(BenchmarkId::new("append_vec", accounts_count), |b| { group.bench_function(BenchmarkId::new("append_vec", accounts_count), |b| {

View File

@ -25,8 +25,7 @@ lazy_static! {
/// This struct contains what is needed to store accounts to a storage /// This struct contains what is needed to store accounts to a storage
/// 1. account & pubkey (StorableAccounts) /// 1. account & pubkey (StorableAccounts)
/// 2. hash per account (Maybe in StorableAccounts, otherwise has to be passed in separately) /// 2. hash per account (Maybe in StorableAccounts, otherwise has to be passed in separately)
/// 3. write version per account (Maybe in StorableAccounts, otherwise has to be passed in separately) pub struct StorableAccountsWithHashes<
pub struct StorableAccountsWithHashesAndWriteVersions<
'a: 'b, 'a: 'b,
'b, 'b,
T: ReadableAccount + Sync + 'b, T: ReadableAccount + Sync + 'b,
@ -35,10 +34,10 @@ pub struct StorableAccountsWithHashesAndWriteVersions<
> { > {
/// accounts to store /// accounts to store
/// always has pubkey and account /// always has pubkey and account
/// may also have hash and write_version per account /// may also have hash per account
pub(crate) accounts: &'b U, pub(crate) accounts: &'b U,
/// if accounts does not have hash and write version, this has a hash and write version per account /// if accounts does not have hash, this has a hash per account
hashes_and_write_versions: Option<(Vec<V>, Vec<StoredMetaWriteVersion>)>, hashes: Option<Vec<V>>,
_phantom: PhantomData<&'a T>, _phantom: PhantomData<&'a T>,
} }
@ -48,45 +47,40 @@ impl<
T: ReadableAccount + Sync + 'b, T: ReadableAccount + Sync + 'b,
U: StorableAccounts<'a, T>, U: StorableAccounts<'a, T>,
V: Borrow<AccountHash>, V: Borrow<AccountHash>,
> StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V> > StorableAccountsWithHashes<'a, 'b, T, U, V>
{ {
/// used when accounts contains hash and write version already /// used when accounts contains hash already
pub fn new(accounts: &'b U) -> Self { pub fn new(accounts: &'b U) -> Self {
assert!(accounts.has_hash()); assert!(accounts.has_hash());
Self { Self {
accounts, accounts,
hashes_and_write_versions: None, hashes: None,
_phantom: PhantomData, _phantom: PhantomData,
} }
} }
/// used when accounts does NOT contains hash or write version /// used when accounts does NOT contains hash
/// In this case, hashes and write_versions have to be passed in separately and zipped together. /// In this case, hashes have to be passed in separately.
pub fn new_with_hashes_and_write_versions( pub fn new_with_hashes(accounts: &'b U, hashes: Vec<V>) -> Self {
accounts: &'b U,
hashes: Vec<V>,
write_versions: Vec<StoredMetaWriteVersion>,
) -> Self {
assert!(!accounts.has_hash()); assert!(!accounts.has_hash());
assert_eq!(accounts.len(), hashes.len()); assert_eq!(accounts.len(), hashes.len());
assert_eq!(write_versions.len(), hashes.len());
Self { Self {
accounts, accounts,
hashes_and_write_versions: Some((hashes, write_versions)), hashes: Some(hashes),
_phantom: PhantomData, _phantom: PhantomData,
} }
} }
/// get all account fields at 'index' /// get all account fields at 'index'
pub fn get(&self, index: usize) -> (Option<&T>, &Pubkey, &AccountHash, StoredMetaWriteVersion) { pub fn get(&self, index: usize) -> (Option<&T>, &Pubkey, &AccountHash) {
let account = self.accounts.account_default_if_zero_lamport(index); let account = self.accounts.account_default_if_zero_lamport(index);
let pubkey = self.accounts.pubkey(index); let pubkey = self.accounts.pubkey(index);
let (hash, write_version) = if self.accounts.has_hash() { let hash = if self.accounts.has_hash() {
(self.accounts.hash(index), StoredMetaWriteVersion::default()) self.accounts.hash(index)
} else { } else {
let item = self.hashes_and_write_versions.as_ref().unwrap(); let item = self.hashes.as_ref().unwrap();
(item.0[index].borrow(), item.1[index]) item[index].borrow()
}; };
(account, pubkey, hash, write_version) (account, pubkey, hash)
} }
/// None if account at index has lamports == 0 /// None if account at index has lamports == 0

View File

@ -26,7 +26,7 @@ use {
crate::{ crate::{
account_info::{AccountInfo, StorageLocation}, account_info::{AccountInfo, StorageLocation},
account_storage::{ account_storage::{
meta::{StorableAccountsWithHashesAndWriteVersions, StoredAccountMeta}, meta::{StorableAccountsWithHashes, StoredAccountMeta},
AccountStorage, AccountStorageStatus, ShrinkInProgress, AccountStorage, AccountStorageStatus, ShrinkInProgress,
}, },
accounts_cache::{AccountsCache, CachedAccount, SlotCache}, accounts_cache::{AccountsCache, CachedAccount, SlotCache},
@ -5984,7 +5984,7 @@ impl AccountsDb {
&self, &self,
slot: Slot, slot: Slot,
storage: &AccountStorageEntry, storage: &AccountStorageEntry,
accounts_and_meta_to_store: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>, accounts_and_meta_to_store: &StorableAccountsWithHashes<'a, 'b, T, U, V>,
) -> Vec<AccountInfo> { ) -> Vec<AccountInfo> {
let mut infos: Vec<AccountInfo> = Vec::with_capacity(accounts_and_meta_to_store.len()); let mut infos: Vec<AccountInfo> = Vec::with_capacity(accounts_and_meta_to_store.len());
let mut total_append_accounts_us = 0; let mut total_append_accounts_us = 0;
@ -6491,21 +6491,14 @@ impl AccountsDb {
self.write_accounts_to_storage( self.write_accounts_to_storage(
slot, slot,
storage, storage,
&StorableAccountsWithHashesAndWriteVersions::<'_, '_, _, _, &AccountHash>::new( &StorableAccountsWithHashes::<'_, '_, _, _, &AccountHash>::new(accounts),
accounts,
),
) )
} else { } else {
let write_versions = vec![0; accounts.len()];
match hashes { match hashes {
Some(hashes) => self.write_accounts_to_storage( Some(hashes) => self.write_accounts_to_storage(
slot, slot,
storage, storage,
&StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions( &StorableAccountsWithHashes::new_with_hashes(accounts, hashes),
accounts,
hashes,
write_versions,
),
), ),
None => { None => {
// hash any accounts where we were lazy in calculating the hash // hash any accounts where we were lazy in calculating the hash
@ -6513,11 +6506,9 @@ impl AccountsDb {
let len = accounts.len(); let len = accounts.len();
let mut hashes = Vec::with_capacity(len); let mut hashes = Vec::with_capacity(len);
for index in 0..accounts.len() { for index in 0..accounts.len() {
let (pubkey, account) = (accounts.pubkey(index), accounts.account(index)); let (pubkey, account) =
let hash = Self::hash_account( (accounts.pubkey(index), accounts.account(index));
account, let hash = Self::hash_account(account, pubkey);
pubkey,
);
hashes.push(hash); hashes.push(hash);
} }
hash_time.stop(); hash_time.stop();
@ -6528,7 +6519,7 @@ impl AccountsDb {
self.write_accounts_to_storage( self.write_accounts_to_storage(
slot, slot,
storage, storage,
&StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(accounts, hashes, write_versions), &StorableAccountsWithHashes::new_with_hashes(accounts, hashes),
) )
} }
} }
@ -9543,7 +9534,7 @@ pub mod tests {
super::*, super::*,
crate::{ crate::{
account_info::StoredSize, account_info::StoredSize,
account_storage::meta::{AccountMeta, StoredMeta, StoredMetaWriteVersion}, account_storage::meta::{AccountMeta, StoredMeta},
accounts_file::AccountsFileProvider, accounts_file::AccountsFileProvider,
accounts_hash::MERKLE_FANOUT, accounts_hash::MERKLE_FANOUT,
accounts_index::{tests::*, AccountSecondaryIndexesIncludeExclude}, accounts_index::{tests::*, AccountSecondaryIndexesIncludeExclude},
@ -9689,13 +9680,7 @@ pub mod tests {
.iter() .iter()
.map(|_| AccountHash(Hash::default())) .map(|_| AccountHash(Hash::default()))
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let write_versions = data.iter().map(|_| 0).collect::<Vec<_>>(); let append = StorableAccountsWithHashes::new_with_hashes(&storable, hashes);
let append =
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&storable,
hashes,
write_versions,
);
// construct append vec with account to generate an index from // construct append vec with account to generate an index from
append_vec.accounts.append_accounts(&append, 0); append_vec.accounts.append_accounts(&append, 0);
@ -10174,13 +10159,8 @@ pub mod tests {
let slice = &accounts[..]; let slice = &accounts[..];
let account_data = (slot, slice); let account_data = (slot, slice);
let hashes = (0..account_data.len()).map(|_| &hash).collect(); let hashes = (0..account_data.len()).map(|_| &hash).collect();
let write_versions = (0..account_data.len()).map(|_| 0).collect();
let storable_accounts = let storable_accounts =
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions( StorableAccountsWithHashes::new_with_hashes(&account_data, hashes);
&account_data,
hashes,
write_versions,
);
copied_storage copied_storage
.accounts .accounts
.append_accounts(&storable_accounts, 0); .append_accounts(&storable_accounts, 0);
@ -10220,13 +10200,8 @@ pub mod tests {
let slice = &accounts[..]; let slice = &accounts[..];
let account_data = (slot, slice); let account_data = (slot, slice);
let hashes = (0..account_data.len()).map(|_| &hash).collect(); let hashes = (0..account_data.len()).map(|_| &hash).collect();
let write_versions = (0..account_data.len()).map(|_| 0).collect();
let storable_accounts = let storable_accounts =
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions( StorableAccountsWithHashes::new_with_hashes(&account_data, hashes);
&account_data,
hashes,
write_versions,
);
copied_storage copied_storage
.accounts .accounts
.append_accounts(&storable_accounts, 0); .append_accounts(&storable_accounts, 0);
@ -10631,7 +10606,7 @@ pub mod tests {
let pubkey = solana_sdk::pubkey::new_rand(); let pubkey = solana_sdk::pubkey::new_rand();
let acc = AccountSharedData::new(1, 48, AccountSharedData::default().owner()); let acc = AccountSharedData::new(1, 48, AccountSharedData::default().owner());
let mark_alive = false; let mark_alive = false;
append_single_account_with_default_hash(&storage, &pubkey, &acc, 1, mark_alive, None); append_single_account_with_default_hash(&storage, &pubkey, &acc, mark_alive, None);
let calls = Arc::new(AtomicU64::new(0)); let calls = Arc::new(AtomicU64::new(0));
let temp_dir = TempDir::new().unwrap(); let temp_dir = TempDir::new().unwrap();
@ -10686,7 +10661,6 @@ pub mod tests {
storage: &AccountStorageEntry, storage: &AccountStorageEntry,
pubkey: &Pubkey, pubkey: &Pubkey,
account: &AccountSharedData, account: &AccountSharedData,
write_version: StoredMetaWriteVersion,
mark_alive: bool, mark_alive: bool,
add_to_index: Option<&AccountInfoAccountsIndex>, add_to_index: Option<&AccountInfoAccountsIndex>,
) { ) {
@ -10696,11 +10670,7 @@ pub mod tests {
let account_data = (slot, slice); let account_data = (slot, slice);
let hash = AccountHash(Hash::default()); let hash = AccountHash(Hash::default());
let storable_accounts = let storable_accounts =
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions( StorableAccountsWithHashes::new_with_hashes(&account_data, vec![&hash]);
&account_data,
vec![&hash],
vec![write_version],
);
let stored_accounts_info = storage let stored_accounts_info = storage
.accounts .accounts
.append_accounts(&storable_accounts, 0) .append_accounts(&storable_accounts, 0)
@ -10756,7 +10726,7 @@ pub mod tests {
let pubkey = solana_sdk::pubkey::new_rand(); let pubkey = solana_sdk::pubkey::new_rand();
let acc = AccountSharedData::new(1, 48, AccountSharedData::default().owner()); let acc = AccountSharedData::new(1, 48, AccountSharedData::default().owner());
let mark_alive = false; let mark_alive = false;
append_single_account_with_default_hash(&storage, &pubkey, &acc, 1, mark_alive, None); append_single_account_with_default_hash(&storage, &pubkey, &acc, mark_alive, None);
let calls = Arc::new(AtomicU64::new(0)); let calls = Arc::new(AtomicU64::new(0));
@ -10785,7 +10755,6 @@ pub mod tests {
fn append_sample_data_to_storage( fn append_sample_data_to_storage(
storage: &Arc<AccountStorageEntry>, storage: &Arc<AccountStorageEntry>,
pubkey: &Pubkey, pubkey: &Pubkey,
write_version: StoredMetaWriteVersion,
mark_alive: bool, mark_alive: bool,
account_data_size: Option<u64>, account_data_size: Option<u64>,
) { ) {
@ -10794,29 +10763,20 @@ pub mod tests {
account_data_size.unwrap_or(48) as usize, account_data_size.unwrap_or(48) as usize,
AccountSharedData::default().owner(), AccountSharedData::default().owner(),
); );
append_single_account_with_default_hash( append_single_account_with_default_hash(storage, pubkey, &acc, mark_alive, None);
storage,
pubkey,
&acc,
write_version,
mark_alive,
None,
);
} }
fn sample_storage_with_entries( fn sample_storage_with_entries(
tf: &TempFile, tf: &TempFile,
write_version: StoredMetaWriteVersion,
slot: Slot, slot: Slot,
pubkey: &Pubkey, pubkey: &Pubkey,
mark_alive: bool, mark_alive: bool,
) -> Arc<AccountStorageEntry> { ) -> Arc<AccountStorageEntry> {
sample_storage_with_entries_id(tf, write_version, slot, pubkey, 0, mark_alive, None) sample_storage_with_entries_id(tf, slot, pubkey, 0, mark_alive, None)
} }
fn sample_storage_with_entries_id_fill_percentage( fn sample_storage_with_entries_id_fill_percentage(
tf: &TempFile, tf: &TempFile,
write_version: StoredMetaWriteVersion,
slot: Slot, slot: Slot,
pubkey: &Pubkey, pubkey: &Pubkey,
id: AccountsFileId, id: AccountsFileId,
@ -10842,13 +10802,12 @@ pub mod tests {
data.accounts = av; data.accounts = av;
let arc = Arc::new(data); let arc = Arc::new(data);
append_sample_data_to_storage(&arc, pubkey, write_version, mark_alive, account_data_size); append_sample_data_to_storage(&arc, pubkey, mark_alive, account_data_size);
arc arc
} }
fn sample_storage_with_entries_id( fn sample_storage_with_entries_id(
tf: &TempFile, tf: &TempFile,
write_version: StoredMetaWriteVersion,
slot: Slot, slot: Slot,
pubkey: &Pubkey, pubkey: &Pubkey,
id: AccountsFileId, id: AccountsFileId,
@ -10857,7 +10816,6 @@ pub mod tests {
) -> Arc<AccountStorageEntry> { ) -> Arc<AccountStorageEntry> {
sample_storage_with_entries_id_fill_percentage( sample_storage_with_entries_id_fill_percentage(
tf, tf,
write_version,
slot, slot,
pubkey, pubkey,
id, id,
@ -10875,12 +10833,10 @@ pub mod tests {
let tf = crate::append_vec::test_utils::get_append_vec_path( let tf = crate::append_vec::test_utils::get_append_vec_path(
"test_accountsdb_scan_account_storage_no_bank", "test_accountsdb_scan_account_storage_no_bank",
); );
let write_version1 = 0;
let pubkey1 = solana_sdk::pubkey::new_rand(); let pubkey1 = solana_sdk::pubkey::new_rand();
let pubkey2 = solana_sdk::pubkey::new_rand(); let pubkey2 = solana_sdk::pubkey::new_rand();
let mark_alive = false; let mark_alive = false;
let storage = let storage = sample_storage_with_entries(&tf, slot_expected, &pubkey1, mark_alive);
sample_storage_with_entries(&tf, write_version1, slot_expected, &pubkey1, mark_alive);
let lamports = storage.accounts.account_iter().next().unwrap().lamports(); let lamports = storage.accounts.account_iter().next().unwrap().lamports();
let calls = Arc::new(AtomicU64::new(0)); let calls = Arc::new(AtomicU64::new(0));
let mut scanner = TestScanSimple { let mut scanner = TestScanSimple {
@ -15185,12 +15141,10 @@ pub mod tests {
let storage = accounts.create_and_insert_store(slot0, 4_000, "flush_slot_cache"); let storage = accounts.create_and_insert_store(slot0, 4_000, "flush_slot_cache");
let hashes = vec![AccountHash(Hash::default()); 1]; let hashes = vec![AccountHash(Hash::default()); 1];
let write_version = vec![0; 1];
storage.accounts.append_accounts( storage.accounts.append_accounts(
&StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions( &StorableAccountsWithHashes::new_with_hashes(
&(slot0, &[(&shared_key, &account)][..]), &(slot0, &[(&shared_key, &account)][..]),
hashes, hashes,
write_version,
), ),
0, 0,
); );
@ -15250,12 +15204,10 @@ pub mod tests {
let slot0 = 0; let slot0 = 0;
let storage = accounts.create_and_insert_store(slot0, 4_000, "flush_slot_cache"); let storage = accounts.create_and_insert_store(slot0, 4_000, "flush_slot_cache");
let hashes = vec![AccountHash(Hash::default()); 2]; let hashes = vec![AccountHash(Hash::default()); 2];
let write_version = vec![0; 2];
storage.accounts.append_accounts( storage.accounts.append_accounts(
&StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions( &StorableAccountsWithHashes::new_with_hashes(
&(slot0, &[(&keys[0], &account), (&keys[1], &account_big)][..]), &(slot0, &[(&keys[0], &account), (&keys[1], &account_big)][..]),
hashes, hashes,
write_version,
), ),
0, 0,
); );
@ -16251,11 +16203,9 @@ pub mod tests {
let tf = crate::append_vec::test_utils::get_append_vec_path( let tf = crate::append_vec::test_utils::get_append_vec_path(
"test_accountsdb_scan_account_storage_no_bank", "test_accountsdb_scan_account_storage_no_bank",
); );
let write_version1 = 0;
let pubkey1 = solana_sdk::pubkey::new_rand(); let pubkey1 = solana_sdk::pubkey::new_rand();
let mark_alive = false; let mark_alive = false;
let storage = let storage = sample_storage_with_entries(&tf, slot, &pubkey1, mark_alive);
sample_storage_with_entries(&tf, write_version1, slot, &pubkey1, mark_alive);
let load = AccountsDb::hash_storage_info(&mut hasher, Some(&storage), slot); let load = AccountsDb::hash_storage_info(&mut hasher, Some(&storage), slot);
let hash = hasher.finish(); let hash = hasher.finish();
@ -16269,13 +16219,7 @@ pub mod tests {
// can't assert hash here - it is a function of mod date // can't assert hash here - it is a function of mod date
assert!(load); assert!(load);
let mut hasher = hash_map::DefaultHasher::new(); let mut hasher = hash_map::DefaultHasher::new();
append_sample_data_to_storage( append_sample_data_to_storage(&storage, &solana_sdk::pubkey::new_rand(), false, None);
&storage,
&solana_sdk::pubkey::new_rand(),
write_version1,
false,
None,
);
let load = AccountsDb::hash_storage_info(&mut hasher, Some(&storage), slot); let load = AccountsDb::hash_storage_info(&mut hasher, Some(&storage), slot);
let hash3 = hasher.finish(); let hash3 = hasher.finish();
assert_ne!(hash2, hash3); // moddate and written size changed assert_ne!(hash2, hash3); // moddate and written size changed
@ -17298,7 +17242,6 @@ pub mod tests {
}); });
let tf = tf.unwrap_or_else(|| local_tf.as_ref().unwrap()); let tf = tf.unwrap_or_else(|| local_tf.as_ref().unwrap());
let write_version1 = 0;
let starting_id = db let starting_id = db
.storage .storage
.iter() .iter()
@ -17310,7 +17253,6 @@ pub mod tests {
let pubkey1 = solana_sdk::pubkey::new_rand(); let pubkey1 = solana_sdk::pubkey::new_rand();
let storage = sample_storage_with_entries_id_fill_percentage( let storage = sample_storage_with_entries_id_fill_percentage(
tf, tf,
write_version1,
starting_slot + (i as Slot), starting_slot + (i as Slot),
&pubkey1, &pubkey1,
id, id,
@ -17347,7 +17289,6 @@ pub mod tests {
}); });
let tf = tf.unwrap_or_else(|| local_tf.as_ref().unwrap()); let tf = tf.unwrap_or_else(|| local_tf.as_ref().unwrap());
let write_version1 = 0;
let starting_id = db let starting_id = db
.storage .storage
.iter() .iter()
@ -17359,7 +17300,6 @@ pub mod tests {
let pubkey1 = solana_sdk::pubkey::new_rand(); let pubkey1 = solana_sdk::pubkey::new_rand();
let storage = sample_storage_with_entries_id( let storage = sample_storage_with_entries_id(
tf, tf,
write_version1,
starting_slot + (i as Slot), starting_slot + (i as Slot),
&pubkey1, &pubkey1,
id, id,
@ -17513,9 +17453,8 @@ pub mod tests {
let tf = crate::append_vec::test_utils::get_append_vec_path( let tf = crate::append_vec::test_utils::get_append_vec_path(
"test_should_move_to_ancient_append_vec", "test_should_move_to_ancient_append_vec",
); );
let write_version1 = 0;
let pubkey1 = solana_sdk::pubkey::new_rand(); let pubkey1 = solana_sdk::pubkey::new_rand();
let storage = sample_storage_with_entries(&tf, write_version1, slot5, &pubkey1, false); let storage = sample_storage_with_entries(&tf, slot5, &pubkey1, false);
let mut current_ancient = CurrentAncientAccountsFile::default(); let mut current_ancient = CurrentAncientAccountsFile::default();
let should_move = db.should_move_to_ancient_accounts_file( let should_move = db.should_move_to_ancient_accounts_file(
@ -17659,7 +17598,7 @@ pub mod tests {
fn make_ancient_append_vec_full(ancient: &Arc<AccountStorageEntry>, mark_alive: bool) { fn make_ancient_append_vec_full(ancient: &Arc<AccountStorageEntry>, mark_alive: bool) {
for _ in 0..100 { for _ in 0..100 {
append_sample_data_to_storage(ancient, &Pubkey::default(), 0, mark_alive, None); append_sample_data_to_storage(ancient, &Pubkey::default(), mark_alive, None);
} }
// since we're not adding to the index, this is how we specify that all these accounts are alive // since we're not adding to the index, this is how we specify that all these accounts are alive
adjust_alive_bytes(ancient, ancient.capacity() as usize); adjust_alive_bytes(ancient, ancient.capacity() as usize);

View File

@ -1,9 +1,7 @@
use { use {
crate::{ crate::{
account_info::AccountInfo, account_info::AccountInfo,
account_storage::meta::{ account_storage::meta::{StorableAccountsWithHashes, StoredAccountInfo, StoredAccountMeta},
StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo, StoredAccountMeta,
},
accounts_db::AccountsFileId, accounts_db::AccountsFileId,
accounts_hash::AccountHash, accounts_hash::AccountHash,
append_vec::{AppendVec, AppendVecError, IndexInfo}, append_vec::{AppendVec, AppendVecError, IndexInfo},
@ -229,7 +227,7 @@ impl AccountsFile {
V: Borrow<AccountHash>, V: Borrow<AccountHash>,
>( >(
&self, &self,
accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>, accounts: &StorableAccountsWithHashes<'a, 'b, T, U, V>,
skip: usize, skip: usize,
) -> Option<Vec<StoredAccountInfo>> { ) -> Option<Vec<StoredAccountInfo>> {
match self { match self {

View File

@ -1174,7 +1174,6 @@ pub mod tests {
storage, storage,
&pk, &pk,
&account, &account,
0,
true, true,
Some(&db.accounts_index), Some(&db.accounts_index),
); );
@ -1282,7 +1281,6 @@ pub mod tests {
storage, storage,
&pk, &pk,
&account, &account,
0,
true, true,
Some(&db.accounts_index), Some(&db.accounts_index),
); );
@ -1514,12 +1512,10 @@ pub mod tests {
storages.iter().for_each(|storage| { storages.iter().for_each(|storage| {
let pk = solana_sdk::pubkey::new_rand(); let pk = solana_sdk::pubkey::new_rand();
let alive = false; let alive = false;
let write_version = 0;
append_single_account_with_default_hash( append_single_account_with_default_hash(
storage, storage,
&pk, &pk,
&AccountSharedData::default(), &AccountSharedData::default(),
write_version,
alive, alive,
Some(&db.accounts_index), Some(&db.accounts_index),
); );
@ -1663,7 +1659,6 @@ pub mod tests {
&storage, &storage,
&pk_with_1_ref, &pk_with_1_ref,
&account_with_1_ref, &account_with_1_ref,
0,
true, true,
Some(&db.accounts_index), Some(&db.accounts_index),
); );
@ -1676,7 +1671,6 @@ pub mod tests {
&ignored_storage, &ignored_storage,
pk_with_2_refs, pk_with_2_refs,
&account_with_2_refs.to_account_shared_data(), &account_with_2_refs.to_account_shared_data(),
0,
true, true,
Some(&db.accounts_index), Some(&db.accounts_index),
); );
@ -1843,7 +1837,6 @@ pub mod tests {
&storage, &storage,
&pk_with_1_ref, &pk_with_1_ref,
&account_with_1_ref, &account_with_1_ref,
0,
true, true,
Some(&db.accounts_index), Some(&db.accounts_index),
); );

View File

@ -7,8 +7,8 @@
use { use {
crate::{ crate::{
account_storage::meta::{ account_storage::meta::{
AccountMeta, StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo, AccountMeta, StorableAccountsWithHashes, StoredAccountInfo, StoredAccountMeta,
StoredAccountMeta, StoredMeta, StoredMetaWriteVersion, StoredMeta, StoredMetaWriteVersion,
}, },
accounts_file::{AccountsFileError, MatchAccountOwnerError, Result, ALIGN_BOUNDARY_OFFSET}, accounts_file::{AccountsFileError, MatchAccountOwnerError, Result, ALIGN_BOUNDARY_OFFSET},
accounts_hash::AccountHash, accounts_hash::AccountHash,
@ -719,7 +719,7 @@ impl AppendVec {
V: Borrow<AccountHash>, V: Borrow<AccountHash>,
>( >(
&self, &self,
accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>, accounts: &StorableAccountsWithHashes<'a, 'b, T, U, V>,
skip: usize, skip: usize,
) -> Option<Vec<StoredAccountInfo>> { ) -> Option<Vec<StoredAccountInfo>> {
let _lock = self.append_lock.lock().unwrap(); let _lock = self.append_lock.lock().unwrap();
@ -732,7 +732,7 @@ impl AppendVec {
let offsets_len = len - skip + 1; let offsets_len = len - skip + 1;
let mut offsets = Vec::with_capacity(offsets_len); let mut offsets = Vec::with_capacity(offsets_len);
for i in skip..len { for i in skip..len {
let (account, pubkey, hash, write_version_obsolete) = accounts.get(i); let (account, pubkey, hash) = accounts.get(i);
let account_meta = account let account_meta = account
.map(|account| AccountMeta { .map(|account| AccountMeta {
lamports: account.lamports(), lamports: account.lamports(),
@ -747,7 +747,7 @@ impl AppendVec {
data_len: account data_len: account
.map(|account| account.data().len()) .map(|account| account.data().len())
.unwrap_or_default() as u64, .unwrap_or_default() as u64,
write_version_obsolete, write_version_obsolete: 0,
}; };
let meta_ptr = &stored_meta as *const StoredMeta; let meta_ptr = &stored_meta as *const StoredMeta;
let account_meta_ptr = &account_meta as *const AccountMeta; let account_meta_ptr = &account_meta as *const AccountMeta;
@ -822,11 +822,7 @@ pub mod tests {
let account_data = (slot_ignored, slice); let account_data = (slot_ignored, slice);
let hash = AccountHash(Hash::default()); let hash = AccountHash(Hash::default());
let storable_accounts = let storable_accounts =
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions( StorableAccountsWithHashes::new_with_hashes(&account_data, vec![&hash]);
&account_data,
vec![&hash],
vec![data.0.write_version_obsolete],
);
self.append_accounts(&storable_accounts, 0) self.append_accounts(&storable_accounts, 0)
.map(|res| res[0].offset) .map(|res| res[0].offset)
@ -879,85 +875,55 @@ pub mod tests {
#[test] #[test]
#[should_panic(expected = "accounts.has_hash()")] #[should_panic(expected = "accounts.has_hash()")]
fn test_storable_accounts_with_hashes_and_write_versions_new() { fn test_storable_accounts_with_hashes_new() {
let account = AccountSharedData::default(); let account = AccountSharedData::default();
// for (Slot, &'a [(&'a Pubkey, &'a T)]) // for (Slot, &'a [(&'a Pubkey, &'a T)])
let slot = 0 as Slot; let slot = 0 as Slot;
let pubkey = Pubkey::default(); let pubkey = Pubkey::default();
StorableAccountsWithHashesAndWriteVersions::<'_, '_, _, _, &AccountHash>::new(&( StorableAccountsWithHashes::<'_, '_, _, _, &AccountHash>::new(&(
slot, slot,
&[(&pubkey, &account)][..], &[(&pubkey, &account)][..],
)); ));
} }
fn test_mismatch(correct_hashes: bool, correct_write_versions: bool) { fn test_mismatch(correct_hashes: bool) {
let account = AccountSharedData::default(); let account = AccountSharedData::default();
// for (Slot, &'a [(&'a Pubkey, &'a T)]) // for (Slot, &'a [(&'a Pubkey, &'a T)])
let slot = 0 as Slot; let slot = 0 as Slot;
let pubkey = Pubkey::default(); let pubkey = Pubkey::default();
// mismatch between lens of accounts, hashes, write_versions // mismatch between lens of accounts and hashes
let mut hashes = Vec::default(); let mut hashes = Vec::default();
if correct_hashes { if correct_hashes {
hashes.push(AccountHash(Hash::default())); hashes.push(AccountHash(Hash::default()));
} }
let mut write_versions = Vec::default(); StorableAccountsWithHashes::new_with_hashes(&(slot, &[(&pubkey, &account)][..]), hashes);
if correct_write_versions {
write_versions.push(0);
}
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&(slot, &[(&pubkey, &account)][..]),
hashes,
write_versions,
);
} }
#[test] #[test]
// rust 1.73+ (our as-of-writing nightly version) changed panic message. we're stuck with this // rust 1.73+ (our as-of-writing nightly version) changed panic message. we're stuck with this
// short common substring until the monorepo is fully 1.73+ including stable. // short common substring until the monorepo is fully 1.73+ including stable.
#[should_panic(expected = "left == right")] #[should_panic(expected = "left == right")]
fn test_storable_accounts_with_hashes_and_write_versions_new2() { fn test_storable_accounts_with_hashes_new2() {
test_mismatch(false, false); test_mismatch(false);
} }
#[test] #[test]
// rust 1.73+ (our as-of-writing nightly version) changed panic message. we're stuck with this fn test_storable_accounts_with_hashes_empty() {
// short common substring until the monorepo is fully 1.73+ including stable.
#[should_panic(expected = "left == right")]
fn test_storable_accounts_with_hashes_and_write_versions_new3() {
test_mismatch(false, true);
}
#[test]
// rust 1.73+ (our as-of-writing nightly version) changed panic message. we're stuck with this
// short common substring until the monorepo is fully 1.73+ including stable.
#[should_panic(expected = "left == right")]
fn test_storable_accounts_with_hashes_and_write_versions_new4() {
test_mismatch(true, false);
}
#[test]
fn test_storable_accounts_with_hashes_and_write_versions_empty() {
// for (Slot, &'a [(&'a Pubkey, &'a T)]) // for (Slot, &'a [(&'a Pubkey, &'a T)])
let account = AccountSharedData::default(); let account = AccountSharedData::default();
let slot = 0 as Slot; let slot = 0 as Slot;
let pubkeys = [Pubkey::default()]; let pubkeys = [Pubkey::default()];
let hashes = Vec::<AccountHash>::default(); let hashes = Vec::<AccountHash>::default();
let write_versions = Vec::default();
let mut accounts = vec![(&pubkeys[0], &account)]; let mut accounts = vec![(&pubkeys[0], &account)];
accounts.clear(); accounts.clear();
let accounts2 = (slot, &accounts[..]); let accounts2 = (slot, &accounts[..]);
let storable = let storable = StorableAccountsWithHashes::new_with_hashes(&accounts2, hashes);
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&accounts2,
hashes,
write_versions,
);
assert_eq!(storable.len(), 0); assert_eq!(storable.len(), 0);
assert!(storable.is_empty()); assert!(storable.is_empty());
} }
#[test] #[test]
fn test_storable_accounts_with_hashes_and_write_versions_hash_and_write_version() { fn test_storable_accounts_with_hashes_hash() {
// for (Slot, &'a [(&'a Pubkey, &'a T)]) // for (Slot, &'a [(&'a Pubkey, &'a T)])
let account = AccountSharedData::default(); let account = AccountSharedData::default();
let slot = 0 as Slot; let slot = 0 as Slot;
@ -966,27 +932,20 @@ pub mod tests {
AccountHash(Hash::new(&[3; 32])), AccountHash(Hash::new(&[3; 32])),
AccountHash(Hash::new(&[4; 32])), AccountHash(Hash::new(&[4; 32])),
]; ];
let write_versions = vec![42, 43];
let accounts = [(&pubkeys[0], &account), (&pubkeys[1], &account)]; let accounts = [(&pubkeys[0], &account), (&pubkeys[1], &account)];
let accounts2 = (slot, &accounts[..]); let accounts2 = (slot, &accounts[..]);
let storable = let storable = StorableAccountsWithHashes::new_with_hashes(&accounts2, hashes.clone());
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&accounts2,
hashes.clone(),
write_versions.clone(),
);
assert_eq!(storable.len(), pubkeys.len()); assert_eq!(storable.len(), pubkeys.len());
assert!(!storable.is_empty()); assert!(!storable.is_empty());
(0..2).for_each(|i| { (0..2).for_each(|i| {
let (_, pubkey, hash, write_version) = storable.get(i); let (_, pubkey, hash) = storable.get(i);
assert_eq!(hash, &hashes[i]); assert_eq!(hash, &hashes[i]);
assert_eq!(write_version, write_versions[i]);
assert_eq!(pubkey, &pubkeys[i]); assert_eq!(pubkey, &pubkeys[i]);
}); });
} }
#[test] #[test]
fn test_storable_accounts_with_hashes_and_write_versions_default() { fn test_storable_accounts_with_hashes_default() {
// 0 lamport account, should return default account (or None in this case) // 0 lamport account, should return default account (or None in this case)
let account = Account { let account = Account {
data: vec![0], data: vec![0],
@ -997,15 +956,9 @@ pub mod tests {
let slot = 0 as Slot; let slot = 0 as Slot;
let pubkey = Pubkey::default(); let pubkey = Pubkey::default();
let hashes = vec![AccountHash(Hash::default())]; let hashes = vec![AccountHash(Hash::default())];
let write_versions = vec![0];
let accounts = [(&pubkey, &account)]; let accounts = [(&pubkey, &account)];
let accounts2 = (slot, &accounts[..]); let accounts2 = (slot, &accounts[..]);
let storable = let storable = StorableAccountsWithHashes::new_with_hashes(&accounts2, hashes.clone());
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&accounts2,
hashes.clone(),
write_versions.clone(),
);
let get_account = storable.account(0); let get_account = storable.account(0);
assert!(get_account.is_none()); assert!(get_account.is_none());
@ -1019,12 +972,7 @@ pub mod tests {
// for (Slot, &'a [(&'a Pubkey, &'a T)]) // for (Slot, &'a [(&'a Pubkey, &'a T)])
let accounts = [(&pubkey, &account)]; let accounts = [(&pubkey, &account)];
let accounts2 = (slot, &accounts[..]); let accounts2 = (slot, &accounts[..]);
let storable = let storable = StorableAccountsWithHashes::new_with_hashes(&accounts2, hashes);
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&accounts2,
hashes,
write_versions,
);
let get_account = storable.account(0); let get_account = storable.account(0);
assert!(accounts_equal(&account, get_account.unwrap())); assert!(accounts_equal(&account, get_account.unwrap()));
} }

View File

@ -14,7 +14,7 @@ mod test_utils;
use { use {
crate::{ crate::{
account_storage::meta::{StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo}, account_storage::meta::{StorableAccountsWithHashes, StoredAccountInfo},
accounts_hash::AccountHash, accounts_hash::AccountHash,
storable_accounts::StorableAccounts, storable_accounts::StorableAccounts,
}, },
@ -119,7 +119,7 @@ impl TieredStorage {
V: Borrow<AccountHash>, V: Borrow<AccountHash>,
>( >(
&self, &self,
accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>, accounts: &StorableAccountsWithHashes<'a, 'b, T, U, V>,
skip: usize, skip: usize,
format: &TieredStorageFormat, format: &TieredStorageFormat,
) -> TieredStorageResult<Vec<StoredAccountInfo>> { ) -> TieredStorageResult<Vec<StoredAccountInfo>> {
@ -180,7 +180,6 @@ impl TieredStorage {
mod tests { mod tests {
use { use {
super::*, super::*,
crate::account_storage::meta::StoredMetaWriteVersion,
file::TieredStorageMagicNumber, file::TieredStorageMagicNumber,
footer::TieredStorageFooter, footer::TieredStorageFooter,
hot::HOT_FORMAT, hot::HOT_FORMAT,
@ -213,11 +212,7 @@ mod tests {
let account_refs = Vec::<(&Pubkey, &AccountSharedData)>::new(); let account_refs = Vec::<(&Pubkey, &AccountSharedData)>::new();
let account_data = (slot_ignored, account_refs.as_slice()); let account_data = (slot_ignored, account_refs.as_slice());
let storable_accounts = let storable_accounts =
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions( StorableAccountsWithHashes::new_with_hashes(&account_data, Vec::<AccountHash>::new());
&account_data,
Vec::<AccountHash>::new(),
Vec::<StoredMetaWriteVersion>::new(),
);
let result = tiered_storage.write_accounts(&storable_accounts, 0, &HOT_FORMAT); let result = tiered_storage.write_accounts(&storable_accounts, 0, &HOT_FORMAT);
@ -350,17 +345,8 @@ mod tests {
let hashes: Vec<_> = std::iter::repeat_with(|| AccountHash(Hash::new_unique())) let hashes: Vec<_> = std::iter::repeat_with(|| AccountHash(Hash::new_unique()))
.take(account_data_sizes.len()) .take(account_data_sizes.len())
.collect(); .collect();
let write_versions: Vec<_> = accounts
.iter()
.map(|account| account.0.write_version_obsolete)
.collect();
let storable_accounts = let storable_accounts = StorableAccountsWithHashes::new_with_hashes(&account_data, hashes);
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&account_data,
hashes,
write_versions,
);
let temp_dir = tempdir().unwrap(); let temp_dir = tempdir().unwrap();
let tiered_storage_path = temp_dir.path().join(path_suffix); let tiered_storage_path = temp_dir.path().join(path_suffix);
@ -373,7 +359,7 @@ mod tests {
let mut expected_accounts_map = HashMap::new(); let mut expected_accounts_map = HashMap::new();
for i in 0..num_accounts { for i in 0..num_accounts {
let (account, address, _account_hash, _write_version) = storable_accounts.get(i); let (account, address, _account_hash) = storable_accounts.get(i);
expected_accounts_map.insert(address, account); expected_accounts_map.insert(address, account);
} }

View File

@ -15,8 +15,8 @@ use {
}, },
mmap_utils::{get_pod, get_slice}, mmap_utils::{get_pod, get_slice},
owners::{OwnerOffset, OwnersBlockFormat, OwnersTable, OWNER_NO_OWNER}, owners::{OwnerOffset, OwnersBlockFormat, OwnersTable, OWNER_NO_OWNER},
StorableAccounts, StorableAccountsWithHashesAndWriteVersions, TieredStorageError, StorableAccounts, StorableAccountsWithHashes, TieredStorageError, TieredStorageFormat,
TieredStorageFormat, TieredStorageResult, TieredStorageResult,
}, },
}, },
bytemuck::{Pod, Zeroable}, bytemuck::{Pod, Zeroable},
@ -634,7 +634,7 @@ impl HotStorageWriter {
V: Borrow<AccountHash>, V: Borrow<AccountHash>,
>( >(
&mut self, &mut self,
accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>, accounts: &StorableAccountsWithHashes<'a, 'b, T, U, V>,
skip: usize, skip: usize,
) -> TieredStorageResult<Vec<StoredAccountInfo>> { ) -> TieredStorageResult<Vec<StoredAccountInfo>> {
let mut footer = new_hot_footer(); let mut footer = new_hot_footer();
@ -648,7 +648,7 @@ impl HotStorageWriter {
let total_input_accounts = len - skip; let total_input_accounts = len - skip;
let mut stored_infos = Vec::with_capacity(total_input_accounts); let mut stored_infos = Vec::with_capacity(total_input_accounts);
for i in skip..len { for i in skip..len {
let (account, address, _account_hash, _write_version) = accounts.get(i); let (account, address, _account_hash) = accounts.get(i);
let index_entry = AccountIndexWriterEntry { let index_entry = AccountIndexWriterEntry {
address, address,
offset: HotAccountOffset::new(cursor)?, offset: HotAccountOffset::new(cursor)?,
@ -1372,17 +1372,8 @@ pub mod tests {
.take(account_data_sizes.len()) .take(account_data_sizes.len())
.collect(); .collect();
let write_versions: Vec<_> = accounts
.iter()
.map(|account| account.0.write_version_obsolete)
.collect();
let storable_accounts = let storable_accounts =
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions( StorableAccountsWithHashes::new_with_hashes(&account_data, hashes.clone());
&account_data,
hashes.clone(),
write_versions.clone(),
);
let temp_dir = TempDir::new().unwrap(); let temp_dir = TempDir::new().unwrap();
let path = temp_dir.path().join("test_write_account_and_index_blocks"); let path = temp_dir.path().join("test_write_account_and_index_blocks");
@ -1401,7 +1392,7 @@ pub mod tests {
.unwrap() .unwrap()
.unwrap(); .unwrap();
let (account, address, _account_hash, _write_version) = storable_accounts.get(i); let (account, address, _account_hash) = storable_accounts.get(i);
verify_test_account(&stored_meta, account, address); verify_test_account(&stored_meta, account, address);
assert_eq!(i + 1, next.0 as usize); assert_eq!(i + 1, next.0 as usize);
@ -1419,8 +1410,7 @@ pub mod tests {
.unwrap() .unwrap()
.unwrap(); .unwrap();
let (account, address, _account_hash, _write_version) = let (account, address, _account_hash) = storable_accounts.get(stored_info.offset);
storable_accounts.get(stored_info.offset);
verify_test_account(&stored_meta, account, address); verify_test_account(&stored_meta, account, address);
} }
@ -1429,7 +1419,7 @@ pub mod tests {
// first, we verify everything // first, we verify everything
for (i, stored_meta) in accounts.iter().enumerate() { for (i, stored_meta) in accounts.iter().enumerate() {
let (account, address, _account_hash, _write_version) = storable_accounts.get(i); let (account, address, _account_hash) = storable_accounts.get(i);
verify_test_account(stored_meta, account, address); verify_test_account(stored_meta, account, address);
} }