Removes write version from StorableAccountsWithHashesAndWriteVersions (#561)
This commit is contained in:
parent
7b204e7d01
commit
afa65c6690
|
@ -4,9 +4,7 @@ extern crate test;
|
||||||
use {
|
use {
|
||||||
rand::{thread_rng, Rng},
|
rand::{thread_rng, Rng},
|
||||||
solana_accounts_db::{
|
solana_accounts_db::{
|
||||||
account_storage::meta::{
|
account_storage::meta::{StorableAccountsWithHashes, StoredAccountInfo, StoredMeta},
|
||||||
StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo, StoredMeta,
|
|
||||||
},
|
|
||||||
accounts_hash::AccountHash,
|
accounts_hash::AccountHash,
|
||||||
append_vec::{
|
append_vec::{
|
||||||
test_utils::{create_test_account, get_append_vec_path},
|
test_utils::{create_test_account, get_append_vec_path},
|
||||||
|
@ -39,12 +37,7 @@ fn append_account(
|
||||||
let accounts = [(&storage_meta.pubkey, account)];
|
let accounts = [(&storage_meta.pubkey, account)];
|
||||||
let slice = &accounts[..];
|
let slice = &accounts[..];
|
||||||
let accounts = (slot_ignored, slice);
|
let accounts = (slot_ignored, slice);
|
||||||
let storable_accounts =
|
let storable_accounts = StorableAccountsWithHashes::new_with_hashes(&accounts, vec![&hash]);
|
||||||
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
|
|
||||||
&accounts,
|
|
||||||
vec![&hash],
|
|
||||||
vec![storage_meta.write_version_obsolete],
|
|
||||||
);
|
|
||||||
let res = vec.append_accounts(&storable_accounts, 0);
|
let res = vec.append_accounts(&storable_accounts, 0);
|
||||||
res.and_then(|res| res.first().cloned())
|
res.and_then(|res| res.first().cloned())
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,7 +2,7 @@
|
||||||
use {
|
use {
|
||||||
criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion, Throughput},
|
criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion, Throughput},
|
||||||
solana_accounts_db::{
|
solana_accounts_db::{
|
||||||
account_storage::meta::StorableAccountsWithHashesAndWriteVersions,
|
account_storage::meta::StorableAccountsWithHashes,
|
||||||
accounts_hash::AccountHash,
|
accounts_hash::AccountHash,
|
||||||
append_vec::{self, AppendVec},
|
append_vec::{self, AppendVec},
|
||||||
tiered_storage::hot::HotStorageWriter,
|
tiered_storage::hot::HotStorageWriter,
|
||||||
|
@ -46,12 +46,10 @@ fn bench_write_accounts_file(c: &mut Criterion) {
|
||||||
.collect();
|
.collect();
|
||||||
let accounts_refs: Vec<_> = accounts.iter().collect();
|
let accounts_refs: Vec<_> = accounts.iter().collect();
|
||||||
let accounts_data = (Slot::MAX, accounts_refs.as_slice());
|
let accounts_data = (Slot::MAX, accounts_refs.as_slice());
|
||||||
let storable_accounts =
|
let storable_accounts = StorableAccountsWithHashes::new_with_hashes(
|
||||||
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
|
&accounts_data,
|
||||||
&accounts_data,
|
vec![AccountHash(Hash::default()); accounts_count],
|
||||||
vec![AccountHash(Hash::default()); accounts_count],
|
);
|
||||||
vec![0; accounts_count],
|
|
||||||
);
|
|
||||||
|
|
||||||
group.bench_function(BenchmarkId::new("append_vec", accounts_count), |b| {
|
group.bench_function(BenchmarkId::new("append_vec", accounts_count), |b| {
|
||||||
b.iter_batched_ref(
|
b.iter_batched_ref(
|
||||||
|
|
|
@ -25,8 +25,7 @@ lazy_static! {
|
||||||
/// This struct contains what is needed to store accounts to a storage
|
/// This struct contains what is needed to store accounts to a storage
|
||||||
/// 1. account & pubkey (StorableAccounts)
|
/// 1. account & pubkey (StorableAccounts)
|
||||||
/// 2. hash per account (Maybe in StorableAccounts, otherwise has to be passed in separately)
|
/// 2. hash per account (Maybe in StorableAccounts, otherwise has to be passed in separately)
|
||||||
/// 3. write version per account (Maybe in StorableAccounts, otherwise has to be passed in separately)
|
pub struct StorableAccountsWithHashes<
|
||||||
pub struct StorableAccountsWithHashesAndWriteVersions<
|
|
||||||
'a: 'b,
|
'a: 'b,
|
||||||
'b,
|
'b,
|
||||||
T: ReadableAccount + Sync + 'b,
|
T: ReadableAccount + Sync + 'b,
|
||||||
|
@ -35,10 +34,10 @@ pub struct StorableAccountsWithHashesAndWriteVersions<
|
||||||
> {
|
> {
|
||||||
/// accounts to store
|
/// accounts to store
|
||||||
/// always has pubkey and account
|
/// always has pubkey and account
|
||||||
/// may also have hash and write_version per account
|
/// may also have hash per account
|
||||||
pub(crate) accounts: &'b U,
|
pub(crate) accounts: &'b U,
|
||||||
/// if accounts does not have hash and write version, this has a hash and write version per account
|
/// if accounts does not have hash, this has a hash per account
|
||||||
hashes_and_write_versions: Option<(Vec<V>, Vec<StoredMetaWriteVersion>)>,
|
hashes: Option<Vec<V>>,
|
||||||
_phantom: PhantomData<&'a T>,
|
_phantom: PhantomData<&'a T>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -48,45 +47,40 @@ impl<
|
||||||
T: ReadableAccount + Sync + 'b,
|
T: ReadableAccount + Sync + 'b,
|
||||||
U: StorableAccounts<'a, T>,
|
U: StorableAccounts<'a, T>,
|
||||||
V: Borrow<AccountHash>,
|
V: Borrow<AccountHash>,
|
||||||
> StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>
|
> StorableAccountsWithHashes<'a, 'b, T, U, V>
|
||||||
{
|
{
|
||||||
/// used when accounts contains hash and write version already
|
/// used when accounts contains hash already
|
||||||
pub fn new(accounts: &'b U) -> Self {
|
pub fn new(accounts: &'b U) -> Self {
|
||||||
assert!(accounts.has_hash());
|
assert!(accounts.has_hash());
|
||||||
Self {
|
Self {
|
||||||
accounts,
|
accounts,
|
||||||
hashes_and_write_versions: None,
|
hashes: None,
|
||||||
_phantom: PhantomData,
|
_phantom: PhantomData,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
/// used when accounts does NOT contains hash or write version
|
/// used when accounts does NOT contains hash
|
||||||
/// In this case, hashes and write_versions have to be passed in separately and zipped together.
|
/// In this case, hashes have to be passed in separately.
|
||||||
pub fn new_with_hashes_and_write_versions(
|
pub fn new_with_hashes(accounts: &'b U, hashes: Vec<V>) -> Self {
|
||||||
accounts: &'b U,
|
|
||||||
hashes: Vec<V>,
|
|
||||||
write_versions: Vec<StoredMetaWriteVersion>,
|
|
||||||
) -> Self {
|
|
||||||
assert!(!accounts.has_hash());
|
assert!(!accounts.has_hash());
|
||||||
assert_eq!(accounts.len(), hashes.len());
|
assert_eq!(accounts.len(), hashes.len());
|
||||||
assert_eq!(write_versions.len(), hashes.len());
|
|
||||||
Self {
|
Self {
|
||||||
accounts,
|
accounts,
|
||||||
hashes_and_write_versions: Some((hashes, write_versions)),
|
hashes: Some(hashes),
|
||||||
_phantom: PhantomData,
|
_phantom: PhantomData,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// get all account fields at 'index'
|
/// get all account fields at 'index'
|
||||||
pub fn get(&self, index: usize) -> (Option<&T>, &Pubkey, &AccountHash, StoredMetaWriteVersion) {
|
pub fn get(&self, index: usize) -> (Option<&T>, &Pubkey, &AccountHash) {
|
||||||
let account = self.accounts.account_default_if_zero_lamport(index);
|
let account = self.accounts.account_default_if_zero_lamport(index);
|
||||||
let pubkey = self.accounts.pubkey(index);
|
let pubkey = self.accounts.pubkey(index);
|
||||||
let (hash, write_version) = if self.accounts.has_hash() {
|
let hash = if self.accounts.has_hash() {
|
||||||
(self.accounts.hash(index), StoredMetaWriteVersion::default())
|
self.accounts.hash(index)
|
||||||
} else {
|
} else {
|
||||||
let item = self.hashes_and_write_versions.as_ref().unwrap();
|
let item = self.hashes.as_ref().unwrap();
|
||||||
(item.0[index].borrow(), item.1[index])
|
item[index].borrow()
|
||||||
};
|
};
|
||||||
(account, pubkey, hash, write_version)
|
(account, pubkey, hash)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// None if account at index has lamports == 0
|
/// None if account at index has lamports == 0
|
||||||
|
|
|
@ -26,7 +26,7 @@ use {
|
||||||
crate::{
|
crate::{
|
||||||
account_info::{AccountInfo, StorageLocation},
|
account_info::{AccountInfo, StorageLocation},
|
||||||
account_storage::{
|
account_storage::{
|
||||||
meta::{StorableAccountsWithHashesAndWriteVersions, StoredAccountMeta},
|
meta::{StorableAccountsWithHashes, StoredAccountMeta},
|
||||||
AccountStorage, AccountStorageStatus, ShrinkInProgress,
|
AccountStorage, AccountStorageStatus, ShrinkInProgress,
|
||||||
},
|
},
|
||||||
accounts_cache::{AccountsCache, CachedAccount, SlotCache},
|
accounts_cache::{AccountsCache, CachedAccount, SlotCache},
|
||||||
|
@ -5984,7 +5984,7 @@ impl AccountsDb {
|
||||||
&self,
|
&self,
|
||||||
slot: Slot,
|
slot: Slot,
|
||||||
storage: &AccountStorageEntry,
|
storage: &AccountStorageEntry,
|
||||||
accounts_and_meta_to_store: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>,
|
accounts_and_meta_to_store: &StorableAccountsWithHashes<'a, 'b, T, U, V>,
|
||||||
) -> Vec<AccountInfo> {
|
) -> Vec<AccountInfo> {
|
||||||
let mut infos: Vec<AccountInfo> = Vec::with_capacity(accounts_and_meta_to_store.len());
|
let mut infos: Vec<AccountInfo> = Vec::with_capacity(accounts_and_meta_to_store.len());
|
||||||
let mut total_append_accounts_us = 0;
|
let mut total_append_accounts_us = 0;
|
||||||
|
@ -6491,21 +6491,14 @@ impl AccountsDb {
|
||||||
self.write_accounts_to_storage(
|
self.write_accounts_to_storage(
|
||||||
slot,
|
slot,
|
||||||
storage,
|
storage,
|
||||||
&StorableAccountsWithHashesAndWriteVersions::<'_, '_, _, _, &AccountHash>::new(
|
&StorableAccountsWithHashes::<'_, '_, _, _, &AccountHash>::new(accounts),
|
||||||
accounts,
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
let write_versions = vec![0; accounts.len()];
|
|
||||||
match hashes {
|
match hashes {
|
||||||
Some(hashes) => self.write_accounts_to_storage(
|
Some(hashes) => self.write_accounts_to_storage(
|
||||||
slot,
|
slot,
|
||||||
storage,
|
storage,
|
||||||
&StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
|
&StorableAccountsWithHashes::new_with_hashes(accounts, hashes),
|
||||||
accounts,
|
|
||||||
hashes,
|
|
||||||
write_versions,
|
|
||||||
),
|
|
||||||
),
|
),
|
||||||
None => {
|
None => {
|
||||||
// hash any accounts where we were lazy in calculating the hash
|
// hash any accounts where we were lazy in calculating the hash
|
||||||
|
@ -6513,11 +6506,9 @@ impl AccountsDb {
|
||||||
let len = accounts.len();
|
let len = accounts.len();
|
||||||
let mut hashes = Vec::with_capacity(len);
|
let mut hashes = Vec::with_capacity(len);
|
||||||
for index in 0..accounts.len() {
|
for index in 0..accounts.len() {
|
||||||
let (pubkey, account) = (accounts.pubkey(index), accounts.account(index));
|
let (pubkey, account) =
|
||||||
let hash = Self::hash_account(
|
(accounts.pubkey(index), accounts.account(index));
|
||||||
account,
|
let hash = Self::hash_account(account, pubkey);
|
||||||
pubkey,
|
|
||||||
);
|
|
||||||
hashes.push(hash);
|
hashes.push(hash);
|
||||||
}
|
}
|
||||||
hash_time.stop();
|
hash_time.stop();
|
||||||
|
@ -6526,10 +6517,10 @@ impl AccountsDb {
|
||||||
.fetch_add(hash_time.as_us(), Ordering::Relaxed);
|
.fetch_add(hash_time.as_us(), Ordering::Relaxed);
|
||||||
|
|
||||||
self.write_accounts_to_storage(
|
self.write_accounts_to_storage(
|
||||||
slot,
|
slot,
|
||||||
storage,
|
storage,
|
||||||
&StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(accounts, hashes, write_versions),
|
&StorableAccountsWithHashes::new_with_hashes(accounts, hashes),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -9543,7 +9534,7 @@ pub mod tests {
|
||||||
super::*,
|
super::*,
|
||||||
crate::{
|
crate::{
|
||||||
account_info::StoredSize,
|
account_info::StoredSize,
|
||||||
account_storage::meta::{AccountMeta, StoredMeta, StoredMetaWriteVersion},
|
account_storage::meta::{AccountMeta, StoredMeta},
|
||||||
accounts_file::AccountsFileProvider,
|
accounts_file::AccountsFileProvider,
|
||||||
accounts_hash::MERKLE_FANOUT,
|
accounts_hash::MERKLE_FANOUT,
|
||||||
accounts_index::{tests::*, AccountSecondaryIndexesIncludeExclude},
|
accounts_index::{tests::*, AccountSecondaryIndexesIncludeExclude},
|
||||||
|
@ -9689,13 +9680,7 @@ pub mod tests {
|
||||||
.iter()
|
.iter()
|
||||||
.map(|_| AccountHash(Hash::default()))
|
.map(|_| AccountHash(Hash::default()))
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
let write_versions = data.iter().map(|_| 0).collect::<Vec<_>>();
|
let append = StorableAccountsWithHashes::new_with_hashes(&storable, hashes);
|
||||||
let append =
|
|
||||||
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
|
|
||||||
&storable,
|
|
||||||
hashes,
|
|
||||||
write_versions,
|
|
||||||
);
|
|
||||||
|
|
||||||
// construct append vec with account to generate an index from
|
// construct append vec with account to generate an index from
|
||||||
append_vec.accounts.append_accounts(&append, 0);
|
append_vec.accounts.append_accounts(&append, 0);
|
||||||
|
@ -10174,13 +10159,8 @@ pub mod tests {
|
||||||
let slice = &accounts[..];
|
let slice = &accounts[..];
|
||||||
let account_data = (slot, slice);
|
let account_data = (slot, slice);
|
||||||
let hashes = (0..account_data.len()).map(|_| &hash).collect();
|
let hashes = (0..account_data.len()).map(|_| &hash).collect();
|
||||||
let write_versions = (0..account_data.len()).map(|_| 0).collect();
|
|
||||||
let storable_accounts =
|
let storable_accounts =
|
||||||
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
|
StorableAccountsWithHashes::new_with_hashes(&account_data, hashes);
|
||||||
&account_data,
|
|
||||||
hashes,
|
|
||||||
write_versions,
|
|
||||||
);
|
|
||||||
copied_storage
|
copied_storage
|
||||||
.accounts
|
.accounts
|
||||||
.append_accounts(&storable_accounts, 0);
|
.append_accounts(&storable_accounts, 0);
|
||||||
|
@ -10220,13 +10200,8 @@ pub mod tests {
|
||||||
let slice = &accounts[..];
|
let slice = &accounts[..];
|
||||||
let account_data = (slot, slice);
|
let account_data = (slot, slice);
|
||||||
let hashes = (0..account_data.len()).map(|_| &hash).collect();
|
let hashes = (0..account_data.len()).map(|_| &hash).collect();
|
||||||
let write_versions = (0..account_data.len()).map(|_| 0).collect();
|
|
||||||
let storable_accounts =
|
let storable_accounts =
|
||||||
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
|
StorableAccountsWithHashes::new_with_hashes(&account_data, hashes);
|
||||||
&account_data,
|
|
||||||
hashes,
|
|
||||||
write_versions,
|
|
||||||
);
|
|
||||||
copied_storage
|
copied_storage
|
||||||
.accounts
|
.accounts
|
||||||
.append_accounts(&storable_accounts, 0);
|
.append_accounts(&storable_accounts, 0);
|
||||||
|
@ -10631,7 +10606,7 @@ pub mod tests {
|
||||||
let pubkey = solana_sdk::pubkey::new_rand();
|
let pubkey = solana_sdk::pubkey::new_rand();
|
||||||
let acc = AccountSharedData::new(1, 48, AccountSharedData::default().owner());
|
let acc = AccountSharedData::new(1, 48, AccountSharedData::default().owner());
|
||||||
let mark_alive = false;
|
let mark_alive = false;
|
||||||
append_single_account_with_default_hash(&storage, &pubkey, &acc, 1, mark_alive, None);
|
append_single_account_with_default_hash(&storage, &pubkey, &acc, mark_alive, None);
|
||||||
|
|
||||||
let calls = Arc::new(AtomicU64::new(0));
|
let calls = Arc::new(AtomicU64::new(0));
|
||||||
let temp_dir = TempDir::new().unwrap();
|
let temp_dir = TempDir::new().unwrap();
|
||||||
|
@ -10686,7 +10661,6 @@ pub mod tests {
|
||||||
storage: &AccountStorageEntry,
|
storage: &AccountStorageEntry,
|
||||||
pubkey: &Pubkey,
|
pubkey: &Pubkey,
|
||||||
account: &AccountSharedData,
|
account: &AccountSharedData,
|
||||||
write_version: StoredMetaWriteVersion,
|
|
||||||
mark_alive: bool,
|
mark_alive: bool,
|
||||||
add_to_index: Option<&AccountInfoAccountsIndex>,
|
add_to_index: Option<&AccountInfoAccountsIndex>,
|
||||||
) {
|
) {
|
||||||
|
@ -10696,11 +10670,7 @@ pub mod tests {
|
||||||
let account_data = (slot, slice);
|
let account_data = (slot, slice);
|
||||||
let hash = AccountHash(Hash::default());
|
let hash = AccountHash(Hash::default());
|
||||||
let storable_accounts =
|
let storable_accounts =
|
||||||
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
|
StorableAccountsWithHashes::new_with_hashes(&account_data, vec![&hash]);
|
||||||
&account_data,
|
|
||||||
vec![&hash],
|
|
||||||
vec![write_version],
|
|
||||||
);
|
|
||||||
let stored_accounts_info = storage
|
let stored_accounts_info = storage
|
||||||
.accounts
|
.accounts
|
||||||
.append_accounts(&storable_accounts, 0)
|
.append_accounts(&storable_accounts, 0)
|
||||||
|
@ -10756,7 +10726,7 @@ pub mod tests {
|
||||||
let pubkey = solana_sdk::pubkey::new_rand();
|
let pubkey = solana_sdk::pubkey::new_rand();
|
||||||
let acc = AccountSharedData::new(1, 48, AccountSharedData::default().owner());
|
let acc = AccountSharedData::new(1, 48, AccountSharedData::default().owner());
|
||||||
let mark_alive = false;
|
let mark_alive = false;
|
||||||
append_single_account_with_default_hash(&storage, &pubkey, &acc, 1, mark_alive, None);
|
append_single_account_with_default_hash(&storage, &pubkey, &acc, mark_alive, None);
|
||||||
|
|
||||||
let calls = Arc::new(AtomicU64::new(0));
|
let calls = Arc::new(AtomicU64::new(0));
|
||||||
|
|
||||||
|
@ -10785,7 +10755,6 @@ pub mod tests {
|
||||||
fn append_sample_data_to_storage(
|
fn append_sample_data_to_storage(
|
||||||
storage: &Arc<AccountStorageEntry>,
|
storage: &Arc<AccountStorageEntry>,
|
||||||
pubkey: &Pubkey,
|
pubkey: &Pubkey,
|
||||||
write_version: StoredMetaWriteVersion,
|
|
||||||
mark_alive: bool,
|
mark_alive: bool,
|
||||||
account_data_size: Option<u64>,
|
account_data_size: Option<u64>,
|
||||||
) {
|
) {
|
||||||
|
@ -10794,29 +10763,20 @@ pub mod tests {
|
||||||
account_data_size.unwrap_or(48) as usize,
|
account_data_size.unwrap_or(48) as usize,
|
||||||
AccountSharedData::default().owner(),
|
AccountSharedData::default().owner(),
|
||||||
);
|
);
|
||||||
append_single_account_with_default_hash(
|
append_single_account_with_default_hash(storage, pubkey, &acc, mark_alive, None);
|
||||||
storage,
|
|
||||||
pubkey,
|
|
||||||
&acc,
|
|
||||||
write_version,
|
|
||||||
mark_alive,
|
|
||||||
None,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sample_storage_with_entries(
|
fn sample_storage_with_entries(
|
||||||
tf: &TempFile,
|
tf: &TempFile,
|
||||||
write_version: StoredMetaWriteVersion,
|
|
||||||
slot: Slot,
|
slot: Slot,
|
||||||
pubkey: &Pubkey,
|
pubkey: &Pubkey,
|
||||||
mark_alive: bool,
|
mark_alive: bool,
|
||||||
) -> Arc<AccountStorageEntry> {
|
) -> Arc<AccountStorageEntry> {
|
||||||
sample_storage_with_entries_id(tf, write_version, slot, pubkey, 0, mark_alive, None)
|
sample_storage_with_entries_id(tf, slot, pubkey, 0, mark_alive, None)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sample_storage_with_entries_id_fill_percentage(
|
fn sample_storage_with_entries_id_fill_percentage(
|
||||||
tf: &TempFile,
|
tf: &TempFile,
|
||||||
write_version: StoredMetaWriteVersion,
|
|
||||||
slot: Slot,
|
slot: Slot,
|
||||||
pubkey: &Pubkey,
|
pubkey: &Pubkey,
|
||||||
id: AccountsFileId,
|
id: AccountsFileId,
|
||||||
|
@ -10842,13 +10802,12 @@ pub mod tests {
|
||||||
data.accounts = av;
|
data.accounts = av;
|
||||||
|
|
||||||
let arc = Arc::new(data);
|
let arc = Arc::new(data);
|
||||||
append_sample_data_to_storage(&arc, pubkey, write_version, mark_alive, account_data_size);
|
append_sample_data_to_storage(&arc, pubkey, mark_alive, account_data_size);
|
||||||
arc
|
arc
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sample_storage_with_entries_id(
|
fn sample_storage_with_entries_id(
|
||||||
tf: &TempFile,
|
tf: &TempFile,
|
||||||
write_version: StoredMetaWriteVersion,
|
|
||||||
slot: Slot,
|
slot: Slot,
|
||||||
pubkey: &Pubkey,
|
pubkey: &Pubkey,
|
||||||
id: AccountsFileId,
|
id: AccountsFileId,
|
||||||
|
@ -10857,7 +10816,6 @@ pub mod tests {
|
||||||
) -> Arc<AccountStorageEntry> {
|
) -> Arc<AccountStorageEntry> {
|
||||||
sample_storage_with_entries_id_fill_percentage(
|
sample_storage_with_entries_id_fill_percentage(
|
||||||
tf,
|
tf,
|
||||||
write_version,
|
|
||||||
slot,
|
slot,
|
||||||
pubkey,
|
pubkey,
|
||||||
id,
|
id,
|
||||||
|
@ -10875,12 +10833,10 @@ pub mod tests {
|
||||||
let tf = crate::append_vec::test_utils::get_append_vec_path(
|
let tf = crate::append_vec::test_utils::get_append_vec_path(
|
||||||
"test_accountsdb_scan_account_storage_no_bank",
|
"test_accountsdb_scan_account_storage_no_bank",
|
||||||
);
|
);
|
||||||
let write_version1 = 0;
|
|
||||||
let pubkey1 = solana_sdk::pubkey::new_rand();
|
let pubkey1 = solana_sdk::pubkey::new_rand();
|
||||||
let pubkey2 = solana_sdk::pubkey::new_rand();
|
let pubkey2 = solana_sdk::pubkey::new_rand();
|
||||||
let mark_alive = false;
|
let mark_alive = false;
|
||||||
let storage =
|
let storage = sample_storage_with_entries(&tf, slot_expected, &pubkey1, mark_alive);
|
||||||
sample_storage_with_entries(&tf, write_version1, slot_expected, &pubkey1, mark_alive);
|
|
||||||
let lamports = storage.accounts.account_iter().next().unwrap().lamports();
|
let lamports = storage.accounts.account_iter().next().unwrap().lamports();
|
||||||
let calls = Arc::new(AtomicU64::new(0));
|
let calls = Arc::new(AtomicU64::new(0));
|
||||||
let mut scanner = TestScanSimple {
|
let mut scanner = TestScanSimple {
|
||||||
|
@ -15185,12 +15141,10 @@ pub mod tests {
|
||||||
|
|
||||||
let storage = accounts.create_and_insert_store(slot0, 4_000, "flush_slot_cache");
|
let storage = accounts.create_and_insert_store(slot0, 4_000, "flush_slot_cache");
|
||||||
let hashes = vec![AccountHash(Hash::default()); 1];
|
let hashes = vec![AccountHash(Hash::default()); 1];
|
||||||
let write_version = vec![0; 1];
|
|
||||||
storage.accounts.append_accounts(
|
storage.accounts.append_accounts(
|
||||||
&StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
|
&StorableAccountsWithHashes::new_with_hashes(
|
||||||
&(slot0, &[(&shared_key, &account)][..]),
|
&(slot0, &[(&shared_key, &account)][..]),
|
||||||
hashes,
|
hashes,
|
||||||
write_version,
|
|
||||||
),
|
),
|
||||||
0,
|
0,
|
||||||
);
|
);
|
||||||
|
@ -15250,12 +15204,10 @@ pub mod tests {
|
||||||
let slot0 = 0;
|
let slot0 = 0;
|
||||||
let storage = accounts.create_and_insert_store(slot0, 4_000, "flush_slot_cache");
|
let storage = accounts.create_and_insert_store(slot0, 4_000, "flush_slot_cache");
|
||||||
let hashes = vec![AccountHash(Hash::default()); 2];
|
let hashes = vec![AccountHash(Hash::default()); 2];
|
||||||
let write_version = vec![0; 2];
|
|
||||||
storage.accounts.append_accounts(
|
storage.accounts.append_accounts(
|
||||||
&StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
|
&StorableAccountsWithHashes::new_with_hashes(
|
||||||
&(slot0, &[(&keys[0], &account), (&keys[1], &account_big)][..]),
|
&(slot0, &[(&keys[0], &account), (&keys[1], &account_big)][..]),
|
||||||
hashes,
|
hashes,
|
||||||
write_version,
|
|
||||||
),
|
),
|
||||||
0,
|
0,
|
||||||
);
|
);
|
||||||
|
@ -16251,11 +16203,9 @@ pub mod tests {
|
||||||
let tf = crate::append_vec::test_utils::get_append_vec_path(
|
let tf = crate::append_vec::test_utils::get_append_vec_path(
|
||||||
"test_accountsdb_scan_account_storage_no_bank",
|
"test_accountsdb_scan_account_storage_no_bank",
|
||||||
);
|
);
|
||||||
let write_version1 = 0;
|
|
||||||
let pubkey1 = solana_sdk::pubkey::new_rand();
|
let pubkey1 = solana_sdk::pubkey::new_rand();
|
||||||
let mark_alive = false;
|
let mark_alive = false;
|
||||||
let storage =
|
let storage = sample_storage_with_entries(&tf, slot, &pubkey1, mark_alive);
|
||||||
sample_storage_with_entries(&tf, write_version1, slot, &pubkey1, mark_alive);
|
|
||||||
|
|
||||||
let load = AccountsDb::hash_storage_info(&mut hasher, Some(&storage), slot);
|
let load = AccountsDb::hash_storage_info(&mut hasher, Some(&storage), slot);
|
||||||
let hash = hasher.finish();
|
let hash = hasher.finish();
|
||||||
|
@ -16269,13 +16219,7 @@ pub mod tests {
|
||||||
// can't assert hash here - it is a function of mod date
|
// can't assert hash here - it is a function of mod date
|
||||||
assert!(load);
|
assert!(load);
|
||||||
let mut hasher = hash_map::DefaultHasher::new();
|
let mut hasher = hash_map::DefaultHasher::new();
|
||||||
append_sample_data_to_storage(
|
append_sample_data_to_storage(&storage, &solana_sdk::pubkey::new_rand(), false, None);
|
||||||
&storage,
|
|
||||||
&solana_sdk::pubkey::new_rand(),
|
|
||||||
write_version1,
|
|
||||||
false,
|
|
||||||
None,
|
|
||||||
);
|
|
||||||
let load = AccountsDb::hash_storage_info(&mut hasher, Some(&storage), slot);
|
let load = AccountsDb::hash_storage_info(&mut hasher, Some(&storage), slot);
|
||||||
let hash3 = hasher.finish();
|
let hash3 = hasher.finish();
|
||||||
assert_ne!(hash2, hash3); // moddate and written size changed
|
assert_ne!(hash2, hash3); // moddate and written size changed
|
||||||
|
@ -17298,7 +17242,6 @@ pub mod tests {
|
||||||
});
|
});
|
||||||
let tf = tf.unwrap_or_else(|| local_tf.as_ref().unwrap());
|
let tf = tf.unwrap_or_else(|| local_tf.as_ref().unwrap());
|
||||||
|
|
||||||
let write_version1 = 0;
|
|
||||||
let starting_id = db
|
let starting_id = db
|
||||||
.storage
|
.storage
|
||||||
.iter()
|
.iter()
|
||||||
|
@ -17310,7 +17253,6 @@ pub mod tests {
|
||||||
let pubkey1 = solana_sdk::pubkey::new_rand();
|
let pubkey1 = solana_sdk::pubkey::new_rand();
|
||||||
let storage = sample_storage_with_entries_id_fill_percentage(
|
let storage = sample_storage_with_entries_id_fill_percentage(
|
||||||
tf,
|
tf,
|
||||||
write_version1,
|
|
||||||
starting_slot + (i as Slot),
|
starting_slot + (i as Slot),
|
||||||
&pubkey1,
|
&pubkey1,
|
||||||
id,
|
id,
|
||||||
|
@ -17347,7 +17289,6 @@ pub mod tests {
|
||||||
});
|
});
|
||||||
let tf = tf.unwrap_or_else(|| local_tf.as_ref().unwrap());
|
let tf = tf.unwrap_or_else(|| local_tf.as_ref().unwrap());
|
||||||
|
|
||||||
let write_version1 = 0;
|
|
||||||
let starting_id = db
|
let starting_id = db
|
||||||
.storage
|
.storage
|
||||||
.iter()
|
.iter()
|
||||||
|
@ -17359,7 +17300,6 @@ pub mod tests {
|
||||||
let pubkey1 = solana_sdk::pubkey::new_rand();
|
let pubkey1 = solana_sdk::pubkey::new_rand();
|
||||||
let storage = sample_storage_with_entries_id(
|
let storage = sample_storage_with_entries_id(
|
||||||
tf,
|
tf,
|
||||||
write_version1,
|
|
||||||
starting_slot + (i as Slot),
|
starting_slot + (i as Slot),
|
||||||
&pubkey1,
|
&pubkey1,
|
||||||
id,
|
id,
|
||||||
|
@ -17513,9 +17453,8 @@ pub mod tests {
|
||||||
let tf = crate::append_vec::test_utils::get_append_vec_path(
|
let tf = crate::append_vec::test_utils::get_append_vec_path(
|
||||||
"test_should_move_to_ancient_append_vec",
|
"test_should_move_to_ancient_append_vec",
|
||||||
);
|
);
|
||||||
let write_version1 = 0;
|
|
||||||
let pubkey1 = solana_sdk::pubkey::new_rand();
|
let pubkey1 = solana_sdk::pubkey::new_rand();
|
||||||
let storage = sample_storage_with_entries(&tf, write_version1, slot5, &pubkey1, false);
|
let storage = sample_storage_with_entries(&tf, slot5, &pubkey1, false);
|
||||||
let mut current_ancient = CurrentAncientAccountsFile::default();
|
let mut current_ancient = CurrentAncientAccountsFile::default();
|
||||||
|
|
||||||
let should_move = db.should_move_to_ancient_accounts_file(
|
let should_move = db.should_move_to_ancient_accounts_file(
|
||||||
|
@ -17659,7 +17598,7 @@ pub mod tests {
|
||||||
|
|
||||||
fn make_ancient_append_vec_full(ancient: &Arc<AccountStorageEntry>, mark_alive: bool) {
|
fn make_ancient_append_vec_full(ancient: &Arc<AccountStorageEntry>, mark_alive: bool) {
|
||||||
for _ in 0..100 {
|
for _ in 0..100 {
|
||||||
append_sample_data_to_storage(ancient, &Pubkey::default(), 0, mark_alive, None);
|
append_sample_data_to_storage(ancient, &Pubkey::default(), mark_alive, None);
|
||||||
}
|
}
|
||||||
// since we're not adding to the index, this is how we specify that all these accounts are alive
|
// since we're not adding to the index, this is how we specify that all these accounts are alive
|
||||||
adjust_alive_bytes(ancient, ancient.capacity() as usize);
|
adjust_alive_bytes(ancient, ancient.capacity() as usize);
|
||||||
|
|
|
@ -1,9 +1,7 @@
|
||||||
use {
|
use {
|
||||||
crate::{
|
crate::{
|
||||||
account_info::AccountInfo,
|
account_info::AccountInfo,
|
||||||
account_storage::meta::{
|
account_storage::meta::{StorableAccountsWithHashes, StoredAccountInfo, StoredAccountMeta},
|
||||||
StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo, StoredAccountMeta,
|
|
||||||
},
|
|
||||||
accounts_db::AccountsFileId,
|
accounts_db::AccountsFileId,
|
||||||
accounts_hash::AccountHash,
|
accounts_hash::AccountHash,
|
||||||
append_vec::{AppendVec, AppendVecError, IndexInfo},
|
append_vec::{AppendVec, AppendVecError, IndexInfo},
|
||||||
|
@ -229,7 +227,7 @@ impl AccountsFile {
|
||||||
V: Borrow<AccountHash>,
|
V: Borrow<AccountHash>,
|
||||||
>(
|
>(
|
||||||
&self,
|
&self,
|
||||||
accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>,
|
accounts: &StorableAccountsWithHashes<'a, 'b, T, U, V>,
|
||||||
skip: usize,
|
skip: usize,
|
||||||
) -> Option<Vec<StoredAccountInfo>> {
|
) -> Option<Vec<StoredAccountInfo>> {
|
||||||
match self {
|
match self {
|
||||||
|
|
|
@ -1174,7 +1174,6 @@ pub mod tests {
|
||||||
storage,
|
storage,
|
||||||
&pk,
|
&pk,
|
||||||
&account,
|
&account,
|
||||||
0,
|
|
||||||
true,
|
true,
|
||||||
Some(&db.accounts_index),
|
Some(&db.accounts_index),
|
||||||
);
|
);
|
||||||
|
@ -1282,7 +1281,6 @@ pub mod tests {
|
||||||
storage,
|
storage,
|
||||||
&pk,
|
&pk,
|
||||||
&account,
|
&account,
|
||||||
0,
|
|
||||||
true,
|
true,
|
||||||
Some(&db.accounts_index),
|
Some(&db.accounts_index),
|
||||||
);
|
);
|
||||||
|
@ -1514,12 +1512,10 @@ pub mod tests {
|
||||||
storages.iter().for_each(|storage| {
|
storages.iter().for_each(|storage| {
|
||||||
let pk = solana_sdk::pubkey::new_rand();
|
let pk = solana_sdk::pubkey::new_rand();
|
||||||
let alive = false;
|
let alive = false;
|
||||||
let write_version = 0;
|
|
||||||
append_single_account_with_default_hash(
|
append_single_account_with_default_hash(
|
||||||
storage,
|
storage,
|
||||||
&pk,
|
&pk,
|
||||||
&AccountSharedData::default(),
|
&AccountSharedData::default(),
|
||||||
write_version,
|
|
||||||
alive,
|
alive,
|
||||||
Some(&db.accounts_index),
|
Some(&db.accounts_index),
|
||||||
);
|
);
|
||||||
|
@ -1663,7 +1659,6 @@ pub mod tests {
|
||||||
&storage,
|
&storage,
|
||||||
&pk_with_1_ref,
|
&pk_with_1_ref,
|
||||||
&account_with_1_ref,
|
&account_with_1_ref,
|
||||||
0,
|
|
||||||
true,
|
true,
|
||||||
Some(&db.accounts_index),
|
Some(&db.accounts_index),
|
||||||
);
|
);
|
||||||
|
@ -1676,7 +1671,6 @@ pub mod tests {
|
||||||
&ignored_storage,
|
&ignored_storage,
|
||||||
pk_with_2_refs,
|
pk_with_2_refs,
|
||||||
&account_with_2_refs.to_account_shared_data(),
|
&account_with_2_refs.to_account_shared_data(),
|
||||||
0,
|
|
||||||
true,
|
true,
|
||||||
Some(&db.accounts_index),
|
Some(&db.accounts_index),
|
||||||
);
|
);
|
||||||
|
@ -1843,7 +1837,6 @@ pub mod tests {
|
||||||
&storage,
|
&storage,
|
||||||
&pk_with_1_ref,
|
&pk_with_1_ref,
|
||||||
&account_with_1_ref,
|
&account_with_1_ref,
|
||||||
0,
|
|
||||||
true,
|
true,
|
||||||
Some(&db.accounts_index),
|
Some(&db.accounts_index),
|
||||||
);
|
);
|
||||||
|
|
|
@ -7,8 +7,8 @@
|
||||||
use {
|
use {
|
||||||
crate::{
|
crate::{
|
||||||
account_storage::meta::{
|
account_storage::meta::{
|
||||||
AccountMeta, StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo,
|
AccountMeta, StorableAccountsWithHashes, StoredAccountInfo, StoredAccountMeta,
|
||||||
StoredAccountMeta, StoredMeta, StoredMetaWriteVersion,
|
StoredMeta, StoredMetaWriteVersion,
|
||||||
},
|
},
|
||||||
accounts_file::{AccountsFileError, MatchAccountOwnerError, Result, ALIGN_BOUNDARY_OFFSET},
|
accounts_file::{AccountsFileError, MatchAccountOwnerError, Result, ALIGN_BOUNDARY_OFFSET},
|
||||||
accounts_hash::AccountHash,
|
accounts_hash::AccountHash,
|
||||||
|
@ -719,7 +719,7 @@ impl AppendVec {
|
||||||
V: Borrow<AccountHash>,
|
V: Borrow<AccountHash>,
|
||||||
>(
|
>(
|
||||||
&self,
|
&self,
|
||||||
accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>,
|
accounts: &StorableAccountsWithHashes<'a, 'b, T, U, V>,
|
||||||
skip: usize,
|
skip: usize,
|
||||||
) -> Option<Vec<StoredAccountInfo>> {
|
) -> Option<Vec<StoredAccountInfo>> {
|
||||||
let _lock = self.append_lock.lock().unwrap();
|
let _lock = self.append_lock.lock().unwrap();
|
||||||
|
@ -732,7 +732,7 @@ impl AppendVec {
|
||||||
let offsets_len = len - skip + 1;
|
let offsets_len = len - skip + 1;
|
||||||
let mut offsets = Vec::with_capacity(offsets_len);
|
let mut offsets = Vec::with_capacity(offsets_len);
|
||||||
for i in skip..len {
|
for i in skip..len {
|
||||||
let (account, pubkey, hash, write_version_obsolete) = accounts.get(i);
|
let (account, pubkey, hash) = accounts.get(i);
|
||||||
let account_meta = account
|
let account_meta = account
|
||||||
.map(|account| AccountMeta {
|
.map(|account| AccountMeta {
|
||||||
lamports: account.lamports(),
|
lamports: account.lamports(),
|
||||||
|
@ -747,7 +747,7 @@ impl AppendVec {
|
||||||
data_len: account
|
data_len: account
|
||||||
.map(|account| account.data().len())
|
.map(|account| account.data().len())
|
||||||
.unwrap_or_default() as u64,
|
.unwrap_or_default() as u64,
|
||||||
write_version_obsolete,
|
write_version_obsolete: 0,
|
||||||
};
|
};
|
||||||
let meta_ptr = &stored_meta as *const StoredMeta;
|
let meta_ptr = &stored_meta as *const StoredMeta;
|
||||||
let account_meta_ptr = &account_meta as *const AccountMeta;
|
let account_meta_ptr = &account_meta as *const AccountMeta;
|
||||||
|
@ -822,11 +822,7 @@ pub mod tests {
|
||||||
let account_data = (slot_ignored, slice);
|
let account_data = (slot_ignored, slice);
|
||||||
let hash = AccountHash(Hash::default());
|
let hash = AccountHash(Hash::default());
|
||||||
let storable_accounts =
|
let storable_accounts =
|
||||||
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
|
StorableAccountsWithHashes::new_with_hashes(&account_data, vec![&hash]);
|
||||||
&account_data,
|
|
||||||
vec![&hash],
|
|
||||||
vec![data.0.write_version_obsolete],
|
|
||||||
);
|
|
||||||
|
|
||||||
self.append_accounts(&storable_accounts, 0)
|
self.append_accounts(&storable_accounts, 0)
|
||||||
.map(|res| res[0].offset)
|
.map(|res| res[0].offset)
|
||||||
|
@ -879,85 +875,55 @@ pub mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[should_panic(expected = "accounts.has_hash()")]
|
#[should_panic(expected = "accounts.has_hash()")]
|
||||||
fn test_storable_accounts_with_hashes_and_write_versions_new() {
|
fn test_storable_accounts_with_hashes_new() {
|
||||||
let account = AccountSharedData::default();
|
let account = AccountSharedData::default();
|
||||||
// for (Slot, &'a [(&'a Pubkey, &'a T)])
|
// for (Slot, &'a [(&'a Pubkey, &'a T)])
|
||||||
let slot = 0 as Slot;
|
let slot = 0 as Slot;
|
||||||
let pubkey = Pubkey::default();
|
let pubkey = Pubkey::default();
|
||||||
StorableAccountsWithHashesAndWriteVersions::<'_, '_, _, _, &AccountHash>::new(&(
|
StorableAccountsWithHashes::<'_, '_, _, _, &AccountHash>::new(&(
|
||||||
slot,
|
slot,
|
||||||
&[(&pubkey, &account)][..],
|
&[(&pubkey, &account)][..],
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
fn test_mismatch(correct_hashes: bool, correct_write_versions: bool) {
|
fn test_mismatch(correct_hashes: bool) {
|
||||||
let account = AccountSharedData::default();
|
let account = AccountSharedData::default();
|
||||||
// for (Slot, &'a [(&'a Pubkey, &'a T)])
|
// for (Slot, &'a [(&'a Pubkey, &'a T)])
|
||||||
let slot = 0 as Slot;
|
let slot = 0 as Slot;
|
||||||
let pubkey = Pubkey::default();
|
let pubkey = Pubkey::default();
|
||||||
// mismatch between lens of accounts, hashes, write_versions
|
// mismatch between lens of accounts and hashes
|
||||||
let mut hashes = Vec::default();
|
let mut hashes = Vec::default();
|
||||||
if correct_hashes {
|
if correct_hashes {
|
||||||
hashes.push(AccountHash(Hash::default()));
|
hashes.push(AccountHash(Hash::default()));
|
||||||
}
|
}
|
||||||
let mut write_versions = Vec::default();
|
StorableAccountsWithHashes::new_with_hashes(&(slot, &[(&pubkey, &account)][..]), hashes);
|
||||||
if correct_write_versions {
|
|
||||||
write_versions.push(0);
|
|
||||||
}
|
|
||||||
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
|
|
||||||
&(slot, &[(&pubkey, &account)][..]),
|
|
||||||
hashes,
|
|
||||||
write_versions,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
// rust 1.73+ (our as-of-writing nightly version) changed panic message. we're stuck with this
|
// rust 1.73+ (our as-of-writing nightly version) changed panic message. we're stuck with this
|
||||||
// short common substring until the monorepo is fully 1.73+ including stable.
|
// short common substring until the monorepo is fully 1.73+ including stable.
|
||||||
#[should_panic(expected = "left == right")]
|
#[should_panic(expected = "left == right")]
|
||||||
fn test_storable_accounts_with_hashes_and_write_versions_new2() {
|
fn test_storable_accounts_with_hashes_new2() {
|
||||||
test_mismatch(false, false);
|
test_mismatch(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
// rust 1.73+ (our as-of-writing nightly version) changed panic message. we're stuck with this
|
fn test_storable_accounts_with_hashes_empty() {
|
||||||
// short common substring until the monorepo is fully 1.73+ including stable.
|
|
||||||
#[should_panic(expected = "left == right")]
|
|
||||||
fn test_storable_accounts_with_hashes_and_write_versions_new3() {
|
|
||||||
test_mismatch(false, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
// rust 1.73+ (our as-of-writing nightly version) changed panic message. we're stuck with this
|
|
||||||
// short common substring until the monorepo is fully 1.73+ including stable.
|
|
||||||
#[should_panic(expected = "left == right")]
|
|
||||||
fn test_storable_accounts_with_hashes_and_write_versions_new4() {
|
|
||||||
test_mismatch(true, false);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_storable_accounts_with_hashes_and_write_versions_empty() {
|
|
||||||
// for (Slot, &'a [(&'a Pubkey, &'a T)])
|
// for (Slot, &'a [(&'a Pubkey, &'a T)])
|
||||||
let account = AccountSharedData::default();
|
let account = AccountSharedData::default();
|
||||||
let slot = 0 as Slot;
|
let slot = 0 as Slot;
|
||||||
let pubkeys = [Pubkey::default()];
|
let pubkeys = [Pubkey::default()];
|
||||||
let hashes = Vec::<AccountHash>::default();
|
let hashes = Vec::<AccountHash>::default();
|
||||||
let write_versions = Vec::default();
|
|
||||||
let mut accounts = vec![(&pubkeys[0], &account)];
|
let mut accounts = vec![(&pubkeys[0], &account)];
|
||||||
accounts.clear();
|
accounts.clear();
|
||||||
let accounts2 = (slot, &accounts[..]);
|
let accounts2 = (slot, &accounts[..]);
|
||||||
let storable =
|
let storable = StorableAccountsWithHashes::new_with_hashes(&accounts2, hashes);
|
||||||
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
|
|
||||||
&accounts2,
|
|
||||||
hashes,
|
|
||||||
write_versions,
|
|
||||||
);
|
|
||||||
assert_eq!(storable.len(), 0);
|
assert_eq!(storable.len(), 0);
|
||||||
assert!(storable.is_empty());
|
assert!(storable.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_storable_accounts_with_hashes_and_write_versions_hash_and_write_version() {
|
fn test_storable_accounts_with_hashes_hash() {
|
||||||
// for (Slot, &'a [(&'a Pubkey, &'a T)])
|
// for (Slot, &'a [(&'a Pubkey, &'a T)])
|
||||||
let account = AccountSharedData::default();
|
let account = AccountSharedData::default();
|
||||||
let slot = 0 as Slot;
|
let slot = 0 as Slot;
|
||||||
|
@ -966,27 +932,20 @@ pub mod tests {
|
||||||
AccountHash(Hash::new(&[3; 32])),
|
AccountHash(Hash::new(&[3; 32])),
|
||||||
AccountHash(Hash::new(&[4; 32])),
|
AccountHash(Hash::new(&[4; 32])),
|
||||||
];
|
];
|
||||||
let write_versions = vec![42, 43];
|
|
||||||
let accounts = [(&pubkeys[0], &account), (&pubkeys[1], &account)];
|
let accounts = [(&pubkeys[0], &account), (&pubkeys[1], &account)];
|
||||||
let accounts2 = (slot, &accounts[..]);
|
let accounts2 = (slot, &accounts[..]);
|
||||||
let storable =
|
let storable = StorableAccountsWithHashes::new_with_hashes(&accounts2, hashes.clone());
|
||||||
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
|
|
||||||
&accounts2,
|
|
||||||
hashes.clone(),
|
|
||||||
write_versions.clone(),
|
|
||||||
);
|
|
||||||
assert_eq!(storable.len(), pubkeys.len());
|
assert_eq!(storable.len(), pubkeys.len());
|
||||||
assert!(!storable.is_empty());
|
assert!(!storable.is_empty());
|
||||||
(0..2).for_each(|i| {
|
(0..2).for_each(|i| {
|
||||||
let (_, pubkey, hash, write_version) = storable.get(i);
|
let (_, pubkey, hash) = storable.get(i);
|
||||||
assert_eq!(hash, &hashes[i]);
|
assert_eq!(hash, &hashes[i]);
|
||||||
assert_eq!(write_version, write_versions[i]);
|
|
||||||
assert_eq!(pubkey, &pubkeys[i]);
|
assert_eq!(pubkey, &pubkeys[i]);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_storable_accounts_with_hashes_and_write_versions_default() {
|
fn test_storable_accounts_with_hashes_default() {
|
||||||
// 0 lamport account, should return default account (or None in this case)
|
// 0 lamport account, should return default account (or None in this case)
|
||||||
let account = Account {
|
let account = Account {
|
||||||
data: vec![0],
|
data: vec![0],
|
||||||
|
@ -997,15 +956,9 @@ pub mod tests {
|
||||||
let slot = 0 as Slot;
|
let slot = 0 as Slot;
|
||||||
let pubkey = Pubkey::default();
|
let pubkey = Pubkey::default();
|
||||||
let hashes = vec![AccountHash(Hash::default())];
|
let hashes = vec![AccountHash(Hash::default())];
|
||||||
let write_versions = vec![0];
|
|
||||||
let accounts = [(&pubkey, &account)];
|
let accounts = [(&pubkey, &account)];
|
||||||
let accounts2 = (slot, &accounts[..]);
|
let accounts2 = (slot, &accounts[..]);
|
||||||
let storable =
|
let storable = StorableAccountsWithHashes::new_with_hashes(&accounts2, hashes.clone());
|
||||||
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
|
|
||||||
&accounts2,
|
|
||||||
hashes.clone(),
|
|
||||||
write_versions.clone(),
|
|
||||||
);
|
|
||||||
let get_account = storable.account(0);
|
let get_account = storable.account(0);
|
||||||
assert!(get_account.is_none());
|
assert!(get_account.is_none());
|
||||||
|
|
||||||
|
@ -1019,12 +972,7 @@ pub mod tests {
|
||||||
// for (Slot, &'a [(&'a Pubkey, &'a T)])
|
// for (Slot, &'a [(&'a Pubkey, &'a T)])
|
||||||
let accounts = [(&pubkey, &account)];
|
let accounts = [(&pubkey, &account)];
|
||||||
let accounts2 = (slot, &accounts[..]);
|
let accounts2 = (slot, &accounts[..]);
|
||||||
let storable =
|
let storable = StorableAccountsWithHashes::new_with_hashes(&accounts2, hashes);
|
||||||
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
|
|
||||||
&accounts2,
|
|
||||||
hashes,
|
|
||||||
write_versions,
|
|
||||||
);
|
|
||||||
let get_account = storable.account(0);
|
let get_account = storable.account(0);
|
||||||
assert!(accounts_equal(&account, get_account.unwrap()));
|
assert!(accounts_equal(&account, get_account.unwrap()));
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,7 +14,7 @@ mod test_utils;
|
||||||
|
|
||||||
use {
|
use {
|
||||||
crate::{
|
crate::{
|
||||||
account_storage::meta::{StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo},
|
account_storage::meta::{StorableAccountsWithHashes, StoredAccountInfo},
|
||||||
accounts_hash::AccountHash,
|
accounts_hash::AccountHash,
|
||||||
storable_accounts::StorableAccounts,
|
storable_accounts::StorableAccounts,
|
||||||
},
|
},
|
||||||
|
@ -119,7 +119,7 @@ impl TieredStorage {
|
||||||
V: Borrow<AccountHash>,
|
V: Borrow<AccountHash>,
|
||||||
>(
|
>(
|
||||||
&self,
|
&self,
|
||||||
accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>,
|
accounts: &StorableAccountsWithHashes<'a, 'b, T, U, V>,
|
||||||
skip: usize,
|
skip: usize,
|
||||||
format: &TieredStorageFormat,
|
format: &TieredStorageFormat,
|
||||||
) -> TieredStorageResult<Vec<StoredAccountInfo>> {
|
) -> TieredStorageResult<Vec<StoredAccountInfo>> {
|
||||||
|
@ -180,7 +180,6 @@ impl TieredStorage {
|
||||||
mod tests {
|
mod tests {
|
||||||
use {
|
use {
|
||||||
super::*,
|
super::*,
|
||||||
crate::account_storage::meta::StoredMetaWriteVersion,
|
|
||||||
file::TieredStorageMagicNumber,
|
file::TieredStorageMagicNumber,
|
||||||
footer::TieredStorageFooter,
|
footer::TieredStorageFooter,
|
||||||
hot::HOT_FORMAT,
|
hot::HOT_FORMAT,
|
||||||
|
@ -213,11 +212,7 @@ mod tests {
|
||||||
let account_refs = Vec::<(&Pubkey, &AccountSharedData)>::new();
|
let account_refs = Vec::<(&Pubkey, &AccountSharedData)>::new();
|
||||||
let account_data = (slot_ignored, account_refs.as_slice());
|
let account_data = (slot_ignored, account_refs.as_slice());
|
||||||
let storable_accounts =
|
let storable_accounts =
|
||||||
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
|
StorableAccountsWithHashes::new_with_hashes(&account_data, Vec::<AccountHash>::new());
|
||||||
&account_data,
|
|
||||||
Vec::<AccountHash>::new(),
|
|
||||||
Vec::<StoredMetaWriteVersion>::new(),
|
|
||||||
);
|
|
||||||
|
|
||||||
let result = tiered_storage.write_accounts(&storable_accounts, 0, &HOT_FORMAT);
|
let result = tiered_storage.write_accounts(&storable_accounts, 0, &HOT_FORMAT);
|
||||||
|
|
||||||
|
@ -350,17 +345,8 @@ mod tests {
|
||||||
let hashes: Vec<_> = std::iter::repeat_with(|| AccountHash(Hash::new_unique()))
|
let hashes: Vec<_> = std::iter::repeat_with(|| AccountHash(Hash::new_unique()))
|
||||||
.take(account_data_sizes.len())
|
.take(account_data_sizes.len())
|
||||||
.collect();
|
.collect();
|
||||||
let write_versions: Vec<_> = accounts
|
|
||||||
.iter()
|
|
||||||
.map(|account| account.0.write_version_obsolete)
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
let storable_accounts =
|
let storable_accounts = StorableAccountsWithHashes::new_with_hashes(&account_data, hashes);
|
||||||
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
|
|
||||||
&account_data,
|
|
||||||
hashes,
|
|
||||||
write_versions,
|
|
||||||
);
|
|
||||||
|
|
||||||
let temp_dir = tempdir().unwrap();
|
let temp_dir = tempdir().unwrap();
|
||||||
let tiered_storage_path = temp_dir.path().join(path_suffix);
|
let tiered_storage_path = temp_dir.path().join(path_suffix);
|
||||||
|
@ -373,7 +359,7 @@ mod tests {
|
||||||
|
|
||||||
let mut expected_accounts_map = HashMap::new();
|
let mut expected_accounts_map = HashMap::new();
|
||||||
for i in 0..num_accounts {
|
for i in 0..num_accounts {
|
||||||
let (account, address, _account_hash, _write_version) = storable_accounts.get(i);
|
let (account, address, _account_hash) = storable_accounts.get(i);
|
||||||
expected_accounts_map.insert(address, account);
|
expected_accounts_map.insert(address, account);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -15,8 +15,8 @@ use {
|
||||||
},
|
},
|
||||||
mmap_utils::{get_pod, get_slice},
|
mmap_utils::{get_pod, get_slice},
|
||||||
owners::{OwnerOffset, OwnersBlockFormat, OwnersTable, OWNER_NO_OWNER},
|
owners::{OwnerOffset, OwnersBlockFormat, OwnersTable, OWNER_NO_OWNER},
|
||||||
StorableAccounts, StorableAccountsWithHashesAndWriteVersions, TieredStorageError,
|
StorableAccounts, StorableAccountsWithHashes, TieredStorageError, TieredStorageFormat,
|
||||||
TieredStorageFormat, TieredStorageResult,
|
TieredStorageResult,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
bytemuck::{Pod, Zeroable},
|
bytemuck::{Pod, Zeroable},
|
||||||
|
@ -634,7 +634,7 @@ impl HotStorageWriter {
|
||||||
V: Borrow<AccountHash>,
|
V: Borrow<AccountHash>,
|
||||||
>(
|
>(
|
||||||
&mut self,
|
&mut self,
|
||||||
accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>,
|
accounts: &StorableAccountsWithHashes<'a, 'b, T, U, V>,
|
||||||
skip: usize,
|
skip: usize,
|
||||||
) -> TieredStorageResult<Vec<StoredAccountInfo>> {
|
) -> TieredStorageResult<Vec<StoredAccountInfo>> {
|
||||||
let mut footer = new_hot_footer();
|
let mut footer = new_hot_footer();
|
||||||
|
@ -648,7 +648,7 @@ impl HotStorageWriter {
|
||||||
let total_input_accounts = len - skip;
|
let total_input_accounts = len - skip;
|
||||||
let mut stored_infos = Vec::with_capacity(total_input_accounts);
|
let mut stored_infos = Vec::with_capacity(total_input_accounts);
|
||||||
for i in skip..len {
|
for i in skip..len {
|
||||||
let (account, address, _account_hash, _write_version) = accounts.get(i);
|
let (account, address, _account_hash) = accounts.get(i);
|
||||||
let index_entry = AccountIndexWriterEntry {
|
let index_entry = AccountIndexWriterEntry {
|
||||||
address,
|
address,
|
||||||
offset: HotAccountOffset::new(cursor)?,
|
offset: HotAccountOffset::new(cursor)?,
|
||||||
|
@ -1372,17 +1372,8 @@ pub mod tests {
|
||||||
.take(account_data_sizes.len())
|
.take(account_data_sizes.len())
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let write_versions: Vec<_> = accounts
|
|
||||||
.iter()
|
|
||||||
.map(|account| account.0.write_version_obsolete)
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
let storable_accounts =
|
let storable_accounts =
|
||||||
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
|
StorableAccountsWithHashes::new_with_hashes(&account_data, hashes.clone());
|
||||||
&account_data,
|
|
||||||
hashes.clone(),
|
|
||||||
write_versions.clone(),
|
|
||||||
);
|
|
||||||
|
|
||||||
let temp_dir = TempDir::new().unwrap();
|
let temp_dir = TempDir::new().unwrap();
|
||||||
let path = temp_dir.path().join("test_write_account_and_index_blocks");
|
let path = temp_dir.path().join("test_write_account_and_index_blocks");
|
||||||
|
@ -1401,7 +1392,7 @@ pub mod tests {
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let (account, address, _account_hash, _write_version) = storable_accounts.get(i);
|
let (account, address, _account_hash) = storable_accounts.get(i);
|
||||||
verify_test_account(&stored_meta, account, address);
|
verify_test_account(&stored_meta, account, address);
|
||||||
|
|
||||||
assert_eq!(i + 1, next.0 as usize);
|
assert_eq!(i + 1, next.0 as usize);
|
||||||
|
@ -1419,8 +1410,7 @@ pub mod tests {
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let (account, address, _account_hash, _write_version) =
|
let (account, address, _account_hash) = storable_accounts.get(stored_info.offset);
|
||||||
storable_accounts.get(stored_info.offset);
|
|
||||||
verify_test_account(&stored_meta, account, address);
|
verify_test_account(&stored_meta, account, address);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1429,7 +1419,7 @@ pub mod tests {
|
||||||
|
|
||||||
// first, we verify everything
|
// first, we verify everything
|
||||||
for (i, stored_meta) in accounts.iter().enumerate() {
|
for (i, stored_meta) in accounts.iter().enumerate() {
|
||||||
let (account, address, _account_hash, _write_version) = storable_accounts.get(i);
|
let (account, address, _account_hash) = storable_accounts.get(i);
|
||||||
verify_test_account(stored_meta, account, address);
|
verify_test_account(stored_meta, account, address);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue