[TieredStorage] Store account address range (#172)
#### Problem The TieredStorageFooter has the min_account_address and max_account_address fields to describe the account address range in its file. But the current implementation hasn't updated the fields yet. #### Summary of Changes This PR enables the TieredStorage to persist address range information into its footer via min_account_address and max_account_address. #### Test Plan Updated tiered-storage test to verify persisted account address range.
This commit is contained in:
parent
e97d3590c7
commit
2273098c55
|
@ -183,7 +183,7 @@ mod tests {
|
|||
mem::ManuallyDrop,
|
||||
},
|
||||
tempfile::tempdir,
|
||||
test_utils::{create_test_account, verify_test_account},
|
||||
test_utils::{create_test_account, verify_test_account_with_footer},
|
||||
};
|
||||
|
||||
impl TieredStorage {
|
||||
|
@ -368,13 +368,33 @@ mod tests {
|
|||
|
||||
let mut index_offset = IndexOffset(0);
|
||||
let mut verified_accounts = HashSet::new();
|
||||
let footer = reader.footer();
|
||||
|
||||
const MIN_PUBKEY: Pubkey = Pubkey::new_from_array([0x00u8; 32]);
|
||||
const MAX_PUBKEY: Pubkey = Pubkey::new_from_array([0xFFu8; 32]);
|
||||
let mut min_pubkey_ref = &MAX_PUBKEY;
|
||||
let mut max_pubkey_ref = &MIN_PUBKEY;
|
||||
|
||||
while let Some((stored_meta, next)) = reader.get_account(index_offset).unwrap() {
|
||||
if let Some(account) = expected_accounts_map.get(stored_meta.pubkey()) {
|
||||
verify_test_account(&stored_meta, *account, stored_meta.pubkey());
|
||||
verify_test_account_with_footer(
|
||||
&stored_meta,
|
||||
*account,
|
||||
stored_meta.pubkey(),
|
||||
footer,
|
||||
);
|
||||
verified_accounts.insert(stored_meta.pubkey());
|
||||
if *min_pubkey_ref > *stored_meta.pubkey() {
|
||||
min_pubkey_ref = stored_meta.pubkey();
|
||||
}
|
||||
if *max_pubkey_ref < *stored_meta.pubkey() {
|
||||
max_pubkey_ref = stored_meta.pubkey();
|
||||
}
|
||||
}
|
||||
index_offset = next;
|
||||
}
|
||||
assert_eq!(footer.min_account_address, *min_pubkey_ref);
|
||||
assert_eq!(footer.max_account_address, *max_pubkey_ref);
|
||||
assert!(!verified_accounts.is_empty());
|
||||
assert_eq!(verified_accounts.len(), expected_accounts_map.len())
|
||||
}
|
||||
|
|
|
@ -10,7 +10,9 @@ use {
|
|||
file::{TieredReadableFile, TieredWritableFile},
|
||||
footer::{AccountBlockFormat, AccountMetaFormat, TieredStorageFooter},
|
||||
index::{AccountIndexWriterEntry, AccountOffset, IndexBlockFormat, IndexOffset},
|
||||
meta::{AccountMetaFlags, AccountMetaOptionalFields, TieredAccountMeta},
|
||||
meta::{
|
||||
AccountAddressRange, AccountMetaFlags, AccountMetaOptionalFields, TieredAccountMeta,
|
||||
},
|
||||
mmap_utils::{get_pod, get_slice},
|
||||
owners::{OwnerOffset, OwnersBlockFormat, OwnersTable, OWNER_NO_OWNER},
|
||||
StorableAccounts, StorableAccountsWithHashesAndWriteVersions, TieredStorageError,
|
||||
|
@ -620,6 +622,7 @@ impl HotStorageWriter {
|
|||
let mut index = vec![];
|
||||
let mut owners_table = OwnersTable::default();
|
||||
let mut cursor = 0;
|
||||
let mut address_range = AccountAddressRange::default();
|
||||
|
||||
// writing accounts blocks
|
||||
let len = accounts.accounts.len();
|
||||
|
@ -631,6 +634,7 @@ impl HotStorageWriter {
|
|||
address,
|
||||
offset: HotAccountOffset::new(cursor)?,
|
||||
};
|
||||
address_range.update(address);
|
||||
|
||||
// Obtain necessary fields from the account, or default fields
|
||||
// for a zero-lamport account in the None case.
|
||||
|
@ -691,7 +695,8 @@ impl HotStorageWriter {
|
|||
footer
|
||||
.owners_block_format
|
||||
.write_owners_block(&mut self.storage, &owners_table)?;
|
||||
|
||||
footer.min_account_address = *address_range.min;
|
||||
footer.max_account_address = *address_range.max;
|
||||
footer.write_footer_block(&mut self.storage)?;
|
||||
|
||||
Ok(stored_infos)
|
||||
|
|
|
@ -4,7 +4,7 @@ use {
|
|||
crate::tiered_storage::owners::OwnerOffset,
|
||||
bytemuck::{Pod, Zeroable},
|
||||
modular_bitfield::prelude::*,
|
||||
solana_sdk::stake_history::Epoch,
|
||||
solana_sdk::{pubkey::Pubkey, stake_history::Epoch},
|
||||
};
|
||||
|
||||
/// The struct that handles the account meta flags.
|
||||
|
@ -124,6 +124,38 @@ impl AccountMetaOptionalFields {
|
|||
}
|
||||
}
|
||||
|
||||
const MIN_ACCOUNT_ADDRESS: Pubkey = Pubkey::new_from_array([0x00u8; 32]);
|
||||
const MAX_ACCOUNT_ADDRESS: Pubkey = Pubkey::new_from_array([0xFFu8; 32]);
|
||||
|
||||
#[derive(Debug)]
|
||||
/// A struct that maintains an address-range using its min and max fields.
|
||||
pub struct AccountAddressRange<'a> {
|
||||
/// The minimum address observed via update()
|
||||
pub min: &'a Pubkey,
|
||||
/// The maximum address observed via update()
|
||||
pub max: &'a Pubkey,
|
||||
}
|
||||
|
||||
impl Default for AccountAddressRange<'_> {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
min: &MAX_ACCOUNT_ADDRESS,
|
||||
max: &MIN_ACCOUNT_ADDRESS,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> AccountAddressRange<'a> {
|
||||
pub fn update(&mut self, address: &'a Pubkey) {
|
||||
if *self.min > *address {
|
||||
self.min = address;
|
||||
}
|
||||
if *self.max < *address {
|
||||
self.max = address;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub mod tests {
|
||||
use super::*;
|
||||
|
@ -221,4 +253,47 @@ pub mod tests {
|
|||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pubkey_range_update_single() {
|
||||
let address = solana_sdk::pubkey::new_rand();
|
||||
let mut address_range = AccountAddressRange::default();
|
||||
|
||||
address_range.update(&address);
|
||||
// For a single update, the min and max should equal to the address
|
||||
assert_eq!(*address_range.min, address);
|
||||
assert_eq!(*address_range.max, address);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pubkey_range_update_multiple() {
|
||||
const NUM_PUBKEYS: usize = 20;
|
||||
|
||||
let mut address_range = AccountAddressRange::default();
|
||||
let mut addresses = Vec::with_capacity(NUM_PUBKEYS);
|
||||
|
||||
let mut min_index = 0;
|
||||
let mut max_index = 0;
|
||||
|
||||
// Generate random addresses and track expected min and max indices
|
||||
for i in 0..NUM_PUBKEYS {
|
||||
let address = solana_sdk::pubkey::new_rand();
|
||||
addresses.push(address);
|
||||
|
||||
// Update expected min and max indices
|
||||
if address < addresses[min_index] {
|
||||
min_index = i;
|
||||
}
|
||||
if address > addresses[max_index] {
|
||||
max_index = i;
|
||||
}
|
||||
}
|
||||
|
||||
addresses
|
||||
.iter()
|
||||
.for_each(|address| address_range.update(address));
|
||||
|
||||
assert_eq!(*address_range.min, addresses[min_index]);
|
||||
assert_eq!(*address_range.max, addresses[max_index]);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
#![cfg(test)]
|
||||
//! Helper functions for TieredStorage tests
|
||||
use {
|
||||
super::footer::TieredStorageFooter,
|
||||
crate::{
|
||||
account_storage::meta::{StoredAccountMeta, StoredMeta},
|
||||
accounts_hash::AccountHash,
|
||||
|
@ -61,3 +62,14 @@ pub(super) fn verify_test_account(
|
|||
assert_eq!(stored_meta.pubkey(), address);
|
||||
assert_eq!(*stored_meta.hash(), AccountHash(Hash::default()));
|
||||
}
|
||||
|
||||
pub(super) fn verify_test_account_with_footer(
|
||||
stored_meta: &StoredAccountMeta<'_>,
|
||||
account: Option<&impl ReadableAccount>,
|
||||
address: &Pubkey,
|
||||
footer: &TieredStorageFooter,
|
||||
) {
|
||||
verify_test_account(stored_meta, account, address);
|
||||
assert!(footer.min_account_address <= *address);
|
||||
assert!(footer.max_account_address >= *address);
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue