[TieredStorage] Make IndexBlock persist u32 offsets (#34133)

#### Problem
In TieredStorage, we want to make AccountOffset and IndexOffset u32 instead
of usize just like OwnerOffset.  However, we need to first change what we persist
in the storage.  Currently, IndexBlock persists offsets as u64 instead of u32. 

#### Summary of Changes
This PR makes IndexBlock persist u32 offsets.

#### Test Plan
Existing test cases.
This commit is contained in:
Yueh-Hsuan Chiang 2023-11-17 11:35:32 -08:00 committed by GitHub
parent e23bfb5167
commit 625e898c63
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 23 additions and 20 deletions

View File

@ -243,7 +243,7 @@ mod tests {
let expected_footer = TieredStorageFooter { let expected_footer = TieredStorageFooter {
account_meta_format: AccountMetaFormat::Hot, account_meta_format: AccountMetaFormat::Hot,
owners_block_format: OwnersBlockFormat::LocalIndex, owners_block_format: OwnersBlockFormat::LocalIndex,
index_block_format: IndexBlockFormat::AddressAndOffset, index_block_format: IndexBlockFormat::AddressAndBlockOffsetOnly,
account_block_format: AccountBlockFormat::AlignedRaw, account_block_format: AccountBlockFormat::AlignedRaw,
account_entry_count: 300, account_entry_count: 300,
account_meta_entry_size: 24, account_meta_entry_size: 24,

View File

@ -26,7 +26,7 @@ pub const HOT_FORMAT: TieredStorageFormat = TieredStorageFormat {
meta_entry_size: std::mem::size_of::<HotAccountMeta>(), meta_entry_size: std::mem::size_of::<HotAccountMeta>(),
account_meta_format: AccountMetaFormat::Hot, account_meta_format: AccountMetaFormat::Hot,
owners_block_format: OwnersBlockFormat::LocalIndex, owners_block_format: OwnersBlockFormat::LocalIndex,
index_block_format: IndexBlockFormat::AddressAndOffset, index_block_format: IndexBlockFormat::AddressAndBlockOffsetOnly,
account_block_format: AccountBlockFormat::AlignedRaw, account_block_format: AccountBlockFormat::AlignedRaw,
}; };
@ -407,7 +407,7 @@ pub mod tests {
let expected_footer = TieredStorageFooter { let expected_footer = TieredStorageFooter {
account_meta_format: AccountMetaFormat::Hot, account_meta_format: AccountMetaFormat::Hot,
owners_block_format: OwnersBlockFormat::LocalIndex, owners_block_format: OwnersBlockFormat::LocalIndex,
index_block_format: IndexBlockFormat::AddressAndOffset, index_block_format: IndexBlockFormat::AddressAndBlockOffsetOnly,
account_block_format: AccountBlockFormat::AlignedRaw, account_block_format: AccountBlockFormat::AlignedRaw,
account_entry_count: 300, account_entry_count: 300,
account_meta_entry_size: 16, account_meta_entry_size: 16,
@ -503,7 +503,7 @@ pub mod tests {
.iter() .iter()
.map(|address| AccountIndexWriterEntry { .map(|address| AccountIndexWriterEntry {
address, address,
block_offset: rng.gen_range(0..u64::MAX), block_offset: rng.gen_range(0..u32::MAX),
intra_block_offset: rng.gen_range(0..4096), intra_block_offset: rng.gen_range(0..4096),
}) })
.collect(); .collect();
@ -532,7 +532,7 @@ pub mod tests {
let hot_storage = HotStorageReader::new_from_path(&path).unwrap(); let hot_storage = HotStorageReader::new_from_path(&path).unwrap();
for (i, index_writer_entry) in index_writer_entries.iter().enumerate() { for (i, index_writer_entry) in index_writer_entries.iter().enumerate() {
let account_offset = hot_storage.get_account_offset(IndexOffset(i)).unwrap(); let account_offset = hot_storage.get_account_offset(IndexOffset(i)).unwrap();
assert_eq!(account_offset.block as u64, index_writer_entry.block_offset); assert_eq!(account_offset.block as u32, index_writer_entry.block_offset);
let account_address = hot_storage.get_account_address(IndexOffset(i)).unwrap(); let account_address = hot_storage.get_account_address(IndexOffset(i)).unwrap();
assert_eq!(account_address, index_writer_entry.address); assert_eq!(account_address, index_writer_entry.address);

View File

@ -13,8 +13,8 @@ use {
#[derive(Debug)] #[derive(Debug)]
pub struct AccountIndexWriterEntry<'a> { pub struct AccountIndexWriterEntry<'a> {
pub address: &'a Pubkey, pub address: &'a Pubkey,
pub block_offset: u64, pub block_offset: u32,
pub intra_block_offset: u64, pub intra_block_offset: u32,
} }
/// The offset to an account stored inside its accounts block. /// The offset to an account stored inside its accounts block.
@ -47,10 +47,10 @@ pub struct IndexOffset(pub usize);
)] )]
pub enum IndexBlockFormat { pub enum IndexBlockFormat {
/// This format optimizes the storage size by storing only account addresses /// This format optimizes the storage size by storing only account addresses
/// and offsets. It skips storing the size of account data by storing account /// and block offsets. It skips storing the size of account data by storing
/// block entries and index block entries in the same order. /// account block entries and index block entries in the same order.
#[default] #[default]
AddressAndOffset = 0, AddressAndBlockOffsetOnly = 0,
} }
impl IndexBlockFormat { impl IndexBlockFormat {
@ -62,7 +62,7 @@ impl IndexBlockFormat {
index_entries: &[AccountIndexWriterEntry], index_entries: &[AccountIndexWriterEntry],
) -> TieredStorageResult<usize> { ) -> TieredStorageResult<usize> {
match self { match self {
Self::AddressAndOffset => { Self::AddressAndBlockOffsetOnly => {
let mut bytes_written = 0; let mut bytes_written = 0;
for index_entry in index_entries { for index_entry in index_entries {
bytes_written += file.write_type(index_entry.address)?; bytes_written += file.write_type(index_entry.address)?;
@ -83,7 +83,7 @@ impl IndexBlockFormat {
index_offset: IndexOffset, index_offset: IndexOffset,
) -> TieredStorageResult<&'a Pubkey> { ) -> TieredStorageResult<&'a Pubkey> {
let account_offset = match self { let account_offset = match self {
Self::AddressAndOffset => { Self::AddressAndBlockOffsetOnly => {
footer.index_block_offset as usize + std::mem::size_of::<Pubkey>() * index_offset.0 footer.index_block_offset as usize + std::mem::size_of::<Pubkey>() * index_offset.0
} }
}; };
@ -99,13 +99,14 @@ impl IndexBlockFormat {
index_offset: IndexOffset, index_offset: IndexOffset,
) -> TieredStorageResult<AccountOffset> { ) -> TieredStorageResult<AccountOffset> {
match self { match self {
Self::AddressAndOffset => { Self::AddressAndBlockOffsetOnly => {
let account_offset = footer.index_block_offset as usize let account_offset = footer.index_block_offset as usize
+ std::mem::size_of::<Pubkey>() * footer.account_entry_count as usize + std::mem::size_of::<Pubkey>() * footer.account_entry_count as usize
+ index_offset.0 * std::mem::size_of::<u64>(); + index_offset.0 * std::mem::size_of::<u32>();
let (account_block_offset, _) = get_type(mmap, account_offset)?; let (block_offset, _) = get_type::<u32>(mmap, account_offset)?;
Ok(AccountOffset { Ok(AccountOffset {
block: *account_block_offset, block: *block_offset as usize,
}) })
} }
} }
@ -114,7 +115,9 @@ impl IndexBlockFormat {
/// Returns the size of one index entry. /// Returns the size of one index entry.
pub fn entry_size(&self) -> usize { pub fn entry_size(&self) -> usize {
match self { match self {
Self::AddressAndOffset => std::mem::size_of::<Pubkey>() + std::mem::size_of::<u64>(), Self::AddressAndBlockOffsetOnly => {
std::mem::size_of::<Pubkey>() + std::mem::size_of::<u32>()
}
} }
} }
} }
@ -150,11 +153,11 @@ mod tests {
{ {
let file = TieredStorageFile::new_writable(&path).unwrap(); let file = TieredStorageFile::new_writable(&path).unwrap();
let indexer = IndexBlockFormat::AddressAndOffset; let indexer = IndexBlockFormat::AddressAndBlockOffsetOnly;
indexer.write_index_block(&file, &index_entries).unwrap(); indexer.write_index_block(&file, &index_entries).unwrap();
} }
let indexer = IndexBlockFormat::AddressAndOffset; let indexer = IndexBlockFormat::AddressAndBlockOffsetOnly;
let file = OpenOptions::new() let file = OpenOptions::new()
.read(true) .read(true)
.create(false) .create(false)
@ -165,7 +168,7 @@ mod tests {
let account_offset = indexer let account_offset = indexer
.get_account_offset(&mmap, &footer, IndexOffset(i)) .get_account_offset(&mmap, &footer, IndexOffset(i))
.unwrap(); .unwrap();
assert_eq!(index_entry.block_offset, account_offset.block as u64); assert_eq!(index_entry.block_offset, account_offset.block as u32);
let address = indexer let address = indexer
.get_account_address(&mmap, &footer, IndexOffset(i)) .get_account_address(&mmap, &footer, IndexOffset(i))
.unwrap(); .unwrap();