From 625e898c6304f7d81d98d4bd0ee46b24beed2019 Mon Sep 17 00:00:00 2001 From: Yueh-Hsuan Chiang <93241502+yhchiang-sol@users.noreply.github.com> Date: Fri, 17 Nov 2023 11:35:32 -0800 Subject: [PATCH] [TieredStorage] Make IndexBlock persist u32 offsets (#34133) #### Problem In TieredStorage, we want to make AccountOffset and IndexOffset u32 instead of usize just like OwnerOffset. However, we need to first change what we persist in the storage. Currently, IndexBlock persists offsets as u64 instead of u32. #### Summary of Changes This PR makes IndexBlock persist u32 offsets. #### Test Plan Existing test cases. --- accounts-db/src/tiered_storage/footer.rs | 2 +- accounts-db/src/tiered_storage/hot.rs | 8 +++--- accounts-db/src/tiered_storage/index.rs | 33 +++++++++++++----------- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/accounts-db/src/tiered_storage/footer.rs b/accounts-db/src/tiered_storage/footer.rs index 0cbf39ba8..12fbf667d 100644 --- a/accounts-db/src/tiered_storage/footer.rs +++ b/accounts-db/src/tiered_storage/footer.rs @@ -243,7 +243,7 @@ mod tests { let expected_footer = TieredStorageFooter { account_meta_format: AccountMetaFormat::Hot, owners_block_format: OwnersBlockFormat::LocalIndex, - index_block_format: IndexBlockFormat::AddressAndOffset, + index_block_format: IndexBlockFormat::AddressAndBlockOffsetOnly, account_block_format: AccountBlockFormat::AlignedRaw, account_entry_count: 300, account_meta_entry_size: 24, diff --git a/accounts-db/src/tiered_storage/hot.rs b/accounts-db/src/tiered_storage/hot.rs index 023f1d683..6cab467fd 100644 --- a/accounts-db/src/tiered_storage/hot.rs +++ b/accounts-db/src/tiered_storage/hot.rs @@ -26,7 +26,7 @@ pub const HOT_FORMAT: TieredStorageFormat = TieredStorageFormat { meta_entry_size: std::mem::size_of::(), account_meta_format: AccountMetaFormat::Hot, owners_block_format: OwnersBlockFormat::LocalIndex, - index_block_format: IndexBlockFormat::AddressAndOffset, + index_block_format: IndexBlockFormat::AddressAndBlockOffsetOnly, account_block_format: AccountBlockFormat::AlignedRaw, }; @@ -407,7 +407,7 @@ pub mod tests { let expected_footer = TieredStorageFooter { account_meta_format: AccountMetaFormat::Hot, owners_block_format: OwnersBlockFormat::LocalIndex, - index_block_format: IndexBlockFormat::AddressAndOffset, + index_block_format: IndexBlockFormat::AddressAndBlockOffsetOnly, account_block_format: AccountBlockFormat::AlignedRaw, account_entry_count: 300, account_meta_entry_size: 16, @@ -503,7 +503,7 @@ pub mod tests { .iter() .map(|address| AccountIndexWriterEntry { address, - block_offset: rng.gen_range(0..u64::MAX), + block_offset: rng.gen_range(0..u32::MAX), intra_block_offset: rng.gen_range(0..4096), }) .collect(); @@ -532,7 +532,7 @@ pub mod tests { let hot_storage = HotStorageReader::new_from_path(&path).unwrap(); for (i, index_writer_entry) in index_writer_entries.iter().enumerate() { let account_offset = hot_storage.get_account_offset(IndexOffset(i)).unwrap(); - assert_eq!(account_offset.block as u64, index_writer_entry.block_offset); + assert_eq!(account_offset.block as u32, index_writer_entry.block_offset); let account_address = hot_storage.get_account_address(IndexOffset(i)).unwrap(); assert_eq!(account_address, index_writer_entry.address); diff --git a/accounts-db/src/tiered_storage/index.rs b/accounts-db/src/tiered_storage/index.rs index acd406d1c..cdaa828e9 100644 --- a/accounts-db/src/tiered_storage/index.rs +++ b/accounts-db/src/tiered_storage/index.rs @@ -13,8 +13,8 @@ use { #[derive(Debug)] pub struct AccountIndexWriterEntry<'a> { pub address: &'a Pubkey, - pub block_offset: u64, - pub intra_block_offset: u64, + pub block_offset: u32, + pub intra_block_offset: u32, } /// The offset to an account stored inside its accounts block. @@ -47,10 +47,10 @@ pub struct IndexOffset(pub usize); )] pub enum IndexBlockFormat { /// This format optimizes the storage size by storing only account addresses - /// and offsets. It skips storing the size of account data by storing account - /// block entries and index block entries in the same order. + /// and block offsets. It skips storing the size of account data by storing + /// account block entries and index block entries in the same order. #[default] - AddressAndOffset = 0, + AddressAndBlockOffsetOnly = 0, } impl IndexBlockFormat { @@ -62,7 +62,7 @@ impl IndexBlockFormat { index_entries: &[AccountIndexWriterEntry], ) -> TieredStorageResult { match self { - Self::AddressAndOffset => { + Self::AddressAndBlockOffsetOnly => { let mut bytes_written = 0; for index_entry in index_entries { bytes_written += file.write_type(index_entry.address)?; @@ -83,7 +83,7 @@ impl IndexBlockFormat { index_offset: IndexOffset, ) -> TieredStorageResult<&'a Pubkey> { let account_offset = match self { - Self::AddressAndOffset => { + Self::AddressAndBlockOffsetOnly => { footer.index_block_offset as usize + std::mem::size_of::() * index_offset.0 } }; @@ -99,13 +99,14 @@ impl IndexBlockFormat { index_offset: IndexOffset, ) -> TieredStorageResult { match self { - Self::AddressAndOffset => { + Self::AddressAndBlockOffsetOnly => { let account_offset = footer.index_block_offset as usize + std::mem::size_of::() * footer.account_entry_count as usize - + index_offset.0 * std::mem::size_of::(); - let (account_block_offset, _) = get_type(mmap, account_offset)?; + + index_offset.0 * std::mem::size_of::(); + let (block_offset, _) = get_type::(mmap, account_offset)?; + Ok(AccountOffset { - block: *account_block_offset, + block: *block_offset as usize, }) } } @@ -114,7 +115,9 @@ impl IndexBlockFormat { /// Returns the size of one index entry. pub fn entry_size(&self) -> usize { match self { - Self::AddressAndOffset => std::mem::size_of::() + std::mem::size_of::(), + Self::AddressAndBlockOffsetOnly => { + std::mem::size_of::() + std::mem::size_of::() + } } } } @@ -150,11 +153,11 @@ mod tests { { let file = TieredStorageFile::new_writable(&path).unwrap(); - let indexer = IndexBlockFormat::AddressAndOffset; + let indexer = IndexBlockFormat::AddressAndBlockOffsetOnly; indexer.write_index_block(&file, &index_entries).unwrap(); } - let indexer = IndexBlockFormat::AddressAndOffset; + let indexer = IndexBlockFormat::AddressAndBlockOffsetOnly; let file = OpenOptions::new() .read(true) .create(false) @@ -165,7 +168,7 @@ mod tests { let account_offset = indexer .get_account_offset(&mmap, &footer, IndexOffset(i)) .unwrap(); - assert_eq!(index_entry.block_offset, account_offset.block as u64); + assert_eq!(index_entry.block_offset, account_offset.block as u32); let address = indexer .get_account_address(&mmap, &footer, IndexOffset(i)) .unwrap();