[TieredStorage] Make IndexBlock persist u32 offsets (#34133)
#### Problem In TieredStorage, we want to make AccountOffset and IndexOffset u32 instead of usize just like OwnerOffset. However, we need to first change what we persist in the storage. Currently, IndexBlock persists offsets as u64 instead of u32. #### Summary of Changes This PR makes IndexBlock persist u32 offsets. #### Test Plan Existing test cases.
This commit is contained in:
parent
e23bfb5167
commit
625e898c63
|
@ -243,7 +243,7 @@ mod tests {
|
||||||
let expected_footer = TieredStorageFooter {
|
let expected_footer = TieredStorageFooter {
|
||||||
account_meta_format: AccountMetaFormat::Hot,
|
account_meta_format: AccountMetaFormat::Hot,
|
||||||
owners_block_format: OwnersBlockFormat::LocalIndex,
|
owners_block_format: OwnersBlockFormat::LocalIndex,
|
||||||
index_block_format: IndexBlockFormat::AddressAndOffset,
|
index_block_format: IndexBlockFormat::AddressAndBlockOffsetOnly,
|
||||||
account_block_format: AccountBlockFormat::AlignedRaw,
|
account_block_format: AccountBlockFormat::AlignedRaw,
|
||||||
account_entry_count: 300,
|
account_entry_count: 300,
|
||||||
account_meta_entry_size: 24,
|
account_meta_entry_size: 24,
|
||||||
|
|
|
@ -26,7 +26,7 @@ pub const HOT_FORMAT: TieredStorageFormat = TieredStorageFormat {
|
||||||
meta_entry_size: std::mem::size_of::<HotAccountMeta>(),
|
meta_entry_size: std::mem::size_of::<HotAccountMeta>(),
|
||||||
account_meta_format: AccountMetaFormat::Hot,
|
account_meta_format: AccountMetaFormat::Hot,
|
||||||
owners_block_format: OwnersBlockFormat::LocalIndex,
|
owners_block_format: OwnersBlockFormat::LocalIndex,
|
||||||
index_block_format: IndexBlockFormat::AddressAndOffset,
|
index_block_format: IndexBlockFormat::AddressAndBlockOffsetOnly,
|
||||||
account_block_format: AccountBlockFormat::AlignedRaw,
|
account_block_format: AccountBlockFormat::AlignedRaw,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -407,7 +407,7 @@ pub mod tests {
|
||||||
let expected_footer = TieredStorageFooter {
|
let expected_footer = TieredStorageFooter {
|
||||||
account_meta_format: AccountMetaFormat::Hot,
|
account_meta_format: AccountMetaFormat::Hot,
|
||||||
owners_block_format: OwnersBlockFormat::LocalIndex,
|
owners_block_format: OwnersBlockFormat::LocalIndex,
|
||||||
index_block_format: IndexBlockFormat::AddressAndOffset,
|
index_block_format: IndexBlockFormat::AddressAndBlockOffsetOnly,
|
||||||
account_block_format: AccountBlockFormat::AlignedRaw,
|
account_block_format: AccountBlockFormat::AlignedRaw,
|
||||||
account_entry_count: 300,
|
account_entry_count: 300,
|
||||||
account_meta_entry_size: 16,
|
account_meta_entry_size: 16,
|
||||||
|
@ -503,7 +503,7 @@ pub mod tests {
|
||||||
.iter()
|
.iter()
|
||||||
.map(|address| AccountIndexWriterEntry {
|
.map(|address| AccountIndexWriterEntry {
|
||||||
address,
|
address,
|
||||||
block_offset: rng.gen_range(0..u64::MAX),
|
block_offset: rng.gen_range(0..u32::MAX),
|
||||||
intra_block_offset: rng.gen_range(0..4096),
|
intra_block_offset: rng.gen_range(0..4096),
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
@ -532,7 +532,7 @@ pub mod tests {
|
||||||
let hot_storage = HotStorageReader::new_from_path(&path).unwrap();
|
let hot_storage = HotStorageReader::new_from_path(&path).unwrap();
|
||||||
for (i, index_writer_entry) in index_writer_entries.iter().enumerate() {
|
for (i, index_writer_entry) in index_writer_entries.iter().enumerate() {
|
||||||
let account_offset = hot_storage.get_account_offset(IndexOffset(i)).unwrap();
|
let account_offset = hot_storage.get_account_offset(IndexOffset(i)).unwrap();
|
||||||
assert_eq!(account_offset.block as u64, index_writer_entry.block_offset);
|
assert_eq!(account_offset.block as u32, index_writer_entry.block_offset);
|
||||||
|
|
||||||
let account_address = hot_storage.get_account_address(IndexOffset(i)).unwrap();
|
let account_address = hot_storage.get_account_address(IndexOffset(i)).unwrap();
|
||||||
assert_eq!(account_address, index_writer_entry.address);
|
assert_eq!(account_address, index_writer_entry.address);
|
||||||
|
|
|
@ -13,8 +13,8 @@ use {
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct AccountIndexWriterEntry<'a> {
|
pub struct AccountIndexWriterEntry<'a> {
|
||||||
pub address: &'a Pubkey,
|
pub address: &'a Pubkey,
|
||||||
pub block_offset: u64,
|
pub block_offset: u32,
|
||||||
pub intra_block_offset: u64,
|
pub intra_block_offset: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The offset to an account stored inside its accounts block.
|
/// The offset to an account stored inside its accounts block.
|
||||||
|
@ -47,10 +47,10 @@ pub struct IndexOffset(pub usize);
|
||||||
)]
|
)]
|
||||||
pub enum IndexBlockFormat {
|
pub enum IndexBlockFormat {
|
||||||
/// This format optimizes the storage size by storing only account addresses
|
/// This format optimizes the storage size by storing only account addresses
|
||||||
/// and offsets. It skips storing the size of account data by storing account
|
/// and block offsets. It skips storing the size of account data by storing
|
||||||
/// block entries and index block entries in the same order.
|
/// account block entries and index block entries in the same order.
|
||||||
#[default]
|
#[default]
|
||||||
AddressAndOffset = 0,
|
AddressAndBlockOffsetOnly = 0,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl IndexBlockFormat {
|
impl IndexBlockFormat {
|
||||||
|
@ -62,7 +62,7 @@ impl IndexBlockFormat {
|
||||||
index_entries: &[AccountIndexWriterEntry],
|
index_entries: &[AccountIndexWriterEntry],
|
||||||
) -> TieredStorageResult<usize> {
|
) -> TieredStorageResult<usize> {
|
||||||
match self {
|
match self {
|
||||||
Self::AddressAndOffset => {
|
Self::AddressAndBlockOffsetOnly => {
|
||||||
let mut bytes_written = 0;
|
let mut bytes_written = 0;
|
||||||
for index_entry in index_entries {
|
for index_entry in index_entries {
|
||||||
bytes_written += file.write_type(index_entry.address)?;
|
bytes_written += file.write_type(index_entry.address)?;
|
||||||
|
@ -83,7 +83,7 @@ impl IndexBlockFormat {
|
||||||
index_offset: IndexOffset,
|
index_offset: IndexOffset,
|
||||||
) -> TieredStorageResult<&'a Pubkey> {
|
) -> TieredStorageResult<&'a Pubkey> {
|
||||||
let account_offset = match self {
|
let account_offset = match self {
|
||||||
Self::AddressAndOffset => {
|
Self::AddressAndBlockOffsetOnly => {
|
||||||
footer.index_block_offset as usize + std::mem::size_of::<Pubkey>() * index_offset.0
|
footer.index_block_offset as usize + std::mem::size_of::<Pubkey>() * index_offset.0
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -99,13 +99,14 @@ impl IndexBlockFormat {
|
||||||
index_offset: IndexOffset,
|
index_offset: IndexOffset,
|
||||||
) -> TieredStorageResult<AccountOffset> {
|
) -> TieredStorageResult<AccountOffset> {
|
||||||
match self {
|
match self {
|
||||||
Self::AddressAndOffset => {
|
Self::AddressAndBlockOffsetOnly => {
|
||||||
let account_offset = footer.index_block_offset as usize
|
let account_offset = footer.index_block_offset as usize
|
||||||
+ std::mem::size_of::<Pubkey>() * footer.account_entry_count as usize
|
+ std::mem::size_of::<Pubkey>() * footer.account_entry_count as usize
|
||||||
+ index_offset.0 * std::mem::size_of::<u64>();
|
+ index_offset.0 * std::mem::size_of::<u32>();
|
||||||
let (account_block_offset, _) = get_type(mmap, account_offset)?;
|
let (block_offset, _) = get_type::<u32>(mmap, account_offset)?;
|
||||||
|
|
||||||
Ok(AccountOffset {
|
Ok(AccountOffset {
|
||||||
block: *account_block_offset,
|
block: *block_offset as usize,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -114,7 +115,9 @@ impl IndexBlockFormat {
|
||||||
/// Returns the size of one index entry.
|
/// Returns the size of one index entry.
|
||||||
pub fn entry_size(&self) -> usize {
|
pub fn entry_size(&self) -> usize {
|
||||||
match self {
|
match self {
|
||||||
Self::AddressAndOffset => std::mem::size_of::<Pubkey>() + std::mem::size_of::<u64>(),
|
Self::AddressAndBlockOffsetOnly => {
|
||||||
|
std::mem::size_of::<Pubkey>() + std::mem::size_of::<u32>()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -150,11 +153,11 @@ mod tests {
|
||||||
|
|
||||||
{
|
{
|
||||||
let file = TieredStorageFile::new_writable(&path).unwrap();
|
let file = TieredStorageFile::new_writable(&path).unwrap();
|
||||||
let indexer = IndexBlockFormat::AddressAndOffset;
|
let indexer = IndexBlockFormat::AddressAndBlockOffsetOnly;
|
||||||
indexer.write_index_block(&file, &index_entries).unwrap();
|
indexer.write_index_block(&file, &index_entries).unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
let indexer = IndexBlockFormat::AddressAndOffset;
|
let indexer = IndexBlockFormat::AddressAndBlockOffsetOnly;
|
||||||
let file = OpenOptions::new()
|
let file = OpenOptions::new()
|
||||||
.read(true)
|
.read(true)
|
||||||
.create(false)
|
.create(false)
|
||||||
|
@ -165,7 +168,7 @@ mod tests {
|
||||||
let account_offset = indexer
|
let account_offset = indexer
|
||||||
.get_account_offset(&mmap, &footer, IndexOffset(i))
|
.get_account_offset(&mmap, &footer, IndexOffset(i))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(index_entry.block_offset, account_offset.block as u64);
|
assert_eq!(index_entry.block_offset, account_offset.block as u32);
|
||||||
let address = indexer
|
let address = indexer
|
||||||
.get_account_address(&mmap, &footer, IndexOffset(i))
|
.get_account_address(&mmap, &footer, IndexOffset(i))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
Loading…
Reference in New Issue