blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Removes write version from tiered storage (#33566)

This commit is contained in:
Brooks 2023-10-06 16:19:35 -04:00 committed by GitHub
parent ecb1f8a9d7
commit bb27bd88d4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 69 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
accounts-db/src/account_storage/meta.rs
View File

@ -157,7 +157,9 @@ impl<'storage> StoredAccountMeta<'storage> {
pub fn write_version(&self) -> StoredMetaWriteVersion {
match self {
Self::AppendVec(av) => av.write_version(),
Self::Hot(hot) => hot.write_version().unwrap_or_default(),
// Hot account does not support this API as it does not
// use a write version.
Self::Hot(_) => StoredMetaWriteVersion::default(),
}
}

4
accounts-db/src/tiered_storage.rs
View File

@ -300,8 +300,6 @@ mod tests {
}
/// Create a test account based on the specified seed.
/// The created test account might have default rent_epoch
/// and write_version.
fn create_account(seed: u64) -> (StoredMeta, AccountSharedData) {
let data_byte = seed as u8;
let account = Account {
@ -317,7 +315,7 @@ mod tests {
};
let stored_meta = StoredMeta {
write_version_obsolete: u64::MAX,
write_version_obsolete: StoredMetaWriteVersion::default(),
pubkey: Pubkey::new_unique(),
data_len: seed,
};

28
accounts-db/src/tiered_storage/byte_block.rs
View File

@ -78,9 +78,6 @@ impl ByteBlockWriter {
if let Some(hash) = opt_fields.account_hash {
size += self.write_type(&hash)?;
}
if let Some(write_version) = opt_fields.write_version {
size += self.write_type(&write_version)?;
}
debug_assert_eq!(size, opt_fields.size());
@ -154,7 +151,6 @@ impl ByteBlockReader {
mod tests {
use {
super::*,
crate::account_storage::meta::StoredMetaWriteVersion,
solana_sdk::{hash::Hash, stake_history::Epoch},
};
@ -307,7 +303,6 @@ mod tests {
fn write_optional_fields(format: AccountBlockFormat) {
let mut test_epoch = 5432312;
let mut test_write_version = 231;
let mut writer = ByteBlockWriter::new(format);
let mut opt_fields_vec = vec![];
@ -317,18 +312,12 @@ mod tests {
// of Some and None.
for rent_epoch in [None, Some(test_epoch)] {
for account_hash in [None, Some(Hash::new_unique())] {
for write_version in [None, Some(test_write_version)] {
some_count += rent_epoch.map_or(0, |_| 1)
+ account_hash.map_or(0, |_| 1)
+ write_version.map_or(0, |_| 1);
some_count += rent_epoch.iter().count() + account_hash.iter().count();
opt_fields_vec.push(AccountMetaOptionalFields {
rent_epoch,
account_hash,
write_version,
});
test_write_version += 1;
}
opt_fields_vec.push(AccountMetaOptionalFields {
rent_epoch,
account_hash,
});
}
test_epoch += 1;
}
@ -367,13 +356,6 @@ mod tests {
verified_count += 1;
offset += std::mem::size_of::<Hash>();
}
if let Some(expected_write_version) = opt_fields.write_version {
let write_version =
read_type::<StoredMetaWriteVersion>(&decoded_buffer, offset).unwrap();
assert_eq!(*write_version, expected_write_version);
verified_count += 1;
offset += std::mem::size_of::<StoredMetaWriteVersion>();
}
}
// make sure the number of Some fields matches the number of fields we

49
accounts-db/src/tiered_storage/hot.rs
View File

@ -2,17 +2,12 @@
//! The account meta and related structs for hot accounts.
use {
crate::{
account_storage::meta::StoredMetaWriteVersion,
tiered_storage::{
byte_block,
footer::{
AccountBlockFormat, AccountMetaFormat, OwnersBlockFormat, TieredStorageFooter,
},
index::AccountIndexFormat,
meta::{AccountMetaFlags, AccountMetaOptionalFields, TieredAccountMeta},
TieredStorageFormat, TieredStorageResult,
},
crate::tiered_storage::{
byte_block,
footer::{AccountBlockFormat, AccountMetaFormat, OwnersBlockFormat, TieredStorageFooter},
index::AccountIndexFormat,
meta::{AccountMetaFlags, AccountMetaOptionalFields, TieredAccountMeta},
TieredStorageFormat, TieredStorageResult,
},
memmap2::{Mmap, MmapOptions},
modular_bitfield::prelude::*,
@ -167,19 +162,6 @@ impl TieredAccountMeta for HotAccountMeta {
.flatten()
}
/// Returns the write version by parsing the specified account block. None
/// will be returned if this account does not persist this optional field.
fn write_version(&self, account_block: &[u8]) -> Option<StoredMetaWriteVersion> {
self.flags
.has_write_version()
.then(|| {
let offset = self.optional_fields_offset(account_block)
+ AccountMetaOptionalFields::write_version_offset(self.flags());
byte_block::read_type::<StoredMetaWriteVersion>(account_block, offset).copied()
})
.flatten()
}
/// Returns the offset of the optional fields based on the specified account
/// block.
fn optional_fields_offset(&self, account_block: &[u8]) -> usize {
@ -239,13 +221,10 @@ impl HotStorageReader {
pub mod tests {
use {
super::*,
crate::{
account_storage::meta::StoredMetaWriteVersion,
tiered_storage::{
byte_block::ByteBlockWriter,
footer::AccountBlockFormat,
meta::{AccountMetaFlags, AccountMetaOptionalFields, TieredAccountMeta},
},
crate::tiered_storage::{
byte_block::ByteBlockWriter,
footer::AccountBlockFormat,
meta::{AccountMetaFlags, AccountMetaOptionalFields, TieredAccountMeta},
},
::solana_sdk::{hash::Hash, stake_history::Epoch},
memoffset::offset_of,
@ -311,7 +290,6 @@ pub mod tests {
let optional_fields = AccountMetaOptionalFields {
rent_epoch: Some(TEST_RENT_EPOCH),
account_hash: Some(Hash::new_unique()),
write_version: None,
};
let flags = AccountMetaFlags::new_from(&optional_fields);
@ -335,12 +313,10 @@ pub mod tests {
const TEST_LAMPORT: u64 = 2314232137;
const OWNER_INDEX: u32 = 0x1fef_1234;
const TEST_RENT_EPOCH: Epoch = 7;
const TEST_WRITE_VERSION: StoredMetaWriteVersion = 0;
let optional_fields = AccountMetaOptionalFields {
rent_epoch: Some(TEST_RENT_EPOCH),
account_hash: Some(Hash::new_unique()),
write_version: Some(TEST_WRITE_VERSION),
};
let flags = AccountMetaFlags::new_from(&optional_fields);
@ -361,7 +337,6 @@ pub mod tests {
assert_eq!(expected_meta, *meta);
assert!(meta.flags().has_rent_epoch());
assert!(meta.flags().has_account_hash());
assert!(meta.flags().has_write_version());
assert_eq!(meta.account_data_padding() as usize, padding.len());
let account_block = &buffer[std::mem::size_of::<HotAccountMeta>()..];
@ -378,9 +353,5 @@ pub mod tests {
*(meta.account_hash(account_block).unwrap()),
optional_fields.account_hash.unwrap()
);
assert_eq!(
meta.write_version(account_block),
optional_fields.write_version
);
}
}

155
accounts-db/src/tiered_storage/meta.rs
View File

@ -1,7 +1,6 @@
#![allow(dead_code)]
//! The account meta and related structs for the tiered storage.
use {
crate::account_storage::meta::StoredMetaWriteVersion,
::solana_sdk::{hash::Hash, stake_history::Epoch},
modular_bitfield::prelude::*,
};
@ -15,10 +14,8 @@ pub struct AccountMetaFlags {
pub has_rent_epoch: bool,
/// whether the account meta has account hash
pub has_account_hash: bool,
/// whether the account meta has write version
pub has_write_version: bool,
/// the reserved bits.
reserved: B29,
reserved: B30,
}
/// A trait that allows different implementations of the account meta that
@ -70,10 +67,6 @@ pub trait TieredAccountMeta: Sized {
/// will be returned if this account does not persist this optional field.
fn account_hash<'a>(&self, _account_block: &'a [u8]) -> Option<&'a Hash>;
/// Returns the write version by parsing the specified account block. None
/// will be returned if this account does not persist this optional field.
fn write_version(&self, _account_block: &[u8]) -> Option<StoredMetaWriteVersion>;
/// Returns the offset of the optional fields based on the specified account
/// block.
fn optional_fields_offset(&self, _account_block: &[u8]) -> usize;
@ -92,7 +85,6 @@ impl AccountMetaFlags {
let mut flags = AccountMetaFlags::default();
flags.set_has_rent_epoch(optional_fields.rent_epoch.is_some());
flags.set_has_account_hash(optional_fields.account_hash.is_some());
flags.set_has_write_version(optional_fields.write_version.is_some());
flags
}
}
@ -107,9 +99,6 @@ pub struct AccountMetaOptionalFields {
pub rent_epoch: Option<Epoch>,
/// the hash of its associated account
pub account_hash: Option<Hash>,
/// Order of stores of its associated account to an accounts file will
/// determine 'latest' account data per pubkey.
pub write_version: Option<StoredMetaWriteVersion>,
}
impl AccountMetaOptionalFields {
@ -117,9 +106,6 @@ impl AccountMetaOptionalFields {
pub fn size(&self) -> usize {
self.rent_epoch.map_or(0, |_| std::mem::size_of::<Epoch>())
+ self.account_hash.map_or(0, |_| std::mem::size_of::<Hash>())
+ self
.write_version
.map_or(0, |_| std::mem::size_of::<StoredMetaWriteVersion>())
}
/// Given the specified AccountMetaFlags, returns the size of its
@ -132,9 +118,6 @@ impl AccountMetaOptionalFields {
if flags.has_account_hash() {
fields_size += std::mem::size_of::<Hash>();
}
if flags.has_write_version() {
fields_size += std::mem::size_of::<StoredMetaWriteVersion>();
}
fields_size
}
@ -155,17 +138,6 @@ impl AccountMetaOptionalFields {
}
offset
}
/// Given the specified AccountMetaFlags, returns the relative offset
/// of its write_version field to the offset of its optional fields entry.
pub fn write_version_offset(flags: &AccountMetaFlags) -> usize {
let mut offset = Self::account_hash_offset(flags);
// account hash is the previous field to write version
if flags.has_account_hash() {
offset += std::mem::size_of::<Hash>();
}
offset
}
}
#[cfg(test)]
@ -178,7 +150,6 @@ pub mod tests {
assert!(!flags.has_rent_epoch());
assert!(!flags.has_account_hash());
assert!(!flags.has_write_version());
assert_eq!(flags.reserved(), 0u32);
assert_eq!(
@ -199,21 +170,12 @@ pub mod tests {
assert!(flags.has_rent_epoch());
assert!(!flags.has_account_hash());
assert!(!flags.has_write_version());
verify_flags_serialization(&flags);
flags.set_has_account_hash(true);
assert!(flags.has_rent_epoch());
assert!(flags.has_account_hash());
assert!(!flags.has_write_version());
verify_flags_serialization(&flags);
flags.set_has_write_version(true);
assert!(flags.has_rent_epoch());
assert!(flags.has_account_hash());
assert!(flags.has_write_version());
verify_flags_serialization(&flags);
// make sure the reserved bits are untouched.
@ -224,27 +186,19 @@ pub mod tests {
let flags: AccountMetaFlags = AccountMetaFlags::new_from(opt_fields);
assert_eq!(flags.has_rent_epoch(), opt_fields.rent_epoch.is_some());
assert_eq!(flags.has_account_hash(), opt_fields.account_hash.is_some());
assert_eq!(
flags.has_write_version(),
opt_fields.write_version.is_some()
);
assert_eq!(flags.reserved(), 0u32);
}
#[test]
fn test_optional_fields_update_flags() {
let test_epoch = 5432312;
let test_write_version = 231;
for rent_epoch in [None, Some(test_epoch)] {
for account_hash in [None, Some(Hash::new_unique())] {
for write_version in [None, Some(test_write_version)] {
update_and_verify_flags(&AccountMetaOptionalFields {
rent_epoch,
account_hash,
write_version,
});
}
update_and_verify_flags(&AccountMetaOptionalFields {
rent_epoch,
account_hash,
});
}
}
}
@ -252,30 +206,24 @@ pub mod tests {
#[test]
fn test_optional_fields_size() {
let test_epoch = 5432312;
let test_write_version = 231;
for rent_epoch in [None, Some(test_epoch)] {
for account_hash in [None, Some(Hash::new_unique())] {
for write_version in [None, Some(test_write_version)] {
let opt_fields = AccountMetaOptionalFields {
rent_epoch,
account_hash,
write_version,
};
assert_eq!(
opt_fields.size(),
rent_epoch.map_or(0, |_| std::mem::size_of::<Epoch>())
+ account_hash.map_or(0, |_| std::mem::size_of::<Hash>())
+ write_version
.map_or(0, |_| std::mem::size_of::<StoredMetaWriteVersion>())
);
assert_eq!(
opt_fields.size(),
AccountMetaOptionalFields::size_from_flags(&AccountMetaFlags::new_from(
&opt_fields
))
);
}
let opt_fields = AccountMetaOptionalFields {
rent_epoch,
account_hash,
};
assert_eq!(
opt_fields.size(),
rent_epoch.map_or(0, |_| std::mem::size_of::<Epoch>())
+ account_hash.map_or(0, |_| std::mem::size_of::<Hash>())
);
assert_eq!(
opt_fields.size(),
AccountMetaOptionalFields::size_from_flags(&AccountMetaFlags::new_from(
&opt_fields
))
);
}
}
}
@ -283,47 +231,34 @@ pub mod tests {
#[test]
fn test_optional_fields_offset() {
let test_epoch = 5432312;
let test_write_version = 231;
for rent_epoch in [None, Some(test_epoch)] {
let rent_epoch_offset = 0;
for account_hash in [None, Some(Hash::new_unique())] {
let mut account_hash_offset = rent_epoch_offset;
if rent_epoch.is_some() {
account_hash_offset += std::mem::size_of::<Epoch>();
}
for write_version in [None, Some(test_write_version)] {
let mut write_version_offset = account_hash_offset;
if account_hash.is_some() {
write_version_offset += std::mem::size_of::<Hash>();
}
let opt_fields = AccountMetaOptionalFields {
rent_epoch,
account_hash,
write_version,
};
let flags = AccountMetaFlags::new_from(&opt_fields);
assert_eq!(
AccountMetaOptionalFields::rent_epoch_offset(&flags),
rent_epoch_offset
);
assert_eq!(
AccountMetaOptionalFields::account_hash_offset(&flags),
account_hash_offset
);
assert_eq!(
AccountMetaOptionalFields::write_version_offset(&flags),
write_version_offset
);
let mut derived_size = AccountMetaOptionalFields::write_version_offset(&flags);
if flags.has_write_version() {
derived_size += std::mem::size_of::<StoredMetaWriteVersion>();
}
assert_eq!(
AccountMetaOptionalFields::size_from_flags(&flags),
derived_size
);
}
let rent_epoch_offset = 0;
let account_hash_offset =
rent_epoch_offset + rent_epoch.as_ref().map(std::mem::size_of_val).unwrap_or(0);
let derived_size = account_hash_offset
+ account_hash
.as_ref()
.map(std::mem::size_of_val)
.unwrap_or(0);
let opt_fields = AccountMetaOptionalFields {
rent_epoch,
account_hash,
};
let flags = AccountMetaFlags::new_from(&opt_fields);
assert_eq!(
AccountMetaOptionalFields::rent_epoch_offset(&flags),
rent_epoch_offset
);
assert_eq!(
AccountMetaOptionalFields::account_hash_offset(&flags),
account_hash_offset
);
assert_eq!(
AccountMetaOptionalFields::size_from_flags(&flags),
derived_size
);
}
}
}

18
accounts-db/src/tiered_storage/readable.rs
View File

@ -1,12 +1,9 @@
use {
crate::{
account_storage::meta::StoredMetaWriteVersion,
tiered_storage::{
footer::{AccountMetaFormat, TieredStorageFooter},
hot::HotStorageReader,
meta::TieredAccountMeta,
TieredStorageResult,
},
crate::tiered_storage::{
footer::{AccountMetaFormat, TieredStorageFooter},
hot::HotStorageReader,
meta::TieredAccountMeta,
TieredStorageResult,
},
solana_sdk::{account::ReadableAccount, hash::Hash, pubkey::Pubkey, stake_history::Epoch},
std::path::Path,
@ -44,11 +41,6 @@ impl<'accounts_file, M: TieredAccountMeta> TieredReadableAccount<'accounts_file,
self.index
}
/// Returns the write version of the account.
pub fn write_version(&self) -> Option<StoredMetaWriteVersion> {
self.meta.write_version(self.account_block)
}
/// Returns the data associated to this account.
pub fn data(&self) -> &'accounts_file [u8] {
self.meta.account_data(self.account_block)