From 4e5c545e23fb9a74c2fed5b5ea49edb02f64744e Mon Sep 17 00:00:00 2001 From: Brooks Date: Thu, 19 Oct 2023 11:52:36 -0400 Subject: [PATCH] Uses AccountHash in tiered storage (#33763) --- accounts-db/src/account_storage/meta.rs | 9 +++--- accounts-db/src/tiered_storage/byte_block.rs | 7 +++-- accounts-db/src/tiered_storage/hot.rs | 31 ++++++++++++-------- accounts-db/src/tiered_storage/meta.rs | 24 ++++++++------- accounts-db/src/tiered_storage/readable.rs | 17 ++++++----- 5 files changed, 49 insertions(+), 39 deletions(-) diff --git a/accounts-db/src/account_storage/meta.rs b/accounts-db/src/account_storage/meta.rs index 4f6a40a92d..0cdf200f70 100644 --- a/accounts-db/src/account_storage/meta.rs +++ b/accounts-db/src/account_storage/meta.rs @@ -126,11 +126,10 @@ impl<'storage> StoredAccountMeta<'storage> { } pub fn hash(&self) -> &'storage AccountHash { - let hash = match self { - Self::AppendVec(av) => av.hash(), - Self::Hot(hot) => hot.hash().unwrap_or(&DEFAULT_ACCOUNT_HASH.0), - }; - bytemuck::cast_ref(hash) + match self { + Self::AppendVec(av) => bytemuck::cast_ref(av.hash()), + Self::Hot(hot) => hot.hash().unwrap_or(&DEFAULT_ACCOUNT_HASH), + } } pub fn stored_size(&self) -> usize { diff --git a/accounts-db/src/tiered_storage/byte_block.rs b/accounts-db/src/tiered_storage/byte_block.rs index 53af0a7137..e0fa8b4b13 100644 --- a/accounts-db/src/tiered_storage/byte_block.rs +++ b/accounts-db/src/tiered_storage/byte_block.rs @@ -151,6 +151,7 @@ impl ByteBlockReader { mod tests { use { super::*, + crate::accounts_hash::AccountHash, solana_sdk::{hash::Hash, stake_history::Epoch}, }; @@ -311,7 +312,7 @@ mod tests { // prepare a vector of optional fields that contains all combinations // of Some and None. for rent_epoch in [None, Some(test_epoch)] { - for account_hash in [None, Some(Hash::new_unique())] { + for account_hash in [None, Some(AccountHash(Hash::new_unique()))] { some_count += rent_epoch.iter().count() + account_hash.iter().count(); opt_fields_vec.push(AccountMetaOptionalFields { @@ -351,10 +352,10 @@ mod tests { offset += std::mem::size_of::(); } if let Some(expected_hash) = opt_fields.account_hash { - let hash = read_type::(&decoded_buffer, offset).unwrap(); + let hash = read_type::(&decoded_buffer, offset).unwrap(); assert_eq!(hash, &expected_hash); verified_count += 1; - offset += std::mem::size_of::(); + offset += std::mem::size_of::(); } } diff --git a/accounts-db/src/tiered_storage/hot.rs b/accounts-db/src/tiered_storage/hot.rs index 0ae2a597ab..7827170068 100644 --- a/accounts-db/src/tiered_storage/hot.rs +++ b/accounts-db/src/tiered_storage/hot.rs @@ -2,17 +2,22 @@ //! The account meta and related structs for hot accounts. use { - crate::tiered_storage::{ - byte_block, - footer::{AccountBlockFormat, AccountMetaFormat, OwnersBlockFormat, TieredStorageFooter}, - index::AccountIndexFormat, - meta::{AccountMetaFlags, AccountMetaOptionalFields, TieredAccountMeta}, - mmap_utils::get_type, - TieredStorageFormat, TieredStorageResult, + crate::{ + accounts_hash::AccountHash, + tiered_storage::{ + byte_block, + footer::{ + AccountBlockFormat, AccountMetaFormat, OwnersBlockFormat, TieredStorageFooter, + }, + index::AccountIndexFormat, + meta::{AccountMetaFlags, AccountMetaOptionalFields, TieredAccountMeta}, + mmap_utils::get_type, + TieredStorageFormat, TieredStorageResult, + }, }, memmap2::{Mmap, MmapOptions}, modular_bitfield::prelude::*, - solana_sdk::{hash::Hash, stake_history::Epoch}, + solana_sdk::stake_history::Epoch, std::{fs::OpenOptions, option::Option, path::Path}, }; @@ -152,13 +157,13 @@ impl TieredAccountMeta for HotAccountMeta { /// Returns the account hash by parsing the specified account block. None /// will be returned if this account does not persist this optional field. - fn account_hash<'a>(&self, account_block: &'a [u8]) -> Option<&'a Hash> { + fn account_hash<'a>(&self, account_block: &'a [u8]) -> Option<&'a AccountHash> { self.flags() .has_account_hash() .then(|| { let offset = self.optional_fields_offset(account_block) + AccountMetaOptionalFields::account_hash_offset(self.flags()); - byte_block::read_type::(account_block, offset) + byte_block::read_type::(account_block, offset) }) .flatten() } @@ -239,9 +244,9 @@ pub mod tests { index::AccountIndexFormat, meta::{AccountMetaFlags, AccountMetaOptionalFields, TieredAccountMeta}, }, - ::solana_sdk::{hash::Hash, pubkey::Pubkey, stake_history::Epoch}, memoffset::offset_of, rand::Rng, + solana_sdk::{hash::Hash, pubkey::Pubkey, stake_history::Epoch}, tempfile::TempDir, }; @@ -304,7 +309,7 @@ pub mod tests { let optional_fields = AccountMetaOptionalFields { rent_epoch: Some(TEST_RENT_EPOCH), - account_hash: Some(Hash::new_unique()), + account_hash: Some(AccountHash(Hash::new_unique())), }; let flags = AccountMetaFlags::new_from(&optional_fields); @@ -331,7 +336,7 @@ pub mod tests { let optional_fields = AccountMetaOptionalFields { rent_epoch: Some(TEST_RENT_EPOCH), - account_hash: Some(Hash::new_unique()), + account_hash: Some(AccountHash(Hash::new_unique())), }; let flags = AccountMetaFlags::new_from(&optional_fields); diff --git a/accounts-db/src/tiered_storage/meta.rs b/accounts-db/src/tiered_storage/meta.rs index 20147bdaf1..668c6ab93d 100644 --- a/accounts-db/src/tiered_storage/meta.rs +++ b/accounts-db/src/tiered_storage/meta.rs @@ -1,8 +1,8 @@ #![allow(dead_code)] //! The account meta and related structs for the tiered storage. use { - ::solana_sdk::{hash::Hash, stake_history::Epoch}, - modular_bitfield::prelude::*, + crate::accounts_hash::AccountHash, modular_bitfield::prelude::*, + solana_sdk::stake_history::Epoch, }; /// The struct that handles the account meta flags. @@ -65,7 +65,7 @@ pub trait TieredAccountMeta: Sized { /// Returns the account hash by parsing the specified account block. None /// will be returned if this account does not persist this optional field. - fn account_hash<'a>(&self, _account_block: &'a [u8]) -> Option<&'a Hash>; + fn account_hash<'a>(&self, _account_block: &'a [u8]) -> Option<&'a AccountHash>; /// Returns the offset of the optional fields based on the specified account /// block. @@ -98,14 +98,16 @@ pub struct AccountMetaOptionalFields { /// the epoch at which its associated account will next owe rent pub rent_epoch: Option, /// the hash of its associated account - pub account_hash: Option, + pub account_hash: Option, } impl AccountMetaOptionalFields { /// The size of the optional fields in bytes (excluding the boolean flags). pub fn size(&self) -> usize { self.rent_epoch.map_or(0, |_| std::mem::size_of::()) - + self.account_hash.map_or(0, |_| std::mem::size_of::()) + + self + .account_hash + .map_or(0, |_| std::mem::size_of::()) } /// Given the specified AccountMetaFlags, returns the size of its @@ -116,7 +118,7 @@ impl AccountMetaOptionalFields { fields_size += std::mem::size_of::(); } if flags.has_account_hash() { - fields_size += std::mem::size_of::(); + fields_size += std::mem::size_of::(); } fields_size @@ -142,7 +144,7 @@ impl AccountMetaOptionalFields { #[cfg(test)] pub mod tests { - use super::*; + use {super::*, solana_sdk::hash::Hash}; #[test] fn test_account_meta_flags_new() { @@ -194,7 +196,7 @@ pub mod tests { let test_epoch = 5432312; for rent_epoch in [None, Some(test_epoch)] { - for account_hash in [None, Some(Hash::new_unique())] { + for account_hash in [None, Some(AccountHash(Hash::new_unique()))] { update_and_verify_flags(&AccountMetaOptionalFields { rent_epoch, account_hash, @@ -208,7 +210,7 @@ pub mod tests { let test_epoch = 5432312; for rent_epoch in [None, Some(test_epoch)] { - for account_hash in [None, Some(Hash::new_unique())] { + for account_hash in [None, Some(AccountHash(Hash::new_unique()))] { let opt_fields = AccountMetaOptionalFields { rent_epoch, account_hash, @@ -216,7 +218,7 @@ pub mod tests { assert_eq!( opt_fields.size(), rent_epoch.map_or(0, |_| std::mem::size_of::()) - + account_hash.map_or(0, |_| std::mem::size_of::()) + + account_hash.map_or(0, |_| std::mem::size_of::()) ); assert_eq!( opt_fields.size(), @@ -233,7 +235,7 @@ pub mod tests { let test_epoch = 5432312; for rent_epoch in [None, Some(test_epoch)] { - for account_hash in [None, Some(Hash::new_unique())] { + for account_hash in [None, Some(AccountHash(Hash::new_unique()))] { let rent_epoch_offset = 0; let account_hash_offset = rent_epoch_offset + rent_epoch.as_ref().map(std::mem::size_of_val).unwrap_or(0); diff --git a/accounts-db/src/tiered_storage/readable.rs b/accounts-db/src/tiered_storage/readable.rs index 426da02ccb..629f08fa1d 100644 --- a/accounts-db/src/tiered_storage/readable.rs +++ b/accounts-db/src/tiered_storage/readable.rs @@ -1,11 +1,14 @@ use { - crate::tiered_storage::{ - footer::{AccountMetaFormat, TieredStorageFooter}, - hot::HotStorageReader, - meta::TieredAccountMeta, - TieredStorageResult, + crate::{ + accounts_hash::AccountHash, + tiered_storage::{ + footer::{AccountMetaFormat, TieredStorageFooter}, + hot::HotStorageReader, + meta::TieredAccountMeta, + TieredStorageResult, + }, }, - solana_sdk::{account::ReadableAccount, hash::Hash, pubkey::Pubkey, stake_history::Epoch}, + solana_sdk::{account::ReadableAccount, pubkey::Pubkey, stake_history::Epoch}, std::path::Path, }; @@ -32,7 +35,7 @@ impl<'accounts_file, M: TieredAccountMeta> TieredReadableAccount<'accounts_file, } /// Returns the hash of this account. - pub fn hash(&self) -> Option<&'accounts_file Hash> { + pub fn hash(&self) -> Option<&'accounts_file AccountHash> { self.meta.account_hash(self.account_block) }