blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

add DiskIndexValue trait for disk index (#30503)

This commit is contained in:
Jeff Washington (jwash) 2023-02-24 13:02:06 -06:00 committed by GitHub
parent af59cced5a
commit 7c086ca42d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 41 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
runtime/src/accounts_db.rs
View File

@ -38,8 +38,8 @@ use {
},
accounts_index::{
AccountIndexGetResult, AccountSecondaryIndexes, AccountsIndex, AccountsIndexConfig,
AccountsIndexRootsStats, AccountsIndexScanResult, IndexKey, IndexValue, IsCached,
RefCount, ScanConfig, ScanResult, SlotList, UpsertReclaim, ZeroLamport,
AccountsIndexRootsStats, AccountsIndexScanResult, DiskIndexValue, IndexKey, IndexValue,
IsCached, RefCount, ScanConfig, ScanResult, SlotList, UpsertReclaim, ZeroLamport,
ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS, ACCOUNTS_INDEX_CONFIG_FOR_TESTING,
},
accounts_index_storage::Startup,
@ -676,6 +676,7 @@ impl GenerateIndexTimings {
}
impl IndexValue for AccountInfo {}
impl DiskIndexValue for AccountInfo {}
impl ZeroLamport for AccountSharedData {
fn is_zero_lamport(&self) -> bool {

31
runtime/src/accounts_index.rs
View File

@ -133,8 +133,10 @@ pub trait IsCached {
fn is_cached(&self) -> bool;
}
pub trait IndexValue:
'static + IsCached + Clone + Debug + PartialEq + ZeroLamport + Copy + Default + Sync + Send
pub trait IndexValue: 'static + IsCached + ZeroLamport + DiskIndexValue {}
pub trait DiskIndexValue:
'static + Clone + Debug + PartialEq + Copy + Default + Sync + Send
{
}
@ -232,7 +234,7 @@ pub struct AccountMapEntryMeta {
}
impl AccountMapEntryMeta {
pub fn new_dirty<T: IndexValue, U: IndexValue + From<T> + Into<T>>(
pub fn new_dirty<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>>(
storage: &Arc<BucketMapHolder<T, U>>,
is_cached: bool,
) -> Self {
@ -241,7 +243,7 @@ impl AccountMapEntryMeta {
age: AtomicU8::new(storage.future_age_to_flush(is_cached)),
}
}
pub fn new_clean<T: IndexValue, U: IndexValue + From<T> + Into<T>>(
pub fn new_clean<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>>(
storage: &Arc<BucketMapHolder<T, U>>,
) -> Self {
AccountMapEntryMeta {
@ -402,7 +404,7 @@ impl<T: IndexValue> PreAllocatedAccountMapEntry<T> {
/// 1. new empty (refcount=0, slot_list={})
/// 2. update(slot, account_info)
/// This code is called when the first entry [ie. (slot,account_info)] for a pubkey is inserted into the index.
pub fn new<U: IndexValue + From<T> + Into<T>>(
pub fn new<U: DiskIndexValue + From<T> + Into<T>>(
slot: Slot,
account_info: T,
storage: &Arc<BucketMapHolder<T, U>>,
@ -415,7 +417,7 @@ impl<T: IndexValue> PreAllocatedAccountMapEntry<T> {
}
}
fn allocate<U: IndexValue + From<T> + Into<T>>(
fn allocate<U: DiskIndexValue + From<T> + Into<T>>(
slot: Slot,
account_info: T,
storage: &Arc<BucketMapHolder<T, U>>,
@ -430,7 +432,7 @@ impl<T: IndexValue> PreAllocatedAccountMapEntry<T> {
))
}
pub fn into_account_map_entry<U: IndexValue + From<T> + Into<T>>(
pub fn into_account_map_entry<U: DiskIndexValue + From<T> + Into<T>>(
self,
storage: &Arc<BucketMapHolder<T, U>>,
) -> AccountMapEntry<T> {
@ -495,7 +497,7 @@ pub struct AccountsIndexRootsStats {
pub clean_dead_slot_us: u64,
}
pub struct AccountsIndexIterator<'a, T: IndexValue, U: IndexValue + From<T> + Into<T>> {
pub struct AccountsIndexIterator<'a, T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> {
account_maps: &'a LockMapTypeSlice<T, U>,
bin_calculator: &'a PubkeyBinCalculator24,
start_bound: Bound<Pubkey>,
@ -504,7 +506,7 @@ pub struct AccountsIndexIterator<'a, T: IndexValue, U: IndexValue + From<T> + In
collect_all_unsorted: bool,
}
impl<'a, T: IndexValue, U: IndexValue + From<T> + Into<T>> AccountsIndexIterator<'a, T, U> {
impl<'a, T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> AccountsIndexIterator<'a, T, U> {
fn range<R>(
map: &AccountMaps<T, U>,
range: R,
@ -604,7 +606,7 @@ impl<'a, T: IndexValue, U: IndexValue + From<T> + Into<T>> AccountsIndexIterator
}
}
impl<'a, T: IndexValue, U: IndexValue + From<T> + Into<T>> Iterator
impl<'a, T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> Iterator
for AccountsIndexIterator<'a, T, U>
{
type Item = Vec<(Pubkey, AccountMapEntry<T>)>;
@ -677,7 +679,7 @@ pub enum AccountsIndexScanResult {
#[derive(Debug)]
/// T: account info type to interact in in-memory items
/// U: account info type to be persisted to disk
pub struct AccountsIndex<T: IndexValue, U: IndexValue + From<T> + Into<T>> {
pub struct AccountsIndex<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> {
pub account_maps: LockMapType<T, U>,
pub bin_calculator: PubkeyBinCalculator24,
program_id_index: SecondaryIndex<DashMapSecondaryIndexEntry>,
@ -715,7 +717,7 @@ pub struct AccountsIndex<T: IndexValue, U: IndexValue + From<T> + Into<T>> {
pub rent_paying_accounts_by_partition: OnceCell<RentPayingAccountsByPartition>,
}
impl<T: IndexValue, U: IndexValue + From<T> + Into<T>> AccountsIndex<T, U> {
impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> AccountsIndex<T, U> {
pub fn default_for_tests() -> Self {
Self::new(Some(ACCOUNTS_INDEX_CONFIG_FOR_TESTING), &Arc::default())
}
@ -2144,7 +2146,7 @@ pub mod tests {
}
}
impl<T: IndexValue> AccountsIndex<T, T> {
impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> AccountsIndex<T, U> {
/// provides the ability to refactor this function on the api without bloody changes
pub fn get_for_tests(
&self,
@ -2416,6 +2418,7 @@ pub mod tests {
type AccountInfoTest = f64;
impl IndexValue for AccountInfoTest {}
impl DiskIndexValue for AccountInfoTest {}
impl IsCached for AccountInfoTest {
fn is_cached(&self) -> bool {
true
@ -3930,6 +3933,8 @@ pub mod tests {
impl IndexValue for bool {}
impl IndexValue for u64 {}
impl DiskIndexValue for bool {}
impl DiskIndexValue for u64 {}
impl IsCached for bool {
fn is_cached(&self) -> bool {
false

10
runtime/src/accounts_index_storage.rs
View File

@ -1,6 +1,6 @@
use {
crate::{
accounts_index::{AccountsIndexConfig, IndexValue},
accounts_index::{AccountsIndexConfig, DiskIndexValue, IndexValue},
bucket_map_holder::BucketMapHolder,
in_mem_accounts_index::InMemAccountsIndex,
waitable_condvar::WaitableCondvar,
@ -16,7 +16,7 @@ use {
};
/// Manages the lifetime of the background processing threads.
pub struct AccountsIndexStorage<T: IndexValue, U: IndexValue + From<T> + Into<T>> {
pub struct AccountsIndexStorage<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> {
_bg_threads: BgThreads,
pub storage: Arc<BucketMapHolder<T, U>>,
@ -27,7 +27,7 @@ pub struct AccountsIndexStorage<T: IndexValue, U: IndexValue + From<T> + Into<T>
startup_worker_threads: Mutex<Option<BgThreads>>,
}
impl<T: IndexValue, U: IndexValue + From<T> + Into<T>> Debug for AccountsIndexStorage<T, U> {
impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> Debug for AccountsIndexStorage<T, U> {
fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
Ok(())
}
@ -53,7 +53,7 @@ impl Drop for BgThreads {
}
impl BgThreads {
fn new<T: IndexValue, U: IndexValue + From<T> + Into<T>>(
fn new<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>>(
storage: &Arc<BucketMapHolder<T, U>>,
in_mem: &[Arc<InMemAccountsIndex<T, U>>],
threads: usize,
@ -109,7 +109,7 @@ pub enum Startup {
StartupWithExtraThreads,
}
impl<T: IndexValue, U: IndexValue + From<T> + Into<T>> AccountsIndexStorage<T, U> {
impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> AccountsIndexStorage<T, U> {
/// startup=true causes:
/// in mem to act in a way that flushes to disk asap
/// also creates some additional bg threads to facilitate flushing to disk asap

8
runtime/src/bucket_map_holder.rs
View File

@ -1,6 +1,6 @@
use {
crate::{
accounts_index::{AccountsIndexConfig, IndexLimitMb, IndexValue},
accounts_index::{AccountsIndexConfig, DiskIndexValue, IndexLimitMb, IndexValue},
bucket_map_holder_stats::BucketMapHolderStats,
in_mem_accounts_index::InMemAccountsIndex,
waitable_condvar::WaitableCondvar,
@ -28,7 +28,7 @@ const AGE_MS: u64 = DEFAULT_MS_PER_SLOT; // match one age per slot time
// 10 GB limit for in-mem idx. In practice, we don't get this high. This tunes how aggressively to save items we expect to use soon.
pub const DEFAULT_DISK_INDEX: Option<usize> = Some(10_000);
pub struct BucketMapHolder<T: IndexValue, U: IndexValue + From<T> + Into<T>> {
pub struct BucketMapHolder<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> {
pub disk: Option<BucketMap<(Slot, U)>>,
pub count_buckets_flushed: AtomicUsize,
@ -70,14 +70,14 @@ pub struct BucketMapHolder<T: IndexValue, U: IndexValue + From<T> + Into<T>> {
_phantom: PhantomData<T>,
}
impl<T: IndexValue, U: IndexValue + From<T> + Into<T>> Debug for BucketMapHolder<T, U> {
impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> Debug for BucketMapHolder<T, U> {
fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
Ok(())
}
}
#[allow(clippy::mutex_atomic)]
impl<T: IndexValue, U: IndexValue + From<T> + Into<T>> BucketMapHolder<T, U> {
impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> BucketMapHolder<T, U> {
/// is the accounts index using disk as a backing store
pub fn is_disk_index_enabled(&self) -> bool {
self.disk.is_some()

9
runtime/src/bucket_map_holder_stats.rs
View File

@ -1,5 +1,8 @@
use {
crate::{accounts_index::IndexValue, bucket_map_holder::BucketMapHolder},
crate::{
accounts_index::{DiskIndexValue, IndexValue},
bucket_map_holder::BucketMapHolder,
},
solana_sdk::timing::AtomicInterval,
std::{
fmt::Debug,
@ -99,7 +102,7 @@ impl BucketMapHolderStats {
per_bucket.map(|stat| stat.fetch_sub(count, Ordering::Relaxed));
}
fn ms_per_age<T: IndexValue, U: IndexValue + From<T> + Into<T>>(
fn ms_per_age<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>>(
&self,
storage: &BucketMapHolder<T, U>,
elapsed_ms: u64,
@ -176,7 +179,7 @@ impl BucketMapHolderStats {
in_mem.saturating_sub(held_in_mem) as usize
}
pub fn report_stats<T: IndexValue, U: IndexValue + From<T> + Into<T>>(
pub fn report_stats<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>>(
&self,
storage: &BucketMapHolder<T, U>,
) {

10
runtime/src/in_mem_accounts_index.rs
View File

@ -1,7 +1,7 @@
use {
crate::{
accounts_index::{
AccountMapEntry, AccountMapEntryInner, AccountMapEntryMeta, IndexValue,
AccountMapEntry, AccountMapEntryInner, AccountMapEntryMeta, DiskIndexValue, IndexValue,
PreAllocatedAccountMapEntry, RefCount, SlotList, UpsertReclaim, ZeroLamport,
},
bucket_map_holder::{Age, BucketMapHolder},
@ -83,7 +83,7 @@ impl<T: IndexValue> PossibleEvictions<T> {
}
// one instance of this represents one bin of the accounts index.
pub struct InMemAccountsIndex<T: IndexValue, U: IndexValue + From<T> + Into<T>> {
pub struct InMemAccountsIndex<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> {
last_age_flushed: AtomicU8,
// backing store
@ -112,7 +112,7 @@ pub struct InMemAccountsIndex<T: IndexValue, U: IndexValue + From<T> + Into<T>>
age_to_flush_bin_mod: Age,
}
impl<T: IndexValue, U: IndexValue + From<T> + Into<T>> Debug for InMemAccountsIndex<T, U> {
impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> Debug for InMemAccountsIndex<T, U> {
fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
Ok(())
}
@ -141,7 +141,7 @@ struct FlushScanResult<T> {
evictions_random: Vec<(Pubkey, AccountMapEntry<T>)>,
}
impl<T: IndexValue, U: IndexValue + From<T> + Into<T>> InMemAccountsIndex<T, U> {
impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> InMemAccountsIndex<T, U> {
pub fn new(storage: &Arc<BucketMapHolder<T, U>>, bin: usize) -> Self {
let ages_to_stay_in_cache = storage.ages_to_stay_in_cache;
Self {
@ -1397,7 +1397,7 @@ struct EvictionsGuard<'a> {
impl<'a> EvictionsGuard<'a> {
#[must_use = "if unused, this evictions lock will be immediately unlocked"]
fn lock<T: IndexValue, U: IndexValue + From<T> + Into<T>>(
fn lock<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>>(
in_mem_accounts_index: &'a InMemAccountsIndex<T, U>,
) -> Self {
Self::lock_with(