blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

disk index: include T on a few structs (#30983)

This commit is contained in:
Jeff Washington (jwash) 2023-03-30 09:44:50 -05:00 committed by GitHub
parent 9f9d6f8a89
commit e6ca734ac4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 49 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
bucket_map/src/bucket.rs
View File

@ -78,10 +78,10 @@ impl<I: BucketOccupied, D: BucketOccupied> Reallocated<I, D> {
}
// >= 2 instances of BucketStorage per 'bucket' in the bucket map. 1 for index, >= 1 for data
pub struct Bucket<T> {
pub struct Bucket<T: 'static> {
drives: Arc<Vec<PathBuf>>,
//index
pub index: BucketStorage<IndexBucket>,
pub index: BucketStorage<IndexBucket<T>>,
//random offset for the index
random: u64,
//storage buckets to store SlotSlice up to a power of 2 in len
@ -89,7 +89,7 @@ pub struct Bucket<T> {
_phantom: PhantomData<T>,
stats: Arc<BucketMapStats>,
pub reallocated: Reallocated<IndexBucket, DataBucket>,
pub reallocated: Reallocated<IndexBucket<T>, DataBucket>,
}
impl<'b, T: Clone + Copy + 'static> Bucket<T> {
@ -156,7 +156,7 @@ impl<'b, T: Clone + Copy + 'static> Bucket<T> {
result
}
pub fn find_index_entry(&self, key: &Pubkey) -> Option<(IndexEntryPlaceInBucket, u64)> {
pub fn find_index_entry(&self, key: &Pubkey) -> Option<(IndexEntryPlaceInBucket<T>, u64)> {
Self::bucket_find_index_entry(&self.index, key, self.random)
}
@ -165,10 +165,10 @@ impl<'b, T: Clone + Copy + 'static> Bucket<T> {
/// if entry does not exist, return just the index of an empty entry appropriate for this key
/// returns (existing entry, index of the found or empty entry)
fn find_index_entry_mut(
index: &mut BucketStorage<IndexBucket>,
index: &mut BucketStorage<IndexBucket<T>>,
key: &Pubkey,
random: u64,
) -> Result<(Option<IndexEntryPlaceInBucket>, u64), BucketMapError> {
) -> Result<(Option<IndexEntryPlaceInBucket<T>>, u64), BucketMapError> {
let ix = Self::bucket_index_ix(index, key, random);
let mut first_free = None;
let mut m = Measure::start("bucket_find_index_entry_mut");
@ -204,10 +204,10 @@ impl<'b, T: Clone + Copy + 'static> Bucket<T> {
}
fn bucket_find_index_entry(
index: &BucketStorage<IndexBucket>,
index: &BucketStorage<IndexBucket<T>>,
key: &Pubkey,
random: u64,
) -> Option<(IndexEntryPlaceInBucket, u64)> {
) -> Option<(IndexEntryPlaceInBucket<T>, u64)> {
let ix = Self::bucket_index_ix(index, key, random);
for i in ix..ix + index.max_search() {
let ii = i % index.capacity();
@ -223,7 +223,7 @@ impl<'b, T: Clone + Copy + 'static> Bucket<T> {
}
fn bucket_create_key(
index: &mut BucketStorage<IndexBucket>,
index: &mut BucketStorage<IndexBucket<T>>,
key: &Pubkey,
random: u64,
is_resizing: bool,
@ -421,7 +421,7 @@ impl<'b, T: Clone + Copy + 'static> Bucket<T> {
}
}
pub fn apply_grow_index(&mut self, random: u64, index: BucketStorage<IndexBucket>) {
pub fn apply_grow_index(&mut self, random: u64, index: BucketStorage<IndexBucket<T>>) {
self.stats
.index
.resize_grow(self.index.capacity_bytes(), index.capacity_bytes());
@ -480,7 +480,7 @@ impl<'b, T: Clone + Copy + 'static> Bucket<T> {
items.data = Some((data_index, new_bucket));
}
fn bucket_index_ix(index: &BucketStorage<IndexBucket>, key: &Pubkey, random: u64) -> u64 {
fn bucket_index_ix(index: &BucketStorage<IndexBucket<T>>, key: &Pubkey, random: u64) -> u64 {
let mut s = DefaultHasher::new();
key.hash(&mut s);
//the locally generated random will make it hard for an attacker

2
bucket_map/src/bucket_storage.rs
View File

@ -370,7 +370,7 @@ mod test {
let paths: Vec<PathBuf> = vec![tmpdir.path().to_path_buf()];
assert!(!paths.is_empty());
let mut storage = BucketStorage::<IndexBucket>::new(
let mut storage = BucketStorage::<IndexBucket<u64>>::new(
Arc::new(paths),
1,
1,

62
bucket_map/src/index_entry.rs
View File

@ -8,7 +8,7 @@ use {
bv::BitVec,
modular_bitfield::prelude::*,
solana_sdk::{clock::Slot, pubkey::Pubkey},
std::fmt::Debug,
std::{fmt::Debug, marker::PhantomData},
};
/// in use/occupied
@ -25,11 +25,12 @@ struct OccupiedHeader {
}
/// allocated in `contents` in a BucketStorage
pub struct BucketWithBitVec {
pub struct BucketWithBitVec<T: 'static> {
pub occupied: BitVec,
_phantom: PhantomData<&'static T>,
}
impl BucketOccupied for BucketWithBitVec {
impl<T> BucketOccupied for BucketWithBitVec<T> {
fn occupy(&mut self, element: &mut [u8], ix: usize) {
assert!(self.is_free(element, ix));
self.occupied.set(ix as u64, true);
@ -48,17 +49,19 @@ impl BucketOccupied for BucketWithBitVec {
fn new(num_elements: usize) -> Self {
Self {
occupied: BitVec::new_fill(false, num_elements as u64),
_phantom: PhantomData,
}
}
}
pub type DataBucket = BucketWithBitVec;
pub type IndexBucket = BucketWithBitVec;
pub type DataBucket = BucketWithBitVec<()>;
pub type IndexBucket<T> = BucketWithBitVec<T>;
/// contains the index of an entry in the index bucket.
/// This type allows us to call methods to interact with the index entry on this type.
pub struct IndexEntryPlaceInBucket {
pub struct IndexEntryPlaceInBucket<T: 'static> {
pub ix: u64,
_phantom: PhantomData<&'static T>,
}
#[repr(C)]
@ -98,8 +101,8 @@ impl IndexEntry {
}
}
impl IndexEntryPlaceInBucket {
pub fn init(&self, index_bucket: &mut BucketStorage<IndexBucket>, pubkey: &Pubkey) {
impl<T> IndexEntryPlaceInBucket<T> {
pub fn init(&self, index_bucket: &mut BucketStorage<IndexBucket<T>>, pubkey: &Pubkey) {
let index_entry = index_bucket.get_mut::<IndexEntry>(self.ix);
index_entry.key = *pubkey;
index_entry.ref_count = 0;
@ -109,7 +112,7 @@ impl IndexEntryPlaceInBucket {
pub fn set_storage_capacity_when_created_pow2(
&self,
index_bucket: &mut BucketStorage<IndexBucket>,
index_bucket: &mut BucketStorage<IndexBucket<T>>,
storage_capacity_when_created_pow2: u8,
) {
index_bucket
@ -120,7 +123,7 @@ impl IndexEntryPlaceInBucket {
pub fn set_storage_offset(
&self,
index_bucket: &mut BucketStorage<IndexBucket>,
index_bucket: &mut BucketStorage<IndexBucket<T>>,
storage_offset: u64,
) {
index_bucket
@ -130,23 +133,26 @@ impl IndexEntryPlaceInBucket {
.expect("New storage offset must fit into 7 bytes!");
}
pub fn data_bucket_ix(&self, index_bucket: &BucketStorage<IndexBucket>) -> u64 {
pub fn data_bucket_ix(&self, index_bucket: &BucketStorage<IndexBucket<T>>) -> u64 {
IndexEntry::data_bucket_from_num_slots(self.num_slots(index_bucket))
}
pub fn ref_count(&self, index_bucket: &BucketStorage<IndexBucket>) -> RefCount {
pub fn ref_count(&self, index_bucket: &BucketStorage<IndexBucket<T>>) -> RefCount {
let index_entry = index_bucket.get::<IndexEntry>(self.ix);
index_entry.ref_count
}
fn storage_capacity_when_created_pow2(&self, index_bucket: &BucketStorage<IndexBucket>) -> u8 {
fn storage_capacity_when_created_pow2(
&self,
index_bucket: &BucketStorage<IndexBucket<T>>,
) -> u8 {
let index_entry = index_bucket.get::<IndexEntry>(self.ix);
index_entry
.storage_cap_and_offset
.capacity_when_created_pow2()
}
pub fn storage_offset(&self, index_bucket: &BucketStorage<IndexBucket>) -> u64 {
pub fn storage_offset(&self, index_bucket: &BucketStorage<IndexBucket<T>>) -> u64 {
index_bucket
.get::<IndexEntry>(self.ix)
.storage_cap_and_offset
@ -157,7 +163,7 @@ impl IndexEntryPlaceInBucket {
/// This is coupled with how we resize bucket storages.
pub fn data_loc(
&self,
index_bucket: &BucketStorage<IndexBucket>,
index_bucket: &BucketStorage<IndexBucket<T>>,
storage: &BucketStorage<DataBucket>,
) -> u64 {
let index_entry = index_bucket.get::<IndexEntry>(self.ix);
@ -168,9 +174,9 @@ impl IndexEntryPlaceInBucket {
.capacity_when_created_pow2())
}
pub fn read_value<'a, T>(
pub fn read_value<'a>(
&self,
index_bucket: &BucketStorage<IndexBucket>,
index_bucket: &BucketStorage<IndexBucket<T>>,
data_buckets: &'a [BucketStorage<DataBucket>],
) -> Option<(&'a [T], RefCount)> {
let num_slots = self.num_slots(index_bucket);
@ -188,28 +194,31 @@ impl IndexEntryPlaceInBucket {
}
pub fn new(ix: u64) -> Self {
Self { ix }
Self {
ix,
_phantom: PhantomData,
}
}
pub fn key<'a>(&self, index_bucket: &'a BucketStorage<IndexBucket>) -> &'a Pubkey {
pub fn key<'a>(&self, index_bucket: &'a BucketStorage<IndexBucket<T>>) -> &'a Pubkey {
let entry: &IndexEntry = index_bucket.get(self.ix);
&entry.key
}
pub fn set_ref_count(
&self,
index_bucket: &mut BucketStorage<IndexBucket>,
index_bucket: &mut BucketStorage<IndexBucket<T>>,
ref_count: RefCount,
) {
let index_entry = index_bucket.get_mut::<IndexEntry>(self.ix);
index_entry.ref_count = ref_count;
}
pub fn num_slots(&self, index_bucket: &BucketStorage<IndexBucket>) -> Slot {
pub fn num_slots(&self, index_bucket: &BucketStorage<IndexBucket<T>>) -> Slot {
index_bucket.get::<IndexEntry>(self.ix).num_slots
}
pub fn set_num_slots(&self, index_bucket: &mut BucketStorage<IndexBucket>, num_slots: Slot) {
pub fn set_num_slots(&self, index_bucket: &mut BucketStorage<IndexBucket<T>>, num_slots: Slot) {
index_bucket.get_mut::<IndexEntry>(self.ix).num_slots = num_slots;
}
}
@ -261,13 +270,13 @@ mod tests {
assert_eq!(std::mem::size_of::<IndexEntry>(), 32 + 8 + 8 + 8);
}
fn index_bucket_for_testing() -> BucketStorage<IndexBucket> {
fn index_bucket_for_testing() -> BucketStorage<IndexBucket<u64>> {
let tmpdir = tempdir().unwrap();
let paths: Vec<PathBuf> = vec![tmpdir.path().to_path_buf()];
assert!(!paths.is_empty());
// `new` here creates a file in `tmpdir`. Once the file is created, `tmpdir` can be dropped without issue.
BucketStorage::<IndexBucket>::new(
BucketStorage::<IndexBucket<u64>>::new(
Arc::new(paths),
1,
std::mem::size_of::<IndexEntry>() as u64,
@ -277,7 +286,10 @@ mod tests {
)
}
fn index_entry_for_testing() -> (BucketStorage<IndexBucket>, IndexEntryPlaceInBucket) {
fn index_entry_for_testing() -> (
BucketStorage<IndexBucket<u64>>,
IndexEntryPlaceInBucket<u64>,
) {
(index_bucket_for_testing(), IndexEntryPlaceInBucket::new(0))
}