store data and index occupy in bitvec (#30899)

Cargo.lock

@@ -5031,6 +5031,7 @@ dependencies = [
 name = "solana-bucket-map"
 version = "1.16.0"
 dependencies = [
+ "bv",
  "fs_extra",
  "log",
  "memmap2",

bucket_map/Cargo.toml

@@ -11,6 +11,7 @@ license = { workspace = true }
 edition = { workspace = true }
 
 [dependencies]
+bv = { workspace = true, features = ["serde"] }
 log = { workspace = true }
 memmap2 = { workspace = true }
 modular-bitfield = { workspace = true }

bucket_map/src/bucket_storage.rs

@@ -205,12 +205,6 @@ impl<O: BucketOccupied> BucketStorage<O> {
         unsafe { &mut *item }
     }
 
-    pub(crate) fn get_from_parts<T: Sized>(item_slice: &[u8]) -> &T {
-        assert!(std::mem::size_of::<T>() <= item_slice.len());
-        let item = item_slice.as_ptr() as *const T;
-        unsafe { &*item }
-    }
-
     pub fn get_mut<T: Sized>(&mut self, ix: u64) -> &mut T {
         let start = self.get_start_offset_no_header(ix);
         let item_slice = &mut self.mmap[start..];

bucket_map/src/index_entry.rs

@@ -6,6 +6,7 @@ use {
         bucket_storage::{BucketOccupied, BucketStorage},
         RefCount,
     },
+    bv::BitVec,
     modular_bitfield::prelude::*,
     solana_sdk::{clock::Slot, pubkey::Pubkey},
     std::fmt::Debug,
@@ -25,38 +26,35 @@ struct OccupiedHeader {
 }
 
 /// allocated in `contents` in a BucketStorage
-pub struct BucketWithHeader {}
+pub struct BucketWithBitVec {
+    pub occupied: BitVec,
+}
 
-impl BucketOccupied for BucketWithHeader {
+impl BucketOccupied for BucketWithBitVec {
-    fn occupy(&mut self, element: &mut [u8], _ix: usize) {
+    fn occupy(&mut self, element: &mut [u8], ix: usize) {
-        let entry: &mut OccupiedHeader =
+        assert!(self.is_free(element, ix));
-            BucketStorage::<BucketWithHeader>::get_mut_from_parts(element);
+        self.occupied.set(ix as u64, true);
-        assert_eq!(entry.occupied, OCCUPIED_FREE);
-        entry.occupied = OCCUPIED_OCCUPIED;
     }
-    fn free(&mut self, element: &mut [u8], _ix: usize) {
+    fn free(&mut self, element: &mut [u8], ix: usize) {
-        let entry: &mut OccupiedHeader =
+        assert!(!self.is_free(element, ix));
-            BucketStorage::<BucketWithHeader>::get_mut_from_parts(element);
+        self.occupied.set(ix as u64, false);
-        assert_eq!(entry.occupied, OCCUPIED_OCCUPIED);
-        entry.occupied = OCCUPIED_FREE;
     }
-    fn is_free(&self, element: &[u8], _ix: usize) -> bool {
+    fn is_free(&self, _element: &[u8], ix: usize) -> bool {
-        let entry: &OccupiedHeader = BucketStorage::<BucketWithHeader>::get_from_parts(element);
+        !self.occupied.get(ix as u64)
-        let free = entry.occupied == OCCUPIED_FREE;
-        assert!(free || entry.occupied == OCCUPIED_OCCUPIED);
-        free
     }
     fn offset_to_first_data() -> usize {
-        std::mem::size_of::<OccupiedHeader>()
+        // no header, nothing stored in data stream
+        0
     }
-    /// initialize this struct
+    fn new(num_elements: usize) -> Self {
-    fn new(_num_elements: usize) -> Self {
+        Self {
-        Self {}
+            occupied: BitVec::new_fill(false, num_elements as u64),
+        }
     }
 }
 
-pub type DataBucket = BucketWithHeader;
+pub type DataBucket = BucketWithBitVec;
-pub type IndexBucket = BucketWithHeader;
+pub type IndexBucket = BucketWithBitVec;
 
 #[repr(C)]
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]

programs/sbf/Cargo.lock

@@ -4466,6 +4466,7 @@ dependencies = [
 name = "solana-bucket-map"
 version = "1.16.0"
 dependencies = [
+ "bv",
  "log",
  "memmap2",
  "modular-bitfield",