Add columns for data and code shreds (#5461)

core/src/blocktree.rs

@@ -91,6 +91,8 @@ pub struct Blocktree {
     erasure_meta_cf: LedgerColumn<cf::ErasureMeta>,
     orphans_cf: LedgerColumn<cf::Orphans>,
     index_cf: LedgerColumn<cf::Index>,
+    _data_shred_cf: LedgerColumn<cf::ShredData>,
+    _code_shred_cf: LedgerColumn<cf::ShredCode>,
     batch_processor: Arc<RwLock<BatchProcessor>>,
     pub new_blobs_signals: Vec<SyncSender<bool>>,
     pub completed_slots_senders: Vec<SyncSender<Vec<u64>>>,
@@ -111,6 +113,10 @@ pub const ORPHANS_CF: &str = "orphans";
 pub const ROOT_CF: &str = "root";
 /// Column family for indexes
 pub const INDEX_CF: &str = "index";
+/// Column family for Data Shreds
+pub const DATA_SHRED_CF: &str = "data_shred";
+/// Column family for Code Shreds
+pub const CODE_SHRED_CF: &str = "code_shred";
 
 impl Blocktree {
     /// Opens a Ledger in directory, provides "infinite" window of blobs
@@ -143,6 +149,9 @@ impl Blocktree {
         let orphans_cf = db.column();
         let index_cf = db.column();
 
+        let data_shred_cf = db.column();
+        let code_shred_cf = db.column();
+
         let db = Arc::new(db);
 
         Ok(Blocktree {
@@ -154,6 +163,8 @@ impl Blocktree {
             erasure_meta_cf,
             orphans_cf,
             index_cf,
+            _data_shred_cf: data_shred_cf,
+            _code_shred_cf: code_shred_cf,
             new_blobs_signals: vec![],
             batch_processor,
             completed_slots_senders: vec![],

core/src/blocktree/db.rs

@@ -44,6 +44,14 @@ pub mod columns {
     #[derive(Debug)]
     /// The index column
     pub struct Index;
+
+    #[derive(Debug)]
+    /// The shred data column
+    pub struct ShredData;
+
+    #[derive(Debug)]
+    /// The shred erasure code column
+    pub struct ShredCode;
 }
 
 pub trait Backend: Sized + Send + Sync {

core/src/blocktree/rocks.rs

@@ -33,7 +33,8 @@ impl Backend for Rocks {
 
     fn open(path: &Path) -> Result<Rocks> {
         use crate::blocktree::db::columns::{
-            Coding, Data, DeadSlots, ErasureMeta, Index, Orphans, Root, SlotMeta,
+            Coding, Data, DeadSlots, ErasureMeta, Index, Orphans, Root, ShredCode, ShredData,
+            SlotMeta,
         };
 
         fs::create_dir_all(&path)?;
@@ -58,6 +59,10 @@ impl Backend for Rocks {
             ColumnFamilyDescriptor::new(Root::NAME, get_cf_options(Root::NAME));
         let index_cf_descriptor =
             ColumnFamilyDescriptor::new(Index::NAME, get_cf_options(Index::NAME));
+        let shred_data_cf_descriptor =
+            ColumnFamilyDescriptor::new(ShredData::NAME, get_cf_options(ShredData::NAME));
+        let shred_code_cf_descriptor =
+            ColumnFamilyDescriptor::new(ShredCode::NAME, get_cf_options(ShredCode::NAME));
 
         let cfs = vec![
             meta_cf_descriptor,
@@ -68,6 +73,8 @@ impl Backend for Rocks {
             orphans_cf_descriptor,
             root_cf_descriptor,
             index_cf_descriptor,
+            shred_data_cf_descriptor,
+            shred_code_cf_descriptor,
         ];
 
         // Open the database
@@ -78,7 +85,8 @@ impl Backend for Rocks {
 
     fn columns(&self) -> Vec<&'static str> {
         use crate::blocktree::db::columns::{
-            Coding, Data, DeadSlots, ErasureMeta, Index, Orphans, Root, SlotMeta,
+            Coding, Data, DeadSlots, ErasureMeta, Index, Orphans, Root, ShredCode, ShredData,
+            SlotMeta,
         };
 
         vec![
@@ -90,6 +98,8 @@ impl Backend for Rocks {
             Orphans::NAME,
             Root::NAME,
             SlotMeta::NAME,
+            ShredData::NAME,
+            ShredCode::NAME,
         ]
     }
 
@@ -196,6 +206,53 @@ impl Column<Rocks> for cf::Data {
     }
 }
 
+impl Column<Rocks> for cf::ShredCode {
+    const NAME: &'static str = super::CODE_SHRED_CF;
+    type Index = (u64, u64);
+
+    fn key(index: (u64, u64)) -> Vec<u8> {
+        cf::ShredData::key(index)
+    }
+
+    fn index(key: &[u8]) -> (u64, u64) {
+        cf::ShredData::index(key)
+    }
+
+    fn slot(index: Self::Index) -> Slot {
+        index.0
+    }
+
+    fn as_index(slot: Slot) -> Self::Index {
+        (slot, 0)
+    }
+}
+
+impl Column<Rocks> for cf::ShredData {
+    const NAME: &'static str = super::DATA_SHRED_CF;
+    type Index = (u64, u64);
+
+    fn key((slot, index): (u64, u64)) -> Vec<u8> {
+        let mut key = vec![0; 16];
+        BigEndian::write_u64(&mut key[..8], slot);
+        BigEndian::write_u64(&mut key[8..16], index);
+        key
+    }
+
+    fn index(key: &[u8]) -> (u64, u64) {
+        let slot = BigEndian::read_u64(&key[..8]);
+        let index = BigEndian::read_u64(&key[8..16]);
+        (slot, index)
+    }
+
+    fn slot(index: Self::Index) -> Slot {
+        index.0
+    }
+
+    fn as_index(slot: Slot) -> Self::Index {
+        (slot, 0)
+    }
+}
+
 impl Column<Rocks> for cf::Index {
     const NAME: &'static str = super::INDEX_CF;
     type Index = u64;