Reduce locking in Blocktree (#4075)
* Reduce lock contention in blocktree * Store root slot in separate column
This commit is contained in:
parent
f91627a230
commit
ed48d8323c
|
@ -49,6 +49,7 @@ macro_rules! db_imports {
|
||||||
pub type Cursor<C> = db::Cursor<$db, C>;
|
pub type Cursor<C> = db::Cursor<$db, C>;
|
||||||
pub type LedgerColumn<C> = db::LedgerColumn<$db, C>;
|
pub type LedgerColumn<C> = db::LedgerColumn<$db, C>;
|
||||||
pub type WriteBatch = db::WriteBatch<$db>;
|
pub type WriteBatch = db::WriteBatch<$db>;
|
||||||
|
type BatchProcessor = db::BatchProcessor<$db>;
|
||||||
|
|
||||||
pub trait Column: db::Column<$db> {}
|
pub trait Column: db::Column<$db> {}
|
||||||
impl<C: db::Column<$db>> Column for C {}
|
impl<C: db::Column<$db>> Column for C {}
|
||||||
|
@ -73,15 +74,15 @@ pub enum BlocktreeError {
|
||||||
|
|
||||||
// ledger window
|
// ledger window
|
||||||
pub struct Blocktree {
|
pub struct Blocktree {
|
||||||
db: Arc<RwLock<Database>>,
|
db: Arc<Database>,
|
||||||
meta_cf: LedgerColumn<cf::SlotMeta>,
|
meta_cf: LedgerColumn<cf::SlotMeta>,
|
||||||
data_cf: LedgerColumn<cf::Data>,
|
data_cf: LedgerColumn<cf::Data>,
|
||||||
erasure_cf: LedgerColumn<cf::Coding>,
|
erasure_cf: LedgerColumn<cf::Coding>,
|
||||||
erasure_meta_cf: LedgerColumn<cf::ErasureMeta>,
|
erasure_meta_cf: LedgerColumn<cf::ErasureMeta>,
|
||||||
orphans_cf: LedgerColumn<cf::Orphans>,
|
orphans_cf: LedgerColumn<cf::Orphans>,
|
||||||
|
batch_processor: Arc<RwLock<BatchProcessor>>,
|
||||||
session: Arc<erasure::Session>,
|
session: Arc<erasure::Session>,
|
||||||
pub new_blobs_signals: Vec<SyncSender<bool>>,
|
pub new_blobs_signals: Vec<SyncSender<bool>>,
|
||||||
pub root_slot: RwLock<u64>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Column family for metadata about a leader slot
|
// Column family for metadata about a leader slot
|
||||||
|
@ -93,6 +94,8 @@ pub const ERASURE_CF: &str = "erasure";
|
||||||
pub const ERASURE_META_CF: &str = "erasure_meta";
|
pub const ERASURE_META_CF: &str = "erasure_meta";
|
||||||
// Column family for orphans data
|
// Column family for orphans data
|
||||||
pub const ORPHANS_CF: &str = "orphans";
|
pub const ORPHANS_CF: &str = "orphans";
|
||||||
|
// Column family for root data
|
||||||
|
pub const ROOT_CF: &str = "root";
|
||||||
|
|
||||||
impl Blocktree {
|
impl Blocktree {
|
||||||
/// Opens a Ledger in directory, provides "infinite" window of blobs
|
/// Opens a Ledger in directory, provides "infinite" window of blobs
|
||||||
|
@ -105,6 +108,8 @@ impl Blocktree {
|
||||||
// Open the database
|
// Open the database
|
||||||
let db = Database::open(&ledger_path)?;
|
let db = Database::open(&ledger_path)?;
|
||||||
|
|
||||||
|
let batch_processor = Arc::new(RwLock::new(db.batch_processor()));
|
||||||
|
|
||||||
// Create the metadata column family
|
// Create the metadata column family
|
||||||
let meta_cf = db.column();
|
let meta_cf = db.column();
|
||||||
|
|
||||||
|
@ -124,7 +129,7 @@ impl Blocktree {
|
||||||
// setup erasure
|
// setup erasure
|
||||||
let session = Arc::new(erasure::Session::default());
|
let session = Arc::new(erasure::Session::default());
|
||||||
|
|
||||||
let db = Arc::new(RwLock::new(db));
|
let db = Arc::new(db);
|
||||||
|
|
||||||
Ok(Blocktree {
|
Ok(Blocktree {
|
||||||
db,
|
db,
|
||||||
|
@ -135,7 +140,7 @@ impl Blocktree {
|
||||||
orphans_cf,
|
orphans_cf,
|
||||||
session,
|
session,
|
||||||
new_blobs_signals: vec![],
|
new_blobs_signals: vec![],
|
||||||
root_slot: RwLock::new(0),
|
batch_processor,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -155,21 +160,19 @@ impl Blocktree {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn meta(&self, slot: u64) -> Result<Option<SlotMeta>> {
|
pub fn meta(&self, slot: u64) -> Result<Option<SlotMeta>> {
|
||||||
self.meta_cf.get(&*self.db.read().unwrap(), slot)
|
self.meta_cf.get(slot)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn erasure_meta(&self, slot: u64, set_index: u64) -> Result<Option<ErasureMeta>> {
|
pub fn erasure_meta(&self, slot: u64, set_index: u64) -> Result<Option<ErasureMeta>> {
|
||||||
self.erasure_meta_cf
|
self.erasure_meta_cf.get((slot, set_index))
|
||||||
.get(&*self.db.read().unwrap(), (slot, set_index))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn orphan(&self, slot: u64) -> Result<Option<bool>> {
|
pub fn orphan(&self, slot: u64) -> Result<Option<bool>> {
|
||||||
self.orphans_cf.get(&*self.db.read().unwrap(), slot)
|
self.orphans_cf.get(slot)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_next_slot(&self, slot: u64) -> Result<Option<u64>> {
|
pub fn get_next_slot(&self, slot: u64) -> Result<Option<u64>> {
|
||||||
let db = self.db.read().unwrap();
|
let mut db_iterator = self.db.cursor::<cf::SlotMeta>()?;
|
||||||
let mut db_iterator = db.cursor::<cf::SlotMeta>()?;
|
|
||||||
|
|
||||||
db_iterator.seek(slot + 1);
|
db_iterator.seek(slot + 1);
|
||||||
if !db_iterator.valid() {
|
if !db_iterator.valid() {
|
||||||
|
@ -264,8 +267,9 @@ impl Blocktree {
|
||||||
I: IntoIterator,
|
I: IntoIterator,
|
||||||
I::Item: Borrow<Blob>,
|
I::Item: Borrow<Blob>,
|
||||||
{
|
{
|
||||||
let mut db = self.db.write().unwrap();
|
let db = &*self.db;
|
||||||
let mut write_batch = db.batch()?;
|
let mut batch_processor = self.batch_processor.write().unwrap();
|
||||||
|
let mut write_batch = batch_processor.batch()?;
|
||||||
|
|
||||||
let new_blobs: Vec<_> = new_blobs.into_iter().collect();
|
let new_blobs: Vec<_> = new_blobs.into_iter().collect();
|
||||||
let mut recovered_data = vec![];
|
let mut recovered_data = vec![];
|
||||||
|
@ -285,7 +289,7 @@ impl Blocktree {
|
||||||
.entry((blob_slot, set_index))
|
.entry((blob_slot, set_index))
|
||||||
.or_insert_with(|| {
|
.or_insert_with(|| {
|
||||||
self.erasure_meta_cf
|
self.erasure_meta_cf
|
||||||
.get(&db, (blob_slot, set_index))
|
.get((blob_slot, set_index))
|
||||||
.expect("Expect database get to succeed")
|
.expect("Expect database get to succeed")
|
||||||
.unwrap_or_else(|| ErasureMeta::new(set_index))
|
.unwrap_or_else(|| ErasureMeta::new(set_index))
|
||||||
});
|
});
|
||||||
|
@ -358,7 +362,7 @@ impl Blocktree {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
db.write(write_batch)?;
|
batch_processor.write(write_batch)?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
@ -374,8 +378,7 @@ impl Blocktree {
|
||||||
buf: &mut [u8],
|
buf: &mut [u8],
|
||||||
slot: u64,
|
slot: u64,
|
||||||
) -> Result<(u64, u64)> {
|
) -> Result<(u64, u64)> {
|
||||||
let db = self.db.read().unwrap();
|
let mut db_iterator = self.db.cursor::<cf::Data>()?;
|
||||||
let mut db_iterator = db.cursor::<cf::Data>()?;
|
|
||||||
|
|
||||||
db_iterator.seek((slot, start_index));
|
db_iterator.seek((slot, start_index));
|
||||||
let mut total_blobs = 0;
|
let mut total_blobs = 0;
|
||||||
|
@ -427,69 +430,65 @@ impl Blocktree {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_coding_blob_bytes(&self, slot: u64, index: u64) -> Result<Option<Vec<u8>>> {
|
pub fn get_coding_blob_bytes(&self, slot: u64, index: u64) -> Result<Option<Vec<u8>>> {
|
||||||
let db = self.db.read().unwrap();
|
self.erasure_cf.get_bytes((slot, index))
|
||||||
self.erasure_cf.get_bytes(&db, (slot, index))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn delete_coding_blob(&self, slot: u64, index: u64) -> Result<()> {
|
pub fn delete_coding_blob(&self, slot: u64, index: u64) -> Result<()> {
|
||||||
let set_index = ErasureMeta::set_index_for(index);
|
let set_index = ErasureMeta::set_index_for(index);
|
||||||
let mut db = self.db.write().unwrap();
|
let mut batch_processor = self.batch_processor.write().unwrap();
|
||||||
|
|
||||||
let mut erasure_meta = self
|
let mut erasure_meta = self
|
||||||
.erasure_meta_cf
|
.erasure_meta_cf
|
||||||
.get(&db, (slot, set_index))?
|
.get((slot, set_index))?
|
||||||
.unwrap_or_else(|| ErasureMeta::new(set_index));
|
.unwrap_or_else(|| ErasureMeta::new(set_index));
|
||||||
|
|
||||||
erasure_meta.set_coding_present(index, false);
|
erasure_meta.set_coding_present(index, false);
|
||||||
|
|
||||||
let mut batch = db.batch()?;
|
let mut batch = batch_processor.batch()?;
|
||||||
|
|
||||||
batch.delete::<cf::Coding>((slot, index))?;
|
batch.delete::<cf::Coding>((slot, index))?;
|
||||||
batch.put::<cf::ErasureMeta>((slot, set_index), &erasure_meta)?;
|
batch.put::<cf::ErasureMeta>((slot, set_index), &erasure_meta)?;
|
||||||
|
|
||||||
db.write(batch)?;
|
batch_processor.write(batch)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_data_blob_bytes(&self, slot: u64, index: u64) -> Result<Option<Vec<u8>>> {
|
pub fn get_data_blob_bytes(&self, slot: u64, index: u64) -> Result<Option<Vec<u8>>> {
|
||||||
let db = self.db.read().unwrap();
|
self.data_cf.get_bytes((slot, index))
|
||||||
self.data_cf.get_bytes(&db, (slot, index))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// For benchmarks, testing, and setup.
|
/// For benchmarks, testing, and setup.
|
||||||
/// Does no metadata tracking. Use with care.
|
/// Does no metadata tracking. Use with care.
|
||||||
pub fn put_data_blob_bytes(&self, slot: u64, index: u64, bytes: &[u8]) -> Result<()> {
|
pub fn put_data_blob_bytes(&self, slot: u64, index: u64, bytes: &[u8]) -> Result<()> {
|
||||||
let mut db = self.db.write().unwrap();
|
self.data_cf.put_bytes((slot, index), bytes)
|
||||||
self.data_cf.put_bytes(&mut db, (slot, index), bytes)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// For benchmarks, testing, and setup.
|
/// For benchmarks, testing, and setup.
|
||||||
/// Does no metadata tracking. Use with care.
|
/// Does no metadata tracking. Use with care.
|
||||||
pub fn put_coding_blob_bytes_raw(&self, slot: u64, index: u64, bytes: &[u8]) -> Result<()> {
|
pub fn put_coding_blob_bytes_raw(&self, slot: u64, index: u64, bytes: &[u8]) -> Result<()> {
|
||||||
let mut db = self.db.write().unwrap();
|
self.erasure_cf.put_bytes((slot, index), bytes)
|
||||||
self.erasure_cf.put_bytes(&mut db, (slot, index), bytes)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// this function will insert coding blobs and also automatically track erasure-related
|
/// this function will insert coding blobs and also automatically track erasure-related
|
||||||
/// metadata. If recovery is available it will be done
|
/// metadata. If recovery is available it will be done
|
||||||
pub fn put_coding_blob_bytes(&self, slot: u64, index: u64, bytes: &[u8]) -> Result<()> {
|
pub fn put_coding_blob_bytes(&self, slot: u64, index: u64, bytes: &[u8]) -> Result<()> {
|
||||||
let set_index = ErasureMeta::set_index_for(index);
|
let set_index = ErasureMeta::set_index_for(index);
|
||||||
let mut db = self.db.write().unwrap();
|
let mut batch_processor = self.batch_processor.write().unwrap();
|
||||||
|
|
||||||
let mut erasure_meta = self
|
let mut erasure_meta = self
|
||||||
.erasure_meta_cf
|
.erasure_meta_cf
|
||||||
.get(&db, (slot, set_index))?
|
.get((slot, set_index))?
|
||||||
.unwrap_or_else(|| ErasureMeta::new(set_index));
|
.unwrap_or_else(|| ErasureMeta::new(set_index));
|
||||||
|
|
||||||
erasure_meta.set_coding_present(index, true);
|
erasure_meta.set_coding_present(index, true);
|
||||||
erasure_meta.set_size(bytes.len() - BLOB_HEADER_SIZE);
|
erasure_meta.set_size(bytes.len() - BLOB_HEADER_SIZE);
|
||||||
|
|
||||||
let mut writebatch = db.batch()?;
|
let mut writebatch = batch_processor.batch()?;
|
||||||
|
|
||||||
writebatch.put_bytes::<cf::Coding>((slot, index), bytes)?;
|
writebatch.put_bytes::<cf::Coding>((slot, index), bytes)?;
|
||||||
|
|
||||||
if let Some((data, coding)) = try_erasure_recover(
|
if let Some((data, coding)) = try_erasure_recover(
|
||||||
&db,
|
&self.db,
|
||||||
&self.session,
|
&self.session,
|
||||||
&erasure_meta,
|
&erasure_meta,
|
||||||
slot,
|
slot,
|
||||||
|
@ -501,7 +500,7 @@ impl Blocktree {
|
||||||
|
|
||||||
insert_data_blob_batch(
|
insert_data_blob_batch(
|
||||||
&data[..],
|
&data[..],
|
||||||
&db,
|
&self.db,
|
||||||
&mut HashMap::new(),
|
&mut HashMap::new(),
|
||||||
&mut erasure_meta_working_set,
|
&mut erasure_meta_working_set,
|
||||||
&mut HashMap::new(),
|
&mut HashMap::new(),
|
||||||
|
@ -522,7 +521,7 @@ impl Blocktree {
|
||||||
|
|
||||||
writebatch.put::<cf::ErasureMeta>((slot, set_index), &erasure_meta)?;
|
writebatch.put::<cf::ErasureMeta>((slot, set_index), &erasure_meta)?;
|
||||||
|
|
||||||
db.write(writebatch)?;
|
batch_processor.write(writebatch)?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
@ -619,8 +618,7 @@ impl Blocktree {
|
||||||
end_index: u64,
|
end_index: u64,
|
||||||
max_missing: usize,
|
max_missing: usize,
|
||||||
) -> Vec<u64> {
|
) -> Vec<u64> {
|
||||||
let db = self.db.read().unwrap();
|
if let Ok(mut db_iterator) = self.db.cursor::<cf::Data>() {
|
||||||
if let Ok(mut db_iterator) = db.cursor::<cf::Data>() {
|
|
||||||
Self::find_missing_indexes(&mut db_iterator, slot, start_index, end_index, max_missing)
|
Self::find_missing_indexes(&mut db_iterator, slot, start_index, end_index, max_missing)
|
||||||
} else {
|
} else {
|
||||||
vec![]
|
vec![]
|
||||||
|
@ -639,9 +637,7 @@ impl Blocktree {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn read_ledger_blobs(&self) -> impl Iterator<Item = Blob> + '_ {
|
pub fn read_ledger_blobs(&self) -> impl Iterator<Item = Blob> + '_ {
|
||||||
let db = self.db.read().unwrap();
|
let iter = self.db.iter::<cf::Data>().unwrap();
|
||||||
|
|
||||||
let iter = db.iter::<cf::Data>().unwrap();
|
|
||||||
iter.map(|(_, blob_data)| Blob::new(&blob_data))
|
iter.map(|(_, blob_data)| Blob::new(&blob_data))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -699,7 +695,7 @@ impl Blocktree {
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let mut db_iterator = self.db.read().unwrap().cursor::<cf::Data>()?;
|
let mut db_iterator = self.db.cursor::<cf::Data>()?;
|
||||||
|
|
||||||
db_iterator.seek_to_first();
|
db_iterator.seek_to_first();
|
||||||
Ok(EntryIterator {
|
Ok(EntryIterator {
|
||||||
|
@ -718,7 +714,7 @@ impl Blocktree {
|
||||||
// Find the next consecutive block of blobs.
|
// Find the next consecutive block of blobs.
|
||||||
let consecutive_blobs = get_slot_consecutive_blobs(
|
let consecutive_blobs = get_slot_consecutive_blobs(
|
||||||
slot,
|
slot,
|
||||||
&self.db.read().unwrap(),
|
&self.db,
|
||||||
&HashMap::new(),
|
&HashMap::new(),
|
||||||
blob_start_index,
|
blob_start_index,
|
||||||
max_entries,
|
max_entries,
|
||||||
|
@ -750,30 +746,28 @@ impl Blocktree {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn is_root(&self, slot: u64) -> bool {
|
pub fn is_root(&self, slot: u64) -> bool {
|
||||||
if let Ok(Some(meta)) = self.meta(slot) {
|
if let Ok(Some(root_slot)) = self.db.get::<cf::Root>(()) {
|
||||||
meta.is_root
|
root_slot == slot
|
||||||
} else {
|
} else {
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn set_root(&self, slot: u64) -> Result<()> {
|
pub fn set_root(&self, slot: u64) -> Result<()> {
|
||||||
let mut root_slot = self.root_slot.write().unwrap();
|
self.db.put::<cf::Root>((), &slot)?;
|
||||||
let mut db = self.db.write().unwrap();
|
|
||||||
|
|
||||||
*root_slot = slot;
|
|
||||||
if let Some(mut meta) = self.meta_cf.get(&db, slot)? {
|
|
||||||
meta.is_root = true;
|
|
||||||
self.meta_cf.put(&mut db, slot, &meta)?;
|
|
||||||
}
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn get_root(&self) -> Result<u64> {
|
||||||
|
let root_opt = self.db.get::<cf::Root>(())?;
|
||||||
|
|
||||||
|
Ok(root_opt.unwrap_or(0))
|
||||||
|
}
|
||||||
|
|
||||||
pub fn get_orphans(&self, max: Option<usize>) -> Vec<u64> {
|
pub fn get_orphans(&self, max: Option<usize>) -> Vec<u64> {
|
||||||
let mut results = vec![];
|
let mut results = vec![];
|
||||||
let db = self.db.read().unwrap();
|
|
||||||
|
|
||||||
let mut iter = db.cursor::<cf::Orphans>().unwrap();
|
let mut iter = self.db.cursor::<cf::Orphans>().unwrap();
|
||||||
iter.seek_to_first();
|
iter.seek_to_first();
|
||||||
while iter.valid() {
|
while iter.valid() {
|
||||||
if let Some(max) = max {
|
if let Some(max) = max {
|
||||||
|
@ -794,19 +788,19 @@ impl Blocktree {
|
||||||
let mut bootstrap_meta = SlotMeta::new(0, 1);
|
let mut bootstrap_meta = SlotMeta::new(0, 1);
|
||||||
let last = blobs.last().unwrap();
|
let last = blobs.last().unwrap();
|
||||||
|
|
||||||
let mut db = self.db.write().unwrap();
|
let mut batch_processor = self.batch_processor.write().unwrap();
|
||||||
|
|
||||||
bootstrap_meta.consumed = last.index() + 1;
|
bootstrap_meta.consumed = last.index() + 1;
|
||||||
bootstrap_meta.received = last.index() + 1;
|
bootstrap_meta.received = last.index() + 1;
|
||||||
bootstrap_meta.is_connected = true;
|
bootstrap_meta.is_connected = true;
|
||||||
|
|
||||||
let mut batch = db.batch()?;
|
let mut batch = batch_processor.batch()?;
|
||||||
batch.put::<cf::SlotMeta>(0, &bootstrap_meta)?;
|
batch.put::<cf::SlotMeta>(0, &bootstrap_meta)?;
|
||||||
for blob in blobs {
|
for blob in blobs {
|
||||||
let serialized_blob_datas = &blob.data[..BLOB_HEADER_SIZE + blob.size()];
|
let serialized_blob_datas = &blob.data[..BLOB_HEADER_SIZE + blob.size()];
|
||||||
batch.put_bytes::<cf::Data>((blob.slot(), blob.index()), serialized_blob_datas)?;
|
batch.put_bytes::<cf::Data>((blob.slot(), blob.index()), serialized_blob_datas)?;
|
||||||
}
|
}
|
||||||
db.write(batch)?;
|
batch_processor.write(batch)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -918,7 +912,7 @@ fn check_insert_data_blob<'a>(
|
||||||
let entry = slot_meta_working_set.entry(blob_slot).or_insert_with(|| {
|
let entry = slot_meta_working_set.entry(blob_slot).or_insert_with(|| {
|
||||||
// Store a 2-tuple of the metadata (working copy, backup copy)
|
// Store a 2-tuple of the metadata (working copy, backup copy)
|
||||||
if let Some(mut meta) = meta_cf
|
if let Some(mut meta) = meta_cf
|
||||||
.get(db, blob_slot)
|
.get(blob_slot)
|
||||||
.expect("Expect database get to succeed")
|
.expect("Expect database get to succeed")
|
||||||
{
|
{
|
||||||
let backup = Some(meta.clone());
|
let backup = Some(meta.clone());
|
||||||
|
@ -965,7 +959,7 @@ fn should_insert_blob(
|
||||||
if blob_index < slot.consumed
|
if blob_index < slot.consumed
|
||||||
|| prev_inserted_blob_datas.contains_key(&(blob_slot, blob_index))
|
|| prev_inserted_blob_datas.contains_key(&(blob_slot, blob_index))
|
||||||
|| data_cf
|
|| data_cf
|
||||||
.get_bytes(db, (blob_slot, blob_index))
|
.get_bytes((blob_slot, blob_index))
|
||||||
.map(|opt| opt.is_some())
|
.map(|opt| opt.is_some())
|
||||||
.unwrap_or(false)
|
.unwrap_or(false)
|
||||||
{
|
{
|
||||||
|
@ -1035,7 +1029,7 @@ fn find_slot_meta_in_db_else_create<'a>(
|
||||||
slot: u64,
|
slot: u64,
|
||||||
insert_map: &'a mut HashMap<u64, Rc<RefCell<SlotMeta>>>,
|
insert_map: &'a mut HashMap<u64, Rc<RefCell<SlotMeta>>>,
|
||||||
) -> Result<Rc<RefCell<SlotMeta>>> {
|
) -> Result<Rc<RefCell<SlotMeta>>> {
|
||||||
if let Some(slot_meta) = db.column::<cf::SlotMeta>().get(db, slot)? {
|
if let Some(slot_meta) = db.column::<cf::SlotMeta>().get(slot)? {
|
||||||
insert_map.insert(slot, Rc::new(RefCell::new(slot_meta)));
|
insert_map.insert(slot, Rc::new(RefCell::new(slot_meta)));
|
||||||
Ok(insert_map.get(&slot).unwrap().clone())
|
Ok(insert_map.get(&slot).unwrap().clone())
|
||||||
} else {
|
} else {
|
||||||
|
@ -1084,7 +1078,7 @@ fn get_slot_consecutive_blobs<'a>(
|
||||||
// Try to find the next blob we're looking for in the prev_inserted_blob_datas
|
// Try to find the next blob we're looking for in the prev_inserted_blob_datas
|
||||||
if let Some(prev_blob_data) = prev_inserted_blob_datas.get(&(slot, current_index)) {
|
if let Some(prev_blob_data) = prev_inserted_blob_datas.get(&(slot, current_index)) {
|
||||||
blobs.push(Cow::Borrowed(*prev_blob_data));
|
blobs.push(Cow::Borrowed(*prev_blob_data));
|
||||||
} else if let Some(blob_data) = data_cf.get_bytes(db, (slot, current_index))? {
|
} else if let Some(blob_data) = data_cf.get_bytes((slot, current_index))? {
|
||||||
// Try to find the next blob we're looking for in the database
|
// Try to find the next blob we're looking for in the database
|
||||||
blobs.push(Cow::Owned(blob_data));
|
blobs.push(Cow::Owned(blob_data));
|
||||||
} else {
|
} else {
|
||||||
|
@ -1345,7 +1339,7 @@ fn recover(
|
||||||
let mut blob_bytes = match new_coding {
|
let mut blob_bytes = match new_coding {
|
||||||
Some((new_coding_index, bytes)) if new_coding_index == i => bytes.to_vec(),
|
Some((new_coding_index, bytes)) if new_coding_index == i => bytes.to_vec(),
|
||||||
_ => erasure_cf
|
_ => erasure_cf
|
||||||
.get_bytes(db, (slot, i))?
|
.get_bytes((slot, i))?
|
||||||
.expect("ErasureMeta must have no false positives"),
|
.expect("ErasureMeta must have no false positives"),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -1368,7 +1362,7 @@ fn recover(
|
||||||
let mut blob_bytes = match prev_inserted_blob_datas.get(&(slot, i)) {
|
let mut blob_bytes = match prev_inserted_blob_datas.get(&(slot, i)) {
|
||||||
Some(bytes) => bytes.to_vec(),
|
Some(bytes) => bytes.to_vec(),
|
||||||
None => data_cf
|
None => data_cf
|
||||||
.get_bytes(db, (slot, i))?
|
.get_bytes((slot, i))?
|
||||||
.expect("erasure_meta must have no false positives"),
|
.expect("erasure_meta must have no false positives"),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -1631,14 +1625,13 @@ pub mod tests {
|
||||||
fn test_put_get_simple() {
|
fn test_put_get_simple() {
|
||||||
let ledger_path = get_tmp_ledger_path("test_put_get_simple");
|
let ledger_path = get_tmp_ledger_path("test_put_get_simple");
|
||||||
let ledger = Blocktree::open(&ledger_path).unwrap();
|
let ledger = Blocktree::open(&ledger_path).unwrap();
|
||||||
let mut db = ledger.db.write().unwrap();
|
|
||||||
|
|
||||||
// Test meta column family
|
// Test meta column family
|
||||||
let meta = SlotMeta::new(0, 1);
|
let meta = SlotMeta::new(0, 1);
|
||||||
ledger.meta_cf.put(&mut db, 0, &meta).unwrap();
|
ledger.meta_cf.put(0, &meta).unwrap();
|
||||||
let result = ledger
|
let result = ledger
|
||||||
.meta_cf
|
.meta_cf
|
||||||
.get(&db, 0)
|
.get(0)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.expect("Expected meta object to exist");
|
.expect("Expected meta object to exist");
|
||||||
|
|
||||||
|
@ -1647,14 +1640,11 @@ pub mod tests {
|
||||||
// Test erasure column family
|
// Test erasure column family
|
||||||
let erasure = vec![1u8; 16];
|
let erasure = vec![1u8; 16];
|
||||||
let erasure_key = (0, 0);
|
let erasure_key = (0, 0);
|
||||||
ledger
|
ledger.erasure_cf.put_bytes(erasure_key, &erasure).unwrap();
|
||||||
.erasure_cf
|
|
||||||
.put_bytes(&mut db, erasure_key, &erasure)
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
let result = ledger
|
let result = ledger
|
||||||
.erasure_cf
|
.erasure_cf
|
||||||
.get_bytes(&db, erasure_key)
|
.get_bytes(erasure_key)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.expect("Expected erasure object to exist");
|
.expect("Expected erasure object to exist");
|
||||||
|
|
||||||
|
@ -1663,18 +1653,17 @@ pub mod tests {
|
||||||
// Test data column family
|
// Test data column family
|
||||||
let data = vec![2u8; 16];
|
let data = vec![2u8; 16];
|
||||||
let data_key = (0, 0);
|
let data_key = (0, 0);
|
||||||
ledger.data_cf.put_bytes(&mut db, data_key, &data).unwrap();
|
ledger.data_cf.put_bytes(data_key, &data).unwrap();
|
||||||
|
|
||||||
let result = ledger
|
let result = ledger
|
||||||
.data_cf
|
.data_cf
|
||||||
.get_bytes(&db, data_key)
|
.get_bytes(data_key)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.expect("Expected data object to exist");
|
.expect("Expected data object to exist");
|
||||||
|
|
||||||
assert_eq!(result, data);
|
assert_eq!(result, data);
|
||||||
|
|
||||||
// Destroying database without closing it first is undefined behavior
|
// Destroying database without closing it first is undefined behavior
|
||||||
drop(db);
|
|
||||||
drop(ledger);
|
drop(ledger);
|
||||||
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
||||||
}
|
}
|
||||||
|
@ -1853,8 +1842,6 @@ pub mod tests {
|
||||||
|
|
||||||
let mut db_iterator = blocktree
|
let mut db_iterator = blocktree
|
||||||
.db
|
.db
|
||||||
.read()
|
|
||||||
.unwrap()
|
|
||||||
.cursor::<cf::Data>()
|
.cursor::<cf::Data>()
|
||||||
.expect("Expected to be able to open database iterator");
|
.expect("Expected to be able to open database iterator");
|
||||||
|
|
||||||
|
@ -2504,10 +2491,7 @@ pub mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
// No orphan slots should exist
|
// No orphan slots should exist
|
||||||
assert!(blocktree
|
assert!(blocktree.orphans_cf.is_empty().unwrap())
|
||||||
.orphans_cf
|
|
||||||
.is_empty(&blocktree.db.read().unwrap())
|
|
||||||
.unwrap())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
||||||
|
@ -2524,20 +2508,14 @@ pub mod tests {
|
||||||
assert!(blocktree.get_slots_since(&vec![0]).unwrap().is_empty());
|
assert!(blocktree.get_slots_since(&vec![0]).unwrap().is_empty());
|
||||||
|
|
||||||
let mut meta0 = SlotMeta::new(0, 0);
|
let mut meta0 = SlotMeta::new(0, 0);
|
||||||
blocktree
|
blocktree.meta_cf.put(0, &meta0).unwrap();
|
||||||
.meta_cf
|
|
||||||
.put(&mut blocktree.db.write().unwrap(), 0, &meta0)
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
// Slot exists, chains to nothing
|
// Slot exists, chains to nothing
|
||||||
let expected: HashMap<u64, Vec<u64>> =
|
let expected: HashMap<u64, Vec<u64>> =
|
||||||
HashMap::from_iter(vec![(0, vec![])].into_iter());
|
HashMap::from_iter(vec![(0, vec![])].into_iter());
|
||||||
assert_eq!(blocktree.get_slots_since(&vec![0]).unwrap(), expected);
|
assert_eq!(blocktree.get_slots_since(&vec![0]).unwrap(), expected);
|
||||||
meta0.next_slots = vec![1, 2];
|
meta0.next_slots = vec![1, 2];
|
||||||
blocktree
|
blocktree.meta_cf.put(0, &meta0).unwrap();
|
||||||
.meta_cf
|
|
||||||
.put(&mut blocktree.db.write().unwrap(), 0, &meta0)
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
// Slot exists, chains to some other slots
|
// Slot exists, chains to some other slots
|
||||||
let expected: HashMap<u64, Vec<u64>> =
|
let expected: HashMap<u64, Vec<u64>> =
|
||||||
|
@ -2547,10 +2525,7 @@ pub mod tests {
|
||||||
|
|
||||||
let mut meta3 = SlotMeta::new(3, 1);
|
let mut meta3 = SlotMeta::new(3, 1);
|
||||||
meta3.next_slots = vec![10, 5];
|
meta3.next_slots = vec![10, 5];
|
||||||
blocktree
|
blocktree.meta_cf.put(3, &meta3).unwrap();
|
||||||
.meta_cf
|
|
||||||
.put(&mut blocktree.db.write().unwrap(), 3, &meta3)
|
|
||||||
.unwrap();
|
|
||||||
let expected: HashMap<u64, Vec<u64>> =
|
let expected: HashMap<u64, Vec<u64>> =
|
||||||
HashMap::from_iter(vec![(0, vec![1, 2]), (3, vec![10, 5])].into_iter());
|
HashMap::from_iter(vec![(0, vec![1, 2]), (3, vec![10, 5])].into_iter());
|
||||||
assert_eq!(blocktree.get_slots_since(&vec![0, 1, 3]).unwrap(), expected);
|
assert_eq!(blocktree.get_slots_since(&vec![0, 1, 3]).unwrap(), expected);
|
||||||
|
@ -2611,10 +2586,7 @@ pub mod tests {
|
||||||
assert!(!is_orphan(&meta));
|
assert!(!is_orphan(&meta));
|
||||||
}
|
}
|
||||||
// Orphans cf is empty
|
// Orphans cf is empty
|
||||||
assert!(blocktree
|
assert!(blocktree.orphans_cf.is_empty().unwrap())
|
||||||
.orphans_cf
|
|
||||||
.is_empty(&blocktree.db.read().unwrap())
|
|
||||||
.unwrap())
|
|
||||||
}
|
}
|
||||||
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
||||||
}
|
}
|
||||||
|
@ -2847,7 +2819,7 @@ pub mod tests {
|
||||||
assert_eq!(slot_meta.consumed, 5);
|
assert_eq!(slot_meta.consumed, 5);
|
||||||
assert!(!should_insert_blob(
|
assert!(!should_insert_blob(
|
||||||
&slot_meta,
|
&slot_meta,
|
||||||
&blocktree.db.read().unwrap(),
|
&blocktree.db,
|
||||||
&HashMap::new(),
|
&HashMap::new(),
|
||||||
&blobs[4].clone()
|
&blobs[4].clone()
|
||||||
));
|
));
|
||||||
|
@ -2857,7 +2829,7 @@ pub mod tests {
|
||||||
let slot_meta = blocktree.meta(0).unwrap().unwrap();
|
let slot_meta = blocktree.meta(0).unwrap().unwrap();
|
||||||
assert!(!should_insert_blob(
|
assert!(!should_insert_blob(
|
||||||
&slot_meta,
|
&slot_meta,
|
||||||
&blocktree.db.read().unwrap(),
|
&blocktree.db,
|
||||||
&HashMap::new(),
|
&HashMap::new(),
|
||||||
&blobs[7].clone()
|
&blobs[7].clone()
|
||||||
));
|
));
|
||||||
|
@ -2870,7 +2842,7 @@ pub mod tests {
|
||||||
blobs[8].set_is_last_in_slot();
|
blobs[8].set_is_last_in_slot();
|
||||||
assert!(!should_insert_blob(
|
assert!(!should_insert_blob(
|
||||||
&slot_meta,
|
&slot_meta,
|
||||||
&blocktree.db.read().unwrap(),
|
&blocktree.db,
|
||||||
&HashMap::new(),
|
&HashMap::new(),
|
||||||
&blobs[8].clone()
|
&blobs[8].clone()
|
||||||
));
|
));
|
||||||
|
@ -2883,7 +2855,7 @@ pub mod tests {
|
||||||
// Trying to insert a blob with index > the "is_last" blob should fail
|
// Trying to insert a blob with index > the "is_last" blob should fail
|
||||||
assert!(!should_insert_blob(
|
assert!(!should_insert_blob(
|
||||||
&slot_meta,
|
&slot_meta,
|
||||||
&blocktree.db.read().unwrap(),
|
&blocktree.db,
|
||||||
&HashMap::new(),
|
&HashMap::new(),
|
||||||
&blobs[10].clone()
|
&blobs[10].clone()
|
||||||
));
|
));
|
||||||
|
@ -3092,11 +3064,9 @@ pub mod tests {
|
||||||
(slot, blob.index());
|
(slot, blob.index());
|
||||||
}
|
}
|
||||||
|
|
||||||
let db = blocktree.db.read().unwrap();
|
|
||||||
|
|
||||||
let erasure_meta = blocktree
|
let erasure_meta = blocktree
|
||||||
.erasure_meta_cf
|
.erasure_meta_cf
|
||||||
.get(&db, (slot, set_index as u64))
|
.get((slot, set_index as u64))
|
||||||
.expect("Erasure Meta should be present")
|
.expect("Erasure Meta should be present")
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
|
@ -3104,7 +3074,7 @@ pub mod tests {
|
||||||
|
|
||||||
let retrieved_data = blocktree
|
let retrieved_data = blocktree
|
||||||
.data_cf
|
.data_cf
|
||||||
.get_bytes(&db, (slot, focused_index as u64))
|
.get_bytes((slot, focused_index as u64))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
assert!(retrieved_data.is_some());
|
assert!(retrieved_data.is_some());
|
||||||
|
@ -3151,7 +3121,7 @@ pub mod tests {
|
||||||
// try recovery even though there aren't enough blobs
|
// try recovery even though there aren't enough blobs
|
||||||
let erasure_meta = blocktree
|
let erasure_meta = blocktree
|
||||||
.erasure_meta_cf
|
.erasure_meta_cf
|
||||||
.get(&blocktree.db.read().unwrap(), (SLOT, SET_INDEX))
|
.get((SLOT, SET_INDEX))
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
|
@ -3160,7 +3130,7 @@ pub mod tests {
|
||||||
let prev_inserted_blob_datas = HashMap::new();
|
let prev_inserted_blob_datas = HashMap::new();
|
||||||
|
|
||||||
let attempt_result = try_erasure_recover(
|
let attempt_result = try_erasure_recover(
|
||||||
&blocktree.db.read().unwrap(),
|
&blocktree.db,
|
||||||
&blocktree.session,
|
&blocktree.session,
|
||||||
&erasure_meta,
|
&erasure_meta,
|
||||||
SLOT,
|
SLOT,
|
||||||
|
@ -3303,10 +3273,7 @@ pub mod tests {
|
||||||
// triggering recovery.
|
// triggering recovery.
|
||||||
let erasure_meta = blocktree
|
let erasure_meta = blocktree
|
||||||
.erasure_meta_cf
|
.erasure_meta_cf
|
||||||
.get(
|
.get((slot, erasure_set.set_index))
|
||||||
&blocktree.db.read().unwrap(),
|
|
||||||
(slot, erasure_set.set_index),
|
|
||||||
)
|
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
|
@ -3340,7 +3307,7 @@ pub mod tests {
|
||||||
|
|
||||||
let erasure_meta = blocktree
|
let erasure_meta = blocktree
|
||||||
.erasure_meta_cf
|
.erasure_meta_cf
|
||||||
.get(&blocktree.db.read().unwrap(), (slot, set_index))
|
.get((slot, set_index))
|
||||||
.expect("DB get must succeed")
|
.expect("DB get must succeed")
|
||||||
.expect("ErasureMeta must be present for each erasure set");
|
.expect("ErasureMeta must be present for each erasure set");
|
||||||
|
|
||||||
|
|
|
@ -9,6 +9,7 @@ use std::borrow::Borrow;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::marker::PhantomData;
|
use std::marker::PhantomData;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
pub mod columns {
|
pub mod columns {
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
|
@ -30,6 +31,10 @@ pub mod columns {
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
/// The erasure meta column
|
/// The erasure meta column
|
||||||
pub struct ErasureMeta;
|
pub struct ErasureMeta;
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
/// The root column
|
||||||
|
pub struct Root;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub trait Backend: Sized + Send + Sync {
|
pub trait Backend: Sized + Send + Sync {
|
||||||
|
@ -112,7 +117,15 @@ pub struct Database<B>
|
||||||
where
|
where
|
||||||
B: Backend,
|
B: Backend,
|
||||||
{
|
{
|
||||||
backend: B,
|
backend: Arc<B>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct BatchProcessor<B>
|
||||||
|
where
|
||||||
|
B: Backend,
|
||||||
|
{
|
||||||
|
backend: Arc<B>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
|
@ -132,7 +145,7 @@ where
|
||||||
B: Backend,
|
B: Backend,
|
||||||
C: Column<B>,
|
C: Column<B>,
|
||||||
{
|
{
|
||||||
backend: PhantomData<B>,
|
backend: Arc<B>,
|
||||||
column: PhantomData<C>,
|
column: PhantomData<C>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -151,7 +164,7 @@ where
|
||||||
B: Backend,
|
B: Backend,
|
||||||
{
|
{
|
||||||
pub fn open(path: &Path) -> Result<Self> {
|
pub fn open(path: &Path) -> Result<Self> {
|
||||||
let backend = B::open(path)?;
|
let backend = Arc::new(B::open(path)?);
|
||||||
|
|
||||||
Ok(Database { backend })
|
Ok(Database { backend })
|
||||||
}
|
}
|
||||||
|
@ -170,7 +183,7 @@ where
|
||||||
.get_cf(self.cf_handle::<C>(), C::key(key).borrow())
|
.get_cf(self.cf_handle::<C>(), C::key(key).borrow())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn put_bytes<C>(&mut self, key: C::Index, data: &[u8]) -> Result<()>
|
pub fn put_bytes<C>(&self, key: C::Index, data: &[u8]) -> Result<()>
|
||||||
where
|
where
|
||||||
C: Column<B>,
|
C: Column<B>,
|
||||||
{
|
{
|
||||||
|
@ -178,7 +191,7 @@ where
|
||||||
.put_cf(self.cf_handle::<C>(), C::key(key).borrow(), data)
|
.put_cf(self.cf_handle::<C>(), C::key(key).borrow(), data)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn delete<C>(&mut self, key: C::Index) -> Result<()>
|
pub fn delete<C>(&self, key: C::Index) -> Result<()>
|
||||||
where
|
where
|
||||||
C: Column<B>,
|
C: Column<B>,
|
||||||
{
|
{
|
||||||
|
@ -202,7 +215,7 @@ where
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn put<C>(&mut self, key: C::Index, value: &C::Type) -> Result<()>
|
pub fn put<C>(&self, key: C::Index, value: &C::Type) -> Result<()>
|
||||||
where
|
where
|
||||||
C: TypedColumn<B>,
|
C: TypedColumn<B>,
|
||||||
{
|
{
|
||||||
|
@ -240,6 +253,35 @@ where
|
||||||
Ok(iter)
|
Ok(iter)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
pub fn cf_handle<C>(&self) -> B::ColumnFamily
|
||||||
|
where
|
||||||
|
C: Column<B>,
|
||||||
|
{
|
||||||
|
self.backend.cf_handle(C::NAME).clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn column<C>(&self) -> LedgerColumn<B, C>
|
||||||
|
where
|
||||||
|
C: Column<B>,
|
||||||
|
{
|
||||||
|
LedgerColumn {
|
||||||
|
backend: Arc::clone(&self.backend),
|
||||||
|
column: PhantomData,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn batch_processor(&self) -> BatchProcessor<B> {
|
||||||
|
BatchProcessor {
|
||||||
|
backend: Arc::clone(&self.backend),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<B> BatchProcessor<B>
|
||||||
|
where
|
||||||
|
B: Backend,
|
||||||
|
{
|
||||||
pub fn batch(&mut self) -> Result<WriteBatch<B>> {
|
pub fn batch(&mut self) -> Result<WriteBatch<B>> {
|
||||||
let db_write_batch = self.backend.batch()?;
|
let db_write_batch = self.backend.batch()?;
|
||||||
let map = self
|
let map = self
|
||||||
|
@ -259,24 +301,6 @@ where
|
||||||
pub fn write(&mut self, batch: WriteBatch<B>) -> Result<()> {
|
pub fn write(&mut self, batch: WriteBatch<B>) -> Result<()> {
|
||||||
self.backend.write(batch.write_batch)
|
self.backend.write(batch.write_batch)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
|
||||||
pub fn cf_handle<C>(&self) -> B::ColumnFamily
|
|
||||||
where
|
|
||||||
C: Column<B>,
|
|
||||||
{
|
|
||||||
self.backend.cf_handle(C::NAME).clone()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn column<C>(&self) -> LedgerColumn<B, C>
|
|
||||||
where
|
|
||||||
C: Column<B>,
|
|
||||||
{
|
|
||||||
LedgerColumn {
|
|
||||||
backend: PhantomData,
|
|
||||||
column: PhantomData,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<B, C> Cursor<B, C>
|
impl<B, C> Cursor<B, C>
|
||||||
|
@ -333,41 +357,47 @@ where
|
||||||
B: Backend,
|
B: Backend,
|
||||||
C: Column<B>,
|
C: Column<B>,
|
||||||
{
|
{
|
||||||
pub fn get_bytes(&self, db: &Database<B>, key: C::Index) -> Result<Option<Vec<u8>>> {
|
pub fn get_bytes(&self, key: C::Index) -> Result<Option<Vec<u8>>> {
|
||||||
db.backend.get_cf(self.handle(db), C::key(key).borrow())
|
self.backend.get_cf(self.handle(), C::key(key).borrow())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn cursor(&self, db: &Database<B>) -> Result<Cursor<B, C>> {
|
pub fn cursor(&self) -> Result<Cursor<B, C>> {
|
||||||
db.cursor()
|
let db_cursor = self.backend.raw_iterator_cf(self.handle())?;
|
||||||
|
|
||||||
|
Ok(Cursor {
|
||||||
|
db_cursor,
|
||||||
|
column: PhantomData,
|
||||||
|
backend: PhantomData,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn iter(&self, db: &Database<B>) -> Result<impl Iterator<Item = (C::Index, Vec<u8>)>> {
|
pub fn iter(&self) -> Result<impl Iterator<Item = (C::Index, Vec<u8>)>> {
|
||||||
db.iter::<C>()
|
let iter = self
|
||||||
|
.backend
|
||||||
|
.iterator_cf(self.handle())?
|
||||||
|
.map(|(key, value)| (C::index(&key), value.into()));
|
||||||
|
|
||||||
|
Ok(iter)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn handle(&self, db: &Database<B>) -> B::ColumnFamily {
|
#[inline]
|
||||||
db.cf_handle::<C>()
|
pub fn handle(&self) -> B::ColumnFamily {
|
||||||
|
self.backend.cf_handle(C::NAME).clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn is_empty(&self, db: &Database<B>) -> Result<bool> {
|
pub fn is_empty(&self) -> Result<bool> {
|
||||||
let mut cursor = self.cursor(db)?;
|
let mut cursor = self.cursor()?;
|
||||||
cursor.seek_to_first();
|
cursor.seek_to_first();
|
||||||
Ok(!cursor.valid())
|
Ok(!cursor.valid())
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
impl<B, C> LedgerColumn<B, C>
|
pub fn put_bytes(&self, key: C::Index, value: &[u8]) -> Result<()> {
|
||||||
where
|
self.backend
|
||||||
B: Backend,
|
.put_cf(self.handle(), C::key(key).borrow(), value)
|
||||||
C: Column<B>,
|
|
||||||
{
|
|
||||||
pub fn put_bytes(&self, db: &mut Database<B>, key: C::Index, value: &[u8]) -> Result<()> {
|
|
||||||
db.backend
|
|
||||||
.put_cf(self.handle(db), C::key(key).borrow(), value)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn delete(&self, db: &mut Database<B>, key: C::Index) -> Result<()> {
|
pub fn delete(&self, key: C::Index) -> Result<()> {
|
||||||
db.backend.delete_cf(self.handle(db), C::key(key).borrow())
|
self.backend.delete_cf(self.handle(), C::key(key).borrow())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -376,18 +406,21 @@ where
|
||||||
B: Backend,
|
B: Backend,
|
||||||
C: TypedColumn<B>,
|
C: TypedColumn<B>,
|
||||||
{
|
{
|
||||||
pub fn get(&self, db: &Database<B>, key: C::Index) -> Result<Option<C::Type>> {
|
pub fn get(&self, key: C::Index) -> Result<Option<C::Type>> {
|
||||||
db.get::<C>(key)
|
if let Some(serialized_value) = self.backend.get_cf(self.handle(), C::key(key).borrow())? {
|
||||||
}
|
let value = deserialize(&serialized_value)?;
|
||||||
}
|
|
||||||
|
|
||||||
impl<B, C> LedgerColumn<B, C>
|
Ok(Some(value))
|
||||||
where
|
} else {
|
||||||
B: Backend,
|
Ok(None)
|
||||||
C: TypedColumn<B>,
|
}
|
||||||
{
|
}
|
||||||
pub fn put(&self, db: &mut Database<B>, key: C::Index, value: &C::Type) -> Result<()> {
|
|
||||||
db.put::<C>(key, value)
|
pub fn put(&self, key: C::Index, value: &C::Type) -> Result<()> {
|
||||||
|
let serialized_value = serialize(value)?;
|
||||||
|
|
||||||
|
self.backend
|
||||||
|
.put_cf(self.handle(), C::key(key).borrow(), &serialized_value)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -119,6 +119,36 @@ impl TypedColumn<Kvs> for cf::Orphans {
|
||||||
type Type = bool;
|
type Type = bool;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Column<Kvs> for cf::Root {
|
||||||
|
const NAME: &'static str = super::ROOT_CF;
|
||||||
|
type Index = ();
|
||||||
|
|
||||||
|
fn key(_: ()) -> Key {
|
||||||
|
Key::default()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn index(_: &Key) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TypedColumn<Kvs> for cf::Root {
|
||||||
|
type Type = u64;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Column<Kvs> for cf::SlotMeta {
|
||||||
|
const NAME: &'static str = super::META_CF;
|
||||||
|
type Index = u64;
|
||||||
|
|
||||||
|
fn key(slot: u64) -> Key {
|
||||||
|
let mut key = Key::default();
|
||||||
|
BigEndian::write_u64(&mut key.0[8..16], slot);
|
||||||
|
key
|
||||||
|
}
|
||||||
|
|
||||||
|
fn index(key: &Key) -> u64 {
|
||||||
|
BigEndian::read_u64(&key.0[8..16])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl Column<Kvs> for cf::SlotMeta {
|
impl Column<Kvs> for cf::SlotMeta {
|
||||||
const NAME: &'static str = super::META_CF;
|
const NAME: &'static str = super::META_CF;
|
||||||
type Index = u64;
|
type Index = u64;
|
||||||
|
|
|
@ -24,8 +24,6 @@ pub struct SlotMeta {
|
||||||
// True if this slot is full (consumed == last_index + 1) and if every
|
// True if this slot is full (consumed == last_index + 1) and if every
|
||||||
// slot that is a parent of this slot is also connected.
|
// slot that is a parent of this slot is also connected.
|
||||||
pub is_connected: bool,
|
pub is_connected: bool,
|
||||||
// True if this slot is a root
|
|
||||||
pub is_root: bool,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SlotMeta {
|
impl SlotMeta {
|
||||||
|
@ -68,7 +66,6 @@ impl SlotMeta {
|
||||||
parent_slot,
|
parent_slot,
|
||||||
next_slots: vec![],
|
next_slots: vec![],
|
||||||
is_connected: slot == 0,
|
is_connected: slot == 0,
|
||||||
is_root: false,
|
|
||||||
last_index: std::u64::MAX,
|
last_index: std::u64::MAX,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -30,7 +30,7 @@ impl Backend for Rocks {
|
||||||
type Error = rocksdb::Error;
|
type Error = rocksdb::Error;
|
||||||
|
|
||||||
fn open(path: &Path) -> Result<Rocks> {
|
fn open(path: &Path) -> Result<Rocks> {
|
||||||
use crate::blocktree::db::columns::{Coding, Data, ErasureMeta, Orphans, SlotMeta};
|
use crate::blocktree::db::columns::{Coding, Data, ErasureMeta, Orphans, Root, SlotMeta};
|
||||||
|
|
||||||
fs::create_dir_all(&path)?;
|
fs::create_dir_all(&path)?;
|
||||||
|
|
||||||
|
@ -44,6 +44,7 @@ impl Backend for Rocks {
|
||||||
let erasure_meta_cf_descriptor =
|
let erasure_meta_cf_descriptor =
|
||||||
ColumnFamilyDescriptor::new(ErasureMeta::NAME, get_cf_options());
|
ColumnFamilyDescriptor::new(ErasureMeta::NAME, get_cf_options());
|
||||||
let orphans_cf_descriptor = ColumnFamilyDescriptor::new(Orphans::NAME, get_cf_options());
|
let orphans_cf_descriptor = ColumnFamilyDescriptor::new(Orphans::NAME, get_cf_options());
|
||||||
|
let root_cf_descriptor = ColumnFamilyDescriptor::new(Root::NAME, get_cf_options());
|
||||||
|
|
||||||
let cfs = vec![
|
let cfs = vec![
|
||||||
meta_cf_descriptor,
|
meta_cf_descriptor,
|
||||||
|
@ -51,6 +52,7 @@ impl Backend for Rocks {
|
||||||
erasure_cf_descriptor,
|
erasure_cf_descriptor,
|
||||||
erasure_meta_cf_descriptor,
|
erasure_meta_cf_descriptor,
|
||||||
orphans_cf_descriptor,
|
orphans_cf_descriptor,
|
||||||
|
root_cf_descriptor,
|
||||||
];
|
];
|
||||||
|
|
||||||
// Open the database
|
// Open the database
|
||||||
|
@ -60,13 +62,14 @@ impl Backend for Rocks {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn columns(&self) -> Vec<&'static str> {
|
fn columns(&self) -> Vec<&'static str> {
|
||||||
use crate::blocktree::db::columns::{Coding, Data, ErasureMeta, Orphans, SlotMeta};
|
use crate::blocktree::db::columns::{Coding, Data, ErasureMeta, Orphans, Root, SlotMeta};
|
||||||
|
|
||||||
vec![
|
vec![
|
||||||
Coding::NAME,
|
Coding::NAME,
|
||||||
ErasureMeta::NAME,
|
ErasureMeta::NAME,
|
||||||
Data::NAME,
|
Data::NAME,
|
||||||
Orphans::NAME,
|
Orphans::NAME,
|
||||||
|
Root::NAME,
|
||||||
SlotMeta::NAME,
|
SlotMeta::NAME,
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
@ -170,6 +173,21 @@ impl TypedColumn<Rocks> for cf::Orphans {
|
||||||
type Type = bool;
|
type Type = bool;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Column<Rocks> for cf::Root {
|
||||||
|
const NAME: &'static str = super::ROOT_CF;
|
||||||
|
type Index = ();
|
||||||
|
|
||||||
|
fn key(_: ()) -> Vec<u8> {
|
||||||
|
vec![0; 8]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn index(_: &[u8]) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TypedColumn<Rocks> for cf::Root {
|
||||||
|
type Type = u64;
|
||||||
|
}
|
||||||
|
|
||||||
impl Column<Rocks> for cf::SlotMeta {
|
impl Column<Rocks> for cf::SlotMeta {
|
||||||
const NAME: &'static str = super::META_CF;
|
const NAME: &'static str = super::META_CF;
|
||||||
type Index = u64;
|
type Index = u64;
|
||||||
|
|
|
@ -201,7 +201,7 @@ impl RepairService {
|
||||||
fn generate_repairs(blocktree: &Blocktree, max_repairs: usize) -> Result<(Vec<RepairType>)> {
|
fn generate_repairs(blocktree: &Blocktree, max_repairs: usize) -> Result<(Vec<RepairType>)> {
|
||||||
// Slot height and blob indexes for blobs we want to repair
|
// Slot height and blob indexes for blobs we want to repair
|
||||||
let mut repairs: Vec<RepairType> = vec![];
|
let mut repairs: Vec<RepairType> = vec![];
|
||||||
let slot = *blocktree.root_slot.read().unwrap();
|
let slot = blocktree.get_root()?;
|
||||||
Self::generate_repairs_for_fork(blocktree, &mut repairs, max_repairs, slot);
|
Self::generate_repairs_for_fork(blocktree, &mut repairs, max_repairs, slot);
|
||||||
|
|
||||||
// TODO: Incorporate gossip to determine priorities for repair?
|
// TODO: Incorporate gossip to determine priorities for repair?
|
||||||
|
|
Loading…
Reference in New Issue