2019-02-07 20:52:39 -08:00
|
|
|
//! The `block_tree` module provides functions for parallel verification of the
|
2018-11-15 15:53:31 -08:00
|
|
|
//! Proof of History ledger as well as iterative read, append write, and random
|
|
|
|
//! access read to a persistent file-based ledger.
|
|
|
|
|
2018-12-07 19:16:27 -08:00
|
|
|
use crate::entry::Entry;
|
2019-04-26 08:52:10 -07:00
|
|
|
use crate::erasure::{self, Session};
|
2018-12-07 19:16:27 -08:00
|
|
|
use crate::packet::{Blob, SharedBlob, BLOB_HEADER_SIZE};
|
|
|
|
use crate::result::{Error, Result};
|
2019-03-15 16:42:47 -07:00
|
|
|
#[cfg(feature = "kvstore")]
|
|
|
|
use solana_kvstore as kvstore;
|
2019-03-11 15:53:14 -07:00
|
|
|
|
2019-04-02 14:58:07 -07:00
|
|
|
use bincode::deserialize;
|
2019-03-11 15:53:14 -07:00
|
|
|
|
2019-02-07 15:10:54 -08:00
|
|
|
use hashbrown::HashMap;
|
2019-03-11 15:53:14 -07:00
|
|
|
|
|
|
|
#[cfg(not(feature = "kvstore"))]
|
|
|
|
use rocksdb;
|
|
|
|
|
2019-04-11 14:14:57 -07:00
|
|
|
use solana_metrics::counter::Counter;
|
2019-03-11 15:53:14 -07:00
|
|
|
|
2019-02-18 22:26:22 -08:00
|
|
|
use solana_sdk::genesis_block::GenesisBlock;
|
2019-01-22 15:50:36 -08:00
|
|
|
use solana_sdk::hash::Hash;
|
2018-12-11 09:14:23 -08:00
|
|
|
use solana_sdk::signature::{Keypair, KeypairUtil};
|
2019-03-11 15:53:14 -07:00
|
|
|
|
2019-02-07 15:10:54 -08:00
|
|
|
use std::borrow::{Borrow, Cow};
|
|
|
|
use std::cell::RefCell;
|
2019-01-08 15:53:44 -08:00
|
|
|
use std::cmp;
|
2019-01-24 12:04:04 -08:00
|
|
|
use std::fs;
|
2018-11-15 15:53:31 -08:00
|
|
|
use std::io;
|
2019-02-07 15:10:54 -08:00
|
|
|
use std::rc::Rc;
|
2019-02-04 15:33:43 -08:00
|
|
|
use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
|
2019-04-06 19:41:22 -07:00
|
|
|
use std::sync::{Arc, RwLock};
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-04-11 14:14:57 -07:00
|
|
|
pub use self::meta::*;
|
|
|
|
|
2019-03-11 15:53:14 -07:00
|
|
|
mod db;
|
2019-04-11 14:14:57 -07:00
|
|
|
mod meta;
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-03-26 23:36:39 -07:00
|
|
|
macro_rules! db_imports {
|
|
|
|
{ $mod:ident, $db:ident, $db_path:expr } => {
|
|
|
|
mod $mod;
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-04-02 14:58:07 -07:00
|
|
|
use $mod::$db;
|
|
|
|
use db::columns as cf;
|
|
|
|
|
|
|
|
pub use db::columns;
|
|
|
|
|
|
|
|
pub type Database = db::Database<$db>;
|
|
|
|
pub type Cursor<C> = db::Cursor<$db, C>;
|
|
|
|
pub type LedgerColumn<C> = db::LedgerColumn<$db, C>;
|
|
|
|
pub type WriteBatch = db::WriteBatch<$db>;
|
2019-05-03 14:46:02 -07:00
|
|
|
type BatchProcessor = db::BatchProcessor<$db>;
|
2019-04-02 14:58:07 -07:00
|
|
|
|
|
|
|
pub trait Column: db::Column<$db> {}
|
|
|
|
impl<C: db::Column<$db>> Column for C {}
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-03-26 23:36:39 -07:00
|
|
|
pub const BLOCKTREE_DIRECTORY: &str = $db_path;
|
|
|
|
};
|
|
|
|
}
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-03-11 15:53:14 -07:00
|
|
|
#[cfg(not(feature = "kvstore"))]
|
2019-03-26 23:36:39 -07:00
|
|
|
db_imports! {rocks, Rocks, "rocksdb"}
|
2019-03-11 15:53:14 -07:00
|
|
|
#[cfg(feature = "kvstore")]
|
2019-03-26 23:36:39 -07:00
|
|
|
db_imports! {kvs, Kvs, "kvstore"}
|
2018-12-20 11:16:07 -08:00
|
|
|
|
2019-03-11 15:53:14 -07:00
|
|
|
#[derive(Debug)]
|
|
|
|
pub enum BlocktreeError {
|
|
|
|
BlobForIndexExists,
|
|
|
|
InvalidBlobData,
|
|
|
|
RocksDb(rocksdb::Error),
|
|
|
|
#[cfg(feature = "kvstore")]
|
|
|
|
KvsDb(kvstore::Error),
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
// ledger window
|
2019-02-07 20:52:39 -08:00
|
|
|
pub struct Blocktree {
|
2019-05-03 14:46:02 -07:00
|
|
|
db: Arc<Database>,
|
2019-04-02 14:58:07 -07:00
|
|
|
meta_cf: LedgerColumn<cf::SlotMeta>,
|
|
|
|
data_cf: LedgerColumn<cf::Data>,
|
|
|
|
erasure_cf: LedgerColumn<cf::Coding>,
|
2019-04-11 14:14:57 -07:00
|
|
|
erasure_meta_cf: LedgerColumn<cf::ErasureMeta>,
|
2019-04-06 19:41:22 -07:00
|
|
|
orphans_cf: LedgerColumn<cf::Orphans>,
|
2019-05-03 14:46:02 -07:00
|
|
|
batch_processor: Arc<RwLock<BatchProcessor>>,
|
2019-04-18 21:56:43 -07:00
|
|
|
session: Arc<erasure::Session>,
|
2019-03-11 13:58:23 -07:00
|
|
|
pub new_blobs_signals: Vec<SyncSender<bool>>,
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Column family for metadata about a leader slot
|
|
|
|
pub const META_CF: &str = "meta";
|
|
|
|
// Column family for the data in a leader slot
|
|
|
|
pub const DATA_CF: &str = "data";
|
|
|
|
// Column family for erasure data
|
|
|
|
pub const ERASURE_CF: &str = "erasure";
|
2019-04-11 14:14:57 -07:00
|
|
|
pub const ERASURE_META_CF: &str = "erasure_meta";
|
2019-04-06 19:41:22 -07:00
|
|
|
// Column family for orphans data
|
|
|
|
pub const ORPHANS_CF: &str = "orphans";
|
2019-05-03 14:46:02 -07:00
|
|
|
// Column family for root data
|
|
|
|
pub const ROOT_CF: &str = "root";
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
impl Blocktree {
|
2019-04-02 14:58:07 -07:00
|
|
|
/// Opens a Ledger in directory, provides "infinite" window of blobs
|
|
|
|
pub fn open(ledger_path: &str) -> Result<Blocktree> {
|
|
|
|
use std::path::Path;
|
|
|
|
|
|
|
|
fs::create_dir_all(&ledger_path)?;
|
|
|
|
let ledger_path = Path::new(&ledger_path).join(BLOCKTREE_DIRECTORY);
|
|
|
|
|
|
|
|
// Open the database
|
2019-04-26 08:52:10 -07:00
|
|
|
let db = Database::open(&ledger_path)?;
|
2019-04-02 14:58:07 -07:00
|
|
|
|
2019-05-03 14:46:02 -07:00
|
|
|
let batch_processor = Arc::new(RwLock::new(db.batch_processor()));
|
|
|
|
|
2019-04-02 14:58:07 -07:00
|
|
|
// Create the metadata column family
|
2019-04-26 08:52:10 -07:00
|
|
|
let meta_cf = db.column();
|
2019-04-02 14:58:07 -07:00
|
|
|
|
|
|
|
// Create the data column family
|
2019-04-26 08:52:10 -07:00
|
|
|
let data_cf = db.column();
|
2019-04-02 14:58:07 -07:00
|
|
|
|
|
|
|
// Create the erasure column family
|
2019-04-26 08:52:10 -07:00
|
|
|
let erasure_cf = db.column();
|
2019-04-18 21:56:43 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
let erasure_meta_cf = db.column();
|
2019-04-02 14:58:07 -07:00
|
|
|
|
2019-04-06 19:41:22 -07:00
|
|
|
// Create the orphans column family. An "orphan" is defined as
|
|
|
|
// the head of a detached chain of slots, i.e. a slot with no
|
|
|
|
// known parent
|
2019-04-26 08:52:10 -07:00
|
|
|
let orphans_cf = db.column();
|
2019-04-02 14:58:07 -07:00
|
|
|
|
2019-04-18 21:56:43 -07:00
|
|
|
// setup erasure
|
|
|
|
let session = Arc::new(erasure::Session::default());
|
|
|
|
|
2019-05-03 14:46:02 -07:00
|
|
|
let db = Arc::new(db);
|
2019-04-26 08:52:10 -07:00
|
|
|
|
2019-04-02 14:58:07 -07:00
|
|
|
Ok(Blocktree {
|
|
|
|
db,
|
|
|
|
meta_cf,
|
|
|
|
data_cf,
|
|
|
|
erasure_cf,
|
2019-04-11 14:14:57 -07:00
|
|
|
erasure_meta_cf,
|
2019-04-06 19:41:22 -07:00
|
|
|
orphans_cf,
|
2019-04-18 21:56:43 -07:00
|
|
|
session,
|
2019-04-02 14:58:07 -07:00
|
|
|
new_blobs_signals: vec![],
|
2019-05-03 14:46:02 -07:00
|
|
|
batch_processor,
|
2019-04-02 14:58:07 -07:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2019-02-17 13:05:47 -08:00
|
|
|
pub fn open_with_signal(ledger_path: &str) -> Result<(Self, Receiver<bool>)> {
|
2019-02-07 20:52:39 -08:00
|
|
|
let mut blocktree = Self::open(ledger_path)?;
|
2019-02-04 15:33:43 -08:00
|
|
|
let (signal_sender, signal_receiver) = sync_channel(1);
|
2019-02-17 13:05:47 -08:00
|
|
|
blocktree.new_blobs_signals = vec![signal_sender];
|
2019-02-04 15:33:43 -08:00
|
|
|
|
2019-02-17 13:05:47 -08:00
|
|
|
Ok((blocktree, signal_receiver))
|
2019-02-04 15:33:43 -08:00
|
|
|
}
|
|
|
|
|
2019-04-02 14:58:07 -07:00
|
|
|
pub fn destroy(ledger_path: &str) -> Result<()> {
|
|
|
|
// Database::destroy() fails is the path doesn't exist
|
|
|
|
fs::create_dir_all(ledger_path)?;
|
|
|
|
let path = std::path::Path::new(ledger_path).join(BLOCKTREE_DIRECTORY);
|
|
|
|
Database::destroy(&path)
|
|
|
|
}
|
|
|
|
|
2019-03-05 14:18:29 -08:00
|
|
|
pub fn meta(&self, slot: u64) -> Result<Option<SlotMeta>> {
|
2019-05-03 14:46:02 -07:00
|
|
|
self.meta_cf.get(slot)
|
2019-04-26 08:52:10 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
pub fn erasure_meta(&self, slot: u64, set_index: u64) -> Result<Option<ErasureMeta>> {
|
2019-05-03 14:46:02 -07:00
|
|
|
self.erasure_meta_cf.get((slot, set_index))
|
2019-01-08 11:41:55 -08:00
|
|
|
}
|
|
|
|
|
2019-04-06 19:41:22 -07:00
|
|
|
pub fn orphan(&self, slot: u64) -> Result<Option<bool>> {
|
2019-05-03 14:46:02 -07:00
|
|
|
self.orphans_cf.get(slot)
|
2019-03-29 16:07:24 -07:00
|
|
|
}
|
|
|
|
|
2019-03-05 14:18:29 -08:00
|
|
|
pub fn get_next_slot(&self, slot: u64) -> Result<Option<u64>> {
|
2019-05-03 14:46:02 -07:00
|
|
|
let mut db_iterator = self.db.cursor::<cf::SlotMeta>()?;
|
2019-04-26 08:52:10 -07:00
|
|
|
|
2019-04-02 14:58:07 -07:00
|
|
|
db_iterator.seek(slot + 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
if !db_iterator.valid() {
|
|
|
|
Ok(None)
|
|
|
|
} else {
|
2019-04-02 14:58:07 -07:00
|
|
|
let next_slot = db_iterator.key().expect("Expected valid key");
|
|
|
|
Ok(Some(next_slot))
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-02-08 14:19:28 -08:00
|
|
|
pub fn write_shared_blobs<I>(&self, shared_blobs: I) -> Result<()>
|
2018-11-24 19:32:33 -08:00
|
|
|
where
|
|
|
|
I: IntoIterator,
|
|
|
|
I::Item: Borrow<SharedBlob>,
|
|
|
|
{
|
2018-12-19 16:11:47 -08:00
|
|
|
let c_blobs: Vec<_> = shared_blobs
|
|
|
|
.into_iter()
|
|
|
|
.map(move |s| s.borrow().clone())
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
let r_blobs: Vec<_> = c_blobs.iter().map(move |b| b.read().unwrap()).collect();
|
|
|
|
|
|
|
|
let blobs = r_blobs.iter().map(|s| &**s);
|
|
|
|
|
2019-02-08 14:19:28 -08:00
|
|
|
self.insert_data_blobs(blobs)
|
2018-11-19 23:20:18 -08:00
|
|
|
}
|
|
|
|
|
2019-02-08 14:19:28 -08:00
|
|
|
pub fn write_blobs<I>(&self, blobs: I) -> Result<()>
|
2018-11-15 15:53:31 -08:00
|
|
|
where
|
2018-12-22 19:30:30 -08:00
|
|
|
I: IntoIterator,
|
2019-02-07 15:10:54 -08:00
|
|
|
I::Item: Borrow<Blob>,
|
2018-11-15 15:53:31 -08:00
|
|
|
{
|
2019-02-08 14:19:28 -08:00
|
|
|
self.insert_data_blobs(blobs)
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
|
|
|
|
2019-02-12 13:14:33 -08:00
|
|
|
pub fn write_entries<I>(
|
|
|
|
&self,
|
|
|
|
start_slot: u64,
|
|
|
|
num_ticks_in_start_slot: u64,
|
|
|
|
start_index: u64,
|
2019-03-14 15:18:37 -07:00
|
|
|
ticks_per_slot: u64,
|
2019-02-12 13:14:33 -08:00
|
|
|
entries: I,
|
|
|
|
) -> Result<()>
|
2018-11-24 19:32:33 -08:00
|
|
|
where
|
|
|
|
I: IntoIterator,
|
|
|
|
I::Item: Borrow<Entry>,
|
|
|
|
{
|
2019-02-12 13:14:33 -08:00
|
|
|
assert!(num_ticks_in_start_slot < ticks_per_slot);
|
|
|
|
let mut remaining_ticks_in_slot = ticks_per_slot - num_ticks_in_start_slot;
|
|
|
|
|
|
|
|
let mut blobs = vec![];
|
|
|
|
let mut current_index = start_index;
|
|
|
|
let mut current_slot = start_slot;
|
2019-02-17 04:36:49 -08:00
|
|
|
let mut parent_slot = {
|
|
|
|
if current_slot == 0 {
|
|
|
|
current_slot
|
|
|
|
} else {
|
|
|
|
current_slot - 1
|
|
|
|
}
|
|
|
|
};
|
2019-02-12 13:14:33 -08:00
|
|
|
// Find all the entries for start_slot
|
|
|
|
for entry in entries {
|
|
|
|
if remaining_ticks_in_slot == 0 {
|
|
|
|
current_slot += 1;
|
|
|
|
current_index = 0;
|
2019-02-17 04:36:49 -08:00
|
|
|
parent_slot = current_slot - 1;
|
2019-02-12 13:14:33 -08:00
|
|
|
remaining_ticks_in_slot = ticks_per_slot;
|
|
|
|
}
|
|
|
|
|
|
|
|
let mut b = entry.borrow().to_blob();
|
2019-02-12 16:06:23 -08:00
|
|
|
|
|
|
|
if entry.borrow().is_tick() {
|
|
|
|
remaining_ticks_in_slot -= 1;
|
|
|
|
if remaining_ticks_in_slot == 0 {
|
|
|
|
b.set_is_last_in_slot();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-02-12 13:14:33 -08:00
|
|
|
b.set_index(current_index);
|
|
|
|
b.set_slot(current_slot);
|
2019-02-17 04:36:49 -08:00
|
|
|
b.set_parent(parent_slot);
|
2019-02-12 13:14:33 -08:00
|
|
|
blobs.push(b);
|
|
|
|
|
|
|
|
current_index += 1;
|
|
|
|
}
|
2018-12-12 20:42:12 -08:00
|
|
|
|
2018-12-22 19:30:30 -08:00
|
|
|
self.write_blobs(&blobs)
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
|
|
|
|
2019-02-08 14:19:28 -08:00
|
|
|
pub fn insert_data_blobs<I>(&self, new_blobs: I) -> Result<()>
|
2018-12-19 16:11:47 -08:00
|
|
|
where
|
|
|
|
I: IntoIterator,
|
|
|
|
I::Item: Borrow<Blob>,
|
|
|
|
{
|
2019-05-03 14:46:02 -07:00
|
|
|
let db = &*self.db;
|
|
|
|
let mut batch_processor = self.batch_processor.write().unwrap();
|
|
|
|
let mut write_batch = batch_processor.batch()?;
|
2019-04-26 08:52:10 -07:00
|
|
|
|
2019-04-24 15:53:01 -07:00
|
|
|
let new_blobs: Vec<_> = new_blobs.into_iter().collect();
|
|
|
|
let mut recovered_data = vec![];
|
|
|
|
|
|
|
|
let mut prev_inserted_blob_datas = HashMap::new();
|
2019-03-05 14:18:29 -08:00
|
|
|
// A map from slot to a 2-tuple of metadata: (working copy, backup copy),
|
2019-02-07 15:10:54 -08:00
|
|
|
// so we can detect changes to the slot metadata later
|
|
|
|
let mut slot_meta_working_set = HashMap::new();
|
2019-04-11 14:14:57 -07:00
|
|
|
let mut erasure_meta_working_set = HashMap::new();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
for blob in new_blobs.iter() {
|
|
|
|
let blob = blob.borrow();
|
|
|
|
let blob_slot = blob.slot();
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-04-18 21:56:43 -07:00
|
|
|
let set_index = ErasureMeta::set_index_for(blob.index());
|
2019-04-24 15:53:01 -07:00
|
|
|
erasure_meta_working_set
|
2019-04-18 21:56:43 -07:00
|
|
|
.entry((blob_slot, set_index))
|
|
|
|
.or_insert_with(|| {
|
|
|
|
self.erasure_meta_cf
|
2019-05-03 14:46:02 -07:00
|
|
|
.get((blob_slot, set_index))
|
2019-04-18 21:56:43 -07:00
|
|
|
.expect("Expect database get to succeed")
|
|
|
|
.unwrap_or_else(|| ErasureMeta::new(set_index))
|
|
|
|
});
|
2019-04-24 15:53:01 -07:00
|
|
|
}
|
2019-04-18 21:56:43 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
insert_data_blob_batch(
|
2019-04-24 15:53:01 -07:00
|
|
|
new_blobs.iter().map(Borrow::borrow),
|
2019-04-26 08:52:10 -07:00
|
|
|
&db,
|
2019-04-24 15:53:01 -07:00
|
|
|
&mut slot_meta_working_set,
|
|
|
|
&mut erasure_meta_working_set,
|
|
|
|
&mut prev_inserted_blob_datas,
|
|
|
|
&mut write_batch,
|
|
|
|
)?;
|
|
|
|
|
|
|
|
for (&(slot, _), erasure_meta) in erasure_meta_working_set.iter_mut() {
|
2019-04-26 08:52:10 -07:00
|
|
|
if let Some((data, coding)) = try_erasure_recover(
|
|
|
|
&db,
|
|
|
|
&self.session,
|
|
|
|
&erasure_meta,
|
|
|
|
slot,
|
|
|
|
&prev_inserted_blob_datas,
|
|
|
|
None,
|
|
|
|
)? {
|
2019-04-24 15:53:01 -07:00
|
|
|
for data_blob in data {
|
|
|
|
recovered_data.push(data_blob);
|
|
|
|
}
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-04-24 15:53:01 -07:00
|
|
|
for coding_blob in coding {
|
|
|
|
erasure_meta.set_coding_present(coding_blob.index(), true);
|
|
|
|
|
|
|
|
write_batch.put_bytes::<cf::Coding>(
|
|
|
|
(coding_blob.slot(), coding_blob.index()),
|
|
|
|
&coding_blob.data[..BLOB_HEADER_SIZE + coding_blob.size()],
|
|
|
|
)?;
|
|
|
|
}
|
|
|
|
}
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
insert_data_blob_batch(
|
2019-04-24 15:53:01 -07:00
|
|
|
recovered_data.iter(),
|
2019-04-26 08:52:10 -07:00
|
|
|
&db,
|
2019-04-24 15:53:01 -07:00
|
|
|
&mut slot_meta_working_set,
|
|
|
|
&mut erasure_meta_working_set,
|
|
|
|
&mut prev_inserted_blob_datas,
|
|
|
|
&mut write_batch,
|
|
|
|
)?;
|
|
|
|
|
2019-02-07 15:10:54 -08:00
|
|
|
// Handle chaining for the working set
|
2019-04-26 08:52:10 -07:00
|
|
|
handle_chaining(&db, &mut write_batch, &slot_meta_working_set)?;
|
2019-02-07 15:10:54 -08:00
|
|
|
let mut should_signal = false;
|
|
|
|
|
|
|
|
// Check if any metadata was changed, if so, insert the new version of the
|
|
|
|
// metadata into the write batch
|
2019-03-29 16:07:24 -07:00
|
|
|
for (slot, (meta, meta_backup)) in slot_meta_working_set.iter() {
|
|
|
|
let meta: &SlotMeta = &RefCell::borrow(&*meta);
|
2019-02-07 15:10:54 -08:00
|
|
|
// Check if the working copy of the metadata has changed
|
|
|
|
if Some(meta) != meta_backup.as_ref() {
|
2019-04-26 08:52:10 -07:00
|
|
|
should_signal = should_signal || slot_has_updates(meta, &meta_backup);
|
2019-04-02 14:58:07 -07:00
|
|
|
write_batch.put::<cf::SlotMeta>(*slot, &meta)?;
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
2018-12-19 16:11:47 -08:00
|
|
|
}
|
|
|
|
|
2019-04-24 15:53:01 -07:00
|
|
|
for ((slot, set_index), erasure_meta) in erasure_meta_working_set {
|
|
|
|
write_batch.put::<cf::ErasureMeta>((slot, set_index), &erasure_meta)?;
|
2019-04-11 14:14:57 -07:00
|
|
|
}
|
|
|
|
|
2019-02-07 15:10:54 -08:00
|
|
|
if should_signal {
|
2019-02-04 15:33:43 -08:00
|
|
|
for signal in self.new_blobs_signals.iter() {
|
|
|
|
let _ = signal.try_send(true);
|
|
|
|
}
|
|
|
|
}
|
2018-12-19 16:11:47 -08:00
|
|
|
|
2019-05-03 14:46:02 -07:00
|
|
|
batch_processor.write(write_batch)?;
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-02-08 14:19:28 -08:00
|
|
|
Ok(())
|
2018-12-19 16:11:47 -08:00
|
|
|
}
|
|
|
|
|
2018-11-15 15:53:31 -08:00
|
|
|
// Fill 'buf' with num_blobs or most number of consecutive
|
|
|
|
// whole blobs that fit into buf.len()
|
|
|
|
//
|
|
|
|
// Return tuple of (number of blob read, total size of blobs read)
|
2019-01-08 15:23:45 -08:00
|
|
|
pub fn read_blobs_bytes(
|
2018-12-18 15:18:57 -08:00
|
|
|
&self,
|
2018-11-15 15:53:31 -08:00
|
|
|
start_index: u64,
|
|
|
|
num_blobs: u64,
|
|
|
|
buf: &mut [u8],
|
2019-03-05 14:18:29 -08:00
|
|
|
slot: u64,
|
2018-11-15 15:53:31 -08:00
|
|
|
) -> Result<(u64, u64)> {
|
2019-05-03 14:46:02 -07:00
|
|
|
let mut db_iterator = self.db.cursor::<cf::Data>()?;
|
2019-04-26 08:52:10 -07:00
|
|
|
|
2019-04-02 14:58:07 -07:00
|
|
|
db_iterator.seek((slot, start_index));
|
2018-11-15 15:53:31 -08:00
|
|
|
let mut total_blobs = 0;
|
|
|
|
let mut total_current_size = 0;
|
|
|
|
for expected_index in start_index..start_index + num_blobs {
|
|
|
|
if !db_iterator.valid() {
|
|
|
|
if expected_index == start_index {
|
|
|
|
return Err(Error::IO(io::Error::new(
|
|
|
|
io::ErrorKind::NotFound,
|
|
|
|
"Blob at start_index not found",
|
|
|
|
)));
|
|
|
|
} else {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check key is the next sequential key based on
|
|
|
|
// blob index
|
2019-04-02 14:58:07 -07:00
|
|
|
let (_, index) = db_iterator.key().expect("Expected valid key");
|
2018-11-15 15:53:31 -08:00
|
|
|
if index != expected_index {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get the blob data
|
2019-04-02 14:58:07 -07:00
|
|
|
let value = &db_iterator.value_bytes();
|
2018-11-15 15:53:31 -08:00
|
|
|
|
|
|
|
if value.is_none() {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
let value = value.as_ref().unwrap();
|
|
|
|
let blob_data_len = value.len();
|
|
|
|
|
|
|
|
if total_current_size + blob_data_len > buf.len() {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
buf[total_current_size..total_current_size + value.len()].copy_from_slice(value);
|
|
|
|
total_current_size += blob_data_len;
|
|
|
|
total_blobs += 1;
|
|
|
|
|
|
|
|
// TODO: Change this logic to support looking for data
|
|
|
|
// that spans multiple leader slots, once we support
|
|
|
|
// a window that knows about different leader slots
|
|
|
|
db_iterator.next();
|
|
|
|
}
|
|
|
|
|
|
|
|
Ok((total_blobs, total_current_size as u64))
|
|
|
|
}
|
2018-12-11 09:14:23 -08:00
|
|
|
|
2019-01-08 15:53:44 -08:00
|
|
|
pub fn get_coding_blob_bytes(&self, slot: u64, index: u64) -> Result<Option<Vec<u8>>> {
|
2019-05-03 14:46:02 -07:00
|
|
|
self.erasure_cf.get_bytes((slot, index))
|
2019-01-08 15:53:44 -08:00
|
|
|
}
|
2019-04-18 21:56:43 -07:00
|
|
|
|
2019-01-08 15:53:44 -08:00
|
|
|
pub fn delete_coding_blob(&self, slot: u64, index: u64) -> Result<()> {
|
2019-04-18 21:56:43 -07:00
|
|
|
let set_index = ErasureMeta::set_index_for(index);
|
2019-05-03 14:46:02 -07:00
|
|
|
let mut batch_processor = self.batch_processor.write().unwrap();
|
2019-04-18 21:56:43 -07:00
|
|
|
|
|
|
|
let mut erasure_meta = self
|
|
|
|
.erasure_meta_cf
|
2019-05-03 14:46:02 -07:00
|
|
|
.get((slot, set_index))?
|
2019-04-18 21:56:43 -07:00
|
|
|
.unwrap_or_else(|| ErasureMeta::new(set_index));
|
|
|
|
|
|
|
|
erasure_meta.set_coding_present(index, false);
|
|
|
|
|
2019-05-03 14:46:02 -07:00
|
|
|
let mut batch = batch_processor.batch()?;
|
2019-04-18 21:56:43 -07:00
|
|
|
|
|
|
|
batch.delete::<cf::Coding>((slot, index))?;
|
|
|
|
batch.put::<cf::ErasureMeta>((slot, set_index), &erasure_meta)?;
|
|
|
|
|
2019-05-03 14:46:02 -07:00
|
|
|
batch_processor.write(batch)?;
|
2019-04-18 21:56:43 -07:00
|
|
|
Ok(())
|
2019-01-08 15:53:44 -08:00
|
|
|
}
|
2019-04-18 21:56:43 -07:00
|
|
|
|
2019-01-08 15:53:44 -08:00
|
|
|
pub fn get_data_blob_bytes(&self, slot: u64, index: u64) -> Result<Option<Vec<u8>>> {
|
2019-05-03 14:46:02 -07:00
|
|
|
self.data_cf.get_bytes((slot, index))
|
2019-01-08 15:53:44 -08:00
|
|
|
}
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-04-18 21:56:43 -07:00
|
|
|
/// For benchmarks, testing, and setup.
|
|
|
|
/// Does no metadata tracking. Use with care.
|
|
|
|
pub fn put_data_blob_bytes(&self, slot: u64, index: u64, bytes: &[u8]) -> Result<()> {
|
2019-05-03 14:46:02 -07:00
|
|
|
self.data_cf.put_bytes((slot, index), bytes)
|
2019-04-17 12:52:12 -07:00
|
|
|
}
|
|
|
|
|
2019-04-24 15:53:01 -07:00
|
|
|
/// For benchmarks, testing, and setup.
|
|
|
|
/// Does no metadata tracking. Use with care.
|
2019-04-18 21:56:43 -07:00
|
|
|
pub fn put_coding_blob_bytes_raw(&self, slot: u64, index: u64, bytes: &[u8]) -> Result<()> {
|
2019-05-03 14:46:02 -07:00
|
|
|
self.erasure_cf.put_bytes((slot, index), bytes)
|
2019-04-17 18:04:30 -07:00
|
|
|
}
|
|
|
|
|
2019-04-11 14:14:57 -07:00
|
|
|
/// this function will insert coding blobs and also automatically track erasure-related
|
|
|
|
/// metadata. If recovery is available it will be done
|
|
|
|
pub fn put_coding_blob_bytes(&self, slot: u64, index: u64, bytes: &[u8]) -> Result<()> {
|
|
|
|
let set_index = ErasureMeta::set_index_for(index);
|
2019-05-03 14:46:02 -07:00
|
|
|
let mut batch_processor = self.batch_processor.write().unwrap();
|
2019-04-26 08:52:10 -07:00
|
|
|
|
2019-04-11 14:14:57 -07:00
|
|
|
let mut erasure_meta = self
|
|
|
|
.erasure_meta_cf
|
2019-05-03 14:46:02 -07:00
|
|
|
.get((slot, set_index))?
|
2019-04-11 14:14:57 -07:00
|
|
|
.unwrap_or_else(|| ErasureMeta::new(set_index));
|
|
|
|
|
2019-04-18 21:56:43 -07:00
|
|
|
erasure_meta.set_coding_present(index, true);
|
2019-04-19 20:22:51 -07:00
|
|
|
erasure_meta.set_size(bytes.len() - BLOB_HEADER_SIZE);
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-05-03 14:46:02 -07:00
|
|
|
let mut writebatch = batch_processor.batch()?;
|
2019-04-11 14:14:57 -07:00
|
|
|
|
|
|
|
writebatch.put_bytes::<cf::Coding>((slot, index), bytes)?;
|
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
if let Some((data, coding)) = try_erasure_recover(
|
2019-05-03 14:46:02 -07:00
|
|
|
&self.db,
|
2019-04-26 08:52:10 -07:00
|
|
|
&self.session,
|
|
|
|
&erasure_meta,
|
|
|
|
slot,
|
|
|
|
&HashMap::new(),
|
|
|
|
Some((index, bytes)),
|
|
|
|
)? {
|
2019-04-24 15:53:01 -07:00
|
|
|
let mut erasure_meta_working_set = HashMap::new();
|
|
|
|
erasure_meta_working_set.insert((slot, set_index), erasure_meta);
|
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
insert_data_blob_batch(
|
2019-04-24 15:53:01 -07:00
|
|
|
&data[..],
|
2019-05-03 14:46:02 -07:00
|
|
|
&self.db,
|
2019-04-24 15:53:01 -07:00
|
|
|
&mut HashMap::new(),
|
|
|
|
&mut erasure_meta_working_set,
|
|
|
|
&mut HashMap::new(),
|
|
|
|
&mut writebatch,
|
|
|
|
)?;
|
|
|
|
|
|
|
|
erasure_meta = *erasure_meta_working_set.values().next().unwrap();
|
|
|
|
|
|
|
|
for coding_blob in coding {
|
|
|
|
erasure_meta.set_coding_present(coding_blob.index(), true);
|
|
|
|
|
|
|
|
writebatch.put_bytes::<cf::Coding>(
|
|
|
|
(coding_blob.slot(), coding_blob.index()),
|
|
|
|
&coding_blob.data[..BLOB_HEADER_SIZE + coding_blob.size()],
|
|
|
|
)?;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-04-11 14:14:57 -07:00
|
|
|
writebatch.put::<cf::ErasureMeta>((slot, set_index), &erasure_meta)?;
|
|
|
|
|
2019-05-03 14:46:02 -07:00
|
|
|
batch_processor.write(writebatch)?;
|
2019-04-24 15:53:01 -07:00
|
|
|
|
2019-04-11 14:14:57 -07:00
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2019-03-05 14:18:29 -08:00
|
|
|
pub fn get_data_blob(&self, slot: u64, blob_index: u64) -> Result<Option<Blob>> {
|
|
|
|
let bytes = self.get_data_blob_bytes(slot, blob_index)?;
|
2019-01-08 15:53:44 -08:00
|
|
|
Ok(bytes.map(|bytes| {
|
|
|
|
let blob = Blob::new(&bytes);
|
2019-03-05 14:18:29 -08:00
|
|
|
assert!(blob.slot() == slot);
|
2019-02-07 15:10:54 -08:00
|
|
|
assert!(blob.index() == blob_index);
|
2019-01-08 15:53:44 -08:00
|
|
|
blob
|
|
|
|
}))
|
|
|
|
}
|
|
|
|
|
2019-01-03 21:29:21 -08:00
|
|
|
pub fn get_entries_bytes(
|
|
|
|
&self,
|
|
|
|
_start_index: u64,
|
|
|
|
_num_entries: u64,
|
|
|
|
_buf: &mut [u8],
|
|
|
|
) -> io::Result<(u64, u64)> {
|
|
|
|
Err(io::Error::new(io::ErrorKind::Other, "TODO"))
|
|
|
|
}
|
|
|
|
|
2019-01-08 15:53:44 -08:00
|
|
|
// Given a start and end entry index, find all the missing
|
|
|
|
// indexes in the ledger in the range [start_index, end_index)
|
2019-03-05 14:18:29 -08:00
|
|
|
// for the slot with the specified slot
|
2019-04-02 14:58:07 -07:00
|
|
|
fn find_missing_indexes<C>(
|
|
|
|
db_iterator: &mut Cursor<C>,
|
2019-01-08 15:53:44 -08:00
|
|
|
slot: u64,
|
|
|
|
start_index: u64,
|
|
|
|
end_index: u64,
|
|
|
|
max_missing: usize,
|
2019-04-02 14:58:07 -07:00
|
|
|
) -> Vec<u64>
|
|
|
|
where
|
|
|
|
C: Column<Index = (u64, u64)>,
|
|
|
|
{
|
2019-01-08 15:53:44 -08:00
|
|
|
if start_index >= end_index || max_missing == 0 {
|
|
|
|
return vec![];
|
|
|
|
}
|
|
|
|
|
|
|
|
let mut missing_indexes = vec![];
|
|
|
|
|
|
|
|
// Seek to the first blob with index >= start_index
|
2019-04-02 14:58:07 -07:00
|
|
|
db_iterator.seek((slot, start_index));
|
2019-01-08 15:53:44 -08:00
|
|
|
|
|
|
|
// The index of the first missing blob in the slot
|
|
|
|
let mut prev_index = start_index;
|
|
|
|
'outer: loop {
|
|
|
|
if !db_iterator.valid() {
|
|
|
|
for i in prev_index..end_index {
|
|
|
|
missing_indexes.push(i);
|
|
|
|
if missing_indexes.len() == max_missing {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
2019-04-02 14:58:07 -07:00
|
|
|
let (current_slot, index) = db_iterator.key().expect("Expect a valid key");
|
2019-02-07 15:10:54 -08:00
|
|
|
let current_index = {
|
|
|
|
if current_slot > slot {
|
|
|
|
end_index
|
|
|
|
} else {
|
2019-04-02 14:58:07 -07:00
|
|
|
index
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
};
|
2019-01-08 15:53:44 -08:00
|
|
|
let upper_index = cmp::min(current_index, end_index);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-01-08 15:53:44 -08:00
|
|
|
for i in prev_index..upper_index {
|
|
|
|
missing_indexes.push(i);
|
|
|
|
if missing_indexes.len() == max_missing {
|
|
|
|
break 'outer;
|
|
|
|
}
|
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
if current_slot > slot {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2019-01-08 15:53:44 -08:00
|
|
|
if current_index >= end_index {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
prev_index = current_index + 1;
|
|
|
|
db_iterator.next();
|
|
|
|
}
|
|
|
|
|
|
|
|
missing_indexes
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn find_missing_data_indexes(
|
|
|
|
&self,
|
|
|
|
slot: u64,
|
|
|
|
start_index: u64,
|
|
|
|
end_index: u64,
|
|
|
|
max_missing: usize,
|
|
|
|
) -> Vec<u64> {
|
2019-05-03 14:46:02 -07:00
|
|
|
if let Ok(mut db_iterator) = self.db.cursor::<cf::Data>() {
|
2019-04-02 14:58:07 -07:00
|
|
|
Self::find_missing_indexes(&mut db_iterator, slot, start_index, end_index, max_missing)
|
|
|
|
} else {
|
|
|
|
vec![]
|
|
|
|
}
|
2019-01-08 15:53:44 -08:00
|
|
|
}
|
|
|
|
|
2019-02-07 15:10:54 -08:00
|
|
|
/// Returns the entry vector for the slot starting with `blob_start_index`
|
|
|
|
pub fn get_slot_entries(
|
|
|
|
&self,
|
2019-03-05 14:18:29 -08:00
|
|
|
slot: u64,
|
2019-02-07 15:10:54 -08:00
|
|
|
blob_start_index: u64,
|
|
|
|
max_entries: Option<u64>,
|
|
|
|
) -> Result<Vec<Entry>> {
|
2019-03-05 14:18:29 -08:00
|
|
|
self.get_slot_entries_with_blob_count(slot, blob_start_index, max_entries)
|
2019-02-26 21:57:45 -08:00
|
|
|
.map(|x| x.0)
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
pub fn read_ledger_blobs(&self) -> impl Iterator<Item = Blob> + '_ {
|
2019-05-03 14:46:02 -07:00
|
|
|
let iter = self.db.iter::<cf::Data>().unwrap();
|
2019-04-26 08:52:10 -07:00
|
|
|
iter.map(|(_, blob_data)| Blob::new(&blob_data))
|
2019-04-02 14:58:07 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/// Return an iterator for all the entries in the given file.
|
|
|
|
pub fn read_ledger(&self) -> Result<impl Iterator<Item = Entry>> {
|
|
|
|
use crate::entry::EntrySlice;
|
|
|
|
use std::collections::VecDeque;
|
|
|
|
|
|
|
|
struct EntryIterator {
|
|
|
|
db_iterator: Cursor<cf::Data>,
|
|
|
|
|
|
|
|
// TODO: remove me when replay_stage is iterating by block (Blocktree)
|
|
|
|
// this verification is duplicating that of replay_stage, which
|
|
|
|
// can do this in parallel
|
|
|
|
blockhash: Option<Hash>,
|
|
|
|
// https://github.com/rust-rocksdb/rust-rocksdb/issues/234
|
|
|
|
// rocksdb issue: the _blocktree member must be lower in the struct to prevent a crash
|
|
|
|
// when the db_iterator member above is dropped.
|
|
|
|
// _blocktree is unused, but dropping _blocktree results in a broken db_iterator
|
|
|
|
// you have to hold the database open in order to iterate over it, and in order
|
|
|
|
// for db_iterator to be able to run Drop
|
|
|
|
// _blocktree: Blocktree,
|
|
|
|
entries: VecDeque<Entry>,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Iterator for EntryIterator {
|
|
|
|
type Item = Entry;
|
|
|
|
|
|
|
|
fn next(&mut self) -> Option<Entry> {
|
|
|
|
if !self.entries.is_empty() {
|
|
|
|
return Some(self.entries.pop_front().unwrap());
|
|
|
|
}
|
|
|
|
|
|
|
|
if self.db_iterator.valid() {
|
|
|
|
if let Some(value) = self.db_iterator.value_bytes() {
|
|
|
|
if let Ok(next_entries) =
|
|
|
|
deserialize::<Vec<Entry>>(&value[BLOB_HEADER_SIZE..])
|
|
|
|
{
|
|
|
|
if let Some(blockhash) = self.blockhash {
|
|
|
|
if !next_entries.verify(&blockhash) {
|
|
|
|
return None;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
self.db_iterator.next();
|
|
|
|
if next_entries.is_empty() {
|
|
|
|
return None;
|
|
|
|
}
|
|
|
|
self.entries = VecDeque::from(next_entries);
|
|
|
|
let entry = self.entries.pop_front().unwrap();
|
|
|
|
self.blockhash = Some(entry.hash);
|
|
|
|
return Some(entry);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
None
|
|
|
|
}
|
|
|
|
}
|
2019-05-03 14:46:02 -07:00
|
|
|
let mut db_iterator = self.db.cursor::<cf::Data>()?;
|
2019-04-02 14:58:07 -07:00
|
|
|
|
|
|
|
db_iterator.seek_to_first();
|
|
|
|
Ok(EntryIterator {
|
|
|
|
entries: VecDeque::new(),
|
|
|
|
db_iterator,
|
|
|
|
blockhash: None,
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2019-02-28 19:49:22 -08:00
|
|
|
pub fn get_slot_entries_with_blob_count(
|
|
|
|
&self,
|
2019-03-05 14:18:29 -08:00
|
|
|
slot: u64,
|
2019-02-28 19:49:22 -08:00
|
|
|
blob_start_index: u64,
|
|
|
|
max_entries: Option<u64>,
|
|
|
|
) -> Result<(Vec<Entry>, usize)> {
|
|
|
|
// Find the next consecutive block of blobs.
|
2019-04-26 08:52:10 -07:00
|
|
|
let consecutive_blobs = get_slot_consecutive_blobs(
|
|
|
|
slot,
|
2019-05-03 14:46:02 -07:00
|
|
|
&self.db,
|
2019-04-26 08:52:10 -07:00
|
|
|
&HashMap::new(),
|
|
|
|
blob_start_index,
|
|
|
|
max_entries,
|
|
|
|
)?;
|
2019-02-28 19:49:22 -08:00
|
|
|
let num = consecutive_blobs.len();
|
2019-04-26 08:52:10 -07:00
|
|
|
Ok((deserialize_blobs(&consecutive_blobs), num))
|
2019-02-28 19:49:22 -08:00
|
|
|
}
|
|
|
|
|
2019-03-05 14:18:29 -08:00
|
|
|
// Returns slots connecting to any element of the list `slots`.
|
|
|
|
pub fn get_slots_since(&self, slots: &[u64]) -> Result<HashMap<u64, Vec<u64>>> {
|
2019-02-07 15:10:54 -08:00
|
|
|
// Return error if there was a database error during lookup of any of the
|
|
|
|
// slot indexes
|
2019-03-05 14:18:29 -08:00
|
|
|
let slot_metas: Result<Vec<Option<SlotMeta>>> =
|
|
|
|
slots.iter().map(|slot| self.meta(*slot)).collect();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-02-28 19:49:22 -08:00
|
|
|
let slot_metas = slot_metas?;
|
2019-03-05 14:18:29 -08:00
|
|
|
let result: HashMap<u64, Vec<u64>> = slots
|
2019-02-28 19:49:22 -08:00
|
|
|
.iter()
|
|
|
|
.zip(slot_metas)
|
|
|
|
.filter_map(|(height, meta)| meta.map(|meta| (*height, meta.next_slots)))
|
2019-02-07 15:10:54 -08:00
|
|
|
.collect();
|
|
|
|
|
2019-02-28 19:49:22 -08:00
|
|
|
Ok(result)
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
2019-01-08 15:53:44 -08:00
|
|
|
|
2019-03-18 16:16:06 -07:00
|
|
|
pub fn deserialize_blob_data(data: &[u8]) -> Result<Vec<Entry>> {
|
|
|
|
let entries = deserialize(data)?;
|
|
|
|
Ok(entries)
|
|
|
|
}
|
|
|
|
|
2019-04-15 13:12:28 -07:00
|
|
|
pub fn is_root(&self, slot: u64) -> bool {
|
2019-05-03 14:46:02 -07:00
|
|
|
if let Ok(Some(root_slot)) = self.db.get::<cf::Root>(()) {
|
|
|
|
root_slot == slot
|
2019-04-15 13:12:28 -07:00
|
|
|
} else {
|
|
|
|
false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn set_root(&self, slot: u64) -> Result<()> {
|
2019-05-03 14:46:02 -07:00
|
|
|
self.db.put::<cf::Root>((), &slot)?;
|
2019-04-15 13:12:28 -07:00
|
|
|
Ok(())
|
2019-04-06 19:41:22 -07:00
|
|
|
}
|
|
|
|
|
2019-05-03 14:46:02 -07:00
|
|
|
pub fn get_root(&self) -> Result<u64> {
|
|
|
|
let root_opt = self.db.get::<cf::Root>(())?;
|
|
|
|
|
|
|
|
Ok(root_opt.unwrap_or(0))
|
|
|
|
}
|
|
|
|
|
2019-04-06 19:41:22 -07:00
|
|
|
pub fn get_orphans(&self, max: Option<usize>) -> Vec<u64> {
|
|
|
|
let mut results = vec![];
|
2019-04-26 08:52:10 -07:00
|
|
|
|
2019-05-03 14:46:02 -07:00
|
|
|
let mut iter = self.db.cursor::<cf::Orphans>().unwrap();
|
2019-04-06 19:41:22 -07:00
|
|
|
iter.seek_to_first();
|
|
|
|
while iter.valid() {
|
|
|
|
if let Some(max) = max {
|
|
|
|
if results.len() > max {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
results.push(iter.key().unwrap());
|
|
|
|
iter.next();
|
|
|
|
}
|
|
|
|
results
|
|
|
|
}
|
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
// Handle special case of writing genesis blobs. For instance, the first two entries
|
|
|
|
// don't count as ticks, even if they're empty entries
|
|
|
|
fn write_genesis_blobs(&self, blobs: &[Blob]) -> Result<()> {
|
|
|
|
// TODO: change bootstrap height to number of slots
|
|
|
|
let mut bootstrap_meta = SlotMeta::new(0, 1);
|
|
|
|
let last = blobs.last().unwrap();
|
2019-02-04 15:33:43 -08:00
|
|
|
|
2019-05-03 14:46:02 -07:00
|
|
|
let mut batch_processor = self.batch_processor.write().unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
bootstrap_meta.consumed = last.index() + 1;
|
|
|
|
bootstrap_meta.received = last.index() + 1;
|
|
|
|
bootstrap_meta.is_connected = true;
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-05-03 14:46:02 -07:00
|
|
|
let mut batch = batch_processor.batch()?;
|
2019-04-26 08:52:10 -07:00
|
|
|
batch.put::<cf::SlotMeta>(0, &bootstrap_meta)?;
|
|
|
|
for blob in blobs {
|
|
|
|
let serialized_blob_datas = &blob.data[..BLOB_HEADER_SIZE + blob.size()];
|
|
|
|
batch.put_bytes::<cf::Data>((blob.slot(), blob.index()), serialized_blob_datas)?;
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
2019-05-03 14:46:02 -07:00
|
|
|
batch_processor.write(batch)?;
|
2019-02-07 15:10:54 -08:00
|
|
|
Ok(())
|
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
fn insert_data_blob_batch<'a, I>(
|
|
|
|
new_blobs: I,
|
|
|
|
db: &Database,
|
|
|
|
slot_meta_working_set: &mut HashMap<u64, (Rc<RefCell<SlotMeta>>, Option<SlotMeta>)>,
|
|
|
|
erasure_meta_working_set: &mut HashMap<(u64, u64), ErasureMeta>,
|
|
|
|
prev_inserted_blob_datas: &mut HashMap<(u64, u64), &'a [u8]>,
|
|
|
|
write_batch: &mut WriteBatch,
|
|
|
|
) -> Result<()>
|
|
|
|
where
|
|
|
|
I: IntoIterator<Item = &'a Blob>,
|
|
|
|
{
|
|
|
|
for blob in new_blobs.into_iter() {
|
|
|
|
let inserted = check_insert_data_blob(
|
|
|
|
blob,
|
|
|
|
db,
|
|
|
|
slot_meta_working_set,
|
|
|
|
prev_inserted_blob_datas,
|
|
|
|
write_batch,
|
|
|
|
);
|
2019-03-29 16:07:24 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
if inserted {
|
|
|
|
erasure_meta_working_set
|
|
|
|
.get_mut(&(blob.slot(), ErasureMeta::set_index_for(blob.index())))
|
|
|
|
.unwrap()
|
|
|
|
.set_data_present(blob.index(), true);
|
|
|
|
}
|
|
|
|
}
|
2019-03-29 16:07:24 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
Ok(())
|
|
|
|
}
|
2019-03-29 16:07:24 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
/// Insert a blob into ledger, updating the slot_meta if necessary
|
|
|
|
fn insert_data_blob<'a>(
|
|
|
|
blob_to_insert: &'a Blob,
|
|
|
|
db: &Database,
|
|
|
|
prev_inserted_blob_datas: &mut HashMap<(u64, u64), &'a [u8]>,
|
|
|
|
slot_meta: &mut SlotMeta,
|
|
|
|
write_batch: &mut WriteBatch,
|
|
|
|
) -> Result<()> {
|
|
|
|
let blob_index = blob_to_insert.index();
|
|
|
|
let blob_slot = blob_to_insert.slot();
|
|
|
|
let blob_size = blob_to_insert.size();
|
|
|
|
|
|
|
|
let new_consumed = {
|
|
|
|
if slot_meta.consumed == blob_index {
|
|
|
|
let blob_datas = get_slot_consecutive_blobs(
|
|
|
|
blob_slot,
|
|
|
|
db,
|
|
|
|
prev_inserted_blob_datas,
|
|
|
|
// Don't start looking for consecutive blobs at blob_index,
|
|
|
|
// because we haven't inserted/committed the new blob_to_insert
|
|
|
|
// into the database or prev_inserted_blob_datas hashmap yet.
|
|
|
|
blob_index + 1,
|
|
|
|
None,
|
|
|
|
)?;
|
|
|
|
|
|
|
|
// Add one because we skipped this current blob when calling
|
|
|
|
// get_slot_consecutive_blobs() earlier
|
|
|
|
slot_meta.consumed + blob_datas.len() as u64 + 1
|
|
|
|
} else {
|
|
|
|
slot_meta.consumed
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
let serialized_blob_data = &blob_to_insert.data[..BLOB_HEADER_SIZE + blob_size];
|
|
|
|
|
|
|
|
// Commit step: commit all changes to the mutable structures at once, or none at all.
|
|
|
|
// We don't want only some of these changes going through.
|
|
|
|
write_batch.put_bytes::<cf::Data>((blob_slot, blob_index), serialized_blob_data)?;
|
|
|
|
prev_inserted_blob_datas.insert((blob_slot, blob_index), serialized_blob_data);
|
|
|
|
// Index is zero-indexed, while the "received" height starts from 1,
|
|
|
|
// so received = index + 1 for the same blob.
|
|
|
|
slot_meta.received = cmp::max(blob_index + 1, slot_meta.received);
|
|
|
|
slot_meta.consumed = new_consumed;
|
|
|
|
slot_meta.last_index = {
|
|
|
|
// If the last slot hasn't been set before, then
|
|
|
|
// set it to this blob index
|
|
|
|
if slot_meta.last_index == std::u64::MAX {
|
|
|
|
if blob_to_insert.is_last_in_slot() {
|
|
|
|
blob_index
|
|
|
|
} else {
|
|
|
|
std::u64::MAX
|
2019-03-29 16:07:24 -07:00
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
} else {
|
|
|
|
slot_meta.last_index
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
};
|
|
|
|
Ok(())
|
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
/// Checks to see if the data blob passes integrity checks for insertion. Proceeds with
|
|
|
|
/// insertion if it does.
|
|
|
|
fn check_insert_data_blob<'a>(
|
|
|
|
blob: &'a Blob,
|
|
|
|
db: &Database,
|
|
|
|
slot_meta_working_set: &mut HashMap<u64, (Rc<RefCell<SlotMeta>>, Option<SlotMeta>)>,
|
|
|
|
prev_inserted_blob_datas: &mut HashMap<(u64, u64), &'a [u8]>,
|
|
|
|
write_batch: &mut WriteBatch,
|
|
|
|
) -> bool {
|
|
|
|
let blob_slot = blob.slot();
|
|
|
|
let parent_slot = blob.parent();
|
|
|
|
let meta_cf = db.column::<cf::SlotMeta>();
|
|
|
|
|
|
|
|
// Check if we've already inserted the slot metadata for this blob's slot
|
|
|
|
let entry = slot_meta_working_set.entry(blob_slot).or_insert_with(|| {
|
|
|
|
// Store a 2-tuple of the metadata (working copy, backup copy)
|
|
|
|
if let Some(mut meta) = meta_cf
|
2019-05-03 14:46:02 -07:00
|
|
|
.get(blob_slot)
|
2019-04-26 08:52:10 -07:00
|
|
|
.expect("Expect database get to succeed")
|
|
|
|
{
|
|
|
|
let backup = Some(meta.clone());
|
|
|
|
// If parent_slot == std::u64::MAX, then this is one of the orphans inserted
|
|
|
|
// during the chaining process, see the function find_slot_meta_in_cached_state()
|
|
|
|
// for details. Slots that are orphans are missing a parent_slot, so we should
|
|
|
|
// fill in the parent now that we know it.
|
|
|
|
if is_orphan(&meta) {
|
|
|
|
meta.parent_slot = parent_slot;
|
|
|
|
}
|
2019-03-29 16:07:24 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
(Rc::new(RefCell::new(meta)), backup)
|
|
|
|
} else {
|
|
|
|
(
|
|
|
|
Rc::new(RefCell::new(SlotMeta::new(blob_slot, parent_slot))),
|
|
|
|
None,
|
|
|
|
)
|
2019-03-29 16:07:24 -07:00
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
});
|
2019-03-29 16:07:24 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
let slot_meta = &mut entry.0.borrow_mut();
|
|
|
|
|
|
|
|
// This slot is full, skip the bogus blob
|
|
|
|
// Check if this blob should be inserted
|
|
|
|
if !should_insert_blob(&slot_meta, db, &prev_inserted_blob_datas, blob) {
|
|
|
|
false
|
|
|
|
} else {
|
|
|
|
let _ = insert_data_blob(blob, db, prev_inserted_blob_datas, slot_meta, write_batch);
|
|
|
|
true
|
2019-03-29 16:07:24 -07:00
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
fn should_insert_blob(
|
|
|
|
slot: &SlotMeta,
|
|
|
|
db: &Database,
|
|
|
|
prev_inserted_blob_datas: &HashMap<(u64, u64), &[u8]>,
|
|
|
|
blob: &Blob,
|
|
|
|
) -> bool {
|
|
|
|
let blob_index = blob.index();
|
|
|
|
let blob_slot = blob.slot();
|
|
|
|
let data_cf = db.column::<cf::Data>();
|
|
|
|
|
|
|
|
// Check that the blob doesn't already exist
|
|
|
|
if blob_index < slot.consumed
|
|
|
|
|| prev_inserted_blob_datas.contains_key(&(blob_slot, blob_index))
|
|
|
|
|| data_cf
|
2019-05-03 14:46:02 -07:00
|
|
|
.get_bytes((blob_slot, blob_index))
|
2019-04-26 08:52:10 -07:00
|
|
|
.map(|opt| opt.is_some())
|
|
|
|
.unwrap_or(false)
|
2019-03-29 16:07:24 -07:00
|
|
|
{
|
2019-04-26 08:52:10 -07:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check that we do not receive blobs >= than the last_index
|
|
|
|
// for the slot
|
|
|
|
let last_index = slot.last_index;
|
|
|
|
if blob_index >= last_index {
|
|
|
|
solana_metrics::submit(
|
|
|
|
solana_metrics::influxdb::Point::new("blocktree_error")
|
|
|
|
.add_field(
|
|
|
|
"error",
|
|
|
|
solana_metrics::influxdb::Value::String(format!(
|
|
|
|
"Received last blob with index {} >= slot.last_index {}",
|
|
|
|
blob_index, last_index
|
|
|
|
)),
|
|
|
|
)
|
|
|
|
.to_owned(),
|
|
|
|
);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check that we do not receive a blob with "last_index" true, but index
|
|
|
|
// less than our current received
|
|
|
|
if blob.is_last_in_slot() && blob_index < slot.received {
|
|
|
|
solana_metrics::submit(
|
|
|
|
solana_metrics::influxdb::Point::new("blocktree_error")
|
|
|
|
.add_field(
|
|
|
|
"error",
|
|
|
|
solana_metrics::influxdb::Value::String(format!(
|
|
|
|
"Received last blob with index {} < slot.received {}",
|
|
|
|
blob_index, slot.received
|
|
|
|
)),
|
|
|
|
)
|
|
|
|
.to_owned(),
|
|
|
|
);
|
|
|
|
return false;
|
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
true
|
|
|
|
}
|
|
|
|
|
|
|
|
// 1) Find the slot metadata in the cache of dirty slot metadata we've previously touched,
|
|
|
|
// else:
|
|
|
|
// 2) Search the database for that slot metadata. If still no luck, then:
|
|
|
|
// 3) Create a dummy orphan slot in the database
|
|
|
|
fn find_slot_meta_else_create<'a>(
|
|
|
|
db: &Database,
|
|
|
|
working_set: &'a HashMap<u64, (Rc<RefCell<SlotMeta>>, Option<SlotMeta>)>,
|
|
|
|
chained_slots: &'a mut HashMap<u64, Rc<RefCell<SlotMeta>>>,
|
|
|
|
slot_index: u64,
|
|
|
|
) -> Result<Rc<RefCell<SlotMeta>>> {
|
|
|
|
let result = find_slot_meta_in_cached_state(working_set, chained_slots, slot_index)?;
|
|
|
|
if let Some(slot) = result {
|
|
|
|
Ok(slot)
|
|
|
|
} else {
|
|
|
|
find_slot_meta_in_db_else_create(db, slot_index, chained_slots)
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
// Search the database for that slot metadata. If still no luck, then
|
|
|
|
// create a dummy orphan slot in the database
|
|
|
|
fn find_slot_meta_in_db_else_create<'a>(
|
|
|
|
db: &Database,
|
|
|
|
slot: u64,
|
|
|
|
insert_map: &'a mut HashMap<u64, Rc<RefCell<SlotMeta>>>,
|
|
|
|
) -> Result<Rc<RefCell<SlotMeta>>> {
|
2019-05-03 14:46:02 -07:00
|
|
|
if let Some(slot_meta) = db.column::<cf::SlotMeta>().get(slot)? {
|
2019-04-26 08:52:10 -07:00
|
|
|
insert_map.insert(slot, Rc::new(RefCell::new(slot_meta)));
|
|
|
|
Ok(insert_map.get(&slot).unwrap().clone())
|
|
|
|
} else {
|
|
|
|
// If this slot doesn't exist, make a orphan slot. This way we
|
|
|
|
// remember which slots chained to this one when we eventually get a real blob
|
|
|
|
// for this slot
|
|
|
|
insert_map.insert(
|
|
|
|
slot,
|
|
|
|
Rc::new(RefCell::new(SlotMeta::new(slot, std::u64::MAX))),
|
|
|
|
);
|
|
|
|
Ok(insert_map.get(&slot).unwrap().clone())
|
2019-04-25 00:04:49 -07:00
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
}
|
2019-04-25 00:04:49 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
// Find the slot metadata in the cache of dirty slot metadata we've previously touched
|
|
|
|
fn find_slot_meta_in_cached_state<'a>(
|
|
|
|
working_set: &'a HashMap<u64, (Rc<RefCell<SlotMeta>>, Option<SlotMeta>)>,
|
|
|
|
chained_slots: &'a HashMap<u64, Rc<RefCell<SlotMeta>>>,
|
|
|
|
slot: u64,
|
|
|
|
) -> Result<Option<Rc<RefCell<SlotMeta>>>> {
|
|
|
|
if let Some((entry, _)) = working_set.get(&slot) {
|
|
|
|
Ok(Some(entry.clone()))
|
|
|
|
} else if let Some(entry) = chained_slots.get(&slot) {
|
|
|
|
Ok(Some(entry.clone()))
|
|
|
|
} else {
|
|
|
|
Ok(None)
|
2019-03-29 16:07:24 -07:00
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
}
|
2019-03-29 16:07:24 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
/// Returns the next consumed index and the number of ticks in the new consumed
|
|
|
|
/// range
|
|
|
|
fn get_slot_consecutive_blobs<'a>(
|
|
|
|
slot: u64,
|
|
|
|
db: &Database,
|
|
|
|
prev_inserted_blob_datas: &HashMap<(u64, u64), &'a [u8]>,
|
|
|
|
mut current_index: u64,
|
|
|
|
max_blobs: Option<u64>,
|
|
|
|
) -> Result<Vec<Cow<'a, [u8]>>> {
|
|
|
|
let mut blobs: Vec<Cow<[u8]>> = vec![];
|
|
|
|
let data_cf = db.column::<cf::Data>();
|
|
|
|
|
|
|
|
loop {
|
|
|
|
if Some(blobs.len() as u64) == max_blobs {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
// Try to find the next blob we're looking for in the prev_inserted_blob_datas
|
|
|
|
if let Some(prev_blob_data) = prev_inserted_blob_datas.get(&(slot, current_index)) {
|
|
|
|
blobs.push(Cow::Borrowed(*prev_blob_data));
|
2019-05-03 14:46:02 -07:00
|
|
|
} else if let Some(blob_data) = data_cf.get_bytes((slot, current_index))? {
|
2019-04-26 08:52:10 -07:00
|
|
|
// Try to find the next blob we're looking for in the database
|
|
|
|
blobs.push(Cow::Owned(blob_data));
|
2019-02-07 15:10:54 -08:00
|
|
|
} else {
|
2019-04-26 08:52:10 -07:00
|
|
|
break;
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
|
|
|
|
current_index += 1;
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
Ok(blobs)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Chaining based on latest discussion here: https://github.com/solana-labs/solana/pull/2253
|
|
|
|
fn handle_chaining(
|
|
|
|
db: &Database,
|
|
|
|
write_batch: &mut WriteBatch,
|
|
|
|
working_set: &HashMap<u64, (Rc<RefCell<SlotMeta>>, Option<SlotMeta>)>,
|
|
|
|
) -> Result<()> {
|
|
|
|
let mut new_chained_slots = HashMap::new();
|
|
|
|
let working_set_slots: Vec<_> = working_set.iter().map(|s| *s.0).collect();
|
|
|
|
for slot in working_set_slots {
|
|
|
|
handle_chaining_for_slot(db, write_batch, working_set, &mut new_chained_slots, slot)?;
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
// Write all the newly changed slots in new_chained_slots to the write_batch
|
|
|
|
for (slot, meta) in new_chained_slots.iter() {
|
|
|
|
let meta: &SlotMeta = &RefCell::borrow(&*meta);
|
|
|
|
write_batch.put::<cf::SlotMeta>(*slot, meta)?;
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
Ok(())
|
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
fn handle_chaining_for_slot(
|
|
|
|
db: &Database,
|
|
|
|
write_batch: &mut WriteBatch,
|
|
|
|
working_set: &HashMap<u64, (Rc<RefCell<SlotMeta>>, Option<SlotMeta>)>,
|
|
|
|
new_chained_slots: &mut HashMap<u64, Rc<RefCell<SlotMeta>>>,
|
|
|
|
slot: u64,
|
|
|
|
) -> Result<()> {
|
|
|
|
let (meta, meta_backup) = working_set
|
|
|
|
.get(&slot)
|
|
|
|
.expect("Slot must exist in the working_set hashmap");
|
2019-04-24 15:53:01 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
{
|
|
|
|
let is_orphaned = meta_backup.is_some() && is_orphan(meta_backup.as_ref().unwrap());
|
2019-04-24 15:53:01 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
let mut meta_mut = meta.borrow_mut();
|
2019-04-24 15:53:01 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
// If:
|
|
|
|
// 1) This is a new slot
|
|
|
|
// 2) slot != 0
|
|
|
|
// then try to chain this slot to a previous slot
|
|
|
|
if slot != 0 {
|
|
|
|
let prev_slot = meta_mut.parent_slot;
|
2019-04-24 15:53:01 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
// Check if the slot represented by meta_mut is either a new slot or a orphan.
|
|
|
|
// In both cases we need to run the chaining logic b/c the parent on the slot was
|
|
|
|
// previously unknown.
|
|
|
|
if meta_backup.is_none() || is_orphaned {
|
|
|
|
let prev_slot_meta =
|
|
|
|
find_slot_meta_else_create(db, working_set, new_chained_slots, prev_slot)?;
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
// This is a newly inserted slot so run the chaining logic
|
|
|
|
chain_new_slot_to_prev_slot(&mut prev_slot_meta.borrow_mut(), slot, &mut meta_mut);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
if is_orphan(&RefCell::borrow(&*prev_slot_meta)) {
|
|
|
|
write_batch.put::<cf::Orphans>(prev_slot, &true)?;
|
2019-02-12 16:06:23 -08:00
|
|
|
}
|
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// At this point this slot has received a parent, so no longer a orphan
|
|
|
|
if is_orphaned {
|
|
|
|
write_batch.delete::<cf::Orphans>(slot)?;
|
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
// This is a newly inserted slot and slot.is_connected is true, so update all
|
|
|
|
// child slots so that their `is_connected` = true
|
|
|
|
let should_propagate_is_connected =
|
|
|
|
is_newly_completed_slot(&RefCell::borrow(&*meta), meta_backup)
|
|
|
|
&& RefCell::borrow(&*meta).is_connected;
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
if should_propagate_is_connected {
|
|
|
|
// slot_function returns a boolean indicating whether to explore the children
|
|
|
|
// of the input slot
|
|
|
|
let slot_function = |slot: &mut SlotMeta| {
|
|
|
|
slot.is_connected = true;
|
2019-04-18 21:56:43 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
// We don't want to set the is_connected flag on the children of non-full
|
|
|
|
// slots
|
|
|
|
slot.is_full()
|
|
|
|
};
|
|
|
|
|
|
|
|
traverse_children_mut(
|
|
|
|
db,
|
|
|
|
slot,
|
|
|
|
&meta,
|
|
|
|
working_set,
|
|
|
|
new_chained_slots,
|
|
|
|
slot_function,
|
|
|
|
)?;
|
|
|
|
}
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
|
|
|
fn traverse_children_mut<F>(
|
|
|
|
db: &Database,
|
|
|
|
slot: u64,
|
|
|
|
slot_meta: &Rc<RefCell<(SlotMeta)>>,
|
|
|
|
working_set: &HashMap<u64, (Rc<RefCell<SlotMeta>>, Option<SlotMeta>)>,
|
|
|
|
new_chained_slots: &mut HashMap<u64, Rc<RefCell<SlotMeta>>>,
|
|
|
|
slot_function: F,
|
|
|
|
) -> Result<()>
|
|
|
|
where
|
|
|
|
F: Fn(&mut SlotMeta) -> bool,
|
|
|
|
{
|
|
|
|
let mut next_slots: Vec<(u64, Rc<RefCell<(SlotMeta)>>)> = vec![(slot, slot_meta.clone())];
|
|
|
|
while !next_slots.is_empty() {
|
|
|
|
let (_, current_slot) = next_slots.pop().unwrap();
|
|
|
|
// Check whether we should explore the children of this slot
|
|
|
|
if slot_function(&mut current_slot.borrow_mut()) {
|
|
|
|
let current_slot = &RefCell::borrow(&*current_slot);
|
|
|
|
for next_slot_index in current_slot.next_slots.iter() {
|
|
|
|
let next_slot = find_slot_meta_else_create(
|
|
|
|
db,
|
|
|
|
working_set,
|
|
|
|
new_chained_slots,
|
|
|
|
*next_slot_index,
|
|
|
|
)?;
|
|
|
|
next_slots.push((*next_slot_index, next_slot));
|
2019-04-11 14:14:57 -07:00
|
|
|
}
|
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
}
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
Ok(())
|
|
|
|
}
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
fn is_orphan(meta: &SlotMeta) -> bool {
|
|
|
|
// If we have no parent, then this is the head of a detached chain of
|
|
|
|
// slots
|
|
|
|
!meta.is_parent_set()
|
|
|
|
}
|
2019-04-18 21:56:43 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
// 1) Chain current_slot to the previous slot defined by prev_slot_meta
|
|
|
|
// 2) Determine whether to set the is_connected flag
|
|
|
|
fn chain_new_slot_to_prev_slot(
|
|
|
|
prev_slot_meta: &mut SlotMeta,
|
|
|
|
current_slot: u64,
|
|
|
|
current_slot_meta: &mut SlotMeta,
|
|
|
|
) {
|
|
|
|
prev_slot_meta.next_slots.push(current_slot);
|
|
|
|
current_slot_meta.is_connected = prev_slot_meta.is_connected && prev_slot_meta.is_full();
|
|
|
|
}
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
fn is_newly_completed_slot(slot_meta: &SlotMeta, backup_slot_meta: &Option<SlotMeta>) -> bool {
|
|
|
|
slot_meta.is_full()
|
|
|
|
&& (backup_slot_meta.is_none()
|
|
|
|
|| is_orphan(&backup_slot_meta.as_ref().unwrap())
|
|
|
|
|| slot_meta.consumed != backup_slot_meta.as_ref().unwrap().consumed)
|
|
|
|
}
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
/// Attempts recovery using erasure coding
|
|
|
|
fn try_erasure_recover(
|
|
|
|
db: &Database,
|
|
|
|
session: &Session,
|
|
|
|
erasure_meta: &ErasureMeta,
|
|
|
|
slot: u64,
|
|
|
|
prev_inserted_blob_datas: &HashMap<(u64, u64), &[u8]>,
|
|
|
|
new_coding_blob: Option<(u64, &[u8])>,
|
|
|
|
) -> Result<Option<(Vec<Blob>, Vec<Blob>)>> {
|
|
|
|
use crate::erasure::ERASURE_SET_SIZE;
|
|
|
|
|
2019-05-02 17:04:40 -07:00
|
|
|
let set_index = erasure_meta.set_index;
|
2019-04-26 08:52:10 -07:00
|
|
|
let blobs = match erasure_meta.status() {
|
|
|
|
ErasureMetaStatus::CanRecover => {
|
|
|
|
let erasure_result = recover(
|
|
|
|
db,
|
|
|
|
session,
|
|
|
|
slot,
|
|
|
|
erasure_meta,
|
|
|
|
prev_inserted_blob_datas,
|
|
|
|
new_coding_blob,
|
|
|
|
);
|
|
|
|
|
|
|
|
match erasure_result {
|
|
|
|
Ok((data, coding)) => {
|
|
|
|
let recovered = data.len() + coding.len();
|
|
|
|
assert_eq!(
|
|
|
|
ERASURE_SET_SIZE,
|
2019-05-02 17:04:40 -07:00
|
|
|
recovered + (erasure_meta.num_coding() + erasure_meta.num_data()) as usize,
|
2019-04-26 08:52:10 -07:00
|
|
|
"Recovery should always complete a set"
|
|
|
|
);
|
|
|
|
|
2019-05-02 17:04:40 -07:00
|
|
|
debug!(
|
|
|
|
"[try_erasure] slot: {}, set_index: {}, recovered {} blobs",
|
|
|
|
slot, set_index, recovered
|
|
|
|
);
|
2019-04-26 08:52:10 -07:00
|
|
|
inc_new_counter_info!("blocktree-erasure-blobs_recovered", recovered);
|
|
|
|
Some((data, coding))
|
|
|
|
}
|
|
|
|
Err(Error::ErasureError(e)) => {
|
|
|
|
inc_new_counter_info!("blocktree-erasure-recovery_failed", 1);
|
|
|
|
error!(
|
2019-05-02 17:04:40 -07:00
|
|
|
"[try_erasure] slot: {}, set_index: {}, recovery failed: cause: {}",
|
2019-04-26 08:52:10 -07:00
|
|
|
slot, erasure_meta.set_index, e
|
|
|
|
);
|
|
|
|
None
|
|
|
|
}
|
|
|
|
|
|
|
|
Err(e) => return Err(e),
|
2019-04-11 14:14:57 -07:00
|
|
|
}
|
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
ErasureMetaStatus::StillNeed(needed) => {
|
2019-05-02 17:04:40 -07:00
|
|
|
debug!(
|
|
|
|
"[try_erasure] slot: {}, set_index: {}, still need {} blobs",
|
|
|
|
slot, set_index, needed
|
|
|
|
);
|
2019-04-26 20:37:40 -07:00
|
|
|
inc_new_counter_info!("blocktree-erasure-blobs_needed", needed, 0, 1000);
|
2019-04-26 08:52:10 -07:00
|
|
|
None
|
|
|
|
}
|
|
|
|
ErasureMetaStatus::DataFull => {
|
2019-05-02 17:04:40 -07:00
|
|
|
debug!(
|
|
|
|
"[try_erasure] slot: {}, set_index: {}, set full",
|
|
|
|
slot, set_index,
|
|
|
|
);
|
2019-04-26 20:37:40 -07:00
|
|
|
inc_new_counter_info!("blocktree-erasure-complete", 1, 0, 1000);
|
2019-04-26 08:52:10 -07:00
|
|
|
None
|
|
|
|
}
|
|
|
|
};
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
Ok(blobs)
|
|
|
|
}
|
2019-04-17 18:04:30 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
fn recover(
|
|
|
|
db: &Database,
|
|
|
|
session: &Session,
|
|
|
|
slot: u64,
|
|
|
|
erasure_meta: &ErasureMeta,
|
|
|
|
prev_inserted_blob_datas: &HashMap<(u64, u64), &[u8]>,
|
|
|
|
new_coding: Option<(u64, &[u8])>,
|
|
|
|
) -> Result<(Vec<Blob>, Vec<Blob>)> {
|
|
|
|
use crate::erasure::ERASURE_SET_SIZE;
|
|
|
|
|
|
|
|
let start_idx = erasure_meta.start_index();
|
|
|
|
let size = erasure_meta.size();
|
|
|
|
let data_cf = db.column::<cf::Data>();
|
|
|
|
let erasure_cf = db.column::<cf::Coding>();
|
|
|
|
|
|
|
|
let (data_end_idx, coding_end_idx) = erasure_meta.end_indexes();
|
|
|
|
|
|
|
|
let present = &mut [true; ERASURE_SET_SIZE];
|
|
|
|
let mut blobs = Vec::with_capacity(ERASURE_SET_SIZE);
|
|
|
|
|
|
|
|
for i in start_idx..coding_end_idx {
|
|
|
|
if erasure_meta.is_coding_present(i) {
|
|
|
|
let mut blob_bytes = match new_coding {
|
|
|
|
Some((new_coding_index, bytes)) if new_coding_index == i => bytes.to_vec(),
|
|
|
|
_ => erasure_cf
|
2019-05-03 14:46:02 -07:00
|
|
|
.get_bytes((slot, i))?
|
2019-04-26 08:52:10 -07:00
|
|
|
.expect("ErasureMeta must have no false positives"),
|
|
|
|
};
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
blob_bytes.drain(..BLOB_HEADER_SIZE);
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
blobs.push(blob_bytes);
|
|
|
|
} else {
|
|
|
|
let set_relative_idx = erasure_meta.coding_index_in_set(i).unwrap() as usize;
|
|
|
|
blobs.push(vec![0; size]);
|
|
|
|
present[set_relative_idx] = false;
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
assert_ne!(size, 0);
|
2019-02-12 19:54:18 -08:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
for i in start_idx..data_end_idx {
|
|
|
|
let set_relative_idx = erasure_meta.data_index_in_set(i).unwrap() as usize;
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
if erasure_meta.is_data_present(i) {
|
|
|
|
let mut blob_bytes = match prev_inserted_blob_datas.get(&(slot, i)) {
|
|
|
|
Some(bytes) => bytes.to_vec(),
|
|
|
|
None => data_cf
|
2019-05-03 14:46:02 -07:00
|
|
|
.get_bytes((slot, i))?
|
2019-04-26 08:52:10 -07:00
|
|
|
.expect("erasure_meta must have no false positives"),
|
|
|
|
};
|
|
|
|
|
|
|
|
// If data is too short, extend it with zeroes
|
|
|
|
blob_bytes.resize(size, 0u8);
|
|
|
|
|
|
|
|
blobs.insert(set_relative_idx, blob_bytes);
|
|
|
|
} else {
|
|
|
|
blobs.insert(set_relative_idx, vec![0u8; size]);
|
|
|
|
// data erasures must come before any coding erasures if present
|
|
|
|
present[set_relative_idx] = false;
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
2019-04-25 00:04:49 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
let (recovered_data, recovered_coding) =
|
|
|
|
session.reconstruct_blobs(&mut blobs, present, size, start_idx, slot)?;
|
2019-04-25 00:04:49 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
trace!(
|
|
|
|
"[recover] reconstruction OK slot: {}, indexes: [{},{})",
|
|
|
|
slot,
|
|
|
|
start_idx,
|
|
|
|
data_end_idx
|
|
|
|
);
|
2019-04-25 00:04:49 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
Ok((recovered_data, recovered_coding))
|
|
|
|
}
|
2019-04-25 00:04:49 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
fn deserialize_blobs<I>(blob_datas: &[I]) -> Vec<Entry>
|
|
|
|
where
|
|
|
|
I: Borrow<[u8]>,
|
|
|
|
{
|
|
|
|
blob_datas
|
|
|
|
.iter()
|
|
|
|
.flat_map(|blob_data| {
|
|
|
|
let serialized_entries_data = &blob_data.borrow()[BLOB_HEADER_SIZE..];
|
|
|
|
Blocktree::deserialize_blob_data(serialized_entries_data)
|
|
|
|
.expect("Ledger should only contain well formed data")
|
|
|
|
})
|
|
|
|
.collect()
|
|
|
|
}
|
|
|
|
|
|
|
|
fn slot_has_updates(slot_meta: &SlotMeta, slot_meta_backup: &Option<SlotMeta>) -> bool {
|
|
|
|
// We should signal that there are updates if we extended the chain of consecutive blocks starting
|
|
|
|
// from block 0, which is true iff:
|
|
|
|
// 1) The block with index prev_block_index is itself part of the trunk of consecutive blocks
|
|
|
|
// starting from block 0,
|
|
|
|
slot_meta.is_connected &&
|
|
|
|
// AND either:
|
|
|
|
// 1) The slot didn't exist in the database before, and now we have a consecutive
|
|
|
|
// block for that slot
|
|
|
|
((slot_meta_backup.is_none() && slot_meta.consumed != 0) ||
|
|
|
|
// OR
|
|
|
|
// 2) The slot did exist, but now we have a new consecutive block for that slot
|
|
|
|
(slot_meta_backup.is_some() && slot_meta_backup.as_ref().unwrap().consumed != slot_meta.consumed))
|
2018-12-11 09:14:23 -08:00
|
|
|
}
|
|
|
|
|
2019-02-26 16:35:00 -08:00
|
|
|
// Creates a new ledger with slot 0 full of ticks (and only ticks).
|
|
|
|
//
|
2019-03-02 10:25:16 -08:00
|
|
|
// Returns the blockhash that can be used to append entries with.
|
2019-02-26 16:35:00 -08:00
|
|
|
pub fn create_new_ledger(ledger_path: &str, genesis_block: &GenesisBlock) -> Result<Hash> {
|
2019-02-21 18:46:04 -08:00
|
|
|
let ticks_per_slot = genesis_block.ticks_per_slot;
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(ledger_path)?;
|
2019-01-24 12:04:04 -08:00
|
|
|
genesis_block.write(&ledger_path)?;
|
2019-01-29 15:49:29 -08:00
|
|
|
|
2019-02-26 16:35:00 -08:00
|
|
|
// Fill slot 0 with ticks that link back to the genesis_block to bootstrap the ledger.
|
2019-03-14 15:18:37 -07:00
|
|
|
let blocktree = Blocktree::open(ledger_path)?;
|
2019-03-01 09:49:37 -08:00
|
|
|
let entries = crate::entry::create_ticks(ticks_per_slot, genesis_block.hash());
|
2019-03-14 15:18:37 -07:00
|
|
|
blocktree.write_entries(0, 0, 0, ticks_per_slot, &entries)?;
|
2019-01-29 15:49:29 -08:00
|
|
|
|
2019-03-01 08:57:42 -08:00
|
|
|
Ok(entries.last().unwrap().hash)
|
2019-01-24 12:04:04 -08:00
|
|
|
}
|
|
|
|
|
2019-02-27 13:37:08 -08:00
|
|
|
pub fn genesis<'a, I>(ledger_path: &str, keypair: &Keypair, entries: I) -> Result<()>
|
2018-12-11 09:14:23 -08:00
|
|
|
where
|
|
|
|
I: IntoIterator<Item = &'a Entry>,
|
|
|
|
{
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree = Blocktree::open(ledger_path)?;
|
2018-12-11 09:14:23 -08:00
|
|
|
|
|
|
|
// TODO sign these blobs with keypair
|
2018-12-22 19:30:30 -08:00
|
|
|
let blobs: Vec<_> = entries
|
|
|
|
.into_iter()
|
|
|
|
.enumerate()
|
|
|
|
.map(|(idx, entry)| {
|
|
|
|
let mut b = entry.borrow().to_blob();
|
2019-01-30 20:18:28 -08:00
|
|
|
b.set_index(idx as u64);
|
2019-02-27 13:37:08 -08:00
|
|
|
b.set_id(&keypair.pubkey());
|
2019-02-25 12:48:48 -08:00
|
|
|
b.set_slot(0);
|
2018-12-22 19:30:30 -08:00
|
|
|
b
|
|
|
|
})
|
|
|
|
.collect();
|
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree.write_genesis_blobs(&blobs[..])?;
|
2018-12-11 09:14:23 -08:00
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2019-02-26 17:11:26 -08:00
|
|
|
#[macro_export]
|
|
|
|
macro_rules! tmp_ledger_name {
|
|
|
|
() => {
|
|
|
|
&format!("{}-{}", file!(), line!())
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
#[macro_export]
|
|
|
|
macro_rules! get_tmp_ledger_path {
|
|
|
|
() => {
|
|
|
|
get_tmp_ledger_path(tmp_ledger_name!())
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2019-01-09 14:33:44 -08:00
|
|
|
pub fn get_tmp_ledger_path(name: &str) -> String {
|
|
|
|
use std::env;
|
|
|
|
let out_dir = env::var("OUT_DIR").unwrap_or_else(|_| "target".to_string());
|
|
|
|
let keypair = Keypair::new();
|
|
|
|
|
2019-02-26 17:11:26 -08:00
|
|
|
let path = format!("{}/tmp/ledger/{}-{}", out_dir, name, keypair.pubkey());
|
2019-01-09 14:33:44 -08:00
|
|
|
|
|
|
|
// whack any possible collision
|
2019-01-24 12:04:04 -08:00
|
|
|
let _ignored = fs::remove_dir_all(&path);
|
2019-01-09 14:33:44 -08:00
|
|
|
|
|
|
|
path
|
|
|
|
}
|
|
|
|
|
2019-02-26 19:19:34 -08:00
|
|
|
#[macro_export]
|
|
|
|
macro_rules! create_new_tmp_ledger {
|
|
|
|
($genesis_block:expr) => {
|
|
|
|
create_new_tmp_ledger(tmp_ledger_name!(), $genesis_block)
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2019-02-26 16:35:00 -08:00
|
|
|
// Same as `create_new_ledger()` but use a temporary ledger name based on the provided `name`
|
|
|
|
//
|
|
|
|
// Note: like `create_new_ledger` the returned ledger will have slot 0 full of ticks (and only
|
|
|
|
// ticks)
|
2019-02-26 19:36:46 -08:00
|
|
|
pub fn create_new_tmp_ledger(name: &str, genesis_block: &GenesisBlock) -> (String, Hash) {
|
2019-01-30 10:26:16 -08:00
|
|
|
let ledger_path = get_tmp_ledger_path(name);
|
2019-03-02 10:25:16 -08:00
|
|
|
let blockhash = create_new_ledger(&ledger_path, genesis_block).unwrap();
|
|
|
|
(ledger_path, blockhash)
|
2019-02-21 22:36:01 -08:00
|
|
|
}
|
|
|
|
|
2019-02-26 17:11:26 -08:00
|
|
|
#[macro_export]
|
|
|
|
macro_rules! tmp_copy_blocktree {
|
|
|
|
($from:expr) => {
|
|
|
|
tmp_copy_blocktree($from, tmp_ledger_name!())
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2019-02-23 06:36:23 -08:00
|
|
|
pub fn tmp_copy_blocktree(from: &str, name: &str) -> String {
|
2019-01-24 12:04:04 -08:00
|
|
|
let path = get_tmp_ledger_path(name);
|
2019-01-09 14:33:44 -08:00
|
|
|
|
2019-02-23 06:36:23 -08:00
|
|
|
let blocktree = Blocktree::open(from).unwrap();
|
2019-02-07 20:52:39 -08:00
|
|
|
let blobs = blocktree.read_ledger_blobs();
|
2019-01-24 12:04:04 -08:00
|
|
|
let genesis_block = GenesisBlock::load(from).unwrap();
|
2019-01-09 14:33:44 -08:00
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&path).expect("Expected successful database destruction");
|
2019-02-23 06:36:23 -08:00
|
|
|
let blocktree = Blocktree::open(&path).unwrap();
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree.write_blobs(blobs).unwrap();
|
2019-01-24 12:04:04 -08:00
|
|
|
genesis_block.write(&path).unwrap();
|
2019-01-09 14:33:44 -08:00
|
|
|
|
2019-01-24 12:04:04 -08:00
|
|
|
path
|
2019-01-09 14:33:44 -08:00
|
|
|
}
|
|
|
|
|
2018-11-15 15:53:31 -08:00
|
|
|
#[cfg(test)]
|
2019-02-18 18:41:31 -08:00
|
|
|
pub mod tests {
|
2018-11-15 15:53:31 -08:00
|
|
|
use super::*;
|
2019-02-18 19:49:43 -08:00
|
|
|
use crate::entry::{
|
2019-03-01 08:57:42 -08:00
|
|
|
create_ticks, make_tiny_test_entries, make_tiny_test_entries_from_hash, Entry, EntrySlice,
|
2019-02-18 19:49:43 -08:00
|
|
|
};
|
2019-03-17 18:48:23 -07:00
|
|
|
use crate::packet;
|
2019-02-13 15:01:56 -08:00
|
|
|
use rand::seq::SliceRandom;
|
|
|
|
use rand::thread_rng;
|
2019-01-22 15:50:36 -08:00
|
|
|
use solana_sdk::hash::Hash;
|
2019-03-30 20:37:33 -07:00
|
|
|
use solana_sdk::pubkey::Pubkey;
|
2019-02-13 15:01:56 -08:00
|
|
|
use std::cmp::min;
|
|
|
|
use std::collections::HashSet;
|
2019-02-12 19:54:18 -08:00
|
|
|
use std::iter::once;
|
2019-02-28 19:49:22 -08:00
|
|
|
use std::iter::FromIterator;
|
2019-02-07 15:10:54 -08:00
|
|
|
use std::time::Duration;
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-02-18 19:49:43 -08:00
|
|
|
#[test]
|
|
|
|
fn test_write_entries() {
|
2019-02-26 17:11:26 -08:00
|
|
|
solana_logger::setup();
|
|
|
|
let ledger_path = get_tmp_ledger_path!();
|
2019-02-18 19:49:43 -08:00
|
|
|
{
|
|
|
|
let ticks_per_slot = 10;
|
|
|
|
let num_slots = 10;
|
|
|
|
let num_ticks = ticks_per_slot * num_slots;
|
2019-03-14 15:18:37 -07:00
|
|
|
let ledger = Blocktree::open(&ledger_path).unwrap();
|
2019-02-18 19:59:09 -08:00
|
|
|
|
|
|
|
let ticks = create_ticks(num_ticks, Hash::default());
|
2019-03-14 15:18:37 -07:00
|
|
|
ledger
|
|
|
|
.write_entries(0, 0, 0, ticks_per_slot, ticks.clone())
|
|
|
|
.unwrap();
|
2019-02-18 19:49:43 -08:00
|
|
|
|
|
|
|
for i in 0..num_slots {
|
|
|
|
let meta = ledger.meta(i).unwrap().unwrap();
|
|
|
|
assert_eq!(meta.consumed, ticks_per_slot);
|
|
|
|
assert_eq!(meta.received, ticks_per_slot);
|
|
|
|
assert_eq!(meta.last_index, ticks_per_slot - 1);
|
2019-02-18 19:59:09 -08:00
|
|
|
if i == num_slots - 1 {
|
|
|
|
assert!(meta.next_slots.is_empty());
|
|
|
|
} else {
|
|
|
|
assert_eq!(meta.next_slots, vec![i + 1]);
|
|
|
|
}
|
2019-02-18 19:49:43 -08:00
|
|
|
if i == 0 {
|
|
|
|
assert_eq!(meta.parent_slot, 0);
|
|
|
|
} else {
|
|
|
|
assert_eq!(meta.parent_slot, i - 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
assert_eq!(
|
|
|
|
&ticks[(i * ticks_per_slot) as usize..((i + 1) * ticks_per_slot) as usize],
|
|
|
|
&ledger.get_slot_entries(i, 0, None).unwrap()[..]
|
|
|
|
);
|
|
|
|
}
|
2019-02-18 19:59:09 -08:00
|
|
|
|
|
|
|
// Simulate writing to the end of a slot with existing ticks
|
|
|
|
ledger
|
|
|
|
.write_entries(
|
|
|
|
num_slots,
|
|
|
|
ticks_per_slot - 1,
|
|
|
|
ticks_per_slot - 2,
|
2019-03-14 15:18:37 -07:00
|
|
|
ticks_per_slot,
|
2019-02-18 19:59:09 -08:00
|
|
|
&ticks[0..2],
|
|
|
|
)
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
let meta = ledger.meta(num_slots).unwrap().unwrap();
|
|
|
|
assert_eq!(meta.consumed, 0);
|
|
|
|
// received blob was ticks_per_slot - 2, so received should be ticks_per_slot - 2 + 1
|
|
|
|
assert_eq!(meta.received, ticks_per_slot - 1);
|
|
|
|
// last blob index ticks_per_slot - 2 because that's the blob that made tick_height == ticks_per_slot
|
|
|
|
// for the slot
|
|
|
|
assert_eq!(meta.last_index, ticks_per_slot - 2);
|
|
|
|
assert_eq!(meta.parent_slot, num_slots - 1);
|
|
|
|
assert_eq!(meta.next_slots, vec![num_slots + 1]);
|
|
|
|
assert_eq!(
|
|
|
|
&ticks[0..1],
|
|
|
|
&ledger
|
|
|
|
.get_slot_entries(num_slots, ticks_per_slot - 2, None)
|
|
|
|
.unwrap()[..]
|
|
|
|
);
|
|
|
|
|
|
|
|
// We wrote two entries, the second should spill into slot num_slots + 1
|
|
|
|
let meta = ledger.meta(num_slots + 1).unwrap().unwrap();
|
|
|
|
assert_eq!(meta.consumed, 1);
|
|
|
|
assert_eq!(meta.received, 1);
|
|
|
|
assert_eq!(meta.last_index, std::u64::MAX);
|
|
|
|
assert_eq!(meta.parent_slot, num_slots);
|
|
|
|
assert!(meta.next_slots.is_empty());
|
|
|
|
|
|
|
|
assert_eq!(
|
|
|
|
&ticks[1..2],
|
|
|
|
&ledger.get_slot_entries(num_slots + 1, 0, None).unwrap()[..]
|
|
|
|
);
|
2019-02-18 19:49:43 -08:00
|
|
|
}
|
|
|
|
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
2018-11-15 15:53:31 -08:00
|
|
|
#[test]
|
|
|
|
fn test_put_get_simple() {
|
|
|
|
let ledger_path = get_tmp_ledger_path("test_put_get_simple");
|
2019-02-07 20:52:39 -08:00
|
|
|
let ledger = Blocktree::open(&ledger_path).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
|
|
|
|
// Test meta column family
|
2019-02-25 12:48:48 -08:00
|
|
|
let meta = SlotMeta::new(0, 1);
|
2019-05-03 14:46:02 -07:00
|
|
|
ledger.meta_cf.put(0, &meta).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
let result = ledger
|
|
|
|
.meta_cf
|
2019-05-03 14:46:02 -07:00
|
|
|
.get(0)
|
2018-11-15 15:53:31 -08:00
|
|
|
.unwrap()
|
|
|
|
.expect("Expected meta object to exist");
|
|
|
|
|
|
|
|
assert_eq!(result, meta);
|
|
|
|
|
|
|
|
// Test erasure column family
|
|
|
|
let erasure = vec![1u8; 16];
|
2019-04-02 14:58:07 -07:00
|
|
|
let erasure_key = (0, 0);
|
2019-05-03 14:46:02 -07:00
|
|
|
ledger.erasure_cf.put_bytes(erasure_key, &erasure).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
|
|
|
|
let result = ledger
|
|
|
|
.erasure_cf
|
2019-05-03 14:46:02 -07:00
|
|
|
.get_bytes(erasure_key)
|
2018-11-15 15:53:31 -08:00
|
|
|
.unwrap()
|
|
|
|
.expect("Expected erasure object to exist");
|
|
|
|
|
|
|
|
assert_eq!(result, erasure);
|
|
|
|
|
|
|
|
// Test data column family
|
|
|
|
let data = vec![2u8; 16];
|
2019-04-02 14:58:07 -07:00
|
|
|
let data_key = (0, 0);
|
2019-05-03 14:46:02 -07:00
|
|
|
ledger.data_cf.put_bytes(data_key, &data).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
|
|
|
|
let result = ledger
|
|
|
|
.data_cf
|
2019-05-03 14:46:02 -07:00
|
|
|
.get_bytes(data_key)
|
2018-11-15 15:53:31 -08:00
|
|
|
.unwrap()
|
|
|
|
.expect("Expected data object to exist");
|
|
|
|
|
|
|
|
assert_eq!(result, data);
|
|
|
|
|
|
|
|
// Destroying database without closing it first is undefined behavior
|
|
|
|
drop(ledger);
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
2019-01-08 15:23:45 -08:00
|
|
|
fn test_read_blobs_bytes() {
|
2019-03-17 18:48:23 -07:00
|
|
|
let shared_blobs = make_tiny_test_entries(10).to_single_entry_shared_blobs();
|
2019-02-25 12:48:48 -08:00
|
|
|
let slot = 0;
|
2019-03-30 20:37:33 -07:00
|
|
|
packet::index_blobs(&shared_blobs, &Pubkey::new_rand(), 0, slot, 0);
|
2018-12-12 20:42:12 -08:00
|
|
|
|
2018-11-15 15:53:31 -08:00
|
|
|
let blob_locks: Vec<_> = shared_blobs.iter().map(|b| b.read().unwrap()).collect();
|
|
|
|
let blobs: Vec<&Blob> = blob_locks.iter().map(|b| &**b).collect();
|
|
|
|
|
2019-01-08 15:23:45 -08:00
|
|
|
let ledger_path = get_tmp_ledger_path("test_read_blobs_bytes");
|
2019-02-07 20:52:39 -08:00
|
|
|
let ledger = Blocktree::open(&ledger_path).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
ledger.write_blobs(blobs.clone()).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
|
|
|
|
let mut buf = [0; 1024];
|
2019-01-08 15:23:45 -08:00
|
|
|
let (num_blobs, bytes) = ledger.read_blobs_bytes(0, 1, &mut buf, slot).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
let bytes = bytes as usize;
|
|
|
|
assert_eq!(num_blobs, 1);
|
|
|
|
{
|
|
|
|
let blob_data = &buf[..bytes];
|
|
|
|
assert_eq!(blob_data, &blobs[0].data[..bytes]);
|
|
|
|
}
|
|
|
|
|
2019-01-08 15:23:45 -08:00
|
|
|
let (num_blobs, bytes2) = ledger.read_blobs_bytes(0, 2, &mut buf, slot).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
let bytes2 = bytes2 as usize;
|
|
|
|
assert_eq!(num_blobs, 2);
|
|
|
|
assert!(bytes2 > bytes);
|
|
|
|
{
|
|
|
|
let blob_data_1 = &buf[..bytes];
|
|
|
|
assert_eq!(blob_data_1, &blobs[0].data[..bytes]);
|
|
|
|
|
|
|
|
let blob_data_2 = &buf[bytes..bytes2];
|
|
|
|
assert_eq!(blob_data_2, &blobs[1].data[..bytes2 - bytes]);
|
|
|
|
}
|
|
|
|
|
|
|
|
// buf size part-way into blob[1], should just return blob[0]
|
|
|
|
let mut buf = vec![0; bytes + 1];
|
2019-01-08 15:23:45 -08:00
|
|
|
let (num_blobs, bytes3) = ledger.read_blobs_bytes(0, 2, &mut buf, slot).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
assert_eq!(num_blobs, 1);
|
|
|
|
let bytes3 = bytes3 as usize;
|
|
|
|
assert_eq!(bytes3, bytes);
|
|
|
|
|
|
|
|
let mut buf = vec![0; bytes2 - 1];
|
2019-01-08 15:23:45 -08:00
|
|
|
let (num_blobs, bytes4) = ledger.read_blobs_bytes(0, 2, &mut buf, slot).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
assert_eq!(num_blobs, 1);
|
|
|
|
let bytes4 = bytes4 as usize;
|
|
|
|
assert_eq!(bytes4, bytes);
|
|
|
|
|
|
|
|
let mut buf = vec![0; bytes * 2];
|
2019-01-08 15:23:45 -08:00
|
|
|
let (num_blobs, bytes6) = ledger.read_blobs_bytes(9, 1, &mut buf, slot).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
assert_eq!(num_blobs, 1);
|
|
|
|
let bytes6 = bytes6 as usize;
|
|
|
|
|
|
|
|
{
|
|
|
|
let blob_data = &buf[..bytes6];
|
|
|
|
assert_eq!(blob_data, &blobs[9].data[..bytes6]);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Read out of range
|
2019-01-08 15:23:45 -08:00
|
|
|
assert!(ledger.read_blobs_bytes(20, 2, &mut buf, slot).is_err());
|
2018-11-15 15:53:31 -08:00
|
|
|
|
|
|
|
// Destroying database without closing it first is undefined behavior
|
|
|
|
drop(ledger);
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_insert_data_blobs_basic() {
|
2019-02-12 19:54:18 -08:00
|
|
|
let num_entries = 5;
|
|
|
|
assert!(num_entries > 1);
|
2018-12-19 16:11:47 -08:00
|
|
|
|
2019-02-12 19:54:18 -08:00
|
|
|
let (blobs, entries) = make_slot_entries(0, 0, num_entries);
|
2018-11-15 15:53:31 -08:00
|
|
|
|
|
|
|
let ledger_path = get_tmp_ledger_path("test_insert_data_blobs_basic");
|
2019-02-07 20:52:39 -08:00
|
|
|
let ledger = Blocktree::open(&ledger_path).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-02-12 19:54:18 -08:00
|
|
|
// Insert last blob, we're missing the other blobs, so no consecutive
|
2019-02-07 15:10:54 -08:00
|
|
|
// blobs starting from slot 0, index 0 should exist.
|
2019-02-12 19:54:18 -08:00
|
|
|
ledger
|
|
|
|
.insert_data_blobs(once(&blobs[num_entries as usize - 1]))
|
|
|
|
.unwrap();
|
|
|
|
assert!(ledger.get_slot_entries(0, 0, None).unwrap().is_empty());
|
2019-02-04 15:33:43 -08:00
|
|
|
|
2018-11-15 15:53:31 -08:00
|
|
|
let meta = ledger
|
2019-04-26 08:52:10 -07:00
|
|
|
.meta(0)
|
2018-11-15 15:53:31 -08:00
|
|
|
.unwrap()
|
|
|
|
.expect("Expected new metadata object to be created");
|
2019-02-12 19:54:18 -08:00
|
|
|
assert!(meta.consumed == 0 && meta.received == num_entries);
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-02-12 19:54:18 -08:00
|
|
|
// Insert the other blobs, check for consecutive returned entries
|
|
|
|
ledger
|
|
|
|
.insert_data_blobs(&blobs[0..(num_entries - 1) as usize])
|
|
|
|
.unwrap();
|
|
|
|
let result = ledger.get_slot_entries(0, 0, None).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
|
|
|
|
assert_eq!(result, entries);
|
|
|
|
|
|
|
|
let meta = ledger
|
2019-04-26 08:52:10 -07:00
|
|
|
.meta(0)
|
2018-11-15 15:53:31 -08:00
|
|
|
.unwrap()
|
|
|
|
.expect("Expected new metadata object to exist");
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(meta.consumed, num_entries);
|
|
|
|
assert_eq!(meta.received, num_entries);
|
|
|
|
assert_eq!(meta.parent_slot, 0);
|
|
|
|
assert_eq!(meta.last_index, num_entries - 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
assert!(meta.next_slots.is_empty());
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(meta.is_connected);
|
2018-11-15 15:53:31 -08:00
|
|
|
|
|
|
|
// Destroying database without closing it first is undefined behavior
|
|
|
|
drop(ledger);
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
2019-02-12 19:54:18 -08:00
|
|
|
fn test_insert_data_blobs_reverse() {
|
|
|
|
let num_entries = 10;
|
|
|
|
let (blobs, entries) = make_slot_entries(0, 0, num_entries);
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-02-12 19:54:18 -08:00
|
|
|
let ledger_path = get_tmp_ledger_path("test_insert_data_blobs_reverse");
|
2019-02-07 20:52:39 -08:00
|
|
|
let ledger = Blocktree::open(&ledger_path).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2018-12-12 15:58:29 -08:00
|
|
|
// Insert blobs in reverse, check for consecutive returned blobs
|
2019-02-12 19:54:18 -08:00
|
|
|
for i in (0..num_entries).rev() {
|
|
|
|
ledger.insert_data_blobs(once(&blobs[i as usize])).unwrap();
|
|
|
|
let result = ledger.get_slot_entries(0, 0, None).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2018-11-15 15:53:31 -08:00
|
|
|
let meta = ledger
|
2019-04-26 08:52:10 -07:00
|
|
|
.meta(0)
|
2018-11-15 15:53:31 -08:00
|
|
|
.unwrap()
|
|
|
|
.expect("Expected metadata object to exist");
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(meta.parent_slot, 0);
|
|
|
|
assert_eq!(meta.last_index, num_entries - 1);
|
2018-11-15 15:53:31 -08:00
|
|
|
if i != 0 {
|
|
|
|
assert_eq!(result.len(), 0);
|
2019-02-12 19:54:18 -08:00
|
|
|
assert!(meta.consumed == 0 && meta.received == num_entries as u64);
|
2018-11-15 15:53:31 -08:00
|
|
|
} else {
|
|
|
|
assert_eq!(result, entries);
|
2019-02-12 19:54:18 -08:00
|
|
|
assert!(meta.consumed == num_entries as u64 && meta.received == num_entries as u64);
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Destroying database without closing it first is undefined behavior
|
|
|
|
drop(ledger);
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
2018-11-22 01:35:19 -08:00
|
|
|
|
2018-12-12 15:58:29 -08:00
|
|
|
#[test]
|
2019-02-07 15:10:54 -08:00
|
|
|
fn test_insert_slots() {
|
|
|
|
test_insert_data_blobs_slots("test_insert_data_blobs_slots_single", false);
|
|
|
|
test_insert_data_blobs_slots("test_insert_data_blobs_slots_bulk", true);
|
2018-12-12 15:58:29 -08:00
|
|
|
}
|
|
|
|
|
2018-11-22 01:35:19 -08:00
|
|
|
#[test]
|
|
|
|
pub fn test_iteration_order() {
|
|
|
|
let slot = 0;
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree_path = get_tmp_ledger_path("test_iteration_order");
|
2018-11-22 01:35:19 -08:00
|
|
|
{
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2018-11-22 01:35:19 -08:00
|
|
|
|
|
|
|
// Write entries
|
|
|
|
let num_entries = 8;
|
2019-01-22 15:50:36 -08:00
|
|
|
let entries = make_tiny_test_entries(num_entries);
|
2019-03-17 18:48:23 -07:00
|
|
|
let mut blobs = entries.to_single_entry_blobs();
|
2018-11-22 01:35:19 -08:00
|
|
|
|
2019-03-17 18:48:23 -07:00
|
|
|
for (i, b) in blobs.iter_mut().enumerate() {
|
|
|
|
b.set_index(1 << (i * 8));
|
|
|
|
b.set_slot(0);
|
2018-11-22 01:35:19 -08:00
|
|
|
}
|
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree
|
2019-03-17 18:48:23 -07:00
|
|
|
.write_blobs(&blobs)
|
2019-02-07 15:10:54 -08:00
|
|
|
.expect("Expected successful write of blobs");
|
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
let mut db_iterator = blocktree
|
2018-11-22 01:35:19 -08:00
|
|
|
.db
|
2019-04-02 14:58:07 -07:00
|
|
|
.cursor::<cf::Data>()
|
2018-11-22 01:35:19 -08:00
|
|
|
.expect("Expected to be able to open database iterator");
|
|
|
|
|
2019-04-02 14:58:07 -07:00
|
|
|
db_iterator.seek((slot, 1));
|
2018-11-22 01:35:19 -08:00
|
|
|
|
|
|
|
// Iterate through ledger
|
|
|
|
for i in 0..num_entries {
|
|
|
|
assert!(db_iterator.valid());
|
2019-04-02 14:58:07 -07:00
|
|
|
let (_, current_index) = db_iterator.key().expect("Expected a valid key");
|
2018-11-22 01:35:19 -08:00
|
|
|
assert_eq!(current_index, (1 as u64) << (i * 8));
|
|
|
|
db_iterator.next();
|
|
|
|
}
|
|
|
|
}
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2018-11-22 01:35:19 -08:00
|
|
|
}
|
2018-12-11 09:14:23 -08:00
|
|
|
|
2019-02-04 15:33:43 -08:00
|
|
|
#[test]
|
|
|
|
pub fn test_get_slot_entries1() {
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree_path = get_tmp_ledger_path("test_get_slot_entries1");
|
2019-02-04 15:33:43 -08:00
|
|
|
{
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2019-02-04 15:33:43 -08:00
|
|
|
let entries = make_tiny_test_entries(8);
|
2019-03-17 18:48:23 -07:00
|
|
|
let mut blobs = entries.clone().to_single_entry_blobs();
|
2019-02-04 15:33:43 -08:00
|
|
|
for (i, b) in blobs.iter_mut().enumerate() {
|
|
|
|
b.set_slot(1);
|
|
|
|
if i < 4 {
|
|
|
|
b.set_index(i as u64);
|
|
|
|
} else {
|
|
|
|
b.set_index(8 + i as u64);
|
|
|
|
}
|
|
|
|
}
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree
|
2019-02-04 15:33:43 -08:00
|
|
|
.write_blobs(&blobs)
|
|
|
|
.expect("Expected successful write of blobs");
|
|
|
|
|
|
|
|
assert_eq!(
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree.get_slot_entries(1, 2, None).unwrap()[..],
|
2019-02-04 15:33:43 -08:00
|
|
|
entries[2..4],
|
|
|
|
);
|
|
|
|
|
|
|
|
assert_eq!(
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree.get_slot_entries(1, 12, None).unwrap()[..],
|
2019-02-04 15:33:43 -08:00
|
|
|
entries[4..],
|
|
|
|
);
|
|
|
|
}
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2019-02-04 15:33:43 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
pub fn test_get_slot_entries2() {
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree_path = get_tmp_ledger_path("test_get_slot_entries2");
|
2019-02-04 15:33:43 -08:00
|
|
|
{
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2019-02-04 15:33:43 -08:00
|
|
|
|
|
|
|
// Write entries
|
|
|
|
let num_slots = 5 as u64;
|
|
|
|
let mut index = 0;
|
2019-03-05 14:18:29 -08:00
|
|
|
for slot in 0..num_slots {
|
|
|
|
let entries = make_tiny_test_entries(slot as usize + 1);
|
2019-02-04 15:33:43 -08:00
|
|
|
let last_entry = entries.last().unwrap().clone();
|
2019-03-17 18:48:23 -07:00
|
|
|
let mut blobs = entries.clone().to_single_entry_blobs();
|
2019-02-04 15:33:43 -08:00
|
|
|
for b in blobs.iter_mut() {
|
|
|
|
b.set_index(index);
|
2019-03-05 14:18:29 -08:00
|
|
|
b.set_slot(slot as u64);
|
2019-02-04 15:33:43 -08:00
|
|
|
index += 1;
|
|
|
|
}
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree
|
2019-02-04 15:33:43 -08:00
|
|
|
.write_blobs(&blobs)
|
|
|
|
.expect("Expected successful write of blobs");
|
|
|
|
assert_eq!(
|
2019-03-05 14:18:29 -08:00
|
|
|
blocktree.get_slot_entries(slot, index - 1, None).unwrap(),
|
2019-02-04 15:33:43 -08:00
|
|
|
vec![last_entry],
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2019-02-04 15:33:43 -08:00
|
|
|
}
|
|
|
|
|
2019-03-17 18:48:23 -07:00
|
|
|
#[test]
|
|
|
|
pub fn test_get_slot_entries3() {
|
|
|
|
// Test inserting/fetching blobs which contain multiple entries per blob
|
|
|
|
let blocktree_path = get_tmp_ledger_path("test_get_slot_entries3");
|
|
|
|
{
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
let num_slots = 5 as u64;
|
|
|
|
let blobs_per_slot = 5 as u64;
|
|
|
|
let entry_serialized_size =
|
|
|
|
bincode::serialized_size(&make_tiny_test_entries(1)).unwrap();
|
|
|
|
let entries_per_slot =
|
|
|
|
(blobs_per_slot * packet::BLOB_DATA_SIZE as u64) / entry_serialized_size;
|
|
|
|
|
|
|
|
// Write entries
|
|
|
|
for slot in 0..num_slots {
|
|
|
|
let mut index = 0;
|
|
|
|
let entries = make_tiny_test_entries(entries_per_slot as usize);
|
|
|
|
let mut blobs = entries.clone().to_blobs();
|
|
|
|
assert_eq!(blobs.len() as u64, blobs_per_slot);
|
|
|
|
for b in blobs.iter_mut() {
|
|
|
|
b.set_index(index);
|
|
|
|
b.set_slot(slot as u64);
|
|
|
|
index += 1;
|
|
|
|
}
|
|
|
|
blocktree
|
|
|
|
.write_blobs(&blobs)
|
|
|
|
.expect("Expected successful write of blobs");
|
|
|
|
assert_eq!(blocktree.get_slot_entries(slot, 0, None).unwrap(), entries,);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
2018-12-19 16:11:47 -08:00
|
|
|
#[test]
|
2019-02-07 15:10:54 -08:00
|
|
|
pub fn test_insert_data_blobs_consecutive() {
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree_path = get_tmp_ledger_path("test_insert_data_blobs_consecutive");
|
2018-12-19 16:11:47 -08:00
|
|
|
{
|
2019-03-14 15:18:37 -07:00
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2019-04-18 21:56:43 -07:00
|
|
|
for i in 0..4 {
|
|
|
|
let slot = i;
|
|
|
|
let parent_slot = if i == 0 { 0 } else { i - 1 };
|
|
|
|
// Write entries
|
|
|
|
let num_entries = 21 as u64 * (i + 1);
|
|
|
|
let (blobs, original_entries) = make_slot_entries(slot, parent_slot, num_entries);
|
2019-04-17 12:52:12 -07:00
|
|
|
|
2019-04-18 21:56:43 -07:00
|
|
|
blocktree
|
|
|
|
.write_blobs(blobs.iter().skip(1).step_by(2))
|
|
|
|
.unwrap();
|
2019-04-17 12:52:12 -07:00
|
|
|
|
2019-04-18 21:56:43 -07:00
|
|
|
assert_eq!(blocktree.get_slot_entries(slot, 0, None).unwrap(), vec![]);
|
2019-04-17 12:52:12 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
let meta = blocktree.meta(slot).unwrap().unwrap();
|
2019-04-18 21:56:43 -07:00
|
|
|
if num_entries % 2 == 0 {
|
|
|
|
assert_eq!(meta.received, num_entries);
|
|
|
|
} else {
|
|
|
|
debug!("got here");
|
|
|
|
assert_eq!(meta.received, num_entries - 1);
|
|
|
|
}
|
|
|
|
assert_eq!(meta.consumed, 0);
|
|
|
|
assert_eq!(meta.parent_slot, parent_slot);
|
|
|
|
if num_entries % 2 == 0 {
|
|
|
|
assert_eq!(meta.last_index, num_entries - 1);
|
|
|
|
} else {
|
|
|
|
assert_eq!(meta.last_index, std::u64::MAX);
|
|
|
|
}
|
2019-04-17 18:04:30 -07:00
|
|
|
|
2019-04-18 21:56:43 -07:00
|
|
|
blocktree.write_blobs(blobs.iter().step_by(2)).unwrap();
|
2019-04-17 18:04:30 -07:00
|
|
|
|
2019-04-18 21:56:43 -07:00
|
|
|
assert_eq!(
|
|
|
|
blocktree.get_slot_entries(slot, 0, None).unwrap(),
|
|
|
|
original_entries,
|
|
|
|
);
|
2019-04-17 18:04:30 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
let meta = blocktree.meta(slot).unwrap().unwrap();
|
2019-04-18 21:56:43 -07:00
|
|
|
assert_eq!(meta.received, num_entries);
|
|
|
|
assert_eq!(meta.consumed, num_entries);
|
|
|
|
assert_eq!(meta.parent_slot, parent_slot);
|
|
|
|
assert_eq!(meta.last_index, num_entries - 1);
|
|
|
|
}
|
2018-12-20 12:12:04 -08:00
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2018-12-20 12:12:04 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
pub fn test_insert_data_blobs_duplicate() {
|
|
|
|
// Create RocksDb ledger
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree_path = get_tmp_ledger_path("test_insert_data_blobs_duplicate");
|
2018-12-20 12:12:04 -08:00
|
|
|
{
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2018-12-20 12:12:04 -08:00
|
|
|
|
2019-02-12 19:54:18 -08:00
|
|
|
// Make duplicate entries and blobs
|
2018-12-20 12:12:04 -08:00
|
|
|
let num_duplicates = 2;
|
2019-02-12 19:54:18 -08:00
|
|
|
let num_unique_entries = 10;
|
|
|
|
let (original_entries, blobs) = {
|
|
|
|
let (blobs, entries) = make_slot_entries(0, 0, num_unique_entries);
|
|
|
|
let entries: Vec<_> = entries
|
|
|
|
.into_iter()
|
|
|
|
.flat_map(|e| vec![e.clone(), e])
|
|
|
|
.collect();
|
|
|
|
let blobs: Vec<_> = blobs.into_iter().flat_map(|b| vec![b.clone(), b]).collect();
|
|
|
|
(entries, blobs)
|
|
|
|
};
|
2018-12-20 12:12:04 -08:00
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree
|
2019-02-12 19:54:18 -08:00
|
|
|
.write_blobs(
|
|
|
|
blobs
|
2019-02-07 15:10:54 -08:00
|
|
|
.iter()
|
2019-02-12 19:54:18 -08:00
|
|
|
.skip(num_duplicates as usize)
|
|
|
|
.step_by(num_duplicates as usize * 2),
|
2019-02-07 15:10:54 -08:00
|
|
|
)
|
|
|
|
.unwrap();
|
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
assert_eq!(blocktree.get_slot_entries(0, 0, None).unwrap(), vec![]);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree
|
2019-02-12 19:54:18 -08:00
|
|
|
.write_blobs(blobs.iter().step_by(num_duplicates as usize * 2))
|
2019-02-07 15:10:54 -08:00
|
|
|
.unwrap();
|
2018-12-20 12:12:04 -08:00
|
|
|
|
|
|
|
let expected: Vec<_> = original_entries
|
|
|
|
.into_iter()
|
2019-02-12 19:54:18 -08:00
|
|
|
.step_by(num_duplicates as usize)
|
2018-12-20 12:12:04 -08:00
|
|
|
.collect();
|
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
assert_eq!(blocktree.get_slot_entries(0, 0, None).unwrap(), expected,);
|
2018-12-20 12:12:04 -08:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
let meta = blocktree.meta(0).unwrap().unwrap();
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(meta.consumed, num_unique_entries);
|
|
|
|
assert_eq!(meta.received, num_unique_entries);
|
|
|
|
assert_eq!(meta.parent_slot, 0);
|
|
|
|
assert_eq!(meta.last_index, num_unique_entries - 1);
|
2018-12-19 16:11:47 -08:00
|
|
|
}
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2018-12-19 16:11:47 -08:00
|
|
|
}
|
|
|
|
|
2018-12-11 09:14:23 -08:00
|
|
|
#[test]
|
|
|
|
pub fn test_genesis_and_entry_iterator() {
|
2019-03-01 08:57:42 -08:00
|
|
|
let entries = make_tiny_test_entries_from_hash(&Hash::default(), 10);
|
2019-01-22 15:50:36 -08:00
|
|
|
|
2018-12-12 20:42:12 -08:00
|
|
|
let ledger_path = get_tmp_ledger_path("test_genesis_and_entry_iterator");
|
2018-12-11 09:14:23 -08:00
|
|
|
{
|
2019-02-27 13:37:08 -08:00
|
|
|
genesis(&ledger_path, &Keypair::new(), &entries).unwrap();
|
2018-12-11 09:14:23 -08:00
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
let ledger = Blocktree::open(&ledger_path).expect("open failed");
|
2018-12-11 09:14:23 -08:00
|
|
|
|
|
|
|
let read_entries: Vec<Entry> =
|
|
|
|
ledger.read_ledger().expect("read_ledger failed").collect();
|
2019-01-22 15:50:36 -08:00
|
|
|
assert!(read_entries.verify(&Hash::default()));
|
2018-12-11 09:14:23 -08:00
|
|
|
assert_eq!(entries, read_entries);
|
|
|
|
}
|
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
2018-12-11 09:14:23 -08:00
|
|
|
}
|
2019-01-22 15:50:36 -08:00
|
|
|
#[test]
|
|
|
|
pub fn test_entry_iterator_up_to_consumed() {
|
2019-03-01 08:57:42 -08:00
|
|
|
let entries = make_tiny_test_entries_from_hash(&Hash::default(), 3);
|
2019-01-22 15:50:36 -08:00
|
|
|
let ledger_path = get_tmp_ledger_path("test_genesis_and_entry_iterator");
|
|
|
|
{
|
|
|
|
// put entries except last 2 into ledger
|
2019-02-27 13:37:08 -08:00
|
|
|
genesis(&ledger_path, &Keypair::new(), &entries[..entries.len() - 2]).unwrap();
|
2019-01-22 15:50:36 -08:00
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
let ledger = Blocktree::open(&ledger_path).expect("open failed");
|
2019-01-22 15:50:36 -08:00
|
|
|
|
|
|
|
// now write the last entry, ledger has a hole in it one before the end
|
|
|
|
// +-+-+-+-+-+-+-+ +-+
|
|
|
|
// | | | | | | | | | |
|
|
|
|
// +-+-+-+-+-+-+-+ +-+
|
|
|
|
ledger
|
|
|
|
.write_entries(
|
|
|
|
0u64,
|
2019-02-12 13:14:33 -08:00
|
|
|
0,
|
2019-01-22 15:50:36 -08:00
|
|
|
(entries.len() - 1) as u64,
|
2019-03-14 15:18:37 -07:00
|
|
|
16,
|
2019-01-22 15:50:36 -08:00
|
|
|
&entries[entries.len() - 1..],
|
|
|
|
)
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
let read_entries: Vec<Entry> =
|
|
|
|
ledger.read_ledger().expect("read_ledger failed").collect();
|
|
|
|
assert!(read_entries.verify(&Hash::default()));
|
|
|
|
|
|
|
|
// enumeration should stop at the hole
|
|
|
|
assert_eq!(entries[..entries.len() - 2].to_vec(), read_entries);
|
|
|
|
}
|
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
2019-01-22 15:50:36 -08:00
|
|
|
}
|
2018-12-11 09:14:23 -08:00
|
|
|
|
2019-02-07 15:10:54 -08:00
|
|
|
#[test]
|
|
|
|
pub fn test_new_blobs_signal() {
|
|
|
|
// Initialize ledger
|
|
|
|
let ledger_path = get_tmp_ledger_path("test_new_blobs_signal");
|
2019-02-17 13:05:47 -08:00
|
|
|
let (ledger, recvr) = Blocktree::open_with_signal(&ledger_path).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
let ledger = Arc::new(ledger);
|
|
|
|
|
2019-02-12 19:54:18 -08:00
|
|
|
let entries_per_slot = 10;
|
|
|
|
// Create entries for slot 0
|
|
|
|
let (blobs, _) = make_slot_entries(0, 0, entries_per_slot);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
// Insert second blob, but we're missing the first blob, so no consecutive
|
|
|
|
// blobs starting from slot 0, index 0 should exist.
|
2019-02-12 19:54:18 -08:00
|
|
|
ledger.insert_data_blobs(once(&blobs[1])).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
let timer = Duration::new(1, 0);
|
|
|
|
assert!(recvr.recv_timeout(timer).is_err());
|
|
|
|
// Insert first blob, now we've made a consecutive block
|
2019-02-12 19:54:18 -08:00
|
|
|
ledger.insert_data_blobs(once(&blobs[0])).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
// Wait to get notified of update, should only be one update
|
|
|
|
assert!(recvr.recv_timeout(timer).is_ok());
|
|
|
|
assert!(recvr.try_recv().is_err());
|
|
|
|
// Insert the rest of the ticks
|
|
|
|
ledger
|
2019-02-12 19:54:18 -08:00
|
|
|
.insert_data_blobs(&blobs[1..entries_per_slot as usize])
|
2019-02-07 15:10:54 -08:00
|
|
|
.unwrap();
|
|
|
|
// Wait to get notified of update, should only be one update
|
|
|
|
assert!(recvr.recv_timeout(timer).is_ok());
|
|
|
|
assert!(recvr.try_recv().is_err());
|
|
|
|
|
|
|
|
// Create some other slots, and send batches of ticks for each slot such that each slot
|
|
|
|
// is missing the tick at blob index == slot index - 1. Thus, no consecutive blocks
|
|
|
|
// will be formed
|
2019-02-12 19:54:18 -08:00
|
|
|
let num_slots = entries_per_slot;
|
|
|
|
let mut blobs: Vec<Blob> = vec![];
|
|
|
|
let mut missing_blobs = vec![];
|
2019-03-05 14:18:29 -08:00
|
|
|
for slot in 1..num_slots + 1 {
|
|
|
|
let (mut slot_blobs, _) = make_slot_entries(slot, slot - 1, entries_per_slot);
|
|
|
|
let missing_blob = slot_blobs.remove(slot as usize - 1);
|
2019-02-12 19:54:18 -08:00
|
|
|
blobs.extend(slot_blobs);
|
|
|
|
missing_blobs.push(missing_blob);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Should be no updates, since no new chains from block 0 were formed
|
2019-02-12 19:54:18 -08:00
|
|
|
ledger.insert_data_blobs(blobs.iter()).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
assert!(recvr.recv_timeout(timer).is_err());
|
|
|
|
|
|
|
|
// Insert a blob for each slot that doesn't make a consecutive block, we
|
|
|
|
// should get no updates
|
2019-02-12 19:54:18 -08:00
|
|
|
let blobs: Vec<_> = (1..num_slots + 1)
|
2019-03-05 14:18:29 -08:00
|
|
|
.flat_map(|slot| {
|
|
|
|
let (mut blob, _) = make_slot_entries(slot, slot - 1, 1);
|
2019-02-12 19:54:18 -08:00
|
|
|
blob[0].set_index(2 * num_slots as u64);
|
2019-02-07 15:10:54 -08:00
|
|
|
blob
|
|
|
|
})
|
|
|
|
.collect();
|
|
|
|
|
2019-02-12 19:54:18 -08:00
|
|
|
ledger.insert_data_blobs(blobs.iter()).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
assert!(recvr.recv_timeout(timer).is_err());
|
|
|
|
|
|
|
|
// For slots 1..num_slots/2, fill in the holes in one batch insertion,
|
|
|
|
// so we should only get one signal
|
2019-02-12 19:54:18 -08:00
|
|
|
ledger
|
|
|
|
.insert_data_blobs(&missing_blobs[..(num_slots / 2) as usize])
|
|
|
|
.unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
assert!(recvr.recv_timeout(timer).is_ok());
|
|
|
|
assert!(recvr.try_recv().is_err());
|
|
|
|
|
|
|
|
// Fill in the holes for each of the remaining slots, we should get a single update
|
|
|
|
// for each
|
2019-02-12 19:54:18 -08:00
|
|
|
for missing_blob in &missing_blobs[(num_slots / 2) as usize..] {
|
|
|
|
ledger
|
|
|
|
.insert_data_blobs(vec![missing_blob.clone()])
|
|
|
|
.unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Destroying database without closing it first is undefined behavior
|
|
|
|
drop(ledger);
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
pub fn test_handle_chaining_basic() {
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree_path = get_tmp_ledger_path("test_handle_chaining_basic");
|
2019-02-07 15:10:54 -08:00
|
|
|
{
|
2019-02-12 19:54:18 -08:00
|
|
|
let entries_per_slot = 2;
|
|
|
|
let num_slots = 3;
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
|
|
|
|
// Construct the blobs
|
|
|
|
let (blobs, _) = make_many_slot_entries(0, num_slots, entries_per_slot);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
// 1) Write to the first slot
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree
|
2019-02-12 19:54:18 -08:00
|
|
|
.write_blobs(&blobs[entries_per_slot as usize..2 * entries_per_slot as usize])
|
2019-02-07 15:10:54 -08:00
|
|
|
.unwrap();
|
2019-02-13 15:01:56 -08:00
|
|
|
let s1 = blocktree.meta(1).unwrap().unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
assert!(s1.next_slots.is_empty());
|
|
|
|
// Slot 1 is not trunk because slot 0 hasn't been inserted yet
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(!s1.is_connected);
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(s1.parent_slot, 0);
|
|
|
|
assert_eq!(s1.last_index, entries_per_slot - 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
// 2) Write to the second slot
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree
|
2019-02-12 19:54:18 -08:00
|
|
|
.write_blobs(&blobs[2 * entries_per_slot as usize..3 * entries_per_slot as usize])
|
2019-02-07 15:10:54 -08:00
|
|
|
.unwrap();
|
2019-02-13 15:01:56 -08:00
|
|
|
let s2 = blocktree.meta(2).unwrap().unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
assert!(s2.next_slots.is_empty());
|
|
|
|
// Slot 2 is not trunk because slot 0 hasn't been inserted yet
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(!s2.is_connected);
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(s2.parent_slot, 1);
|
|
|
|
assert_eq!(s2.last_index, entries_per_slot - 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
// Check the first slot again, it should chain to the second slot,
|
|
|
|
// but still isn't part of the trunk
|
2019-02-13 15:01:56 -08:00
|
|
|
let s1 = blocktree.meta(1).unwrap().unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
assert_eq!(s1.next_slots, vec![2]);
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(!s1.is_connected);
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(s1.parent_slot, 0);
|
|
|
|
assert_eq!(s1.last_index, entries_per_slot - 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
// 3) Write to the zeroth slot, check that every slot
|
|
|
|
// is now part of the trunk
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree
|
2019-02-12 19:54:18 -08:00
|
|
|
.write_blobs(&blobs[0..entries_per_slot as usize])
|
2019-02-07 15:10:54 -08:00
|
|
|
.unwrap();
|
|
|
|
for i in 0..3 {
|
2019-02-13 15:01:56 -08:00
|
|
|
let s = blocktree.meta(i).unwrap().unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
// The last slot will not chain to any other slots
|
|
|
|
if i != 2 {
|
|
|
|
assert_eq!(s.next_slots, vec![i + 1]);
|
|
|
|
}
|
2019-02-12 19:54:18 -08:00
|
|
|
if i == 0 {
|
|
|
|
assert_eq!(s.parent_slot, 0);
|
|
|
|
} else {
|
|
|
|
assert_eq!(s.parent_slot, i - 1);
|
|
|
|
}
|
|
|
|
assert_eq!(s.last_index, entries_per_slot - 1);
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(s.is_connected);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
pub fn test_handle_chaining_missing_slots() {
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree_path = get_tmp_ledger_path("test_handle_chaining_missing_slots");
|
2019-02-07 15:10:54 -08:00
|
|
|
{
|
2019-02-12 19:54:18 -08:00
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
let num_slots = 30;
|
2019-02-12 19:54:18 -08:00
|
|
|
let entries_per_slot = 2;
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-02-12 19:54:18 -08:00
|
|
|
// Separate every other slot into two separate vectors
|
|
|
|
let mut slots = vec![];
|
|
|
|
let mut missing_slots = vec![];
|
2019-03-05 14:18:29 -08:00
|
|
|
for slot in 0..num_slots {
|
2019-02-12 19:54:18 -08:00
|
|
|
let parent_slot = {
|
2019-03-05 14:18:29 -08:00
|
|
|
if slot == 0 {
|
2019-02-12 19:54:18 -08:00
|
|
|
0
|
|
|
|
} else {
|
2019-03-05 14:18:29 -08:00
|
|
|
slot - 1
|
2019-02-12 19:54:18 -08:00
|
|
|
}
|
|
|
|
};
|
2019-03-05 14:18:29 -08:00
|
|
|
let (slot_blobs, _) = make_slot_entries(slot, parent_slot, entries_per_slot);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-03-05 14:18:29 -08:00
|
|
|
if slot % 2 == 1 {
|
2019-02-12 19:54:18 -08:00
|
|
|
slots.extend(slot_blobs);
|
|
|
|
} else {
|
|
|
|
missing_slots.extend(slot_blobs);
|
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-02-12 19:54:18 -08:00
|
|
|
// Write the blobs for every other slot
|
|
|
|
blocktree.write_blobs(&slots).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
// Check metadata
|
|
|
|
for i in 0..num_slots {
|
2019-02-12 19:54:18 -08:00
|
|
|
// If "i" is the index of a slot we just inserted, then next_slots should be empty
|
|
|
|
// for slot "i" because no slots chain to that slot, because slot i + 1 is missing.
|
2019-04-06 19:41:22 -07:00
|
|
|
// However, if it's a slot we haven't inserted, aka one of the gaps, then one of the
|
|
|
|
// slots we just inserted will chain to that gap, so next_slots for that orphan slot
|
|
|
|
// won't be empty, but the parent slot is unknown so should equal std::u64::MAX.
|
2019-02-13 15:01:56 -08:00
|
|
|
let s = blocktree.meta(i as u64).unwrap().unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
if i % 2 == 0 {
|
|
|
|
assert_eq!(s.next_slots, vec![i as u64 + 1]);
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(s.parent_slot, std::u64::MAX);
|
2019-02-07 15:10:54 -08:00
|
|
|
} else {
|
|
|
|
assert!(s.next_slots.is_empty());
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(s.parent_slot, i - 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
if i == 0 {
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(s.is_connected);
|
2019-02-07 15:10:54 -08:00
|
|
|
} else {
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(!s.is_connected);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-02-12 19:54:18 -08:00
|
|
|
// Write the blobs for the other half of the slots that we didn't insert earlier
|
|
|
|
blocktree.write_blobs(&missing_slots[..]).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
for i in 0..num_slots {
|
|
|
|
// Check that all the slots chain correctly once the missing slots
|
|
|
|
// have been filled
|
2019-02-13 15:01:56 -08:00
|
|
|
let s = blocktree.meta(i as u64).unwrap().unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
if i != num_slots - 1 {
|
|
|
|
assert_eq!(s.next_slots, vec![i as u64 + 1]);
|
|
|
|
} else {
|
|
|
|
assert!(s.next_slots.is_empty());
|
|
|
|
}
|
2019-02-12 19:54:18 -08:00
|
|
|
|
|
|
|
if i == 0 {
|
|
|
|
assert_eq!(s.parent_slot, 0);
|
|
|
|
} else {
|
|
|
|
assert_eq!(s.parent_slot, i - 1);
|
|
|
|
}
|
|
|
|
assert_eq!(s.last_index, entries_per_slot - 1);
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(s.is_connected);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
2019-03-20 11:19:37 -07:00
|
|
|
pub fn test_forward_chaining_is_connected() {
|
|
|
|
let blocktree_path = get_tmp_ledger_path("test_forward_chaining_is_connected");
|
2019-02-07 15:10:54 -08:00
|
|
|
{
|
2019-02-12 19:54:18 -08:00
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
let num_slots = 15;
|
2019-02-12 19:54:18 -08:00
|
|
|
let entries_per_slot = 2;
|
|
|
|
assert!(entries_per_slot > 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-02-12 19:54:18 -08:00
|
|
|
let (blobs, _) = make_many_slot_entries(0, num_slots, entries_per_slot);
|
|
|
|
|
|
|
|
// Write the blobs such that every 3rd slot has a gap in the beginning
|
2019-03-05 14:18:29 -08:00
|
|
|
for (slot, slot_ticks) in blobs.chunks(entries_per_slot as usize).enumerate() {
|
|
|
|
if slot % 3 == 0 {
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree
|
2019-02-12 19:54:18 -08:00
|
|
|
.write_blobs(&slot_ticks[1..entries_per_slot as usize])
|
2019-02-07 15:10:54 -08:00
|
|
|
.unwrap();
|
|
|
|
} else {
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree
|
2019-02-12 19:54:18 -08:00
|
|
|
.write_blobs(&slot_ticks[..entries_per_slot as usize])
|
2019-02-07 15:10:54 -08:00
|
|
|
.unwrap();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check metadata
|
|
|
|
for i in 0..num_slots {
|
2019-02-13 15:01:56 -08:00
|
|
|
let s = blocktree.meta(i as u64).unwrap().unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
// The last slot will not chain to any other slots
|
|
|
|
if i as u64 != num_slots - 1 {
|
|
|
|
assert_eq!(s.next_slots, vec![i as u64 + 1]);
|
|
|
|
} else {
|
|
|
|
assert!(s.next_slots.is_empty());
|
|
|
|
}
|
2019-02-12 19:54:18 -08:00
|
|
|
|
|
|
|
if i == 0 {
|
|
|
|
assert_eq!(s.parent_slot, 0);
|
2019-02-07 15:10:54 -08:00
|
|
|
} else {
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(s.parent_slot, i - 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(s.last_index, entries_per_slot - 1);
|
|
|
|
|
2019-02-07 15:10:54 -08:00
|
|
|
// Other than slot 0, no slots should be part of the trunk
|
|
|
|
if i != 0 {
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(!s.is_connected);
|
2019-02-07 15:10:54 -08:00
|
|
|
} else {
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(s.is_connected);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Iteratively finish every 3rd slot, and check that all slots up to and including
|
|
|
|
// slot_index + 3 become part of the trunk
|
2019-02-12 19:54:18 -08:00
|
|
|
for (slot_index, slot_ticks) in blobs.chunks(entries_per_slot as usize).enumerate() {
|
2019-02-07 15:10:54 -08:00
|
|
|
if slot_index % 3 == 0 {
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree.write_blobs(&slot_ticks[0..1]).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
for i in 0..num_slots {
|
2019-02-13 15:01:56 -08:00
|
|
|
let s = blocktree.meta(i as u64).unwrap().unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
if i != num_slots - 1 {
|
|
|
|
assert_eq!(s.next_slots, vec![i as u64 + 1]);
|
|
|
|
} else {
|
|
|
|
assert!(s.next_slots.is_empty());
|
|
|
|
}
|
|
|
|
if i <= slot_index as u64 + 3 {
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(s.is_connected);
|
2019-02-07 15:10:54 -08:00
|
|
|
} else {
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(!s.is_connected);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
2019-02-12 19:54:18 -08:00
|
|
|
|
|
|
|
if i == 0 {
|
|
|
|
assert_eq!(s.parent_slot, 0);
|
|
|
|
} else {
|
|
|
|
assert_eq!(s.parent_slot, i - 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
assert_eq!(s.last_index, entries_per_slot - 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-02-13 15:01:56 -08:00
|
|
|
#[test]
|
|
|
|
pub fn test_chaining_tree() {
|
|
|
|
let blocktree_path = get_tmp_ledger_path("test_chaining_forks");
|
|
|
|
{
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
let num_tree_levels = 6;
|
|
|
|
assert!(num_tree_levels > 1);
|
|
|
|
let branching_factor: u64 = 4;
|
|
|
|
// Number of slots that will be in the tree
|
|
|
|
let num_slots = (branching_factor.pow(num_tree_levels) - 1) / (branching_factor - 1);
|
|
|
|
let entries_per_slot = 2;
|
|
|
|
assert!(entries_per_slot > 1);
|
|
|
|
|
|
|
|
let (mut blobs, _) = make_many_slot_entries(0, num_slots, entries_per_slot);
|
|
|
|
|
|
|
|
// Insert tree one slot at a time in a random order
|
|
|
|
let mut slots: Vec<_> = (0..num_slots).collect();
|
|
|
|
|
|
|
|
// Get blobs for the slot
|
|
|
|
slots.shuffle(&mut thread_rng());
|
2019-03-05 14:18:29 -08:00
|
|
|
for slot in slots {
|
|
|
|
// Get blobs for the slot "slot"
|
|
|
|
let slot_blobs = &mut blobs
|
|
|
|
[(slot * entries_per_slot) as usize..((slot + 1) * entries_per_slot) as usize];
|
2019-02-13 15:01:56 -08:00
|
|
|
for blob in slot_blobs.iter_mut() {
|
|
|
|
// Get the parent slot of the slot in the tree
|
|
|
|
let slot_parent = {
|
2019-03-05 14:18:29 -08:00
|
|
|
if slot == 0 {
|
2019-02-13 15:01:56 -08:00
|
|
|
0
|
|
|
|
} else {
|
2019-03-05 14:18:29 -08:00
|
|
|
(slot - 1) / branching_factor
|
2019-02-13 15:01:56 -08:00
|
|
|
}
|
|
|
|
};
|
|
|
|
blob.set_parent(slot_parent);
|
|
|
|
}
|
|
|
|
|
|
|
|
blocktree.write_blobs(slot_blobs).unwrap();
|
|
|
|
}
|
|
|
|
|
|
|
|
// Make sure everything chains correctly
|
|
|
|
let last_level =
|
|
|
|
(branching_factor.pow(num_tree_levels - 1) - 1) / (branching_factor - 1);
|
2019-03-05 14:18:29 -08:00
|
|
|
for slot in 0..num_slots {
|
|
|
|
let slot_meta = blocktree.meta(slot).unwrap().unwrap();
|
2019-02-13 15:01:56 -08:00
|
|
|
assert_eq!(slot_meta.consumed, entries_per_slot);
|
|
|
|
assert_eq!(slot_meta.received, entries_per_slot);
|
2019-03-29 16:07:24 -07:00
|
|
|
assert!(slot_meta.is_connected);
|
2019-02-13 15:01:56 -08:00
|
|
|
let slot_parent = {
|
2019-03-05 14:18:29 -08:00
|
|
|
if slot == 0 {
|
2019-02-13 15:01:56 -08:00
|
|
|
0
|
|
|
|
} else {
|
2019-03-05 14:18:29 -08:00
|
|
|
(slot - 1) / branching_factor
|
2019-02-13 15:01:56 -08:00
|
|
|
}
|
|
|
|
};
|
|
|
|
assert_eq!(slot_meta.parent_slot, slot_parent);
|
|
|
|
|
|
|
|
let expected_children: HashSet<_> = {
|
2019-03-05 14:18:29 -08:00
|
|
|
if slot >= last_level {
|
2019-02-13 15:01:56 -08:00
|
|
|
HashSet::new()
|
|
|
|
} else {
|
2019-03-05 14:18:29 -08:00
|
|
|
let first_child_slot = min(num_slots - 1, slot * branching_factor + 1);
|
|
|
|
let last_child_slot = min(num_slots - 1, (slot + 1) * branching_factor);
|
2019-02-13 15:01:56 -08:00
|
|
|
(first_child_slot..last_child_slot + 1).collect()
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
let result: HashSet<_> = slot_meta.next_slots.iter().cloned().collect();
|
|
|
|
if expected_children.len() != 0 {
|
|
|
|
assert_eq!(slot_meta.next_slots.len(), branching_factor as usize);
|
|
|
|
} else {
|
|
|
|
assert_eq!(slot_meta.next_slots.len(), 0);
|
|
|
|
}
|
|
|
|
assert_eq!(expected_children, result);
|
|
|
|
}
|
2019-03-29 16:07:24 -07:00
|
|
|
|
2019-04-06 19:41:22 -07:00
|
|
|
// No orphan slots should exist
|
2019-05-03 14:46:02 -07:00
|
|
|
assert!(blocktree.orphans_cf.is_empty().unwrap())
|
2019-02-13 15:01:56 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
2019-02-07 15:10:54 -08:00
|
|
|
#[test]
|
|
|
|
pub fn test_get_slots_since() {
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree_path = get_tmp_ledger_path("test_get_slots_since");
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
{
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
// Slot doesn't exist
|
2019-02-07 20:52:39 -08:00
|
|
|
assert!(blocktree.get_slots_since(&vec![0]).unwrap().is_empty());
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-02-28 19:49:22 -08:00
|
|
|
let mut meta0 = SlotMeta::new(0, 0);
|
2019-05-03 14:46:02 -07:00
|
|
|
blocktree.meta_cf.put(0, &meta0).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
// Slot exists, chains to nothing
|
2019-02-28 19:49:22 -08:00
|
|
|
let expected: HashMap<u64, Vec<u64>> =
|
|
|
|
HashMap::from_iter(vec![(0, vec![])].into_iter());
|
|
|
|
assert_eq!(blocktree.get_slots_since(&vec![0]).unwrap(), expected);
|
2019-02-07 15:10:54 -08:00
|
|
|
meta0.next_slots = vec![1, 2];
|
2019-05-03 14:46:02 -07:00
|
|
|
blocktree.meta_cf.put(0, &meta0).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
// Slot exists, chains to some other slots
|
2019-02-28 19:49:22 -08:00
|
|
|
let expected: HashMap<u64, Vec<u64>> =
|
|
|
|
HashMap::from_iter(vec![(0, vec![1, 2])].into_iter());
|
|
|
|
assert_eq!(blocktree.get_slots_since(&vec![0]).unwrap(), expected);
|
|
|
|
assert_eq!(blocktree.get_slots_since(&vec![0, 1]).unwrap(), expected);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
let mut meta3 = SlotMeta::new(3, 1);
|
|
|
|
meta3.next_slots = vec![10, 5];
|
2019-05-03 14:46:02 -07:00
|
|
|
blocktree.meta_cf.put(3, &meta3).unwrap();
|
2019-02-28 19:49:22 -08:00
|
|
|
let expected: HashMap<u64, Vec<u64>> =
|
|
|
|
HashMap::from_iter(vec![(0, vec![1, 2]), (3, vec![10, 5])].into_iter());
|
|
|
|
assert_eq!(blocktree.get_slots_since(&vec![0, 1, 3]).unwrap(), expected);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-03-29 16:07:24 -07:00
|
|
|
#[test]
|
2019-04-06 19:41:22 -07:00
|
|
|
fn test_orphans() {
|
|
|
|
let blocktree_path = get_tmp_ledger_path("test_orphans");
|
2019-03-29 16:07:24 -07:00
|
|
|
{
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
|
|
|
|
// Create blobs and entries
|
|
|
|
let entries_per_slot = 1;
|
|
|
|
let (blobs, _) = make_many_slot_entries(0, 3, entries_per_slot);
|
|
|
|
|
|
|
|
// Write slot 2, which chains to slot 1. We're missing slot 0,
|
2019-04-06 19:41:22 -07:00
|
|
|
// so slot 1 is the orphan
|
2019-03-29 16:07:24 -07:00
|
|
|
blocktree.write_blobs(once(&blobs[2])).unwrap();
|
|
|
|
let meta = blocktree
|
|
|
|
.meta(1)
|
|
|
|
.expect("Expect database get to succeed")
|
|
|
|
.unwrap();
|
2019-04-26 08:52:10 -07:00
|
|
|
assert!(is_orphan(&meta));
|
2019-04-06 19:41:22 -07:00
|
|
|
assert_eq!(blocktree.get_orphans(None), vec![1]);
|
2019-03-29 16:07:24 -07:00
|
|
|
|
|
|
|
// Write slot 1 which chains to slot 0, so now slot 0 is the
|
2019-04-06 19:41:22 -07:00
|
|
|
// orphan, and slot 1 is no longer the orphan.
|
2019-03-29 16:07:24 -07:00
|
|
|
blocktree.write_blobs(once(&blobs[1])).unwrap();
|
|
|
|
let meta = blocktree
|
|
|
|
.meta(1)
|
|
|
|
.expect("Expect database get to succeed")
|
|
|
|
.unwrap();
|
2019-04-26 08:52:10 -07:00
|
|
|
assert!(!is_orphan(&meta));
|
2019-03-29 16:07:24 -07:00
|
|
|
let meta = blocktree
|
|
|
|
.meta(0)
|
|
|
|
.expect("Expect database get to succeed")
|
|
|
|
.unwrap();
|
2019-04-26 08:52:10 -07:00
|
|
|
assert!(is_orphan(&meta));
|
2019-04-06 19:41:22 -07:00
|
|
|
assert_eq!(blocktree.get_orphans(None), vec![0]);
|
2019-03-29 16:07:24 -07:00
|
|
|
|
2019-04-06 19:41:22 -07:00
|
|
|
// Write some slot that also chains to existing slots and orphan,
|
2019-03-29 16:07:24 -07:00
|
|
|
// nothing should change
|
|
|
|
let blob4 = &make_slot_entries(4, 0, 1).0[0];
|
|
|
|
let blob5 = &make_slot_entries(5, 1, 1).0[0];
|
|
|
|
blocktree.write_blobs(vec![blob4, blob5]).unwrap();
|
2019-04-06 19:41:22 -07:00
|
|
|
assert_eq!(blocktree.get_orphans(None), vec![0]);
|
2019-03-29 16:07:24 -07:00
|
|
|
|
2019-04-06 19:41:22 -07:00
|
|
|
// Write zeroth slot, no more orphans
|
2019-03-29 16:07:24 -07:00
|
|
|
blocktree.write_blobs(once(&blobs[0])).unwrap();
|
|
|
|
for i in 0..3 {
|
|
|
|
let meta = blocktree
|
|
|
|
.meta(i)
|
|
|
|
.expect("Expect database get to succeed")
|
|
|
|
.unwrap();
|
2019-04-26 08:52:10 -07:00
|
|
|
assert!(!is_orphan(&meta));
|
2019-03-29 16:07:24 -07:00
|
|
|
}
|
2019-04-06 19:41:22 -07:00
|
|
|
// Orphans cf is empty
|
2019-05-03 14:46:02 -07:00
|
|
|
assert!(blocktree.orphans_cf.is_empty().unwrap())
|
2019-03-29 16:07:24 -07:00
|
|
|
}
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
2019-02-07 15:10:54 -08:00
|
|
|
fn test_insert_data_blobs_slots(name: &str, should_bulk_write: bool) {
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree_path = get_tmp_ledger_path(name);
|
2019-02-07 15:10:54 -08:00
|
|
|
{
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-02-12 19:54:18 -08:00
|
|
|
// Create blobs and entries
|
2019-02-07 15:10:54 -08:00
|
|
|
let num_entries = 20 as u64;
|
2019-02-12 19:54:18 -08:00
|
|
|
let mut entries = vec![];
|
|
|
|
let mut blobs = vec![];
|
2019-03-05 14:18:29 -08:00
|
|
|
for slot in 0..num_entries {
|
2019-02-12 19:54:18 -08:00
|
|
|
let parent_slot = {
|
2019-03-05 14:18:29 -08:00
|
|
|
if slot == 0 {
|
2019-02-12 19:54:18 -08:00
|
|
|
0
|
|
|
|
} else {
|
2019-03-05 14:18:29 -08:00
|
|
|
slot - 1
|
2019-02-12 19:54:18 -08:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2019-03-05 14:18:29 -08:00
|
|
|
let (mut blob, entry) = make_slot_entries(slot, parent_slot, 1);
|
|
|
|
blob[0].set_index(slot);
|
2019-02-12 19:54:18 -08:00
|
|
|
blobs.extend(blob);
|
|
|
|
entries.extend(entry);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-02-12 19:54:18 -08:00
|
|
|
// Write blobs to the database
|
2019-02-07 15:10:54 -08:00
|
|
|
if should_bulk_write {
|
2019-02-12 19:54:18 -08:00
|
|
|
blocktree.write_blobs(blobs.iter()).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
} else {
|
|
|
|
for i in 0..num_entries {
|
|
|
|
let i = i as usize;
|
2019-02-12 19:54:18 -08:00
|
|
|
blocktree.write_blobs(&blobs[i..i + 1]).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for i in 0..num_entries - 1 {
|
|
|
|
assert_eq!(
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree.get_slot_entries(i, i, None).unwrap()[0],
|
2019-02-12 19:54:18 -08:00
|
|
|
entries[i as usize]
|
2019-02-07 15:10:54 -08:00
|
|
|
);
|
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
let meta = blocktree.meta(i).unwrap().unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
assert_eq!(meta.received, i + 1);
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(meta.last_index, i);
|
2019-02-07 15:10:54 -08:00
|
|
|
if i != 0 {
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(meta.parent_slot, i - 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
assert!(meta.consumed == 0);
|
|
|
|
} else {
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(meta.parent_slot, 0);
|
2019-02-07 15:10:54 -08:00
|
|
|
assert!(meta.consumed == 1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
2019-02-12 19:54:18 -08:00
|
|
|
|
2019-03-27 23:55:51 -07:00
|
|
|
#[test]
|
|
|
|
fn test_find_missing_data_indexes() {
|
|
|
|
let slot = 0;
|
|
|
|
let blocktree_path = get_tmp_ledger_path!();
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
|
|
|
|
// Write entries
|
|
|
|
let gap = 10;
|
|
|
|
assert!(gap > 3);
|
|
|
|
let num_entries = 10;
|
|
|
|
let mut blobs = make_tiny_test_entries(num_entries).to_single_entry_blobs();
|
|
|
|
for (i, b) in blobs.iter_mut().enumerate() {
|
|
|
|
b.set_index(i as u64 * gap);
|
|
|
|
b.set_slot(slot);
|
|
|
|
}
|
|
|
|
blocktree.write_blobs(&blobs).unwrap();
|
|
|
|
|
|
|
|
// Index of the first blob is 0
|
|
|
|
// Index of the second blob is "gap"
|
|
|
|
// Thus, the missing indexes should then be [1, gap - 1] for the input index
|
|
|
|
// range of [0, gap)
|
|
|
|
let expected: Vec<u64> = (1..gap).collect();
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(slot, 0, gap, gap as usize),
|
|
|
|
expected
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(slot, 1, gap, (gap - 1) as usize),
|
|
|
|
expected,
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(slot, 0, gap - 1, (gap - 1) as usize),
|
|
|
|
&expected[..expected.len() - 1],
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(slot, gap - 2, gap, gap as usize),
|
|
|
|
vec![gap - 2, gap - 1],
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(slot, gap - 2, gap, 1),
|
|
|
|
vec![gap - 2],
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(slot, 0, gap, 1),
|
|
|
|
vec![1],
|
|
|
|
);
|
|
|
|
|
|
|
|
// Test with end indexes that are greater than the last item in the ledger
|
|
|
|
let mut expected: Vec<u64> = (1..gap).collect();
|
|
|
|
expected.push(gap + 1);
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(slot, 0, gap + 2, (gap + 2) as usize),
|
|
|
|
expected,
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(slot, 0, gap + 2, (gap - 1) as usize),
|
|
|
|
&expected[..expected.len() - 1],
|
|
|
|
);
|
|
|
|
|
|
|
|
for i in 0..num_entries as u64 {
|
|
|
|
for j in 0..i {
|
|
|
|
let expected: Vec<u64> = (j..i)
|
|
|
|
.flat_map(|k| {
|
|
|
|
let begin = k * gap + 1;
|
|
|
|
let end = (k + 1) * gap;
|
|
|
|
(begin..end)
|
|
|
|
})
|
|
|
|
.collect();
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(
|
|
|
|
slot,
|
|
|
|
j * gap,
|
|
|
|
i * gap,
|
|
|
|
((i - j) * gap) as usize
|
|
|
|
),
|
|
|
|
expected,
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
drop(blocktree);
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_find_missing_data_indexes_sanity() {
|
|
|
|
let slot = 0;
|
|
|
|
|
|
|
|
let blocktree_path = get_tmp_ledger_path!();
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
|
|
|
|
// Early exit conditions
|
|
|
|
let empty: Vec<u64> = vec![];
|
|
|
|
assert_eq!(blocktree.find_missing_data_indexes(slot, 0, 0, 1), empty);
|
|
|
|
assert_eq!(blocktree.find_missing_data_indexes(slot, 5, 5, 1), empty);
|
|
|
|
assert_eq!(blocktree.find_missing_data_indexes(slot, 4, 3, 1), empty);
|
|
|
|
assert_eq!(blocktree.find_missing_data_indexes(slot, 1, 2, 0), empty);
|
|
|
|
|
|
|
|
let mut blobs = make_tiny_test_entries(2).to_single_entry_blobs();
|
|
|
|
|
|
|
|
const ONE: u64 = 1;
|
|
|
|
const OTHER: u64 = 4;
|
|
|
|
|
|
|
|
blobs[0].set_index(ONE);
|
|
|
|
blobs[1].set_index(OTHER);
|
|
|
|
|
|
|
|
// Insert one blob at index = first_index
|
|
|
|
blocktree.write_blobs(&blobs).unwrap();
|
|
|
|
|
|
|
|
const STARTS: u64 = OTHER * 2;
|
|
|
|
const END: u64 = OTHER * 3;
|
|
|
|
const MAX: usize = 10;
|
|
|
|
// The first blob has index = first_index. Thus, for i < first_index,
|
|
|
|
// given the input range of [i, first_index], the missing indexes should be
|
|
|
|
// [i, first_index - 1]
|
|
|
|
for start in 0..STARTS {
|
|
|
|
let result = blocktree.find_missing_data_indexes(
|
|
|
|
slot, start, // start
|
|
|
|
END, //end
|
|
|
|
MAX, //max
|
|
|
|
);
|
|
|
|
let expected: Vec<u64> = (start..END).filter(|i| *i != ONE && *i != OTHER).collect();
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
}
|
|
|
|
|
|
|
|
drop(blocktree);
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
pub fn test_no_missing_blob_indexes() {
|
|
|
|
let slot = 0;
|
|
|
|
let blocktree_path = get_tmp_ledger_path!();
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
|
|
|
|
// Write entries
|
|
|
|
let num_entries = 10;
|
|
|
|
let shared_blobs = make_tiny_test_entries(num_entries).to_single_entry_shared_blobs();
|
|
|
|
|
2019-03-30 20:37:33 -07:00
|
|
|
crate::packet::index_blobs(&shared_blobs, &Pubkey::new_rand(), 0, slot, 0);
|
2019-03-27 23:55:51 -07:00
|
|
|
|
|
|
|
let blob_locks: Vec<_> = shared_blobs.iter().map(|b| b.read().unwrap()).collect();
|
|
|
|
let blobs: Vec<&Blob> = blob_locks.iter().map(|b| &**b).collect();
|
|
|
|
blocktree.write_blobs(blobs).unwrap();
|
|
|
|
|
|
|
|
let empty: Vec<u64> = vec![];
|
|
|
|
for i in 0..num_entries as u64 {
|
|
|
|
for j in 0..i {
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(slot, j, i, (i - j) as usize),
|
|
|
|
empty
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
drop(blocktree);
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
2019-04-25 00:04:49 -07:00
|
|
|
#[test]
|
|
|
|
pub fn test_should_insert_blob() {
|
|
|
|
let (mut blobs, _) = make_slot_entries(0, 0, 20);
|
|
|
|
let blocktree_path = get_tmp_ledger_path!();
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
|
|
|
|
// Insert the first 5 blobs, we don't have a "is_last" blob yet
|
|
|
|
blocktree.insert_data_blobs(&blobs[0..5]).unwrap();
|
|
|
|
|
|
|
|
// Trying to insert a blob less than consumed should fail
|
|
|
|
let slot_meta = blocktree.meta(0).unwrap().unwrap();
|
|
|
|
assert_eq!(slot_meta.consumed, 5);
|
2019-04-26 08:52:10 -07:00
|
|
|
assert!(!should_insert_blob(
|
|
|
|
&slot_meta,
|
2019-05-03 14:46:02 -07:00
|
|
|
&blocktree.db,
|
2019-04-26 08:52:10 -07:00
|
|
|
&HashMap::new(),
|
|
|
|
&blobs[4].clone()
|
|
|
|
));
|
2019-04-25 00:04:49 -07:00
|
|
|
|
|
|
|
// Trying to insert the same blob again should fail
|
|
|
|
blocktree.insert_data_blobs(&blobs[7..8]).unwrap();
|
|
|
|
let slot_meta = blocktree.meta(0).unwrap().unwrap();
|
2019-04-26 08:52:10 -07:00
|
|
|
assert!(!should_insert_blob(
|
|
|
|
&slot_meta,
|
2019-05-03 14:46:02 -07:00
|
|
|
&blocktree.db,
|
2019-04-26 08:52:10 -07:00
|
|
|
&HashMap::new(),
|
|
|
|
&blobs[7].clone()
|
|
|
|
));
|
2019-04-25 00:04:49 -07:00
|
|
|
|
|
|
|
// Trying to insert another "is_last" blob with index < the received index
|
|
|
|
// should fail
|
|
|
|
blocktree.insert_data_blobs(&blobs[8..9]).unwrap();
|
|
|
|
let slot_meta = blocktree.meta(0).unwrap().unwrap();
|
|
|
|
assert_eq!(slot_meta.received, 9);
|
|
|
|
blobs[8].set_is_last_in_slot();
|
2019-04-26 08:52:10 -07:00
|
|
|
assert!(!should_insert_blob(
|
|
|
|
&slot_meta,
|
2019-05-03 14:46:02 -07:00
|
|
|
&blocktree.db,
|
2019-04-26 08:52:10 -07:00
|
|
|
&HashMap::new(),
|
|
|
|
&blobs[8].clone()
|
|
|
|
));
|
2019-04-25 00:04:49 -07:00
|
|
|
|
|
|
|
// Insert the 10th blob, which is marked as "is_last"
|
|
|
|
blobs[9].set_is_last_in_slot();
|
|
|
|
blocktree.insert_data_blobs(&blobs[9..10]).unwrap();
|
|
|
|
let slot_meta = blocktree.meta(0).unwrap().unwrap();
|
|
|
|
|
|
|
|
// Trying to insert a blob with index > the "is_last" blob should fail
|
2019-04-26 08:52:10 -07:00
|
|
|
assert!(!should_insert_blob(
|
|
|
|
&slot_meta,
|
2019-05-03 14:46:02 -07:00
|
|
|
&blocktree.db,
|
2019-04-26 08:52:10 -07:00
|
|
|
&HashMap::new(),
|
|
|
|
&blobs[10].clone()
|
|
|
|
));
|
2019-04-25 00:04:49 -07:00
|
|
|
|
|
|
|
drop(blocktree);
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
pub fn test_insert_multiple_is_last() {
|
|
|
|
let (mut blobs, _) = make_slot_entries(0, 0, 20);
|
|
|
|
let blocktree_path = get_tmp_ledger_path!();
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
|
|
|
|
// Inserting multiple blobs with the is_last flag set should only insert
|
|
|
|
// the first blob with the "is_last" flag, and drop the rest
|
|
|
|
for i in 6..20 {
|
|
|
|
blobs[i].set_is_last_in_slot();
|
|
|
|
}
|
|
|
|
|
|
|
|
blocktree.insert_data_blobs(&blobs[..]).unwrap();
|
|
|
|
let slot_meta = blocktree.meta(0).unwrap().unwrap();
|
|
|
|
|
|
|
|
assert_eq!(slot_meta.consumed, 7);
|
|
|
|
assert_eq!(slot_meta.received, 7);
|
|
|
|
assert_eq!(slot_meta.last_index, 6);
|
|
|
|
assert!(slot_meta.is_full());
|
|
|
|
|
|
|
|
drop(blocktree);
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
2019-04-11 14:14:57 -07:00
|
|
|
mod erasure {
|
|
|
|
use super::*;
|
2019-04-24 15:53:01 -07:00
|
|
|
use crate::blocktree::meta::ErasureMetaStatus;
|
2019-04-11 14:14:57 -07:00
|
|
|
use crate::erasure::test::{generate_ledger_model, ErasureSpec, SlotSpec};
|
|
|
|
use crate::erasure::{CodingGenerator, NUM_CODING, NUM_DATA};
|
|
|
|
use rand::{thread_rng, Rng};
|
|
|
|
use std::sync::RwLock;
|
|
|
|
|
|
|
|
impl Into<SharedBlob> for Blob {
|
|
|
|
fn into(self) -> SharedBlob {
|
|
|
|
Arc::new(RwLock::new(self))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_erasure_meta_accuracy() {
|
2019-05-02 17:04:40 -07:00
|
|
|
use crate::erasure::ERASURE_SET_SIZE;
|
2019-04-24 15:53:01 -07:00
|
|
|
use ErasureMetaStatus::{DataFull, StillNeed};
|
|
|
|
|
2019-04-11 14:14:57 -07:00
|
|
|
let path = get_tmp_ledger_path!();
|
|
|
|
let blocktree = Blocktree::open(&path).unwrap();
|
|
|
|
|
2019-04-24 15:53:01 -07:00
|
|
|
// two erasure sets
|
2019-05-02 17:04:40 -07:00
|
|
|
let num_blobs = NUM_DATA as u64 * 2;
|
2019-04-11 14:14:57 -07:00
|
|
|
let slot = 0;
|
|
|
|
|
|
|
|
let (blobs, _) = make_slot_entries(slot, 0, num_blobs);
|
|
|
|
let shared_blobs: Vec<_> = blobs
|
|
|
|
.iter()
|
|
|
|
.cloned()
|
|
|
|
.map(|blob| Arc::new(RwLock::new(blob)))
|
|
|
|
.collect();
|
|
|
|
|
2019-05-02 17:04:40 -07:00
|
|
|
blocktree.write_blobs(&blobs[..2]).unwrap();
|
2019-04-11 14:14:57 -07:00
|
|
|
|
|
|
|
let erasure_meta_opt = blocktree
|
2019-04-26 08:52:10 -07:00
|
|
|
.erasure_meta(slot, 0)
|
2019-04-11 14:14:57 -07:00
|
|
|
.expect("DB get must succeed");
|
|
|
|
|
|
|
|
assert!(erasure_meta_opt.is_some());
|
|
|
|
let erasure_meta = erasure_meta_opt.unwrap();
|
|
|
|
|
2019-05-02 17:04:40 -07:00
|
|
|
let should_need = ERASURE_SET_SIZE - NUM_CODING - 2;
|
|
|
|
match erasure_meta.status() {
|
|
|
|
StillNeed(n) => assert_eq!(n, should_need),
|
|
|
|
_ => panic!("Should still need more blobs"),
|
|
|
|
};
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-05-02 17:04:40 -07:00
|
|
|
blocktree.write_blobs(&blobs[2..NUM_DATA]).unwrap();
|
2019-04-11 14:14:57 -07:00
|
|
|
|
|
|
|
let erasure_meta = blocktree
|
2019-04-26 08:52:10 -07:00
|
|
|
.erasure_meta(slot, 0)
|
2019-04-11 14:14:57 -07:00
|
|
|
.expect("DB get must succeed")
|
|
|
|
.unwrap();
|
|
|
|
|
2019-04-24 15:53:01 -07:00
|
|
|
assert_eq!(erasure_meta.status(), DataFull);
|
|
|
|
|
|
|
|
// insert all coding blobs in first set
|
|
|
|
let mut coding_generator = CodingGenerator::new(Arc::clone(&blocktree.session));
|
|
|
|
let coding_blobs = coding_generator.next(&shared_blobs[..NUM_DATA]);
|
|
|
|
|
|
|
|
for shared_coding_blob in coding_blobs {
|
|
|
|
let blob = shared_coding_blob.read().unwrap();
|
|
|
|
let size = blob.size() + BLOB_HEADER_SIZE;
|
|
|
|
blocktree
|
|
|
|
.put_coding_blob_bytes(blob.slot(), blob.index(), &blob.data[..size])
|
|
|
|
.unwrap();
|
|
|
|
}
|
|
|
|
|
|
|
|
let erasure_meta = blocktree
|
2019-04-26 08:52:10 -07:00
|
|
|
.erasure_meta(slot, 0)
|
2019-04-24 15:53:01 -07:00
|
|
|
.expect("DB get must succeed")
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
assert_eq!(erasure_meta.status(), DataFull);
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-05-02 17:04:40 -07:00
|
|
|
// insert blobs in the 2nd set until recovery should be possible given all coding blobs
|
|
|
|
let set2 = &blobs[NUM_DATA..];
|
|
|
|
let mut end = 1;
|
|
|
|
let blobs_needed = ERASURE_SET_SIZE - NUM_CODING;
|
|
|
|
while end < blobs_needed {
|
|
|
|
blocktree.write_blobs(&set2[end - 1..end]).unwrap();
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-05-02 17:04:40 -07:00
|
|
|
let erasure_meta = blocktree
|
|
|
|
.erasure_meta(slot, 1)
|
|
|
|
.expect("DB get must succeed")
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
match erasure_meta.status() {
|
|
|
|
StillNeed(n) => assert_eq!(n, blobs_needed - end),
|
|
|
|
_ => panic!("Should still need more blobs"),
|
|
|
|
};
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-05-02 17:04:40 -07:00
|
|
|
end += 1;
|
|
|
|
}
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-05-02 17:04:40 -07:00
|
|
|
// insert all coding blobs in 2nd set. Should trigger recovery
|
2019-04-18 21:56:43 -07:00
|
|
|
let mut coding_generator = CodingGenerator::new(Arc::clone(&blocktree.session));
|
2019-04-24 15:53:01 -07:00
|
|
|
let coding_blobs = coding_generator.next(&shared_blobs[NUM_DATA..]);
|
2019-04-11 14:14:57 -07:00
|
|
|
|
|
|
|
for shared_coding_blob in coding_blobs {
|
|
|
|
let blob = shared_coding_blob.read().unwrap();
|
|
|
|
let size = blob.size() + BLOB_HEADER_SIZE;
|
|
|
|
blocktree
|
|
|
|
.put_coding_blob_bytes(blob.slot(), blob.index(), &blob.data[..size])
|
|
|
|
.unwrap();
|
|
|
|
}
|
|
|
|
|
2019-04-24 15:53:01 -07:00
|
|
|
let erasure_meta = blocktree
|
2019-04-26 08:52:10 -07:00
|
|
|
.erasure_meta(slot, 1)
|
2019-04-24 15:53:01 -07:00
|
|
|
.expect("DB get must succeed")
|
|
|
|
.unwrap();
|
2019-04-18 21:56:43 -07:00
|
|
|
|
2019-04-24 15:53:01 -07:00
|
|
|
assert_eq!(erasure_meta.status(), DataFull);
|
|
|
|
|
|
|
|
// remove coding blobs, erasure meta should still report being full
|
2019-04-18 21:56:43 -07:00
|
|
|
let (start_idx, coding_end_idx) =
|
|
|
|
(erasure_meta.start_index(), erasure_meta.end_indexes().1);
|
|
|
|
|
|
|
|
for idx in start_idx..coding_end_idx {
|
|
|
|
blocktree.delete_coding_blob(slot, idx).unwrap();
|
|
|
|
}
|
|
|
|
|
|
|
|
let erasure_meta = blocktree
|
2019-04-26 08:52:10 -07:00
|
|
|
.erasure_meta(slot, 1)
|
2019-04-18 21:56:43 -07:00
|
|
|
.expect("DB get must succeed")
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
assert_eq!(erasure_meta.status(), ErasureMetaStatus::DataFull);
|
2019-04-11 14:14:57 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
pub fn test_recovery_basic() {
|
|
|
|
solana_logger::setup();
|
|
|
|
|
|
|
|
let slot = 0;
|
|
|
|
|
|
|
|
let ledger_path = get_tmp_ledger_path!();
|
|
|
|
|
|
|
|
let blocktree = Blocktree::open(&ledger_path).unwrap();
|
|
|
|
let data_blobs = make_slot_entries(slot, 0, 3 * NUM_DATA as u64)
|
|
|
|
.0
|
|
|
|
.into_iter()
|
|
|
|
.map(Blob::into)
|
|
|
|
.collect::<Vec<_>>();
|
|
|
|
|
2019-04-18 21:56:43 -07:00
|
|
|
let mut coding_generator = CodingGenerator::new(Arc::clone(&blocktree.session));
|
2019-04-11 14:14:57 -07:00
|
|
|
|
|
|
|
for (set_index, data_blobs) in data_blobs.chunks_exact(NUM_DATA).enumerate() {
|
|
|
|
let focused_index = (set_index + 1) * NUM_DATA - 1;
|
2019-04-14 18:12:37 -07:00
|
|
|
let coding_blobs = coding_generator.next(&data_blobs);
|
2019-04-18 21:56:43 -07:00
|
|
|
|
2019-04-11 14:14:57 -07:00
|
|
|
assert_eq!(coding_blobs.len(), NUM_CODING);
|
|
|
|
|
|
|
|
let deleted_data = data_blobs[NUM_DATA - 1].clone();
|
|
|
|
debug!(
|
|
|
|
"deleted: slot: {}, index: {}",
|
|
|
|
deleted_data.read().unwrap().slot(),
|
|
|
|
deleted_data.read().unwrap().index()
|
|
|
|
);
|
|
|
|
|
|
|
|
blocktree
|
|
|
|
.write_shared_blobs(&data_blobs[..NUM_DATA - 1])
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
// this should trigger recovery
|
|
|
|
for shared_coding_blob in coding_blobs {
|
|
|
|
let blob = shared_coding_blob.read().unwrap();
|
|
|
|
let size = blob.size() + BLOB_HEADER_SIZE;
|
|
|
|
|
|
|
|
blocktree
|
|
|
|
.put_coding_blob_bytes(slot, blob.index(), &blob.data[..size])
|
|
|
|
.expect("Inserting coding blobs must succeed");
|
|
|
|
(slot, blob.index());
|
|
|
|
}
|
|
|
|
|
|
|
|
let erasure_meta = blocktree
|
|
|
|
.erasure_meta_cf
|
2019-05-03 14:46:02 -07:00
|
|
|
.get((slot, set_index as u64))
|
2019-04-11 14:14:57 -07:00
|
|
|
.expect("Erasure Meta should be present")
|
|
|
|
.unwrap();
|
|
|
|
|
2019-05-02 17:04:40 -07:00
|
|
|
assert_eq!(erasure_meta.status(), ErasureMetaStatus::DataFull);
|
2019-04-11 14:14:57 -07:00
|
|
|
|
|
|
|
let retrieved_data = blocktree
|
|
|
|
.data_cf
|
2019-05-03 14:46:02 -07:00
|
|
|
.get_bytes((slot, focused_index as u64))
|
2019-04-11 14:14:57 -07:00
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
assert!(retrieved_data.is_some());
|
|
|
|
|
|
|
|
let data_blob = Blob::new(&retrieved_data.unwrap());
|
|
|
|
|
|
|
|
assert_eq!(&data_blob, &*deleted_data.read().unwrap());
|
|
|
|
}
|
|
|
|
|
|
|
|
drop(blocktree);
|
|
|
|
|
|
|
|
Blocktree::destroy(&ledger_path).expect("Expect successful Blocktree destruction");
|
|
|
|
}
|
|
|
|
|
2019-04-24 15:53:01 -07:00
|
|
|
#[test]
|
|
|
|
fn test_recovery_fails_safely() {
|
|
|
|
const SLOT: u64 = 0;
|
|
|
|
const SET_INDEX: u64 = 0;
|
|
|
|
|
|
|
|
solana_logger::setup();
|
|
|
|
let ledger_path = get_tmp_ledger_path!();
|
|
|
|
let blocktree = Blocktree::open(&ledger_path).unwrap();
|
|
|
|
let data_blobs = make_slot_entries(SLOT, 0, NUM_DATA as u64)
|
|
|
|
.0
|
|
|
|
.into_iter()
|
|
|
|
.map(Blob::into)
|
|
|
|
.collect::<Vec<_>>();
|
|
|
|
|
|
|
|
let mut coding_generator = CodingGenerator::new(Arc::clone(&blocktree.session));
|
|
|
|
|
|
|
|
let shared_coding_blobs = coding_generator.next(&data_blobs);
|
|
|
|
assert_eq!(shared_coding_blobs.len(), NUM_CODING);
|
|
|
|
|
2019-05-02 17:04:40 -07:00
|
|
|
// Insert coding blobs except 1 and no data. Not enough to do recovery
|
|
|
|
for shared_blob in shared_coding_blobs.iter().skip(1) {
|
2019-04-24 15:53:01 -07:00
|
|
|
let blob = shared_blob.read().unwrap();
|
|
|
|
let size = blob.size() + BLOB_HEADER_SIZE;
|
|
|
|
|
|
|
|
blocktree
|
|
|
|
.put_coding_blob_bytes(SLOT, blob.index(), &blob.data[..size])
|
|
|
|
.expect("Inserting coding blobs must succeed");
|
|
|
|
}
|
|
|
|
|
|
|
|
// try recovery even though there aren't enough blobs
|
|
|
|
let erasure_meta = blocktree
|
|
|
|
.erasure_meta_cf
|
2019-05-03 14:46:02 -07:00
|
|
|
.get((SLOT, SET_INDEX))
|
2019-04-24 15:53:01 -07:00
|
|
|
.unwrap()
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
assert_eq!(erasure_meta.status(), ErasureMetaStatus::StillNeed(1));
|
|
|
|
|
|
|
|
let prev_inserted_blob_datas = HashMap::new();
|
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
let attempt_result = try_erasure_recover(
|
2019-05-03 14:46:02 -07:00
|
|
|
&blocktree.db,
|
2019-04-26 08:52:10 -07:00
|
|
|
&blocktree.session,
|
|
|
|
&erasure_meta,
|
|
|
|
SLOT,
|
|
|
|
&prev_inserted_blob_datas,
|
|
|
|
None,
|
|
|
|
);
|
2019-04-24 15:53:01 -07:00
|
|
|
|
|
|
|
assert!(attempt_result.is_ok());
|
|
|
|
let recovered_blobs_opt = attempt_result.unwrap();
|
|
|
|
|
|
|
|
assert!(recovered_blobs_opt.is_none());
|
|
|
|
}
|
|
|
|
|
2019-04-11 14:14:57 -07:00
|
|
|
#[test]
|
|
|
|
fn test_recovery_multi_slot_multi_thread() {
|
2019-04-24 15:53:01 -07:00
|
|
|
use rand::{rngs::SmallRng, seq::SliceRandom, SeedableRng};
|
2019-04-11 14:14:57 -07:00
|
|
|
use std::thread;
|
|
|
|
|
2019-04-24 15:53:01 -07:00
|
|
|
const N_THREADS: usize = 3;
|
|
|
|
|
2019-04-11 14:14:57 -07:00
|
|
|
let slots = vec![0, 3, 5, 50, 100];
|
|
|
|
let max_erasure_sets = 16;
|
|
|
|
solana_logger::setup();
|
|
|
|
|
|
|
|
let path = get_tmp_ledger_path!();
|
|
|
|
let mut rng = thread_rng();
|
|
|
|
|
|
|
|
// Specification should generate a ledger where each slot has an random number of
|
2019-04-24 15:53:01 -07:00
|
|
|
// erasure sets. Odd erasure sets will have all coding blobs and between 1-4 data blobs
|
|
|
|
// missing, and even ones will have between 1-2 data blobs missing and 1-2 coding blobs
|
|
|
|
// missing
|
2019-04-11 14:14:57 -07:00
|
|
|
let specs = slots
|
|
|
|
.iter()
|
|
|
|
.map(|&slot| {
|
|
|
|
let num_erasure_sets = rng.gen_range(0, max_erasure_sets);
|
|
|
|
|
|
|
|
let set_specs = (0..num_erasure_sets)
|
|
|
|
.map(|set_index| {
|
|
|
|
let (num_data, num_coding) = if set_index % 2 == 0 {
|
2019-04-24 15:53:01 -07:00
|
|
|
(
|
|
|
|
NUM_DATA - rng.gen_range(1, 3),
|
|
|
|
NUM_CODING - rng.gen_range(1, 3),
|
|
|
|
)
|
2019-04-11 14:14:57 -07:00
|
|
|
} else {
|
2019-04-24 15:53:01 -07:00
|
|
|
(NUM_DATA - rng.gen_range(1, 5), NUM_CODING)
|
2019-04-11 14:14:57 -07:00
|
|
|
};
|
|
|
|
ErasureSpec {
|
|
|
|
set_index,
|
|
|
|
num_data,
|
|
|
|
num_coding,
|
|
|
|
}
|
|
|
|
})
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
SlotSpec { slot, set_specs }
|
|
|
|
})
|
|
|
|
.collect::<Vec<_>>();
|
|
|
|
|
2019-04-24 15:53:01 -07:00
|
|
|
let model = generate_ledger_model(specs);
|
2019-04-11 14:14:57 -07:00
|
|
|
let blocktree = Arc::new(Blocktree::open(&path).unwrap());
|
|
|
|
|
|
|
|
// Write to each slot in a different thread simultaneously.
|
|
|
|
// These writes should trigger the recovery. Every erasure set should have all of its
|
2019-04-24 15:53:01 -07:00
|
|
|
// data blobs and coding_blobs at the end
|
2019-04-11 14:14:57 -07:00
|
|
|
let mut handles = vec![];
|
|
|
|
|
2019-04-24 15:53:01 -07:00
|
|
|
// Each thread will attempt to write to each slot in order. Within a slot, each thread
|
|
|
|
// will try to write each erasure set in a random order. Within each erasure set, there
|
|
|
|
// is a 50/50 chance of attempting to write the coding blobs first or the data blobs
|
|
|
|
// first.
|
|
|
|
// The goal is to be as racey as possible and cover a wide range of situations
|
2019-04-25 00:04:49 -07:00
|
|
|
for thread_id in 0..N_THREADS {
|
2019-04-11 14:14:57 -07:00
|
|
|
let blocktree = Arc::clone(&blocktree);
|
2019-04-18 21:56:43 -07:00
|
|
|
let mut rng = SmallRng::from_rng(&mut rng).unwrap();
|
2019-04-24 15:53:01 -07:00
|
|
|
let model = model.clone();
|
2019-04-18 21:56:43 -07:00
|
|
|
let handle = thread::spawn(move || {
|
2019-04-24 15:53:01 -07:00
|
|
|
for slot_model in model {
|
|
|
|
let slot = slot_model.slot;
|
|
|
|
let num_erasure_sets = slot_model.chunks.len();
|
|
|
|
let unordered_sets = slot_model
|
|
|
|
.chunks
|
|
|
|
.choose_multiple(&mut rng, num_erasure_sets);
|
|
|
|
|
|
|
|
for erasure_set in unordered_sets {
|
2019-04-25 00:04:49 -07:00
|
|
|
let mut attempt = 0;
|
|
|
|
loop {
|
|
|
|
if rng.gen() {
|
2019-04-24 15:53:01 -07:00
|
|
|
blocktree
|
2019-04-25 00:04:49 -07:00
|
|
|
.write_shared_blobs(&erasure_set.data)
|
|
|
|
.expect("Writing data blobs must succeed");
|
|
|
|
debug!(
|
|
|
|
"multislot: wrote data: slot: {}, erasure_set: {}",
|
|
|
|
slot, erasure_set.set_index
|
|
|
|
);
|
|
|
|
|
|
|
|
for shared_coding_blob in &erasure_set.coding {
|
|
|
|
let blob = shared_coding_blob.read().unwrap();
|
|
|
|
let size = blob.size() + BLOB_HEADER_SIZE;
|
|
|
|
blocktree
|
|
|
|
.put_coding_blob_bytes(
|
|
|
|
slot,
|
|
|
|
blob.index(),
|
|
|
|
&blob.data[..size],
|
|
|
|
)
|
|
|
|
.expect("Writing coding blobs must succeed");
|
|
|
|
}
|
|
|
|
debug!(
|
|
|
|
"multislot: wrote coding: slot: {}, erasure_set: {}",
|
|
|
|
slot, erasure_set.set_index
|
|
|
|
);
|
|
|
|
} else {
|
|
|
|
// write coding blobs first, then write the data blobs.
|
|
|
|
for shared_coding_blob in &erasure_set.coding {
|
|
|
|
let blob = shared_coding_blob.read().unwrap();
|
|
|
|
let size = blob.size() + BLOB_HEADER_SIZE;
|
|
|
|
blocktree
|
|
|
|
.put_coding_blob_bytes(
|
|
|
|
slot,
|
|
|
|
blob.index(),
|
|
|
|
&blob.data[..size],
|
|
|
|
)
|
|
|
|
.expect("Writing coding blobs must succeed");
|
|
|
|
}
|
|
|
|
debug!(
|
|
|
|
"multislot: wrote coding: slot: {}, erasure_set: {}",
|
|
|
|
slot, erasure_set.set_index
|
|
|
|
);
|
|
|
|
|
2019-04-24 15:53:01 -07:00
|
|
|
blocktree
|
2019-04-25 00:04:49 -07:00
|
|
|
.write_shared_blobs(&erasure_set.data)
|
|
|
|
.expect("Writing data blobs must succeed");
|
|
|
|
debug!(
|
|
|
|
"multislot: wrote data: slot: {}, erasure_set: {}",
|
|
|
|
slot, erasure_set.set_index
|
|
|
|
);
|
2019-04-24 15:53:01 -07:00
|
|
|
}
|
|
|
|
|
2019-04-25 00:04:49 -07:00
|
|
|
// due to racing, some blobs might not be inserted. don't stop
|
|
|
|
// trying until *some* thread succeeds in writing everything and
|
|
|
|
// triggering recovery.
|
|
|
|
let erasure_meta = blocktree
|
|
|
|
.erasure_meta_cf
|
2019-05-03 14:46:02 -07:00
|
|
|
.get((slot, erasure_set.set_index))
|
2019-04-25 00:04:49 -07:00
|
|
|
.unwrap()
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
let status = erasure_meta.status();
|
|
|
|
attempt += 1;
|
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
debug!(
|
2019-04-25 00:04:49 -07:00
|
|
|
"[multi_slot] thread_id: {}, attempt: {}, slot: {}, set_index: {}, status: {:?}",
|
|
|
|
thread_id, attempt, slot, erasure_set.set_index, status
|
2019-04-24 15:53:01 -07:00
|
|
|
);
|
2019-04-25 00:04:49 -07:00
|
|
|
if status == ErasureMetaStatus::DataFull {
|
|
|
|
break;
|
|
|
|
}
|
2019-04-18 21:56:43 -07:00
|
|
|
}
|
2019-04-11 14:14:57 -07:00
|
|
|
}
|
|
|
|
}
|
2019-04-18 21:56:43 -07:00
|
|
|
});
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-04-18 21:56:43 -07:00
|
|
|
handles.push(handle);
|
2019-04-11 14:14:57 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
handles
|
|
|
|
.into_iter()
|
|
|
|
.for_each(|handle| handle.join().unwrap());
|
|
|
|
|
|
|
|
for slot_model in model {
|
|
|
|
let slot = slot_model.slot;
|
|
|
|
|
|
|
|
for erasure_set_model in slot_model.chunks {
|
|
|
|
let set_index = erasure_set_model.set_index as u64;
|
|
|
|
|
|
|
|
let erasure_meta = blocktree
|
|
|
|
.erasure_meta_cf
|
2019-05-03 14:46:02 -07:00
|
|
|
.get((slot, set_index))
|
2019-04-11 14:14:57 -07:00
|
|
|
.expect("DB get must succeed")
|
|
|
|
.expect("ErasureMeta must be present for each erasure set");
|
|
|
|
|
|
|
|
debug!(
|
|
|
|
"multislot: got erasure_meta: slot: {}, set_index: {}, erasure_meta: {:?}",
|
|
|
|
slot, set_index, erasure_meta
|
|
|
|
);
|
|
|
|
|
|
|
|
// all possibility for recovery should be exhausted
|
2019-04-18 21:56:43 -07:00
|
|
|
assert_eq!(erasure_meta.status(), ErasureMetaStatus::DataFull);
|
2019-04-11 14:14:57 -07:00
|
|
|
// Should have all data
|
2019-05-02 17:04:40 -07:00
|
|
|
assert_eq!(erasure_meta.num_data(), NUM_DATA);
|
2019-04-24 15:53:01 -07:00
|
|
|
// Should have all coding
|
2019-05-02 17:04:40 -07:00
|
|
|
assert_eq!(erasure_meta.num_coding(), NUM_CODING);
|
2019-04-11 14:14:57 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
drop(blocktree);
|
|
|
|
Blocktree::destroy(&path).expect("Blocktree destruction must succeed");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-03-04 19:22:23 -08:00
|
|
|
pub fn entries_to_blobs(
|
|
|
|
entries: &Vec<Entry>,
|
2019-03-05 14:18:29 -08:00
|
|
|
slot: u64,
|
2019-03-04 19:22:23 -08:00
|
|
|
parent_slot: u64,
|
|
|
|
is_full_slot: bool,
|
|
|
|
) -> Vec<Blob> {
|
2019-03-17 18:48:23 -07:00
|
|
|
let mut blobs = entries.clone().to_single_entry_blobs();
|
2019-02-12 19:54:18 -08:00
|
|
|
for (i, b) in blobs.iter_mut().enumerate() {
|
|
|
|
b.set_index(i as u64);
|
2019-03-05 14:18:29 -08:00
|
|
|
b.set_slot(slot);
|
2019-02-12 19:54:18 -08:00
|
|
|
b.set_parent(parent_slot);
|
|
|
|
}
|
2019-03-04 19:22:23 -08:00
|
|
|
if is_full_slot {
|
|
|
|
blobs.last_mut().unwrap().set_is_last_in_slot();
|
|
|
|
}
|
2019-02-19 19:40:23 -08:00
|
|
|
blobs
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn make_slot_entries(
|
2019-03-05 14:18:29 -08:00
|
|
|
slot: u64,
|
2019-02-19 19:40:23 -08:00
|
|
|
parent_slot: u64,
|
|
|
|
num_entries: u64,
|
|
|
|
) -> (Vec<Blob>, Vec<Entry>) {
|
|
|
|
let entries = make_tiny_test_entries(num_entries as usize);
|
2019-03-05 14:18:29 -08:00
|
|
|
let blobs = entries_to_blobs(&entries, slot, parent_slot, true);
|
2019-02-12 19:54:18 -08:00
|
|
|
(blobs, entries)
|
|
|
|
}
|
|
|
|
|
2019-02-18 18:41:31 -08:00
|
|
|
pub fn make_many_slot_entries(
|
2019-03-05 14:18:29 -08:00
|
|
|
start_slot: u64,
|
2019-02-12 19:54:18 -08:00
|
|
|
num_slots: u64,
|
|
|
|
entries_per_slot: u64,
|
|
|
|
) -> (Vec<Blob>, Vec<Entry>) {
|
|
|
|
let mut blobs = vec![];
|
|
|
|
let mut entries = vec![];
|
2019-03-05 14:18:29 -08:00
|
|
|
for slot in start_slot..start_slot + num_slots {
|
|
|
|
let parent_slot = if slot == 0 { 0 } else { slot - 1 };
|
2019-02-12 19:54:18 -08:00
|
|
|
|
2019-03-05 14:18:29 -08:00
|
|
|
let (slot_blobs, slot_entries) = make_slot_entries(slot, parent_slot, entries_per_slot);
|
2019-02-12 19:54:18 -08:00
|
|
|
blobs.extend(slot_blobs);
|
|
|
|
entries.extend(slot_entries);
|
|
|
|
}
|
|
|
|
|
|
|
|
(blobs, entries)
|
|
|
|
}
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|