2019-02-07 20:52:39 -08:00
|
|
|
//! The `block_tree` module provides functions for parallel verification of the
|
2018-11-15 15:53:31 -08:00
|
|
|
//! Proof of History ledger as well as iterative read, append write, and random
|
|
|
|
//! access read to a persistent file-based ledger.
|
2018-12-07 19:16:27 -08:00
|
|
|
use crate::entry::Entry;
|
2019-09-03 21:32:51 -07:00
|
|
|
use crate::erasure::ErasureConfig;
|
2018-12-07 19:16:27 -08:00
|
|
|
use crate::result::{Error, Result};
|
2019-08-20 17:16:06 -07:00
|
|
|
use crate::shred::{Shred, Shredder};
|
2019-05-15 00:28:31 -07:00
|
|
|
|
2019-03-15 16:42:47 -07:00
|
|
|
#[cfg(feature = "kvstore")]
|
|
|
|
use solana_kvstore as kvstore;
|
2019-03-11 15:53:14 -07:00
|
|
|
|
2019-04-02 14:58:07 -07:00
|
|
|
use bincode::deserialize;
|
2019-03-11 15:53:14 -07:00
|
|
|
|
2019-05-30 21:31:35 -07:00
|
|
|
use std::collections::HashMap;
|
2019-03-11 15:53:14 -07:00
|
|
|
|
|
|
|
#[cfg(not(feature = "kvstore"))]
|
|
|
|
use rocksdb;
|
|
|
|
|
2019-06-20 20:15:33 -07:00
|
|
|
use solana_metrics::{datapoint_error, datapoint_info};
|
2019-03-11 15:53:14 -07:00
|
|
|
|
2019-02-18 22:26:22 -08:00
|
|
|
use solana_sdk::genesis_block::GenesisBlock;
|
2019-01-22 15:50:36 -08:00
|
|
|
use solana_sdk::hash::Hash;
|
2018-12-11 09:14:23 -08:00
|
|
|
use solana_sdk::signature::{Keypair, KeypairUtil};
|
2019-03-11 15:53:14 -07:00
|
|
|
|
2019-02-07 15:10:54 -08:00
|
|
|
use std::borrow::{Borrow, Cow};
|
|
|
|
use std::cell::RefCell;
|
2019-01-08 15:53:44 -08:00
|
|
|
use std::cmp;
|
2019-01-24 12:04:04 -08:00
|
|
|
use std::fs;
|
2019-07-30 15:53:41 -07:00
|
|
|
use std::path::{Path, PathBuf};
|
2019-02-07 15:10:54 -08:00
|
|
|
use std::rc::Rc;
|
2019-05-09 14:10:04 -07:00
|
|
|
use std::sync::mpsc::{sync_channel, Receiver, SyncSender, TrySendError};
|
2019-04-06 19:41:22 -07:00
|
|
|
use std::sync::{Arc, RwLock};
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-04-11 14:14:57 -07:00
|
|
|
pub use self::meta::*;
|
2019-05-20 23:09:00 -07:00
|
|
|
pub use self::rooted_slot_iterator::*;
|
2019-09-05 18:20:30 -07:00
|
|
|
use crate::leader_schedule_cache::LeaderScheduleCache;
|
2019-09-06 14:30:56 -07:00
|
|
|
use solana_sdk::clock::Slot;
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-03-11 15:53:14 -07:00
|
|
|
mod db;
|
2019-04-11 14:14:57 -07:00
|
|
|
mod meta;
|
2019-05-20 23:09:00 -07:00
|
|
|
mod rooted_slot_iterator;
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-03-26 23:36:39 -07:00
|
|
|
macro_rules! db_imports {
|
|
|
|
{ $mod:ident, $db:ident, $db_path:expr } => {
|
|
|
|
mod $mod;
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-04-02 14:58:07 -07:00
|
|
|
use $mod::$db;
|
2019-08-27 15:09:41 -07:00
|
|
|
use db::{columns as cf, IteratorMode, IteratorDirection};
|
2019-04-02 14:58:07 -07:00
|
|
|
pub use db::columns;
|
|
|
|
|
|
|
|
pub type Database = db::Database<$db>;
|
2019-05-15 18:28:23 -07:00
|
|
|
pub type Cursor<C> = db::Cursor<$db, C>;
|
2019-04-02 14:58:07 -07:00
|
|
|
pub type LedgerColumn<C> = db::LedgerColumn<$db, C>;
|
|
|
|
pub type WriteBatch = db::WriteBatch<$db>;
|
2019-05-03 14:46:02 -07:00
|
|
|
type BatchProcessor = db::BatchProcessor<$db>;
|
2019-04-02 14:58:07 -07:00
|
|
|
|
|
|
|
pub trait Column: db::Column<$db> {}
|
|
|
|
impl<C: db::Column<$db>> Column for C {}
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-03-26 23:36:39 -07:00
|
|
|
pub const BLOCKTREE_DIRECTORY: &str = $db_path;
|
|
|
|
};
|
|
|
|
}
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-03-11 15:53:14 -07:00
|
|
|
#[cfg(not(feature = "kvstore"))]
|
2019-03-26 23:36:39 -07:00
|
|
|
db_imports! {rocks, Rocks, "rocksdb"}
|
2019-03-11 15:53:14 -07:00
|
|
|
#[cfg(feature = "kvstore")]
|
2019-03-26 23:36:39 -07:00
|
|
|
db_imports! {kvs, Kvs, "kvstore"}
|
2018-12-20 11:16:07 -08:00
|
|
|
|
2019-05-09 14:10:04 -07:00
|
|
|
pub const MAX_COMPLETED_SLOTS_IN_CHANNEL: usize = 100_000;
|
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
pub type SlotMetaWorkingSetEntry = (Rc<RefCell<SlotMeta>>, Option<SlotMeta>);
|
2019-05-09 14:10:04 -07:00
|
|
|
pub type CompletedSlotsReceiver = Receiver<Vec<u64>>;
|
|
|
|
|
2019-03-11 15:53:14 -07:00
|
|
|
#[derive(Debug)]
|
|
|
|
pub enum BlocktreeError {
|
2019-09-04 12:47:09 -07:00
|
|
|
ShredForIndexExists,
|
|
|
|
InvalidShredData(Box<bincode::ErrorKind>),
|
2019-03-11 15:53:14 -07:00
|
|
|
RocksDb(rocksdb::Error),
|
|
|
|
#[cfg(feature = "kvstore")]
|
|
|
|
KvsDb(kvstore::Error),
|
2019-05-20 23:09:00 -07:00
|
|
|
SlotNotRooted,
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
// ledger window
|
2019-02-07 20:52:39 -08:00
|
|
|
pub struct Blocktree {
|
2019-05-03 14:46:02 -07:00
|
|
|
db: Arc<Database>,
|
2019-04-02 14:58:07 -07:00
|
|
|
meta_cf: LedgerColumn<cf::SlotMeta>,
|
2019-06-20 15:50:41 -07:00
|
|
|
dead_slots_cf: LedgerColumn<cf::DeadSlots>,
|
2019-04-11 14:14:57 -07:00
|
|
|
erasure_meta_cf: LedgerColumn<cf::ErasureMeta>,
|
2019-04-06 19:41:22 -07:00
|
|
|
orphans_cf: LedgerColumn<cf::Orphans>,
|
2019-07-10 11:08:17 -07:00
|
|
|
index_cf: LedgerColumn<cf::Index>,
|
2019-08-20 17:16:06 -07:00
|
|
|
data_shred_cf: LedgerColumn<cf::ShredData>,
|
2019-08-26 18:27:45 -07:00
|
|
|
code_shred_cf: LedgerColumn<cf::ShredCode>,
|
2019-05-03 14:46:02 -07:00
|
|
|
batch_processor: Arc<RwLock<BatchProcessor>>,
|
2019-08-27 15:09:41 -07:00
|
|
|
last_root: Arc<RwLock<u64>>,
|
2019-09-03 21:32:51 -07:00
|
|
|
pub new_shreds_signals: Vec<SyncSender<bool>>,
|
2019-05-09 14:10:04 -07:00
|
|
|
pub completed_slots_senders: Vec<SyncSender<Vec<u64>>>,
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Column family for metadata about a leader slot
|
|
|
|
pub const META_CF: &str = "meta";
|
2019-06-20 15:50:41 -07:00
|
|
|
// Column family for slots that have been marked as dead
|
|
|
|
pub const DEAD_SLOTS_CF: &str = "dead_slots";
|
2019-04-11 14:14:57 -07:00
|
|
|
pub const ERASURE_META_CF: &str = "erasure_meta";
|
2019-04-06 19:41:22 -07:00
|
|
|
// Column family for orphans data
|
|
|
|
pub const ORPHANS_CF: &str = "orphans";
|
2019-05-03 14:46:02 -07:00
|
|
|
// Column family for root data
|
|
|
|
pub const ROOT_CF: &str = "root";
|
2019-07-10 11:08:17 -07:00
|
|
|
/// Column family for indexes
|
|
|
|
pub const INDEX_CF: &str = "index";
|
2019-08-12 10:03:57 -07:00
|
|
|
/// Column family for Data Shreds
|
|
|
|
pub const DATA_SHRED_CF: &str = "data_shred";
|
|
|
|
/// Column family for Code Shreds
|
|
|
|
pub const CODE_SHRED_CF: &str = "code_shred";
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
impl Blocktree {
|
2019-09-03 21:32:51 -07:00
|
|
|
/// Opens a Ledger in directory, provides "infinite" window of shreds
|
2019-07-30 15:53:41 -07:00
|
|
|
pub fn open(ledger_path: &Path) -> Result<Blocktree> {
|
2019-04-02 14:58:07 -07:00
|
|
|
fs::create_dir_all(&ledger_path)?;
|
2019-07-30 15:53:41 -07:00
|
|
|
let blocktree_path = ledger_path.join(BLOCKTREE_DIRECTORY);
|
2019-04-02 14:58:07 -07:00
|
|
|
|
|
|
|
// Open the database
|
2019-07-30 15:53:41 -07:00
|
|
|
let db = Database::open(&blocktree_path)?;
|
2019-04-02 14:58:07 -07:00
|
|
|
|
2019-05-20 19:04:18 -07:00
|
|
|
let batch_processor = unsafe { Arc::new(RwLock::new(db.batch_processor())) };
|
2019-05-03 14:46:02 -07:00
|
|
|
|
2019-04-02 14:58:07 -07:00
|
|
|
// Create the metadata column family
|
2019-04-26 08:52:10 -07:00
|
|
|
let meta_cf = db.column();
|
2019-04-02 14:58:07 -07:00
|
|
|
|
2019-06-20 15:50:41 -07:00
|
|
|
// Create the dead slots column family
|
|
|
|
let dead_slots_cf = db.column();
|
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
let erasure_meta_cf = db.column();
|
2019-04-02 14:58:07 -07:00
|
|
|
|
2019-04-06 19:41:22 -07:00
|
|
|
// Create the orphans column family. An "orphan" is defined as
|
|
|
|
// the head of a detached chain of slots, i.e. a slot with no
|
|
|
|
// known parent
|
2019-04-26 08:52:10 -07:00
|
|
|
let orphans_cf = db.column();
|
2019-07-10 11:08:17 -07:00
|
|
|
let index_cf = db.column();
|
2019-04-02 14:58:07 -07:00
|
|
|
|
2019-08-12 10:03:57 -07:00
|
|
|
let data_shred_cf = db.column();
|
|
|
|
let code_shred_cf = db.column();
|
|
|
|
|
2019-05-03 14:46:02 -07:00
|
|
|
let db = Arc::new(db);
|
2019-04-26 08:52:10 -07:00
|
|
|
|
2019-08-27 15:09:41 -07:00
|
|
|
// Get max root or 0 if it doesn't exist
|
|
|
|
let max_root = db
|
|
|
|
.iter::<cf::Root>(IteratorMode::End)?
|
|
|
|
.next()
|
|
|
|
.map(|(slot, _)| slot)
|
|
|
|
.unwrap_or(0);
|
|
|
|
let last_root = Arc::new(RwLock::new(max_root));
|
|
|
|
|
2019-04-02 14:58:07 -07:00
|
|
|
Ok(Blocktree {
|
|
|
|
db,
|
|
|
|
meta_cf,
|
2019-06-20 15:50:41 -07:00
|
|
|
dead_slots_cf,
|
2019-04-11 14:14:57 -07:00
|
|
|
erasure_meta_cf,
|
2019-04-06 19:41:22 -07:00
|
|
|
orphans_cf,
|
2019-07-10 11:08:17 -07:00
|
|
|
index_cf,
|
2019-08-20 17:16:06 -07:00
|
|
|
data_shred_cf,
|
2019-08-26 18:27:45 -07:00
|
|
|
code_shred_cf,
|
2019-09-03 21:32:51 -07:00
|
|
|
new_shreds_signals: vec![],
|
2019-05-03 14:46:02 -07:00
|
|
|
batch_processor,
|
2019-05-09 14:10:04 -07:00
|
|
|
completed_slots_senders: vec![],
|
2019-08-27 15:09:41 -07:00
|
|
|
last_root,
|
2019-04-02 14:58:07 -07:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2019-05-09 14:10:04 -07:00
|
|
|
pub fn open_with_signal(
|
2019-07-30 15:53:41 -07:00
|
|
|
ledger_path: &Path,
|
2019-05-09 14:10:04 -07:00
|
|
|
) -> Result<(Self, Receiver<bool>, CompletedSlotsReceiver)> {
|
2019-02-07 20:52:39 -08:00
|
|
|
let mut blocktree = Self::open(ledger_path)?;
|
2019-02-04 15:33:43 -08:00
|
|
|
let (signal_sender, signal_receiver) = sync_channel(1);
|
2019-05-09 14:10:04 -07:00
|
|
|
let (completed_slots_sender, completed_slots_receiver) =
|
|
|
|
sync_channel(MAX_COMPLETED_SLOTS_IN_CHANNEL);
|
2019-09-03 21:32:51 -07:00
|
|
|
blocktree.new_shreds_signals = vec![signal_sender];
|
2019-05-09 14:10:04 -07:00
|
|
|
blocktree.completed_slots_senders = vec![completed_slots_sender];
|
2019-02-04 15:33:43 -08:00
|
|
|
|
2019-05-09 14:10:04 -07:00
|
|
|
Ok((blocktree, signal_receiver, completed_slots_receiver))
|
2019-02-04 15:33:43 -08:00
|
|
|
}
|
|
|
|
|
2019-07-30 15:53:41 -07:00
|
|
|
pub fn destroy(ledger_path: &Path) -> Result<()> {
|
|
|
|
// Database::destroy() fails if the path doesn't exist
|
2019-04-02 14:58:07 -07:00
|
|
|
fs::create_dir_all(ledger_path)?;
|
2019-07-30 15:53:41 -07:00
|
|
|
let blocktree_path = ledger_path.join(BLOCKTREE_DIRECTORY);
|
|
|
|
Database::destroy(&blocktree_path)
|
2019-04-02 14:58:07 -07:00
|
|
|
}
|
|
|
|
|
2019-03-05 14:18:29 -08:00
|
|
|
pub fn meta(&self, slot: u64) -> Result<Option<SlotMeta>> {
|
2019-05-03 14:46:02 -07:00
|
|
|
self.meta_cf.get(slot)
|
2019-04-26 08:52:10 -07:00
|
|
|
}
|
|
|
|
|
2019-05-29 15:01:20 -07:00
|
|
|
pub fn is_full(&self, slot: u64) -> bool {
|
|
|
|
if let Ok(meta) = self.meta_cf.get(slot) {
|
|
|
|
if let Some(meta) = meta {
|
|
|
|
return meta.is_full();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
false
|
|
|
|
}
|
|
|
|
|
2019-07-20 13:13:55 -07:00
|
|
|
/// Silently deletes all blocktree column families starting at the given slot until the `to` slot
|
2019-07-24 17:28:08 -07:00
|
|
|
/// Dangerous; Use with care:
|
|
|
|
/// Does not check for integrity and does not update slot metas that refer to deleted slots
|
|
|
|
/// Modifies multiple column families simultaneously
|
|
|
|
pub fn purge_slots(&self, mut from_slot: Slot, to_slot: Option<Slot>) {
|
|
|
|
// split the purge request into batches of 1000 slots
|
|
|
|
const PURGE_BATCH_SIZE: u64 = 1000;
|
|
|
|
let mut batch_end = to_slot
|
|
|
|
.unwrap_or(from_slot + PURGE_BATCH_SIZE)
|
|
|
|
.min(from_slot + PURGE_BATCH_SIZE);
|
|
|
|
while from_slot < batch_end {
|
|
|
|
if let Ok(end) = self.run_purge_batch(from_slot, batch_end) {
|
|
|
|
// no more slots to iter or reached the upper bound
|
|
|
|
if end {
|
|
|
|
break;
|
|
|
|
} else {
|
|
|
|
// update the next batch bounds
|
|
|
|
from_slot = batch_end;
|
|
|
|
batch_end = to_slot
|
|
|
|
.unwrap_or(batch_end + PURGE_BATCH_SIZE)
|
|
|
|
.min(batch_end + PURGE_BATCH_SIZE);
|
|
|
|
}
|
|
|
|
}
|
2019-07-17 14:42:29 -07:00
|
|
|
}
|
2019-07-24 17:28:08 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// Returns whether or not all iterators have reached their end
|
|
|
|
fn run_purge_batch(&self, from_slot: Slot, batch_end: Slot) -> Result<bool> {
|
|
|
|
let from_slot = Some(from_slot);
|
|
|
|
let batch_end = Some(batch_end);
|
2019-08-26 09:47:48 -07:00
|
|
|
|
2019-07-24 17:28:08 -07:00
|
|
|
unsafe {
|
|
|
|
let mut batch_processor = self.db.batch_processor();
|
|
|
|
let mut write_batch = batch_processor
|
|
|
|
.batch()
|
|
|
|
.expect("Database Error: Failed to get write batch");
|
2019-08-26 09:47:48 -07:00
|
|
|
let end = self
|
2019-07-24 17:28:08 -07:00
|
|
|
.meta_cf
|
|
|
|
.delete_slot(&mut write_batch, from_slot, batch_end)
|
2019-08-26 09:47:48 -07:00
|
|
|
.unwrap_or(false)
|
|
|
|
&& self
|
|
|
|
.erasure_meta_cf
|
|
|
|
.delete_slot(&mut write_batch, from_slot, batch_end)
|
|
|
|
.unwrap_or(false)
|
2019-08-26 18:27:45 -07:00
|
|
|
&& self
|
|
|
|
.data_shred_cf
|
|
|
|
.delete_slot(&mut write_batch, from_slot, batch_end)
|
|
|
|
.unwrap_or(false)
|
|
|
|
&& self
|
|
|
|
.code_shred_cf
|
|
|
|
.delete_slot(&mut write_batch, from_slot, batch_end)
|
|
|
|
.unwrap_or(false)
|
2019-08-26 09:47:48 -07:00
|
|
|
&& self
|
|
|
|
.orphans_cf
|
|
|
|
.delete_slot(&mut write_batch, from_slot, batch_end)
|
|
|
|
.unwrap_or(false)
|
|
|
|
&& self
|
|
|
|
.index_cf
|
|
|
|
.delete_slot(&mut write_batch, from_slot, batch_end)
|
|
|
|
.unwrap_or(false)
|
|
|
|
&& self
|
|
|
|
.dead_slots_cf
|
|
|
|
.delete_slot(&mut write_batch, from_slot, batch_end)
|
|
|
|
.unwrap_or(false)
|
|
|
|
&& self
|
|
|
|
.db
|
|
|
|
.column::<cf::Root>()
|
|
|
|
.delete_slot(&mut write_batch, from_slot, batch_end)
|
|
|
|
.unwrap_or(false);
|
|
|
|
|
2019-07-24 17:28:08 -07:00
|
|
|
if let Err(e) = batch_processor.write(write_batch) {
|
|
|
|
error!(
|
|
|
|
"Error: {:?} while submitting write batch for slot {:?} retrying...",
|
|
|
|
e, from_slot
|
|
|
|
);
|
|
|
|
Err(e)?;
|
|
|
|
}
|
|
|
|
Ok(end)
|
2019-07-17 14:42:29 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
pub fn erasure_meta(&self, slot: u64, set_index: u64) -> Result<Option<ErasureMeta>> {
|
2019-05-03 14:46:02 -07:00
|
|
|
self.erasure_meta_cf.get((slot, set_index))
|
2019-01-08 11:41:55 -08:00
|
|
|
}
|
|
|
|
|
2019-04-06 19:41:22 -07:00
|
|
|
pub fn orphan(&self, slot: u64) -> Result<Option<bool>> {
|
2019-05-03 14:46:02 -07:00
|
|
|
self.orphans_cf.get(slot)
|
2019-03-29 16:07:24 -07:00
|
|
|
}
|
|
|
|
|
2019-07-17 14:42:29 -07:00
|
|
|
pub fn rooted_slot_iterator(&self, slot: u64) -> Result<RootedSlotIterator> {
|
2019-05-20 23:09:00 -07:00
|
|
|
RootedSlotIterator::new(slot, self)
|
|
|
|
}
|
|
|
|
|
2019-05-15 18:28:23 -07:00
|
|
|
pub fn slot_meta_iterator(&self, slot: u64) -> Result<impl Iterator<Item = (u64, SlotMeta)>> {
|
2019-08-27 15:09:41 -07:00
|
|
|
let meta_iter = self
|
|
|
|
.db
|
|
|
|
.iter::<cf::SlotMeta>(IteratorMode::From(slot, IteratorDirection::Forward))?;
|
2019-05-15 18:28:23 -07:00
|
|
|
Ok(meta_iter.map(|(slot, slot_meta_bytes)| {
|
|
|
|
(
|
|
|
|
slot,
|
|
|
|
deserialize(&slot_meta_bytes)
|
|
|
|
.unwrap_or_else(|_| panic!("Could not deserialize SlotMeta for slot {}", slot)),
|
|
|
|
)
|
|
|
|
}))
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-05-13 22:04:54 -07:00
|
|
|
pub fn slot_data_iterator(
|
|
|
|
&self,
|
|
|
|
slot: u64,
|
2019-05-23 03:10:16 -07:00
|
|
|
) -> Result<impl Iterator<Item = ((u64, u64), Box<[u8]>)>> {
|
2019-08-27 15:09:41 -07:00
|
|
|
let slot_iterator = self
|
|
|
|
.db
|
2019-09-03 21:32:51 -07:00
|
|
|
.iter::<cf::ShredData>(IteratorMode::From((slot, 0), IteratorDirection::Forward))?;
|
|
|
|
Ok(slot_iterator.take_while(move |((shred_slot, _), _)| *shred_slot == slot))
|
2019-05-13 22:04:54 -07:00
|
|
|
}
|
|
|
|
|
2019-08-26 18:27:45 -07:00
|
|
|
fn try_shred_recovery(
|
|
|
|
db: &Database,
|
|
|
|
erasure_metas: &HashMap<(u64, u64), ErasureMeta>,
|
|
|
|
index_working_set: &HashMap<u64, Index>,
|
|
|
|
prev_inserted_datas: &mut HashMap<(u64, u64), Shred>,
|
|
|
|
prev_inserted_codes: &mut HashMap<(u64, u64), Shred>,
|
|
|
|
) -> Vec<Shred> {
|
|
|
|
let data_cf = db.column::<cf::ShredData>();
|
|
|
|
let code_cf = db.column::<cf::ShredCode>();
|
|
|
|
let mut recovered_data_shreds = vec![];
|
|
|
|
// Recovery rules:
|
|
|
|
// 1. Only try recovery around indexes for which new data or coding shreds are received
|
|
|
|
// 2. For new data shreds, check if an erasure set exists. If not, don't try recovery
|
|
|
|
// 3. Before trying recovery, check if enough number of shreds have been received
|
|
|
|
// 3a. Enough number of shreds = (#data + #coding shreds) > erasure.num_data
|
|
|
|
for (&(slot, set_index), erasure_meta) in erasure_metas.iter() {
|
2019-08-27 11:22:06 -07:00
|
|
|
let submit_metrics = |attempted: bool, status: String| {
|
|
|
|
datapoint_info!(
|
|
|
|
"blocktree-erasure",
|
|
|
|
("slot", slot as i64, i64),
|
|
|
|
("start_index", set_index as i64, i64),
|
|
|
|
("end_index", erasure_meta.end_indexes().0 as i64, i64),
|
|
|
|
("recovery_attempted", attempted, bool),
|
|
|
|
("recovery_status", status, String),
|
|
|
|
);
|
|
|
|
};
|
|
|
|
|
2019-08-26 18:27:45 -07:00
|
|
|
let index = index_working_set.get(&slot).expect("Index");
|
2019-08-27 11:22:06 -07:00
|
|
|
match erasure_meta.status(&index) {
|
|
|
|
ErasureMetaStatus::CanRecover => {
|
|
|
|
// Find shreds for this erasure set and try recovery
|
|
|
|
let slot = index.slot;
|
|
|
|
let mut available_shreds = vec![];
|
|
|
|
(set_index..set_index + erasure_meta.config.num_data() as u64).for_each(|i| {
|
|
|
|
if index.data().is_present(i) {
|
|
|
|
if let Some(shred) =
|
|
|
|
prev_inserted_datas.remove(&(slot, i)).or_else(|| {
|
|
|
|
let some_data = data_cf
|
|
|
|
.get_bytes((slot, i))
|
|
|
|
.expect("Database failure, could not fetch data shred");
|
|
|
|
if let Some(data) = some_data {
|
|
|
|
bincode::deserialize(&data).ok()
|
|
|
|
} else {
|
|
|
|
warn!("Data shred deleted while reading for recovery");
|
|
|
|
None
|
|
|
|
}
|
|
|
|
})
|
|
|
|
{
|
|
|
|
available_shreds.push(shred);
|
2019-08-26 18:27:45 -07:00
|
|
|
}
|
|
|
|
}
|
2019-08-27 11:22:06 -07:00
|
|
|
});
|
|
|
|
(set_index..set_index + erasure_meta.config.num_coding() as u64).for_each(
|
|
|
|
|i| {
|
|
|
|
if index.coding().is_present(i) {
|
|
|
|
if let Some(shred) =
|
|
|
|
prev_inserted_codes.remove(&(slot, i)).or_else(|| {
|
|
|
|
let some_code = code_cf
|
|
|
|
.get_bytes((slot, i))
|
|
|
|
.expect("Database failure, could not fetch code shred");
|
|
|
|
if let Some(code) = some_code {
|
|
|
|
bincode::deserialize(&code).ok()
|
|
|
|
} else {
|
|
|
|
warn!("Code shred deleted while reading for recovery");
|
|
|
|
None
|
|
|
|
}
|
|
|
|
})
|
|
|
|
{
|
|
|
|
available_shreds.push(shred);
|
|
|
|
}
|
2019-08-26 18:27:45 -07:00
|
|
|
}
|
2019-08-27 11:22:06 -07:00
|
|
|
},
|
|
|
|
);
|
|
|
|
if let Ok(mut result) = Shredder::try_recovery(
|
|
|
|
&available_shreds,
|
|
|
|
erasure_meta.config.num_data(),
|
|
|
|
erasure_meta.config.num_coding(),
|
|
|
|
set_index as usize,
|
|
|
|
slot,
|
|
|
|
) {
|
|
|
|
submit_metrics(true, "complete".into());
|
|
|
|
recovered_data_shreds.append(&mut result.recovered_data);
|
|
|
|
} else {
|
|
|
|
submit_metrics(true, "incomplete".into());
|
2019-08-26 18:27:45 -07:00
|
|
|
}
|
|
|
|
}
|
2019-08-27 11:22:06 -07:00
|
|
|
ErasureMetaStatus::DataFull => {
|
|
|
|
submit_metrics(false, "complete".into());
|
|
|
|
}
|
|
|
|
ErasureMetaStatus::StillNeed(needed) => {
|
|
|
|
submit_metrics(false, format!("still need: {}", needed));
|
|
|
|
}
|
|
|
|
};
|
2019-08-26 18:27:45 -07:00
|
|
|
}
|
|
|
|
recovered_data_shreds
|
|
|
|
}
|
|
|
|
|
2019-09-05 18:20:30 -07:00
|
|
|
pub fn insert_shreds(
|
|
|
|
&self,
|
|
|
|
shreds: Vec<Shred>,
|
|
|
|
leader_schedule: Option<&Arc<LeaderScheduleCache>>,
|
|
|
|
) -> Result<()> {
|
2019-08-20 17:16:06 -07:00
|
|
|
let db = &*self.db;
|
|
|
|
let mut batch_processor = self.batch_processor.write().unwrap();
|
|
|
|
let mut write_batch = batch_processor.batch()?;
|
|
|
|
|
2019-08-26 18:27:45 -07:00
|
|
|
let mut just_inserted_coding_shreds = HashMap::new();
|
2019-09-04 17:14:42 -07:00
|
|
|
let mut just_inserted_data_shreds = HashMap::new();
|
2019-08-26 18:27:45 -07:00
|
|
|
let mut erasure_metas = HashMap::new();
|
2019-08-20 17:16:06 -07:00
|
|
|
let mut slot_meta_working_set = HashMap::new();
|
|
|
|
let mut index_working_set = HashMap::new();
|
|
|
|
|
2019-08-26 18:27:45 -07:00
|
|
|
shreds.into_iter().for_each(|shred| {
|
2019-09-04 17:14:42 -07:00
|
|
|
if let Shred::Coding(_) = &shred {
|
|
|
|
self.check_insert_coding_shred(
|
|
|
|
shred,
|
|
|
|
&mut erasure_metas,
|
2019-08-27 15:09:41 -07:00
|
|
|
&mut index_working_set,
|
|
|
|
&mut write_batch,
|
2019-09-04 17:14:42 -07:00
|
|
|
&mut just_inserted_coding_shreds,
|
|
|
|
);
|
|
|
|
} else {
|
|
|
|
self.check_insert_data_shred(
|
|
|
|
shred,
|
|
|
|
&mut index_working_set,
|
|
|
|
&mut slot_meta_working_set,
|
|
|
|
&mut write_batch,
|
|
|
|
&mut just_inserted_data_shreds,
|
|
|
|
);
|
2019-08-26 18:27:45 -07:00
|
|
|
}
|
|
|
|
});
|
2019-08-20 17:16:06 -07:00
|
|
|
|
2019-09-05 18:20:30 -07:00
|
|
|
if let Some(leader_schedule_cache) = leader_schedule {
|
|
|
|
let recovered_data = Self::try_shred_recovery(
|
|
|
|
&db,
|
|
|
|
&erasure_metas,
|
|
|
|
&index_working_set,
|
|
|
|
&mut just_inserted_data_shreds,
|
|
|
|
&mut just_inserted_coding_shreds,
|
2019-08-26 18:27:45 -07:00
|
|
|
);
|
2019-09-05 18:20:30 -07:00
|
|
|
|
|
|
|
recovered_data.into_iter().for_each(|shred| {
|
|
|
|
if let Some(leader) = leader_schedule_cache.slot_leader_at(shred.slot(), None) {
|
|
|
|
if shred.verify(&leader) {
|
|
|
|
self.check_insert_data_shred(
|
|
|
|
shred,
|
|
|
|
&mut index_working_set,
|
|
|
|
&mut slot_meta_working_set,
|
|
|
|
&mut write_batch,
|
|
|
|
&mut just_inserted_coding_shreds,
|
|
|
|
)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
});
|
|
|
|
}
|
2019-08-20 17:16:06 -07:00
|
|
|
|
|
|
|
// Handle chaining for the working set
|
2019-08-27 15:09:41 -07:00
|
|
|
handle_chaining(&self.db, &mut write_batch, &slot_meta_working_set)?;
|
2019-08-20 17:16:06 -07:00
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
let (should_signal, newly_completed_slots) = commit_slot_meta_working_set(
|
2019-08-20 17:16:06 -07:00
|
|
|
&slot_meta_working_set,
|
|
|
|
&self.completed_slots_senders,
|
|
|
|
&mut write_batch,
|
|
|
|
)?;
|
|
|
|
|
2019-08-27 11:22:06 -07:00
|
|
|
for ((slot, set_index), erasure_meta) in erasure_metas {
|
|
|
|
write_batch.put::<cf::ErasureMeta>((slot, set_index), &erasure_meta)?;
|
|
|
|
}
|
|
|
|
|
2019-08-20 17:16:06 -07:00
|
|
|
for (&slot, index) in index_working_set.iter() {
|
|
|
|
write_batch.put::<cf::Index>(slot, index)?;
|
|
|
|
}
|
|
|
|
|
|
|
|
batch_processor.write(write_batch)?;
|
|
|
|
|
|
|
|
if should_signal {
|
2019-09-03 21:32:51 -07:00
|
|
|
for signal in &self.new_shreds_signals {
|
2019-08-20 17:16:06 -07:00
|
|
|
let _ = signal.try_send(true);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
send_signals(
|
2019-09-03 21:32:51 -07:00
|
|
|
&self.new_shreds_signals,
|
2019-08-20 17:16:06 -07:00
|
|
|
&self.completed_slots_senders,
|
|
|
|
should_signal,
|
|
|
|
newly_completed_slots,
|
|
|
|
)?;
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
fn check_insert_coding_shred(
|
2019-08-26 18:27:45 -07:00
|
|
|
&self,
|
2019-09-04 17:14:42 -07:00
|
|
|
shred: Shred,
|
2019-08-26 18:27:45 -07:00
|
|
|
erasure_metas: &mut HashMap<(u64, u64), ErasureMeta>,
|
2019-09-04 17:14:42 -07:00
|
|
|
index_working_set: &mut HashMap<u64, Index>,
|
|
|
|
write_batch: &mut WriteBatch,
|
|
|
|
just_inserted_coding_shreds: &mut HashMap<(u64, u64), Shred>,
|
|
|
|
) {
|
|
|
|
let slot = shred.slot();
|
|
|
|
let shred_index = u64::from(shred.index());
|
|
|
|
|
|
|
|
let (index_meta, mut new_index_meta) =
|
|
|
|
get_index_meta_entry(&self.db, slot, index_working_set);
|
|
|
|
|
|
|
|
let index_meta = index_meta.unwrap_or_else(|| new_index_meta.as_mut().unwrap());
|
|
|
|
// This gives the index of first coding shred in this FEC block
|
|
|
|
// So, all coding shreds in a given FEC block will have the same set index
|
|
|
|
if Blocktree::should_insert_coding_shred(&shred, index_meta.coding(), &self.last_root)
|
|
|
|
&& self
|
|
|
|
.insert_coding_shred(erasure_metas, index_meta, &shred, write_batch)
|
|
|
|
.is_ok()
|
|
|
|
{
|
|
|
|
just_inserted_coding_shreds
|
|
|
|
.entry((slot, shred_index))
|
|
|
|
.or_insert_with(|| shred);
|
|
|
|
new_index_meta.map(|n| index_working_set.insert(slot, n));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn check_insert_data_shred(
|
|
|
|
&self,
|
2019-08-26 18:27:45 -07:00
|
|
|
shred: Shred,
|
2019-09-04 17:14:42 -07:00
|
|
|
index_working_set: &mut HashMap<u64, Index>,
|
|
|
|
slot_meta_working_set: &mut HashMap<u64, SlotMetaWorkingSetEntry>,
|
2019-08-26 18:27:45 -07:00
|
|
|
write_batch: &mut WriteBatch,
|
2019-09-04 17:14:42 -07:00
|
|
|
just_inserted_data_shreds: &mut HashMap<(u64, u64), Shred>,
|
2019-08-26 18:27:45 -07:00
|
|
|
) {
|
|
|
|
let slot = shred.slot();
|
|
|
|
let shred_index = u64::from(shred.index());
|
2019-09-04 17:14:42 -07:00
|
|
|
let (index_meta, mut new_index_meta) =
|
|
|
|
get_index_meta_entry(&self.db, slot, index_working_set);
|
|
|
|
let (slot_meta_entry, mut new_slot_meta_entry) =
|
|
|
|
get_slot_meta_entry(&self.db, slot_meta_working_set, slot, shred.parent());
|
|
|
|
|
|
|
|
let insert_success = {
|
|
|
|
let index_meta = index_meta.unwrap_or_else(|| new_index_meta.as_mut().unwrap());
|
|
|
|
let entry = slot_meta_entry.unwrap_or_else(|| new_slot_meta_entry.as_mut().unwrap());
|
|
|
|
let mut slot_meta = entry.0.borrow_mut();
|
2019-08-26 18:27:45 -07:00
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
if Blocktree::should_insert_data_shred(
|
|
|
|
&shred,
|
|
|
|
&slot_meta,
|
|
|
|
index_meta.data(),
|
|
|
|
&self.last_root,
|
|
|
|
) {
|
|
|
|
self.insert_data_shred(&mut slot_meta, index_meta.data_mut(), &shred, write_batch)
|
|
|
|
.is_ok()
|
|
|
|
} else {
|
|
|
|
false
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
if insert_success {
|
|
|
|
just_inserted_data_shreds.insert((slot, shred_index), shred);
|
|
|
|
new_index_meta.map(|n| index_working_set.insert(slot, n));
|
|
|
|
new_slot_meta_entry.map(|n| slot_meta_working_set.insert(slot, n));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn should_insert_coding_shred(
|
|
|
|
shred: &Shred,
|
|
|
|
coding_index: &CodingIndex,
|
|
|
|
last_root: &RwLock<u64>,
|
|
|
|
) -> bool {
|
|
|
|
let slot = shred.slot();
|
|
|
|
let shred_index = shred.index();
|
|
|
|
|
|
|
|
let (pos, num_coding) = {
|
|
|
|
if let Shred::Coding(coding_shred) = &shred {
|
|
|
|
(
|
|
|
|
u32::from(coding_shred.header.position),
|
|
|
|
coding_shred.header.num_coding_shreds,
|
|
|
|
)
|
|
|
|
} else {
|
|
|
|
panic!("should_insert_coding_shred called with non-coding shred")
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
if shred_index < pos {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
let set_index = shred_index - pos;
|
|
|
|
!(num_coding == 0
|
|
|
|
|| pos >= u32::from(num_coding)
|
|
|
|
|| std::u32::MAX - set_index < u32::from(num_coding) - 1
|
|
|
|
|| coding_index.is_present(u64::from(shred_index))
|
|
|
|
|| slot <= *last_root.read().unwrap())
|
|
|
|
}
|
|
|
|
|
|
|
|
fn insert_coding_shred(
|
|
|
|
&self,
|
|
|
|
erasure_metas: &mut HashMap<(u64, u64), ErasureMeta>,
|
|
|
|
index_meta: &mut Index,
|
|
|
|
shred: &Shred,
|
|
|
|
write_batch: &mut WriteBatch,
|
|
|
|
) -> Result<()> {
|
|
|
|
let slot = shred.slot();
|
|
|
|
let shred_index = u64::from(shred.index());
|
|
|
|
let (num_data, num_coding, pos) = {
|
|
|
|
if let Shred::Coding(coding_shred) = &shred {
|
|
|
|
(
|
|
|
|
coding_shred.header.num_data_shreds as usize,
|
|
|
|
coding_shred.header.num_coding_shreds as usize,
|
|
|
|
u64::from(coding_shred.header.position),
|
|
|
|
)
|
|
|
|
} else {
|
|
|
|
panic!("insert_coding_shred called with non-coding shred")
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
// Assert guaranteed by integrity checks on the shred that happen before
|
|
|
|
// `insert_coding_shred` is called
|
|
|
|
if shred_index < pos {
|
|
|
|
error!("Due to earlier validation, shred index must be >= pos");
|
|
|
|
return Err(Error::BlocktreeError(BlocktreeError::InvalidShredData(
|
|
|
|
Box::new(bincode::ErrorKind::Custom("shred index < pos".to_string())),
|
|
|
|
)));
|
|
|
|
}
|
|
|
|
|
|
|
|
let set_index = shred_index - pos;
|
2019-08-26 18:27:45 -07:00
|
|
|
let erasure_config = ErasureConfig::new(num_data, num_coding);
|
|
|
|
|
|
|
|
let erasure_meta = erasure_metas.entry((slot, set_index)).or_insert_with(|| {
|
|
|
|
self.erasure_meta_cf
|
|
|
|
.get((slot, set_index))
|
|
|
|
.expect("Expect database get to succeed")
|
|
|
|
.unwrap_or_else(|| ErasureMeta::new(set_index, &erasure_config))
|
|
|
|
});
|
|
|
|
|
|
|
|
if erasure_config != erasure_meta.config {
|
|
|
|
// ToDo: This is a potential slashing condition
|
|
|
|
warn!("Received multiple erasure configs for the same erasure set!!!");
|
|
|
|
warn!(
|
|
|
|
"Stored config: {:#?}, new config: {:#?}",
|
|
|
|
erasure_meta.config, erasure_config
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
let serialized_shred = bincode::serialize(shred).unwrap();
|
2019-08-26 18:27:45 -07:00
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
// Commit step: commit all changes to the mutable structures at once, or none at all.
|
|
|
|
// We don't want only a subset of these changes going through.
|
|
|
|
write_batch.put_bytes::<cf::ShredCode>((slot, shred_index), &serialized_shred)?;
|
|
|
|
index_meta.coding_mut().set_present(shred_index, true);
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
|
|
|
fn should_insert_data_shred(
|
|
|
|
shred: &Shred,
|
|
|
|
slot_meta: &SlotMeta,
|
|
|
|
data_index: &DataIndex,
|
|
|
|
last_root: &RwLock<u64>,
|
|
|
|
) -> bool {
|
|
|
|
let shred_index = u64::from(shred.index());
|
|
|
|
let slot = shred.slot();
|
|
|
|
let last_in_slot = if let Shred::LastInSlot(_) = shred {
|
|
|
|
debug!("got last in slot");
|
|
|
|
true
|
|
|
|
} else {
|
|
|
|
false
|
|
|
|
};
|
|
|
|
|
|
|
|
// Check that the data shred doesn't already exist in blocktree
|
|
|
|
if shred_index < slot_meta.consumed || data_index.is_present(shred_index) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check that we do not receive shred_index >= than the last_index
|
|
|
|
// for the slot
|
|
|
|
let last_index = slot_meta.last_index;
|
|
|
|
if shred_index >= last_index {
|
|
|
|
datapoint_error!(
|
|
|
|
"blocktree_error",
|
|
|
|
(
|
|
|
|
"error",
|
|
|
|
format!(
|
|
|
|
"Received index {} >= slot.last_index {}",
|
|
|
|
shred_index, last_index
|
|
|
|
),
|
|
|
|
String
|
|
|
|
)
|
|
|
|
);
|
|
|
|
return false;
|
2019-08-26 18:27:45 -07:00
|
|
|
}
|
2019-09-04 17:14:42 -07:00
|
|
|
// Check that we do not receive a blob with "last_index" true, but shred_index
|
|
|
|
// less than our current received
|
|
|
|
if last_in_slot && shred_index < slot_meta.received {
|
|
|
|
datapoint_error!(
|
|
|
|
"blocktree_error",
|
|
|
|
(
|
|
|
|
"error",
|
|
|
|
format!(
|
|
|
|
"Received shred_index {} < slot.received {}",
|
|
|
|
shred_index, slot_meta.received
|
|
|
|
),
|
|
|
|
String
|
|
|
|
)
|
|
|
|
);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
let last_root = *last_root.read().unwrap();
|
|
|
|
if !is_valid_write_to_slot_0(slot, slot_meta.parent_slot, last_root) {
|
|
|
|
// Check that the parent_slot < slot
|
|
|
|
if slot_meta.parent_slot >= slot {
|
|
|
|
datapoint_error!(
|
|
|
|
"blocktree_error",
|
|
|
|
(
|
|
|
|
"error",
|
|
|
|
format!(
|
|
|
|
"Received blob with parent_slot {} >= slot {}",
|
|
|
|
slot_meta.parent_slot, slot
|
|
|
|
),
|
|
|
|
String
|
|
|
|
)
|
|
|
|
);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check that the blob is for a slot that is past the root
|
|
|
|
if slot <= last_root {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Ignore blobs that chain to slots before the last root
|
|
|
|
if slot_meta.parent_slot < last_root {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
true
|
2019-08-26 18:27:45 -07:00
|
|
|
}
|
|
|
|
|
2019-08-20 17:16:06 -07:00
|
|
|
fn insert_data_shred(
|
2019-08-27 15:09:41 -07:00
|
|
|
&self,
|
2019-09-04 17:14:42 -07:00
|
|
|
slot_meta: &mut SlotMeta,
|
|
|
|
data_index: &mut DataIndex,
|
2019-08-20 17:16:06 -07:00
|
|
|
shred: &Shred,
|
|
|
|
write_batch: &mut WriteBatch,
|
2019-09-04 17:14:42 -07:00
|
|
|
) -> Result<()> {
|
2019-08-20 17:16:06 -07:00
|
|
|
let slot = shred.slot();
|
|
|
|
let index = u64::from(shred.index());
|
2019-08-21 15:27:42 -07:00
|
|
|
let parent = shred.parent();
|
2019-08-20 17:16:06 -07:00
|
|
|
|
|
|
|
let last_in_slot = if let Shred::LastInSlot(_) = shred {
|
|
|
|
debug!("got last in slot");
|
|
|
|
true
|
|
|
|
} else {
|
|
|
|
false
|
|
|
|
};
|
|
|
|
|
|
|
|
if is_orphan(slot_meta) {
|
|
|
|
slot_meta.parent_slot = parent;
|
|
|
|
}
|
|
|
|
|
2019-08-27 15:09:41 -07:00
|
|
|
let data_cf = self.db.column::<cf::ShredData>();
|
2019-08-20 17:16:06 -07:00
|
|
|
|
|
|
|
let check_data_cf = |slot, index| {
|
|
|
|
data_cf
|
|
|
|
.get_bytes((slot, index))
|
|
|
|
.map(|opt| opt.is_some())
|
|
|
|
.unwrap_or(false)
|
|
|
|
};
|
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
let new_consumed = if slot_meta.consumed == index {
|
|
|
|
let mut current_index = index + 1;
|
2019-08-26 18:27:45 -07:00
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
while data_index.is_present(current_index) || check_data_cf(slot, current_index) {
|
|
|
|
current_index += 1;
|
|
|
|
}
|
|
|
|
current_index
|
|
|
|
} else {
|
|
|
|
slot_meta.consumed
|
|
|
|
};
|
2019-08-20 17:16:06 -07:00
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
let serialized_shred = bincode::serialize(shred).unwrap();
|
2019-08-20 17:16:06 -07:00
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
// Commit step: commit all changes to the mutable structures at once, or none at all.
|
|
|
|
// We don't want only a subset of these changes going through.
|
|
|
|
write_batch.put_bytes::<cf::ShredData>((slot, index), &serialized_shred)?;
|
|
|
|
update_slot_meta(last_in_slot, slot_meta, index, new_consumed);
|
|
|
|
data_index.set_present(index, true);
|
|
|
|
trace!("inserted shred into slot {:?} and index {:?}", slot, index);
|
|
|
|
Ok(())
|
2019-08-20 17:16:06 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
pub fn get_data_shred(&self, slot: u64, index: u64) -> Result<Option<Vec<u8>>> {
|
|
|
|
self.data_shred_cf.get_bytes((slot, index))
|
|
|
|
}
|
|
|
|
|
2019-08-28 22:34:47 -07:00
|
|
|
pub fn get_data_shreds(
|
|
|
|
&self,
|
|
|
|
slot: u64,
|
|
|
|
from_index: u64,
|
2019-09-03 21:32:51 -07:00
|
|
|
to_index: u64,
|
2019-08-28 22:34:47 -07:00
|
|
|
buffer: &mut [u8],
|
|
|
|
) -> Result<(u64, usize)> {
|
|
|
|
let meta_cf = self.db.column::<cf::SlotMeta>();
|
|
|
|
let mut buffer_offset = 0;
|
|
|
|
let mut last_index = 0;
|
|
|
|
if let Some(meta) = meta_cf.get(slot)? {
|
|
|
|
if !meta.is_full() {
|
|
|
|
warn!("The slot is not yet full. Will not return any shreds");
|
|
|
|
return Ok((last_index, buffer_offset));
|
|
|
|
}
|
2019-09-03 21:32:51 -07:00
|
|
|
let to_index = cmp::min(to_index, meta.consumed);
|
|
|
|
for index in from_index..to_index {
|
2019-08-28 22:34:47 -07:00
|
|
|
if let Some(shred_data) = self.get_data_shred(slot, index)? {
|
|
|
|
let shred_len = shred_data.len();
|
|
|
|
if buffer.len().saturating_sub(buffer_offset) >= shred_len {
|
|
|
|
buffer[buffer_offset..buffer_offset + shred_len]
|
|
|
|
.copy_from_slice(&shred_data[..shred_len]);
|
|
|
|
buffer_offset += shred_len;
|
|
|
|
last_index = index;
|
|
|
|
// All shreds are of the same length.
|
|
|
|
// Let's check if we have scope to accomodate another shred
|
|
|
|
// If not, let's break right away, as it'll save on 1 DB read
|
|
|
|
if buffer.len().saturating_sub(buffer_offset) < shred_len {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
Ok((last_index, buffer_offset))
|
|
|
|
}
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
pub fn get_coding_shred(&self, slot: u64, index: u64) -> Result<Option<Vec<u8>>> {
|
|
|
|
self.code_shred_cf.get_bytes((slot, index))
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
|
|
|
|
2019-02-12 13:14:33 -08:00
|
|
|
pub fn write_entries<I>(
|
2019-08-20 17:16:06 -07:00
|
|
|
&self,
|
|
|
|
start_slot: u64,
|
|
|
|
num_ticks_in_start_slot: u64,
|
|
|
|
start_index: u64,
|
|
|
|
ticks_per_slot: u64,
|
|
|
|
parent: Option<u64>,
|
|
|
|
is_full_slot: bool,
|
2019-08-28 22:34:47 -07:00
|
|
|
keypair: &Arc<Keypair>,
|
2019-08-20 17:16:06 -07:00
|
|
|
entries: I,
|
|
|
|
) -> Result<usize>
|
|
|
|
where
|
|
|
|
I: IntoIterator,
|
|
|
|
I::Item: Borrow<Entry>,
|
|
|
|
{
|
|
|
|
assert!(num_ticks_in_start_slot < ticks_per_slot);
|
|
|
|
let mut remaining_ticks_in_slot = ticks_per_slot - num_ticks_in_start_slot;
|
|
|
|
|
|
|
|
let mut current_slot = start_slot;
|
|
|
|
let mut parent_slot = parent.map_or(
|
|
|
|
if current_slot == 0 {
|
|
|
|
current_slot
|
|
|
|
} else {
|
|
|
|
current_slot - 1
|
|
|
|
},
|
|
|
|
|v| v,
|
|
|
|
);
|
2019-09-04 21:06:47 -07:00
|
|
|
let mut shredder =
|
|
|
|
Shredder::new(current_slot, parent_slot, 0.0, keypair, start_index as u32)
|
|
|
|
.expect("Failed to create entry shredder");
|
2019-08-20 17:16:06 -07:00
|
|
|
let mut all_shreds = vec![];
|
|
|
|
// Find all the entries for start_slot
|
|
|
|
for entry in entries {
|
|
|
|
if remaining_ticks_in_slot == 0 {
|
|
|
|
current_slot += 1;
|
|
|
|
parent_slot = current_slot - 1;
|
|
|
|
remaining_ticks_in_slot = ticks_per_slot;
|
|
|
|
shredder.finalize_slot();
|
|
|
|
let shreds: Vec<Shred> = shredder
|
|
|
|
.shreds
|
|
|
|
.iter()
|
|
|
|
.map(|s| bincode::deserialize(s).unwrap())
|
|
|
|
.collect();
|
|
|
|
all_shreds.extend(shreds);
|
2019-09-04 21:06:47 -07:00
|
|
|
shredder =
|
|
|
|
Shredder::new(current_slot, parent_slot, 0.0, &Arc::new(Keypair::new()), 0)
|
|
|
|
.expect("Failed to create entry shredder");
|
2019-08-20 17:16:06 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
if entry.borrow().is_tick() {
|
|
|
|
remaining_ticks_in_slot -= 1;
|
|
|
|
}
|
|
|
|
|
2019-08-27 17:11:24 -07:00
|
|
|
bincode::serialize_into(&mut shredder, &vec![entry.borrow().clone()])
|
|
|
|
.expect("Expect to write all entries to shreds");
|
2019-08-26 18:27:45 -07:00
|
|
|
if remaining_ticks_in_slot == 0 {
|
|
|
|
shredder.finalize_slot();
|
|
|
|
} else {
|
|
|
|
shredder.finalize_fec_block();
|
|
|
|
}
|
2019-08-20 17:16:06 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
if is_full_slot && remaining_ticks_in_slot != 0 {
|
|
|
|
shredder.finalize_slot();
|
|
|
|
}
|
|
|
|
let shreds: Vec<Shred> = shredder
|
|
|
|
.shreds
|
|
|
|
.iter()
|
|
|
|
.map(|s| bincode::deserialize(s).unwrap())
|
|
|
|
.collect();
|
|
|
|
all_shreds.extend(shreds);
|
|
|
|
|
|
|
|
let num_shreds = all_shreds.len();
|
2019-09-05 18:20:30 -07:00
|
|
|
self.insert_shreds(all_shreds, None)?;
|
2019-08-20 17:16:06 -07:00
|
|
|
Ok(num_shreds)
|
|
|
|
}
|
|
|
|
|
2019-07-10 11:08:17 -07:00
|
|
|
pub fn get_index(&self, slot: u64) -> Result<Option<Index>> {
|
|
|
|
self.index_cf.get(slot)
|
|
|
|
}
|
|
|
|
|
2019-07-17 14:42:29 -07:00
|
|
|
/// Manually update the meta for a slot.
|
|
|
|
/// Can interfere with automatic meta update and potentially break chaining.
|
|
|
|
/// Dangerous. Use with care.
|
|
|
|
pub fn put_meta_bytes(&self, slot: u64, bytes: &[u8]) -> Result<()> {
|
|
|
|
self.meta_cf.put_bytes(slot, bytes)
|
|
|
|
}
|
|
|
|
|
2019-01-08 15:53:44 -08:00
|
|
|
// Given a start and end entry index, find all the missing
|
|
|
|
// indexes in the ledger in the range [start_index, end_index)
|
2019-03-05 14:18:29 -08:00
|
|
|
// for the slot with the specified slot
|
2019-04-02 14:58:07 -07:00
|
|
|
fn find_missing_indexes<C>(
|
|
|
|
db_iterator: &mut Cursor<C>,
|
2019-01-08 15:53:44 -08:00
|
|
|
slot: u64,
|
|
|
|
start_index: u64,
|
|
|
|
end_index: u64,
|
|
|
|
max_missing: usize,
|
2019-04-02 14:58:07 -07:00
|
|
|
) -> Vec<u64>
|
|
|
|
where
|
|
|
|
C: Column<Index = (u64, u64)>,
|
|
|
|
{
|
2019-01-08 15:53:44 -08:00
|
|
|
if start_index >= end_index || max_missing == 0 {
|
|
|
|
return vec![];
|
|
|
|
}
|
|
|
|
|
|
|
|
let mut missing_indexes = vec![];
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Seek to the first shred with index >= start_index
|
2019-04-02 14:58:07 -07:00
|
|
|
db_iterator.seek((slot, start_index));
|
2019-01-08 15:53:44 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// The index of the first missing shred in the slot
|
2019-01-08 15:53:44 -08:00
|
|
|
let mut prev_index = start_index;
|
|
|
|
'outer: loop {
|
|
|
|
if !db_iterator.valid() {
|
|
|
|
for i in prev_index..end_index {
|
|
|
|
missing_indexes.push(i);
|
|
|
|
if missing_indexes.len() == max_missing {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
2019-04-02 14:58:07 -07:00
|
|
|
let (current_slot, index) = db_iterator.key().expect("Expect a valid key");
|
2019-02-07 15:10:54 -08:00
|
|
|
let current_index = {
|
|
|
|
if current_slot > slot {
|
|
|
|
end_index
|
|
|
|
} else {
|
2019-04-02 14:58:07 -07:00
|
|
|
index
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
};
|
2019-01-08 15:53:44 -08:00
|
|
|
let upper_index = cmp::min(current_index, end_index);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-01-08 15:53:44 -08:00
|
|
|
for i in prev_index..upper_index {
|
|
|
|
missing_indexes.push(i);
|
|
|
|
if missing_indexes.len() == max_missing {
|
|
|
|
break 'outer;
|
|
|
|
}
|
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
if current_slot > slot {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2019-01-08 15:53:44 -08:00
|
|
|
if current_index >= end_index {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
prev_index = current_index + 1;
|
|
|
|
db_iterator.next();
|
|
|
|
}
|
|
|
|
|
|
|
|
missing_indexes
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn find_missing_data_indexes(
|
|
|
|
&self,
|
|
|
|
slot: u64,
|
|
|
|
start_index: u64,
|
|
|
|
end_index: u64,
|
|
|
|
max_missing: usize,
|
|
|
|
) -> Vec<u64> {
|
2019-08-21 15:27:42 -07:00
|
|
|
if let Ok(mut db_iterator) = self.db.cursor::<cf::ShredData>() {
|
2019-04-02 14:58:07 -07:00
|
|
|
Self::find_missing_indexes(&mut db_iterator, slot, start_index, end_index, max_missing)
|
|
|
|
} else {
|
|
|
|
vec![]
|
|
|
|
}
|
2019-01-08 15:53:44 -08:00
|
|
|
}
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
/// Returns the entry vector for the slot starting with `shred_start_index`
|
2019-02-07 15:10:54 -08:00
|
|
|
pub fn get_slot_entries(
|
|
|
|
&self,
|
2019-03-05 14:18:29 -08:00
|
|
|
slot: u64,
|
2019-09-03 21:32:51 -07:00
|
|
|
shred_start_index: u64,
|
2019-08-20 17:16:06 -07:00
|
|
|
_max_entries: Option<u64>,
|
2019-02-07 15:10:54 -08:00
|
|
|
) -> Result<Vec<Entry>> {
|
2019-09-03 21:32:51 -07:00
|
|
|
self.get_slot_entries_with_shred_count(slot, shred_start_index)
|
2019-02-26 21:57:45 -08:00
|
|
|
.map(|x| x.0)
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-08-20 17:16:06 -07:00
|
|
|
pub fn get_slot_entries_with_shred_count(
|
|
|
|
&self,
|
|
|
|
slot: u64,
|
|
|
|
start_index: u64,
|
|
|
|
) -> Result<(Vec<Entry>, usize)> {
|
2019-09-03 21:32:51 -07:00
|
|
|
// Find the next consecutive block of shreds.
|
2019-08-20 17:16:06 -07:00
|
|
|
let serialized_shreds = get_slot_consecutive_shreds(slot, &self.db, start_index)?;
|
|
|
|
trace!(
|
|
|
|
"Found {:?} shreds for slot {:?}",
|
|
|
|
serialized_shreds.len(),
|
|
|
|
slot
|
|
|
|
);
|
|
|
|
let mut shreds: Vec<Shred> = serialized_shreds
|
|
|
|
.iter()
|
|
|
|
.map(|serialzied_shred| {
|
|
|
|
let shred: Shred =
|
|
|
|
bincode::deserialize(serialzied_shred).expect("Failed to deserialize shred");
|
|
|
|
shred
|
|
|
|
})
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
let mut all_entries = vec![];
|
|
|
|
let mut num = 0;
|
|
|
|
loop {
|
|
|
|
let mut look_for_last_shred = true;
|
|
|
|
|
|
|
|
let mut shred_chunk = vec![];
|
|
|
|
while look_for_last_shred && !shreds.is_empty() {
|
|
|
|
let shred = shreds.remove(0);
|
|
|
|
if let Shred::LastInFECSet(_) = shred {
|
|
|
|
look_for_last_shred = false;
|
|
|
|
} else if let Shred::LastInSlot(_) = shred {
|
|
|
|
look_for_last_shred = false;
|
|
|
|
}
|
|
|
|
shred_chunk.push(shred);
|
|
|
|
}
|
|
|
|
|
|
|
|
debug!(
|
|
|
|
"{:?} shreds in last FEC set. Looking for last shred {:?}",
|
|
|
|
shred_chunk.len(),
|
|
|
|
look_for_last_shred
|
|
|
|
);
|
|
|
|
|
|
|
|
// Break if we didn't find the last shred (as more data is required)
|
|
|
|
if look_for_last_shred {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2019-08-26 18:27:45 -07:00
|
|
|
if let Ok(deshred_payload) = Shredder::deshred(&shred_chunk) {
|
|
|
|
let entries: Vec<Entry> = bincode::deserialize(&deshred_payload)?;
|
2019-08-20 17:16:06 -07:00
|
|
|
trace!("Found entries: {:#?}", entries);
|
|
|
|
all_entries.extend(entries);
|
|
|
|
num += shred_chunk.len();
|
|
|
|
} else {
|
|
|
|
debug!("Failed in deshredding shred payloads");
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
trace!("Found {:?} entries", all_entries.len());
|
|
|
|
|
|
|
|
Ok((all_entries, num))
|
|
|
|
}
|
|
|
|
|
2019-03-05 14:18:29 -08:00
|
|
|
// Returns slots connecting to any element of the list `slots`.
|
|
|
|
pub fn get_slots_since(&self, slots: &[u64]) -> Result<HashMap<u64, Vec<u64>>> {
|
2019-02-07 15:10:54 -08:00
|
|
|
// Return error if there was a database error during lookup of any of the
|
|
|
|
// slot indexes
|
2019-03-05 14:18:29 -08:00
|
|
|
let slot_metas: Result<Vec<Option<SlotMeta>>> =
|
|
|
|
slots.iter().map(|slot| self.meta(*slot)).collect();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-02-28 19:49:22 -08:00
|
|
|
let slot_metas = slot_metas?;
|
2019-03-05 14:18:29 -08:00
|
|
|
let result: HashMap<u64, Vec<u64>> = slots
|
2019-02-28 19:49:22 -08:00
|
|
|
.iter()
|
|
|
|
.zip(slot_metas)
|
2019-06-20 15:50:41 -07:00
|
|
|
.filter_map(|(height, meta)| {
|
|
|
|
meta.map(|meta| {
|
|
|
|
let valid_next_slots: Vec<u64> = meta
|
|
|
|
.next_slots
|
|
|
|
.iter()
|
|
|
|
.cloned()
|
|
|
|
.filter(|s| !self.is_dead(*s))
|
|
|
|
.collect();
|
|
|
|
(*height, valid_next_slots)
|
|
|
|
})
|
|
|
|
})
|
2019-02-07 15:10:54 -08:00
|
|
|
.collect();
|
|
|
|
|
2019-02-28 19:49:22 -08:00
|
|
|
Ok(result)
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
2019-01-08 15:53:44 -08:00
|
|
|
|
2019-04-15 13:12:28 -07:00
|
|
|
pub fn is_root(&self, slot: u64) -> bool {
|
2019-05-20 19:04:18 -07:00
|
|
|
if let Ok(Some(true)) = self.db.get::<cf::Root>(slot) {
|
|
|
|
true
|
2019-04-15 13:12:28 -07:00
|
|
|
} else {
|
|
|
|
false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-05-29 09:43:22 -07:00
|
|
|
pub fn set_roots(&self, rooted_slots: &[u64]) -> Result<()> {
|
2019-05-20 19:04:18 -07:00
|
|
|
unsafe {
|
|
|
|
let mut batch_processor = self.db.batch_processor();
|
|
|
|
let mut write_batch = batch_processor.batch()?;
|
2019-05-29 09:43:22 -07:00
|
|
|
for slot in rooted_slots {
|
|
|
|
write_batch.put::<cf::Root>(*slot, &true)?;
|
2019-05-20 19:04:18 -07:00
|
|
|
}
|
2019-05-03 14:46:02 -07:00
|
|
|
|
2019-05-20 19:04:18 -07:00
|
|
|
batch_processor.write(write_batch)?;
|
|
|
|
}
|
2019-08-27 15:09:41 -07:00
|
|
|
|
|
|
|
let mut last_root = self.last_root.write().unwrap();
|
|
|
|
if *last_root == std::u64::MAX {
|
|
|
|
*last_root = 0;
|
|
|
|
}
|
|
|
|
*last_root = cmp::max(*rooted_slots.iter().max().unwrap(), *last_root);
|
2019-05-20 19:04:18 -07:00
|
|
|
Ok(())
|
2019-05-03 14:46:02 -07:00
|
|
|
}
|
|
|
|
|
2019-06-20 15:50:41 -07:00
|
|
|
pub fn is_dead(&self, slot: u64) -> bool {
|
|
|
|
if let Some(true) = self
|
|
|
|
.db
|
|
|
|
.get::<cf::DeadSlots>(slot)
|
|
|
|
.expect("fetch from DeadSlots column family failed")
|
|
|
|
{
|
|
|
|
true
|
|
|
|
} else {
|
|
|
|
false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn set_dead_slot(&self, slot: u64) -> Result<()> {
|
|
|
|
self.dead_slots_cf.put(slot, &true)
|
|
|
|
}
|
|
|
|
|
2019-04-06 19:41:22 -07:00
|
|
|
pub fn get_orphans(&self, max: Option<usize>) -> Vec<u64> {
|
|
|
|
let mut results = vec![];
|
2019-04-26 08:52:10 -07:00
|
|
|
|
2019-05-03 14:46:02 -07:00
|
|
|
let mut iter = self.db.cursor::<cf::Orphans>().unwrap();
|
2019-04-06 19:41:22 -07:00
|
|
|
iter.seek_to_first();
|
|
|
|
while iter.valid() {
|
|
|
|
if let Some(max) = max {
|
|
|
|
if results.len() > max {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
results.push(iter.key().unwrap());
|
|
|
|
iter.next();
|
|
|
|
}
|
|
|
|
results
|
|
|
|
}
|
|
|
|
|
2019-07-17 14:42:29 -07:00
|
|
|
/// Prune blocktree such that slots higher than `target_slot` are deleted and all references to
|
|
|
|
/// higher slots are removed
|
|
|
|
pub fn prune(&self, target_slot: u64) {
|
|
|
|
let mut meta = self
|
|
|
|
.meta(target_slot)
|
|
|
|
.expect("couldn't read slot meta")
|
|
|
|
.expect("no meta for target slot");
|
|
|
|
meta.next_slots.clear();
|
|
|
|
self.put_meta_bytes(
|
|
|
|
target_slot,
|
|
|
|
&bincode::serialize(&meta).expect("couldn't get meta bytes"),
|
|
|
|
)
|
|
|
|
.expect("unable to update meta for target slot");
|
|
|
|
|
2019-07-20 13:13:55 -07:00
|
|
|
self.purge_slots(target_slot + 1, None);
|
2019-07-17 14:42:29 -07:00
|
|
|
|
|
|
|
// fixup anything that refers to non-root slots and delete the rest
|
|
|
|
for (slot, mut meta) in self
|
|
|
|
.slot_meta_iterator(0)
|
|
|
|
.expect("unable to iterate over meta")
|
|
|
|
{
|
|
|
|
if slot > target_slot {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
meta.next_slots.retain(|slot| *slot <= target_slot);
|
|
|
|
self.put_meta_bytes(
|
|
|
|
slot,
|
|
|
|
&bincode::serialize(&meta).expect("couldn't update meta"),
|
|
|
|
)
|
|
|
|
.expect("couldn't update meta");
|
|
|
|
}
|
|
|
|
}
|
2019-08-27 15:09:41 -07:00
|
|
|
|
|
|
|
pub fn last_root(&self) -> u64 {
|
|
|
|
*self.last_root.read().unwrap()
|
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-08-20 17:16:06 -07:00
|
|
|
fn update_slot_meta(
|
|
|
|
is_last_in_slot: bool,
|
|
|
|
slot_meta: &mut SlotMeta,
|
|
|
|
index: u64,
|
|
|
|
new_consumed: u64,
|
|
|
|
) {
|
2019-04-26 08:52:10 -07:00
|
|
|
// Index is zero-indexed, while the "received" height starts from 1,
|
2019-09-03 21:32:51 -07:00
|
|
|
// so received = index + 1 for the same shred.
|
2019-08-20 17:16:06 -07:00
|
|
|
slot_meta.received = cmp::max(index + 1, slot_meta.received);
|
2019-04-26 08:52:10 -07:00
|
|
|
slot_meta.consumed = new_consumed;
|
|
|
|
slot_meta.last_index = {
|
2019-05-09 14:10:04 -07:00
|
|
|
// If the last index in the slot hasn't been set before, then
|
2019-09-03 21:32:51 -07:00
|
|
|
// set it to this shred index
|
2019-04-26 08:52:10 -07:00
|
|
|
if slot_meta.last_index == std::u64::MAX {
|
2019-08-20 17:16:06 -07:00
|
|
|
if is_last_in_slot {
|
|
|
|
index
|
2019-04-26 08:52:10 -07:00
|
|
|
} else {
|
|
|
|
std::u64::MAX
|
2019-03-29 16:07:24 -07:00
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
} else {
|
|
|
|
slot_meta.last_index
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
};
|
2019-08-20 17:16:06 -07:00
|
|
|
}
|
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
fn get_index_meta_entry<'a>(
|
2019-08-20 17:16:06 -07:00
|
|
|
db: &Database,
|
|
|
|
slot: u64,
|
2019-09-04 17:14:42 -07:00
|
|
|
index_working_set: &'a mut HashMap<u64, Index>,
|
|
|
|
) -> (Option<&'a mut Index>, Option<Index>) {
|
|
|
|
let index_cf = db.column::<cf::Index>();
|
|
|
|
index_working_set
|
|
|
|
.get_mut(&slot)
|
|
|
|
.map(|i| (Some(i), None))
|
|
|
|
.unwrap_or_else(|| {
|
|
|
|
let newly_inserted_meta = Some(
|
|
|
|
index_cf
|
|
|
|
.get(slot)
|
|
|
|
.unwrap()
|
|
|
|
.unwrap_or_else(|| Index::new(slot)),
|
|
|
|
);
|
2019-03-29 16:07:24 -07:00
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
(None, newly_inserted_meta)
|
|
|
|
})
|
2019-04-26 08:52:10 -07:00
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
fn get_slot_meta_entry<'a>(
|
|
|
|
db: &Database,
|
|
|
|
slot_meta_working_set: &'a mut HashMap<u64, SlotMetaWorkingSetEntry>,
|
2019-08-20 17:16:06 -07:00
|
|
|
slot: u64,
|
2019-09-04 17:14:42 -07:00
|
|
|
parent_slot: u64,
|
|
|
|
) -> (
|
|
|
|
Option<&'a mut SlotMetaWorkingSetEntry>,
|
|
|
|
Option<SlotMetaWorkingSetEntry>,
|
|
|
|
) {
|
|
|
|
let meta_cf = db.column::<cf::SlotMeta>();
|
2019-08-27 15:09:41 -07:00
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
// Check if we've already inserted the slot metadata for this blob's slot
|
|
|
|
slot_meta_working_set
|
|
|
|
.get_mut(&slot)
|
|
|
|
.map(|s| (Some(s), None))
|
|
|
|
.unwrap_or_else(|| {
|
|
|
|
// Store a 2-tuple of the metadata (working copy, backup copy)
|
|
|
|
if let Some(mut meta) = meta_cf.get(slot).expect("Expect database get to succeed") {
|
|
|
|
let backup = Some(meta.clone());
|
|
|
|
// If parent_slot == std::u64::MAX, then this is one of the orphans inserted
|
|
|
|
// during the chaining process, see the function find_slot_meta_in_cached_state()
|
|
|
|
// for details. Slots that are orphans are missing a parent_slot, so we should
|
|
|
|
// fill in the parent now that we know it.
|
|
|
|
if is_orphan(&meta) {
|
|
|
|
meta.parent_slot = parent_slot;
|
|
|
|
}
|
|
|
|
|
|
|
|
(None, Some((Rc::new(RefCell::new(meta)), backup)))
|
|
|
|
} else {
|
2019-08-27 15:09:41 -07:00
|
|
|
(
|
2019-09-04 17:14:42 -07:00
|
|
|
None,
|
|
|
|
Some((
|
|
|
|
Rc::new(RefCell::new(SlotMeta::new(slot, parent_slot))),
|
|
|
|
None,
|
|
|
|
)),
|
2019-08-27 15:09:41 -07:00
|
|
|
)
|
2019-09-04 17:14:42 -07:00
|
|
|
}
|
|
|
|
})
|
2019-04-26 08:52:10 -07:00
|
|
|
}
|
|
|
|
|
2019-08-27 15:09:41 -07:00
|
|
|
fn is_valid_write_to_slot_0(slot_to_write: u64, parent_slot: u64, last_root: u64) -> bool {
|
|
|
|
slot_to_write == 0 && last_root == 0 && parent_slot == 0
|
|
|
|
}
|
|
|
|
|
2019-07-10 11:08:17 -07:00
|
|
|
fn send_signals(
|
2019-09-03 21:32:51 -07:00
|
|
|
new_shreds_signals: &[SyncSender<bool>],
|
2019-07-10 11:08:17 -07:00
|
|
|
completed_slots_senders: &[SyncSender<Vec<u64>>],
|
|
|
|
should_signal: bool,
|
|
|
|
newly_completed_slots: Vec<u64>,
|
|
|
|
) -> Result<()> {
|
|
|
|
if should_signal {
|
2019-09-03 21:32:51 -07:00
|
|
|
for signal in new_shreds_signals {
|
2019-07-10 11:08:17 -07:00
|
|
|
let _ = signal.try_send(true);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if !completed_slots_senders.is_empty() && !newly_completed_slots.is_empty() {
|
|
|
|
let mut slots: Vec<_> = (0..completed_slots_senders.len() - 1)
|
|
|
|
.map(|_| newly_completed_slots.clone())
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
slots.push(newly_completed_slots);
|
|
|
|
|
|
|
|
for (signal, slots) in completed_slots_senders.iter().zip(slots.into_iter()) {
|
|
|
|
let res = signal.try_send(slots);
|
|
|
|
if let Err(TrySendError::Full(_)) = res {
|
2019-09-07 12:48:45 -07:00
|
|
|
datapoint_error!(
|
|
|
|
"blocktree_error",
|
|
|
|
(
|
|
|
|
"error",
|
|
|
|
"Unable to send newly completed slot because channel is full".to_string(),
|
|
|
|
String
|
|
|
|
),
|
2019-07-10 11:08:17 -07:00
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
fn commit_slot_meta_working_set(
|
|
|
|
slot_meta_working_set: &HashMap<u64, SlotMetaWorkingSetEntry>,
|
2019-07-10 11:08:17 -07:00
|
|
|
completed_slots_senders: &[SyncSender<Vec<u64>>],
|
|
|
|
write_batch: &mut WriteBatch,
|
|
|
|
) -> Result<(bool, Vec<u64>)> {
|
|
|
|
let mut should_signal = false;
|
|
|
|
let mut newly_completed_slots = vec![];
|
|
|
|
|
|
|
|
// Check if any metadata was changed, if so, insert the new version of the
|
|
|
|
// metadata into the write batch
|
|
|
|
for (slot, (meta, meta_backup)) in slot_meta_working_set.iter() {
|
|
|
|
let meta: &SlotMeta = &RefCell::borrow(&*meta);
|
|
|
|
if !completed_slots_senders.is_empty() && is_newly_completed_slot(meta, meta_backup) {
|
|
|
|
newly_completed_slots.push(*slot);
|
|
|
|
}
|
|
|
|
// Check if the working copy of the metadata has changed
|
|
|
|
if Some(meta) != meta_backup.as_ref() {
|
|
|
|
should_signal = should_signal || slot_has_updates(meta, &meta_backup);
|
|
|
|
write_batch.put::<cf::SlotMeta>(*slot, &meta)?;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Ok((should_signal, newly_completed_slots))
|
|
|
|
}
|
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
// 1) Find the slot metadata in the cache of dirty slot metadata we've previously touched,
|
|
|
|
// else:
|
|
|
|
// 2) Search the database for that slot metadata. If still no luck, then:
|
|
|
|
// 3) Create a dummy orphan slot in the database
|
|
|
|
fn find_slot_meta_else_create<'a>(
|
|
|
|
db: &Database,
|
2019-09-04 17:14:42 -07:00
|
|
|
working_set: &'a HashMap<u64, SlotMetaWorkingSetEntry>,
|
2019-04-26 08:52:10 -07:00
|
|
|
chained_slots: &'a mut HashMap<u64, Rc<RefCell<SlotMeta>>>,
|
|
|
|
slot_index: u64,
|
|
|
|
) -> Result<Rc<RefCell<SlotMeta>>> {
|
|
|
|
let result = find_slot_meta_in_cached_state(working_set, chained_slots, slot_index)?;
|
|
|
|
if let Some(slot) = result {
|
|
|
|
Ok(slot)
|
|
|
|
} else {
|
|
|
|
find_slot_meta_in_db_else_create(db, slot_index, chained_slots)
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
// Search the database for that slot metadata. If still no luck, then
|
|
|
|
// create a dummy orphan slot in the database
|
|
|
|
fn find_slot_meta_in_db_else_create<'a>(
|
|
|
|
db: &Database,
|
|
|
|
slot: u64,
|
|
|
|
insert_map: &'a mut HashMap<u64, Rc<RefCell<SlotMeta>>>,
|
|
|
|
) -> Result<Rc<RefCell<SlotMeta>>> {
|
2019-05-03 14:46:02 -07:00
|
|
|
if let Some(slot_meta) = db.column::<cf::SlotMeta>().get(slot)? {
|
2019-04-26 08:52:10 -07:00
|
|
|
insert_map.insert(slot, Rc::new(RefCell::new(slot_meta)));
|
|
|
|
Ok(insert_map.get(&slot).unwrap().clone())
|
|
|
|
} else {
|
|
|
|
// If this slot doesn't exist, make a orphan slot. This way we
|
2019-09-03 21:32:51 -07:00
|
|
|
// remember which slots chained to this one when we eventually get a real shred
|
2019-04-26 08:52:10 -07:00
|
|
|
// for this slot
|
|
|
|
insert_map.insert(
|
|
|
|
slot,
|
|
|
|
Rc::new(RefCell::new(SlotMeta::new(slot, std::u64::MAX))),
|
|
|
|
);
|
|
|
|
Ok(insert_map.get(&slot).unwrap().clone())
|
2019-04-25 00:04:49 -07:00
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
}
|
2019-04-25 00:04:49 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
// Find the slot metadata in the cache of dirty slot metadata we've previously touched
|
|
|
|
fn find_slot_meta_in_cached_state<'a>(
|
2019-09-04 17:14:42 -07:00
|
|
|
working_set: &'a HashMap<u64, SlotMetaWorkingSetEntry>,
|
2019-04-26 08:52:10 -07:00
|
|
|
chained_slots: &'a HashMap<u64, Rc<RefCell<SlotMeta>>>,
|
|
|
|
slot: u64,
|
|
|
|
) -> Result<Option<Rc<RefCell<SlotMeta>>>> {
|
|
|
|
if let Some((entry, _)) = working_set.get(&slot) {
|
|
|
|
Ok(Some(entry.clone()))
|
|
|
|
} else if let Some(entry) = chained_slots.get(&slot) {
|
|
|
|
Ok(Some(entry.clone()))
|
|
|
|
} else {
|
|
|
|
Ok(None)
|
2019-03-29 16:07:24 -07:00
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
}
|
2019-03-29 16:07:24 -07:00
|
|
|
|
2019-08-20 17:16:06 -07:00
|
|
|
fn get_slot_consecutive_shreds<'a>(
|
|
|
|
slot: u64,
|
|
|
|
db: &Database,
|
|
|
|
mut current_index: u64,
|
|
|
|
) -> Result<Vec<Cow<'a, [u8]>>> {
|
|
|
|
let mut serialized_shreds: Vec<Cow<[u8]>> = vec![];
|
|
|
|
let data_cf = db.column::<cf::ShredData>();
|
|
|
|
|
|
|
|
while let Some(serialized_shred) = data_cf.get_bytes((slot, current_index))? {
|
|
|
|
serialized_shreds.push(Cow::Owned(serialized_shred));
|
|
|
|
current_index += 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
Ok(serialized_shreds)
|
|
|
|
}
|
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
// Chaining based on latest discussion here: https://github.com/solana-labs/solana/pull/2253
|
|
|
|
fn handle_chaining(
|
|
|
|
db: &Database,
|
|
|
|
write_batch: &mut WriteBatch,
|
2019-09-04 17:14:42 -07:00
|
|
|
working_set: &HashMap<u64, SlotMetaWorkingSetEntry>,
|
2019-04-26 08:52:10 -07:00
|
|
|
) -> Result<()> {
|
|
|
|
let mut new_chained_slots = HashMap::new();
|
|
|
|
let working_set_slots: Vec<_> = working_set.iter().map(|s| *s.0).collect();
|
|
|
|
for slot in working_set_slots {
|
|
|
|
handle_chaining_for_slot(db, write_batch, working_set, &mut new_chained_slots, slot)?;
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
// Write all the newly changed slots in new_chained_slots to the write_batch
|
|
|
|
for (slot, meta) in new_chained_slots.iter() {
|
|
|
|
let meta: &SlotMeta = &RefCell::borrow(&*meta);
|
|
|
|
write_batch.put::<cf::SlotMeta>(*slot, meta)?;
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
Ok(())
|
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
fn handle_chaining_for_slot(
|
|
|
|
db: &Database,
|
|
|
|
write_batch: &mut WriteBatch,
|
2019-09-04 17:14:42 -07:00
|
|
|
working_set: &HashMap<u64, SlotMetaWorkingSetEntry>,
|
2019-04-26 08:52:10 -07:00
|
|
|
new_chained_slots: &mut HashMap<u64, Rc<RefCell<SlotMeta>>>,
|
|
|
|
slot: u64,
|
|
|
|
) -> Result<()> {
|
|
|
|
let (meta, meta_backup) = working_set
|
|
|
|
.get(&slot)
|
|
|
|
.expect("Slot must exist in the working_set hashmap");
|
2019-04-24 15:53:01 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
{
|
|
|
|
let mut meta_mut = meta.borrow_mut();
|
2019-05-09 14:10:04 -07:00
|
|
|
let was_orphan_slot = meta_backup.is_some() && is_orphan(meta_backup.as_ref().unwrap());
|
2019-04-24 15:53:01 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
// If:
|
|
|
|
// 1) This is a new slot
|
|
|
|
// 2) slot != 0
|
|
|
|
// then try to chain this slot to a previous slot
|
|
|
|
if slot != 0 {
|
|
|
|
let prev_slot = meta_mut.parent_slot;
|
2019-04-24 15:53:01 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
// Check if the slot represented by meta_mut is either a new slot or a orphan.
|
|
|
|
// In both cases we need to run the chaining logic b/c the parent on the slot was
|
|
|
|
// previously unknown.
|
2019-05-09 14:10:04 -07:00
|
|
|
if meta_backup.is_none() || was_orphan_slot {
|
2019-04-26 08:52:10 -07:00
|
|
|
let prev_slot_meta =
|
|
|
|
find_slot_meta_else_create(db, working_set, new_chained_slots, prev_slot)?;
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-05-09 14:10:04 -07:00
|
|
|
// This is a newly inserted slot/orphan so run the chaining logic to link it to a
|
|
|
|
// newly discovered parent
|
2019-04-26 08:52:10 -07:00
|
|
|
chain_new_slot_to_prev_slot(&mut prev_slot_meta.borrow_mut(), slot, &mut meta_mut);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-05-09 14:10:04 -07:00
|
|
|
// If the parent of `slot` is a newly inserted orphan, insert it into the orphans
|
|
|
|
// column family
|
2019-04-26 08:52:10 -07:00
|
|
|
if is_orphan(&RefCell::borrow(&*prev_slot_meta)) {
|
|
|
|
write_batch.put::<cf::Orphans>(prev_slot, &true)?;
|
2019-02-12 16:06:23 -08:00
|
|
|
}
|
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
}
|
|
|
|
|
2019-05-09 14:10:04 -07:00
|
|
|
// At this point this slot has received a parent, so it's no longer an orphan
|
|
|
|
if was_orphan_slot {
|
2019-04-26 08:52:10 -07:00
|
|
|
write_batch.delete::<cf::Orphans>(slot)?;
|
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-05-09 14:10:04 -07:00
|
|
|
// If this is a newly inserted slot, then we know the children of this slot were not previously
|
|
|
|
// connected to the trunk of the ledger. Thus if slot.is_connected is now true, we need to
|
|
|
|
// update all child slots with `is_connected` = true because these children are also now newly
|
2019-08-21 17:46:59 -07:00
|
|
|
// connected to trunk of the ledger
|
2019-04-26 08:52:10 -07:00
|
|
|
let should_propagate_is_connected =
|
|
|
|
is_newly_completed_slot(&RefCell::borrow(&*meta), meta_backup)
|
|
|
|
&& RefCell::borrow(&*meta).is_connected;
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
if should_propagate_is_connected {
|
|
|
|
// slot_function returns a boolean indicating whether to explore the children
|
|
|
|
// of the input slot
|
|
|
|
let slot_function = |slot: &mut SlotMeta| {
|
|
|
|
slot.is_connected = true;
|
2019-04-18 21:56:43 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
// We don't want to set the is_connected flag on the children of non-full
|
|
|
|
// slots
|
|
|
|
slot.is_full()
|
|
|
|
};
|
|
|
|
|
|
|
|
traverse_children_mut(
|
|
|
|
db,
|
|
|
|
slot,
|
|
|
|
&meta,
|
|
|
|
working_set,
|
|
|
|
new_chained_slots,
|
|
|
|
slot_function,
|
|
|
|
)?;
|
|
|
|
}
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
|
|
|
fn traverse_children_mut<F>(
|
|
|
|
db: &Database,
|
|
|
|
slot: u64,
|
|
|
|
slot_meta: &Rc<RefCell<(SlotMeta)>>,
|
2019-09-04 17:14:42 -07:00
|
|
|
working_set: &HashMap<u64, SlotMetaWorkingSetEntry>,
|
2019-04-26 08:52:10 -07:00
|
|
|
new_chained_slots: &mut HashMap<u64, Rc<RefCell<SlotMeta>>>,
|
|
|
|
slot_function: F,
|
|
|
|
) -> Result<()>
|
|
|
|
where
|
|
|
|
F: Fn(&mut SlotMeta) -> bool,
|
|
|
|
{
|
|
|
|
let mut next_slots: Vec<(u64, Rc<RefCell<(SlotMeta)>>)> = vec![(slot, slot_meta.clone())];
|
|
|
|
while !next_slots.is_empty() {
|
|
|
|
let (_, current_slot) = next_slots.pop().unwrap();
|
|
|
|
// Check whether we should explore the children of this slot
|
|
|
|
if slot_function(&mut current_slot.borrow_mut()) {
|
|
|
|
let current_slot = &RefCell::borrow(&*current_slot);
|
|
|
|
for next_slot_index in current_slot.next_slots.iter() {
|
|
|
|
let next_slot = find_slot_meta_else_create(
|
|
|
|
db,
|
|
|
|
working_set,
|
|
|
|
new_chained_slots,
|
|
|
|
*next_slot_index,
|
|
|
|
)?;
|
|
|
|
next_slots.push((*next_slot_index, next_slot));
|
2019-04-11 14:14:57 -07:00
|
|
|
}
|
|
|
|
}
|
2019-04-26 08:52:10 -07:00
|
|
|
}
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
Ok(())
|
|
|
|
}
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
fn is_orphan(meta: &SlotMeta) -> bool {
|
|
|
|
// If we have no parent, then this is the head of a detached chain of
|
|
|
|
// slots
|
|
|
|
!meta.is_parent_set()
|
|
|
|
}
|
2019-04-18 21:56:43 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
// 1) Chain current_slot to the previous slot defined by prev_slot_meta
|
|
|
|
// 2) Determine whether to set the is_connected flag
|
|
|
|
fn chain_new_slot_to_prev_slot(
|
|
|
|
prev_slot_meta: &mut SlotMeta,
|
|
|
|
current_slot: u64,
|
|
|
|
current_slot_meta: &mut SlotMeta,
|
|
|
|
) {
|
|
|
|
prev_slot_meta.next_slots.push(current_slot);
|
|
|
|
current_slot_meta.is_connected = prev_slot_meta.is_connected && prev_slot_meta.is_full();
|
|
|
|
}
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
fn is_newly_completed_slot(slot_meta: &SlotMeta, backup_slot_meta: &Option<SlotMeta>) -> bool {
|
|
|
|
slot_meta.is_full()
|
|
|
|
&& (backup_slot_meta.is_none()
|
|
|
|
|| slot_meta.consumed != backup_slot_meta.as_ref().unwrap().consumed)
|
|
|
|
}
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
fn slot_has_updates(slot_meta: &SlotMeta, slot_meta_backup: &Option<SlotMeta>) -> bool {
|
|
|
|
// We should signal that there are updates if we extended the chain of consecutive blocks starting
|
|
|
|
// from block 0, which is true iff:
|
|
|
|
// 1) The block with index prev_block_index is itself part of the trunk of consecutive blocks
|
|
|
|
// starting from block 0,
|
|
|
|
slot_meta.is_connected &&
|
|
|
|
// AND either:
|
|
|
|
// 1) The slot didn't exist in the database before, and now we have a consecutive
|
|
|
|
// block for that slot
|
|
|
|
((slot_meta_backup.is_none() && slot_meta.consumed != 0) ||
|
|
|
|
// OR
|
|
|
|
// 2) The slot did exist, but now we have a new consecutive block for that slot
|
|
|
|
(slot_meta_backup.is_some() && slot_meta_backup.as_ref().unwrap().consumed != slot_meta.consumed))
|
2018-12-11 09:14:23 -08:00
|
|
|
}
|
|
|
|
|
2019-02-26 16:35:00 -08:00
|
|
|
// Creates a new ledger with slot 0 full of ticks (and only ticks).
|
|
|
|
//
|
2019-03-02 10:25:16 -08:00
|
|
|
// Returns the blockhash that can be used to append entries with.
|
2019-07-30 15:53:41 -07:00
|
|
|
pub fn create_new_ledger(ledger_path: &Path, genesis_block: &GenesisBlock) -> Result<Hash> {
|
2019-02-21 18:46:04 -08:00
|
|
|
let ticks_per_slot = genesis_block.ticks_per_slot;
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(ledger_path)?;
|
2019-01-24 12:04:04 -08:00
|
|
|
genesis_block.write(&ledger_path)?;
|
2019-01-29 15:49:29 -08:00
|
|
|
|
2019-02-26 16:35:00 -08:00
|
|
|
// Fill slot 0 with ticks that link back to the genesis_block to bootstrap the ledger.
|
2019-03-14 15:18:37 -07:00
|
|
|
let blocktree = Blocktree::open(ledger_path)?;
|
2019-03-01 09:49:37 -08:00
|
|
|
let entries = crate::entry::create_ticks(ticks_per_slot, genesis_block.hash());
|
2019-01-29 15:49:29 -08:00
|
|
|
|
2019-09-04 21:06:47 -07:00
|
|
|
let mut shredder = Shredder::new(0, 0, 0.0, &Arc::new(Keypair::new()), 0)
|
2019-08-20 17:16:06 -07:00
|
|
|
.expect("Failed to create entry shredder");
|
|
|
|
let last_hash = entries.last().unwrap().hash;
|
2019-08-27 17:11:24 -07:00
|
|
|
bincode::serialize_into(&mut shredder, &entries)
|
|
|
|
.expect("Expect to write all entries to shreds");
|
2019-08-26 18:27:45 -07:00
|
|
|
shredder.finalize_slot();
|
2019-08-20 17:16:06 -07:00
|
|
|
let shreds: Vec<Shred> = shredder
|
|
|
|
.shreds
|
|
|
|
.iter()
|
|
|
|
.map(|s| bincode::deserialize(s).unwrap())
|
|
|
|
.collect();
|
|
|
|
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds, None)?;
|
2019-08-27 15:09:41 -07:00
|
|
|
blocktree.set_roots(&[0])?;
|
2019-08-20 17:16:06 -07:00
|
|
|
|
|
|
|
Ok(last_hash)
|
2019-01-24 12:04:04 -08:00
|
|
|
}
|
|
|
|
|
2019-02-26 17:11:26 -08:00
|
|
|
#[macro_export]
|
|
|
|
macro_rules! tmp_ledger_name {
|
|
|
|
() => {
|
|
|
|
&format!("{}-{}", file!(), line!())
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
#[macro_export]
|
|
|
|
macro_rules! get_tmp_ledger_path {
|
|
|
|
() => {
|
|
|
|
get_tmp_ledger_path(tmp_ledger_name!())
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2019-07-30 15:53:41 -07:00
|
|
|
pub fn get_tmp_ledger_path(name: &str) -> PathBuf {
|
2019-01-09 14:33:44 -08:00
|
|
|
use std::env;
|
2019-07-17 14:27:58 -07:00
|
|
|
let out_dir = env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string());
|
2019-01-09 14:33:44 -08:00
|
|
|
let keypair = Keypair::new();
|
|
|
|
|
2019-07-30 15:53:41 -07:00
|
|
|
let path = [
|
|
|
|
out_dir,
|
|
|
|
"ledger".to_string(),
|
|
|
|
format!("{}-{}", name, keypair.pubkey()),
|
|
|
|
]
|
|
|
|
.iter()
|
|
|
|
.collect();
|
2019-01-09 14:33:44 -08:00
|
|
|
|
|
|
|
// whack any possible collision
|
2019-01-24 12:04:04 -08:00
|
|
|
let _ignored = fs::remove_dir_all(&path);
|
2019-01-09 14:33:44 -08:00
|
|
|
|
|
|
|
path
|
|
|
|
}
|
|
|
|
|
2019-02-26 19:19:34 -08:00
|
|
|
#[macro_export]
|
|
|
|
macro_rules! create_new_tmp_ledger {
|
|
|
|
($genesis_block:expr) => {
|
|
|
|
create_new_tmp_ledger(tmp_ledger_name!(), $genesis_block)
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2019-02-26 16:35:00 -08:00
|
|
|
// Same as `create_new_ledger()` but use a temporary ledger name based on the provided `name`
|
|
|
|
//
|
|
|
|
// Note: like `create_new_ledger` the returned ledger will have slot 0 full of ticks (and only
|
|
|
|
// ticks)
|
2019-07-30 15:53:41 -07:00
|
|
|
pub fn create_new_tmp_ledger(name: &str, genesis_block: &GenesisBlock) -> (PathBuf, Hash) {
|
2019-01-30 10:26:16 -08:00
|
|
|
let ledger_path = get_tmp_ledger_path(name);
|
2019-03-02 10:25:16 -08:00
|
|
|
let blockhash = create_new_ledger(&ledger_path, genesis_block).unwrap();
|
|
|
|
(ledger_path, blockhash)
|
2019-02-21 22:36:01 -08:00
|
|
|
}
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
pub fn entries_to_test_shreds(
|
|
|
|
entries: Vec<Entry>,
|
|
|
|
slot: u64,
|
|
|
|
parent_slot: u64,
|
|
|
|
is_full_slot: bool,
|
|
|
|
) -> Vec<Shred> {
|
2019-09-04 21:06:47 -07:00
|
|
|
let mut shredder = Shredder::new(slot, parent_slot, 0.0, &Arc::new(Keypair::new()), 0 as u32)
|
|
|
|
.expect("Failed to create entry shredder");
|
2019-09-03 21:32:51 -07:00
|
|
|
|
|
|
|
bincode::serialize_into(&mut shredder, &entries)
|
|
|
|
.expect("Expect to write all entries to shreds");
|
|
|
|
if is_full_slot {
|
|
|
|
shredder.finalize_slot();
|
|
|
|
} else {
|
|
|
|
shredder.finalize_fec_block();
|
|
|
|
}
|
|
|
|
|
|
|
|
let shreds: Vec<Shred> = shredder
|
|
|
|
.shreds
|
|
|
|
.iter()
|
|
|
|
.map(|s| bincode::deserialize(s).unwrap())
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
shreds
|
|
|
|
}
|
|
|
|
|
2018-11-15 15:53:31 -08:00
|
|
|
#[cfg(test)]
|
2019-02-18 18:41:31 -08:00
|
|
|
pub mod tests {
|
2018-11-15 15:53:31 -08:00
|
|
|
use super::*;
|
2019-09-03 21:32:51 -07:00
|
|
|
use crate::entry::{create_ticks, make_tiny_test_entries, Entry};
|
2019-09-04 17:14:42 -07:00
|
|
|
use crate::shred::CodingShred;
|
2019-09-03 21:32:51 -07:00
|
|
|
use itertools::Itertools;
|
2019-02-13 15:01:56 -08:00
|
|
|
use rand::seq::SliceRandom;
|
|
|
|
use rand::thread_rng;
|
2019-01-22 15:50:36 -08:00
|
|
|
use solana_sdk::hash::Hash;
|
2019-08-21 20:07:51 -07:00
|
|
|
use solana_sdk::packet::PACKET_DATA_SIZE;
|
2019-02-28 19:49:22 -08:00
|
|
|
use std::iter::FromIterator;
|
2019-02-07 15:10:54 -08:00
|
|
|
use std::time::Duration;
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-02-18 19:49:43 -08:00
|
|
|
#[test]
|
|
|
|
fn test_write_entries() {
|
2019-02-26 17:11:26 -08:00
|
|
|
solana_logger::setup();
|
|
|
|
let ledger_path = get_tmp_ledger_path!();
|
2019-02-18 19:49:43 -08:00
|
|
|
{
|
|
|
|
let ticks_per_slot = 10;
|
|
|
|
let num_slots = 10;
|
2019-03-14 15:18:37 -07:00
|
|
|
let ledger = Blocktree::open(&ledger_path).unwrap();
|
2019-08-20 17:16:06 -07:00
|
|
|
let mut ticks = vec![];
|
|
|
|
//let mut shreds_per_slot = 0 as u64;
|
|
|
|
let mut shreds_per_slot = vec![];
|
2019-02-18 19:59:09 -08:00
|
|
|
|
2019-08-20 17:16:06 -07:00
|
|
|
for i in 0..num_slots {
|
|
|
|
let mut new_ticks = create_ticks(ticks_per_slot, Hash::default());
|
|
|
|
let num_shreds = ledger
|
2019-09-03 21:32:51 -07:00
|
|
|
.write_entries(
|
2019-08-20 17:16:06 -07:00
|
|
|
i,
|
|
|
|
0,
|
|
|
|
0,
|
|
|
|
ticks_per_slot,
|
|
|
|
Some(i.saturating_sub(1)),
|
|
|
|
true,
|
2019-08-28 22:34:47 -07:00
|
|
|
&Arc::new(Keypair::new()),
|
2019-08-20 17:16:06 -07:00
|
|
|
new_ticks.clone(),
|
|
|
|
)
|
|
|
|
.unwrap() as u64;
|
|
|
|
shreds_per_slot.push(num_shreds);
|
|
|
|
ticks.append(&mut new_ticks);
|
|
|
|
}
|
2019-02-18 19:49:43 -08:00
|
|
|
|
|
|
|
for i in 0..num_slots {
|
|
|
|
let meta = ledger.meta(i).unwrap().unwrap();
|
2019-08-20 17:16:06 -07:00
|
|
|
let num_shreds = shreds_per_slot[i as usize];
|
|
|
|
assert_eq!(meta.consumed, num_shreds);
|
|
|
|
assert_eq!(meta.received, num_shreds);
|
|
|
|
assert_eq!(meta.last_index, num_shreds - 1);
|
2019-02-18 19:59:09 -08:00
|
|
|
if i == num_slots - 1 {
|
|
|
|
assert!(meta.next_slots.is_empty());
|
|
|
|
} else {
|
|
|
|
assert_eq!(meta.next_slots, vec![i + 1]);
|
|
|
|
}
|
2019-02-18 19:49:43 -08:00
|
|
|
if i == 0 {
|
|
|
|
assert_eq!(meta.parent_slot, 0);
|
|
|
|
} else {
|
|
|
|
assert_eq!(meta.parent_slot, i - 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
assert_eq!(
|
|
|
|
&ticks[(i * ticks_per_slot) as usize..((i + 1) * ticks_per_slot) as usize],
|
|
|
|
&ledger.get_slot_entries(i, 0, None).unwrap()[..]
|
|
|
|
);
|
|
|
|
}
|
2019-02-18 19:59:09 -08:00
|
|
|
|
2019-08-20 17:16:06 -07:00
|
|
|
/*
|
|
|
|
// Simulate writing to the end of a slot with existing ticks
|
|
|
|
ledger
|
|
|
|
.write_entries(
|
|
|
|
num_slots,
|
|
|
|
ticks_per_slot - 1,
|
|
|
|
ticks_per_slot - 2,
|
|
|
|
ticks_per_slot,
|
|
|
|
&ticks[0..2],
|
|
|
|
)
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
let meta = ledger.meta(num_slots).unwrap().unwrap();
|
|
|
|
assert_eq!(meta.consumed, 0);
|
2019-09-03 21:32:51 -07:00
|
|
|
// received shred was ticks_per_slot - 2, so received should be ticks_per_slot - 2 + 1
|
2019-08-20 17:16:06 -07:00
|
|
|
assert_eq!(meta.received, ticks_per_slot - 1);
|
2019-09-03 21:32:51 -07:00
|
|
|
// last shred index ticks_per_slot - 2 because that's the shred that made tick_height == ticks_per_slot
|
2019-08-20 17:16:06 -07:00
|
|
|
// for the slot
|
|
|
|
assert_eq!(meta.last_index, ticks_per_slot - 2);
|
|
|
|
assert_eq!(meta.parent_slot, num_slots - 1);
|
|
|
|
assert_eq!(meta.next_slots, vec![num_slots + 1]);
|
|
|
|
assert_eq!(
|
|
|
|
&ticks[0..1],
|
|
|
|
&ledger
|
|
|
|
.get_slot_entries(num_slots, ticks_per_slot - 2, None)
|
|
|
|
.unwrap()[..]
|
|
|
|
);
|
|
|
|
|
|
|
|
// We wrote two entries, the second should spill into slot num_slots + 1
|
|
|
|
let meta = ledger.meta(num_slots + 1).unwrap().unwrap();
|
|
|
|
assert_eq!(meta.consumed, 1);
|
|
|
|
assert_eq!(meta.received, 1);
|
|
|
|
assert_eq!(meta.last_index, std::u64::MAX);
|
|
|
|
assert_eq!(meta.parent_slot, num_slots);
|
|
|
|
assert!(meta.next_slots.is_empty());
|
|
|
|
|
|
|
|
assert_eq!(
|
|
|
|
&ticks[1..2],
|
|
|
|
&ledger.get_slot_entries(num_slots + 1, 0, None).unwrap()[..]
|
|
|
|
);
|
|
|
|
*/
|
2019-02-18 19:49:43 -08:00
|
|
|
}
|
|
|
|
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
2018-11-15 15:53:31 -08:00
|
|
|
#[test]
|
|
|
|
fn test_put_get_simple() {
|
|
|
|
let ledger_path = get_tmp_ledger_path("test_put_get_simple");
|
2019-02-07 20:52:39 -08:00
|
|
|
let ledger = Blocktree::open(&ledger_path).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
|
|
|
|
// Test meta column family
|
2019-02-25 12:48:48 -08:00
|
|
|
let meta = SlotMeta::new(0, 1);
|
2019-05-03 14:46:02 -07:00
|
|
|
ledger.meta_cf.put(0, &meta).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
let result = ledger
|
|
|
|
.meta_cf
|
2019-05-03 14:46:02 -07:00
|
|
|
.get(0)
|
2018-11-15 15:53:31 -08:00
|
|
|
.unwrap()
|
|
|
|
.expect("Expected meta object to exist");
|
|
|
|
|
|
|
|
assert_eq!(result, meta);
|
|
|
|
|
|
|
|
// Test erasure column family
|
|
|
|
let erasure = vec![1u8; 16];
|
2019-06-20 20:15:33 -07:00
|
|
|
let erasure_key = (0, 0);
|
2019-09-04 12:47:09 -07:00
|
|
|
ledger
|
|
|
|
.code_shred_cf
|
|
|
|
.put_bytes(erasure_key, &erasure)
|
|
|
|
.unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
|
|
|
|
let result = ledger
|
2019-09-04 12:47:09 -07:00
|
|
|
.code_shred_cf
|
2019-05-03 14:46:02 -07:00
|
|
|
.get_bytes(erasure_key)
|
2018-11-15 15:53:31 -08:00
|
|
|
.unwrap()
|
|
|
|
.expect("Expected erasure object to exist");
|
|
|
|
|
|
|
|
assert_eq!(result, erasure);
|
|
|
|
|
|
|
|
// Test data column family
|
|
|
|
let data = vec![2u8; 16];
|
2019-04-02 14:58:07 -07:00
|
|
|
let data_key = (0, 0);
|
2019-09-04 12:47:09 -07:00
|
|
|
ledger.data_shred_cf.put_bytes(data_key, &data).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
|
|
|
|
let result = ledger
|
2019-09-04 12:47:09 -07:00
|
|
|
.data_shred_cf
|
2019-05-03 14:46:02 -07:00
|
|
|
.get_bytes(data_key)
|
2018-11-15 15:53:31 -08:00
|
|
|
.unwrap()
|
|
|
|
.expect("Expected data object to exist");
|
|
|
|
|
|
|
|
assert_eq!(result, data);
|
|
|
|
|
|
|
|
// Destroying database without closing it first is undefined behavior
|
|
|
|
drop(ledger);
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
2019-09-03 21:32:51 -07:00
|
|
|
fn test_read_shred_bytes() {
|
2019-02-25 12:48:48 -08:00
|
|
|
let slot = 0;
|
2019-09-03 21:32:51 -07:00
|
|
|
let (shreds, _) = make_slot_entries(slot, 0, 100);
|
|
|
|
let num_shreds = shreds.len() as u64;
|
|
|
|
let shred_bufs: Vec<_> = shreds
|
|
|
|
.iter()
|
|
|
|
.map(|shred| bincode::serialize(shred).unwrap())
|
|
|
|
.collect();
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let ledger_path = get_tmp_ledger_path("test_read_shreds_bytes");
|
2019-02-07 20:52:39 -08:00
|
|
|
let ledger = Blocktree::open(&ledger_path).unwrap();
|
2019-09-05 18:20:30 -07:00
|
|
|
ledger.insert_shreds(shreds, None).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let mut buf = [0; 4096];
|
|
|
|
let (_, bytes) = ledger.get_data_shreds(slot, 0, 1, &mut buf).unwrap();
|
|
|
|
assert_eq!(buf[..bytes], shred_bufs[0][..bytes]);
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let (last_index, bytes2) = ledger.get_data_shreds(slot, 0, 2, &mut buf).unwrap();
|
|
|
|
assert_eq!(last_index, 1);
|
2018-11-15 15:53:31 -08:00
|
|
|
assert!(bytes2 > bytes);
|
|
|
|
{
|
2019-09-03 21:32:51 -07:00
|
|
|
let shred_data_1 = &buf[..bytes];
|
|
|
|
assert_eq!(shred_data_1, &shred_bufs[0][..bytes]);
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let shred_data_2 = &buf[bytes..bytes2];
|
|
|
|
assert_eq!(shred_data_2, &shred_bufs[1][..bytes2 - bytes]);
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// buf size part-way into shred[1], should just return shred[0]
|
2018-11-15 15:53:31 -08:00
|
|
|
let mut buf = vec![0; bytes + 1];
|
2019-09-03 21:32:51 -07:00
|
|
|
let (last_index, bytes3) = ledger.get_data_shreds(slot, 0, 2, &mut buf).unwrap();
|
|
|
|
assert_eq!(last_index, 0);
|
2018-11-15 15:53:31 -08:00
|
|
|
assert_eq!(bytes3, bytes);
|
|
|
|
|
|
|
|
let mut buf = vec![0; bytes2 - 1];
|
2019-09-03 21:32:51 -07:00
|
|
|
let (last_index, bytes4) = ledger.get_data_shreds(slot, 0, 2, &mut buf).unwrap();
|
|
|
|
assert_eq!(last_index, 0);
|
2018-11-15 15:53:31 -08:00
|
|
|
assert_eq!(bytes4, bytes);
|
|
|
|
|
|
|
|
let mut buf = vec![0; bytes * 2];
|
2019-09-03 21:32:51 -07:00
|
|
|
let (last_index, bytes6) = ledger.get_data_shreds(slot, 9, 10, &mut buf).unwrap();
|
|
|
|
assert_eq!(last_index, 9);
|
2018-11-15 15:53:31 -08:00
|
|
|
|
|
|
|
{
|
2019-09-03 21:32:51 -07:00
|
|
|
let shred_data = &buf[..bytes6];
|
|
|
|
assert_eq!(shred_data, &shred_bufs[9][..bytes6]);
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Read out of range
|
2019-09-03 21:32:51 -07:00
|
|
|
let (last_index, bytes6) = ledger
|
|
|
|
.get_data_shreds(slot, num_shreds, num_shreds + 2, &mut buf)
|
|
|
|
.unwrap();
|
|
|
|
assert_eq!(last_index, 0);
|
|
|
|
assert_eq!(bytes6, 0);
|
2018-11-15 15:53:31 -08:00
|
|
|
|
|
|
|
// Destroying database without closing it first is undefined behavior
|
|
|
|
drop(ledger);
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
2019-08-20 17:16:06 -07:00
|
|
|
fn test_insert_data_shreds_basic() {
|
2019-02-12 19:54:18 -08:00
|
|
|
let num_entries = 5;
|
|
|
|
assert!(num_entries > 1);
|
2018-12-19 16:11:47 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let (mut shreds, entries) = make_slot_entries(0, 0, num_entries);
|
2019-08-20 17:16:06 -07:00
|
|
|
let num_shreds = shreds.len() as u64;
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-08-20 17:16:06 -07:00
|
|
|
let ledger_path = get_tmp_ledger_path("test_insert_data_shreds_basic");
|
2019-02-07 20:52:39 -08:00
|
|
|
let ledger = Blocktree::open(&ledger_path).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Insert last shred, we're missing the other shreds, so no consecutive
|
|
|
|
// shreds starting from slot 0, index 0 should exist.
|
2019-08-20 17:16:06 -07:00
|
|
|
let last_shred = shreds.pop().unwrap();
|
2019-09-05 18:20:30 -07:00
|
|
|
ledger.insert_shreds(vec![last_shred], None).unwrap();
|
2019-02-12 19:54:18 -08:00
|
|
|
assert!(ledger.get_slot_entries(0, 0, None).unwrap().is_empty());
|
2019-02-04 15:33:43 -08:00
|
|
|
|
2018-11-15 15:53:31 -08:00
|
|
|
let meta = ledger
|
2019-04-26 08:52:10 -07:00
|
|
|
.meta(0)
|
2018-11-15 15:53:31 -08:00
|
|
|
.unwrap()
|
|
|
|
.expect("Expected new metadata object to be created");
|
2019-08-20 17:16:06 -07:00
|
|
|
assert!(meta.consumed == 0 && meta.received == num_shreds);
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Insert the other shreds, check for consecutive returned entries
|
2019-09-05 18:20:30 -07:00
|
|
|
ledger.insert_shreds(shreds, None).unwrap();
|
2019-02-12 19:54:18 -08:00
|
|
|
let result = ledger.get_slot_entries(0, 0, None).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
|
|
|
|
assert_eq!(result, entries);
|
|
|
|
|
|
|
|
let meta = ledger
|
2019-04-26 08:52:10 -07:00
|
|
|
.meta(0)
|
2018-11-15 15:53:31 -08:00
|
|
|
.unwrap()
|
|
|
|
.expect("Expected new metadata object to exist");
|
2019-08-20 17:16:06 -07:00
|
|
|
assert_eq!(meta.consumed, num_shreds);
|
|
|
|
assert_eq!(meta.received, num_shreds);
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(meta.parent_slot, 0);
|
2019-08-20 17:16:06 -07:00
|
|
|
assert_eq!(meta.last_index, num_shreds - 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
assert!(meta.next_slots.is_empty());
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(meta.is_connected);
|
2018-11-15 15:53:31 -08:00
|
|
|
|
|
|
|
// Destroying database without closing it first is undefined behavior
|
|
|
|
drop(ledger);
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
2019-08-20 17:16:06 -07:00
|
|
|
fn test_insert_data_shreds_reverse() {
|
2019-02-12 19:54:18 -08:00
|
|
|
let num_entries = 10;
|
2019-09-03 21:32:51 -07:00
|
|
|
let (mut shreds, entries) = make_slot_entries(0, 0, num_entries);
|
2019-08-20 17:16:06 -07:00
|
|
|
let num_shreds = shreds.len() as u64;
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-08-20 17:16:06 -07:00
|
|
|
let ledger_path = get_tmp_ledger_path("test_insert_data_shreds_reverse");
|
2019-02-07 20:52:39 -08:00
|
|
|
let ledger = Blocktree::open(&ledger_path).unwrap();
|
2018-11-15 15:53:31 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Insert shreds in reverse, check for consecutive returned shreds
|
2019-08-20 17:16:06 -07:00
|
|
|
for i in (0..num_shreds).rev() {
|
|
|
|
let shred = shreds.pop().unwrap();
|
2019-09-05 18:20:30 -07:00
|
|
|
ledger.insert_shreds(vec![shred], None).unwrap();
|
2019-02-12 19:54:18 -08:00
|
|
|
let result = ledger.get_slot_entries(0, 0, None).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2018-11-15 15:53:31 -08:00
|
|
|
let meta = ledger
|
2019-04-26 08:52:10 -07:00
|
|
|
.meta(0)
|
2018-11-15 15:53:31 -08:00
|
|
|
.unwrap()
|
|
|
|
.expect("Expected metadata object to exist");
|
2019-08-20 17:16:06 -07:00
|
|
|
assert_eq!(meta.last_index, num_shreds - 1);
|
2018-11-15 15:53:31 -08:00
|
|
|
if i != 0 {
|
|
|
|
assert_eq!(result.len(), 0);
|
2019-08-20 17:16:06 -07:00
|
|
|
assert!(meta.consumed == 0 && meta.received == num_shreds as u64);
|
2018-11-15 15:53:31 -08:00
|
|
|
} else {
|
2019-08-20 17:16:06 -07:00
|
|
|
assert_eq!(meta.parent_slot, 0);
|
2018-11-15 15:53:31 -08:00
|
|
|
assert_eq!(result, entries);
|
2019-08-20 17:16:06 -07:00
|
|
|
assert!(meta.consumed == num_shreds as u64 && meta.received == num_shreds as u64);
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Destroying database without closing it first is undefined behavior
|
|
|
|
drop(ledger);
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|
2018-11-22 01:35:19 -08:00
|
|
|
|
2018-12-12 15:58:29 -08:00
|
|
|
#[test]
|
2019-02-07 15:10:54 -08:00
|
|
|
fn test_insert_slots() {
|
2019-09-03 21:32:51 -07:00
|
|
|
test_insert_data_shreds_slots("test_insert_data_shreds_slots_single", false);
|
|
|
|
test_insert_data_shreds_slots("test_insert_data_shreds_slots_bulk", true);
|
2018-12-12 15:58:29 -08:00
|
|
|
}
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
/*
|
|
|
|
#[test]
|
|
|
|
pub fn test_iteration_order() {
|
|
|
|
let slot = 0;
|
|
|
|
let blocktree_path = get_tmp_ledger_path("test_iteration_order");
|
|
|
|
{
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2018-11-22 01:35:19 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Write entries
|
|
|
|
let num_entries = 8;
|
|
|
|
let entries = make_tiny_test_entries(num_entries);
|
|
|
|
let mut shreds = entries.to_single_entry_shreds();
|
2018-11-22 01:35:19 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
for (i, b) in shreds.iter_mut().enumerate() {
|
|
|
|
b.set_index(1 << (i * 8));
|
|
|
|
b.set_slot(0);
|
|
|
|
}
|
2018-11-22 01:35:19 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
blocktree
|
|
|
|
.write_shreds(&shreds)
|
|
|
|
.expect("Expected successful write of shreds");
|
|
|
|
|
|
|
|
let mut db_iterator = blocktree
|
|
|
|
.db
|
|
|
|
.cursor::<cf::Data>()
|
|
|
|
.expect("Expected to be able to open database iterator");
|
|
|
|
|
|
|
|
db_iterator.seek((slot, 1));
|
|
|
|
|
|
|
|
// Iterate through ledger
|
|
|
|
for i in 0..num_entries {
|
|
|
|
assert!(db_iterator.valid());
|
|
|
|
let (_, current_index) = db_iterator.key().expect("Expected a valid key");
|
|
|
|
assert_eq!(current_index, (1 as u64) << (i * 8));
|
|
|
|
db_iterator.next();
|
|
|
|
}
|
2018-11-22 01:35:19 -08:00
|
|
|
}
|
2019-09-03 21:32:51 -07:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2018-11-22 01:35:19 -08:00
|
|
|
}
|
2019-09-03 21:32:51 -07:00
|
|
|
*/
|
2018-12-11 09:14:23 -08:00
|
|
|
|
2019-02-04 15:33:43 -08:00
|
|
|
#[test]
|
|
|
|
pub fn test_get_slot_entries1() {
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree_path = get_tmp_ledger_path("test_get_slot_entries1");
|
2019-02-04 15:33:43 -08:00
|
|
|
{
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2019-08-21 20:07:51 -07:00
|
|
|
let entries = make_tiny_test_entries(8);
|
|
|
|
let shreds = entries_to_test_shreds(entries[0..4].to_vec(), 1, 0, false);
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree
|
2019-09-05 18:20:30 -07:00
|
|
|
.insert_shreds(shreds, None)
|
2019-09-03 21:32:51 -07:00
|
|
|
.expect("Expected successful write of shreds");
|
2019-02-04 15:33:43 -08:00
|
|
|
|
2019-08-21 20:07:51 -07:00
|
|
|
let mut shreds1 = entries_to_test_shreds(entries[4..].to_vec(), 1, 0, false);
|
|
|
|
for (i, b) in shreds1.iter_mut().enumerate() {
|
|
|
|
b.set_index(8 + i as u32);
|
|
|
|
}
|
|
|
|
blocktree
|
2019-09-05 18:20:30 -07:00
|
|
|
.insert_shreds(shreds1, None)
|
2019-09-03 21:32:51 -07:00
|
|
|
.expect("Expected successful write of shreds");
|
2019-08-21 20:07:51 -07:00
|
|
|
|
2019-02-04 15:33:43 -08:00
|
|
|
assert_eq!(
|
2019-08-20 17:16:06 -07:00
|
|
|
blocktree.get_slot_entries(1, 0, None).unwrap()[2..4],
|
2019-02-04 15:33:43 -08:00
|
|
|
entries[2..4],
|
|
|
|
);
|
|
|
|
}
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2019-02-04 15:33:43 -08:00
|
|
|
}
|
|
|
|
|
2019-08-21 20:07:51 -07:00
|
|
|
// This test seems to be unnecessary with introduction of data shreds. There are no
|
2019-09-03 21:32:51 -07:00
|
|
|
// guarantees that a particular shred index contains a complete entry
|
2019-02-04 15:33:43 -08:00
|
|
|
#[test]
|
2019-08-20 17:16:06 -07:00
|
|
|
#[ignore]
|
2019-02-04 15:33:43 -08:00
|
|
|
pub fn test_get_slot_entries2() {
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree_path = get_tmp_ledger_path("test_get_slot_entries2");
|
2019-02-04 15:33:43 -08:00
|
|
|
{
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2019-02-04 15:33:43 -08:00
|
|
|
|
|
|
|
// Write entries
|
|
|
|
let num_slots = 5 as u64;
|
|
|
|
let mut index = 0;
|
2019-03-05 14:18:29 -08:00
|
|
|
for slot in 0..num_slots {
|
|
|
|
let entries = make_tiny_test_entries(slot as usize + 1);
|
2019-02-04 15:33:43 -08:00
|
|
|
let last_entry = entries.last().unwrap().clone();
|
2019-08-21 20:07:51 -07:00
|
|
|
let mut shreds =
|
|
|
|
entries_to_test_shreds(entries, slot, slot.saturating_sub(1), false);
|
|
|
|
for b in shreds.iter_mut() {
|
2019-02-04 15:33:43 -08:00
|
|
|
b.set_index(index);
|
2019-03-05 14:18:29 -08:00
|
|
|
b.set_slot(slot as u64);
|
2019-02-04 15:33:43 -08:00
|
|
|
index += 1;
|
|
|
|
}
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree
|
2019-09-05 18:20:30 -07:00
|
|
|
.insert_shreds(shreds, None)
|
2019-08-21 20:07:51 -07:00
|
|
|
.expect("Expected successful write of shreds");
|
2019-02-04 15:33:43 -08:00
|
|
|
assert_eq!(
|
2019-08-21 20:07:51 -07:00
|
|
|
blocktree
|
|
|
|
.get_slot_entries(slot, u64::from(index - 1), None)
|
|
|
|
.unwrap(),
|
2019-02-04 15:33:43 -08:00
|
|
|
vec![last_entry],
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2019-02-04 15:33:43 -08:00
|
|
|
}
|
|
|
|
|
2019-03-17 18:48:23 -07:00
|
|
|
#[test]
|
|
|
|
pub fn test_get_slot_entries3() {
|
2019-09-03 21:32:51 -07:00
|
|
|
// Test inserting/fetching shreds which contain multiple entries per shred
|
2019-03-17 18:48:23 -07:00
|
|
|
let blocktree_path = get_tmp_ledger_path("test_get_slot_entries3");
|
|
|
|
{
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
let num_slots = 5 as u64;
|
2019-08-21 20:07:51 -07:00
|
|
|
let shreds_per_slot = 5 as u64;
|
2019-03-17 18:48:23 -07:00
|
|
|
let entry_serialized_size =
|
|
|
|
bincode::serialized_size(&make_tiny_test_entries(1)).unwrap();
|
|
|
|
let entries_per_slot =
|
2019-08-21 20:07:51 -07:00
|
|
|
(shreds_per_slot * PACKET_DATA_SIZE as u64) / entry_serialized_size;
|
2019-03-17 18:48:23 -07:00
|
|
|
|
|
|
|
// Write entries
|
|
|
|
for slot in 0..num_slots {
|
|
|
|
let entries = make_tiny_test_entries(entries_per_slot as usize);
|
2019-08-21 20:07:51 -07:00
|
|
|
let shreds =
|
|
|
|
entries_to_test_shreds(entries.clone(), slot, slot.saturating_sub(1), false);
|
|
|
|
assert!(shreds.len() as u64 >= shreds_per_slot);
|
2019-03-17 18:48:23 -07:00
|
|
|
blocktree
|
2019-09-05 18:20:30 -07:00
|
|
|
.insert_shreds(shreds, None)
|
2019-08-21 20:07:51 -07:00
|
|
|
.expect("Expected successful write of shreds");
|
|
|
|
assert_eq!(blocktree.get_slot_entries(slot, 0, None).unwrap(), entries);
|
2019-03-17 18:48:23 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
2018-12-19 16:11:47 -08:00
|
|
|
#[test]
|
2019-08-20 17:16:06 -07:00
|
|
|
pub fn test_insert_data_shreds_consecutive() {
|
2019-09-03 21:32:51 -07:00
|
|
|
let blocktree_path = get_tmp_ledger_path("test_insert_data_shreds_consecutive");
|
2018-12-19 16:11:47 -08:00
|
|
|
{
|
2019-03-14 15:18:37 -07:00
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2019-04-18 21:56:43 -07:00
|
|
|
for i in 0..4 {
|
|
|
|
let slot = i;
|
|
|
|
let parent_slot = if i == 0 { 0 } else { i - 1 };
|
|
|
|
// Write entries
|
|
|
|
let num_entries = 21 as u64 * (i + 1);
|
2019-08-20 17:16:06 -07:00
|
|
|
let (mut shreds, original_entries) =
|
2019-09-03 21:32:51 -07:00
|
|
|
make_slot_entries(slot, parent_slot, num_entries);
|
2019-08-20 17:16:06 -07:00
|
|
|
|
|
|
|
let num_shreds = shreds.len() as u64;
|
|
|
|
let mut odd_shreds = vec![];
|
|
|
|
for i in (0..num_shreds).rev() {
|
|
|
|
if i % 2 != 0 {
|
|
|
|
odd_shreds.insert(0, shreds.remove(i as usize));
|
|
|
|
}
|
|
|
|
}
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(odd_shreds, None).unwrap();
|
2019-04-17 12:52:12 -07:00
|
|
|
|
2019-04-18 21:56:43 -07:00
|
|
|
assert_eq!(blocktree.get_slot_entries(slot, 0, None).unwrap(), vec![]);
|
2019-04-17 12:52:12 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
let meta = blocktree.meta(slot).unwrap().unwrap();
|
2019-08-20 17:16:06 -07:00
|
|
|
if num_shreds % 2 == 0 {
|
|
|
|
assert_eq!(meta.received, num_shreds);
|
2019-04-18 21:56:43 -07:00
|
|
|
} else {
|
|
|
|
debug!("got here");
|
2019-08-20 17:16:06 -07:00
|
|
|
assert_eq!(meta.received, num_shreds - 1);
|
2019-04-18 21:56:43 -07:00
|
|
|
}
|
|
|
|
assert_eq!(meta.consumed, 0);
|
2019-08-20 17:16:06 -07:00
|
|
|
if num_shreds % 2 == 0 {
|
|
|
|
assert_eq!(meta.last_index, num_shreds - 1);
|
2019-04-18 21:56:43 -07:00
|
|
|
} else {
|
|
|
|
assert_eq!(meta.last_index, std::u64::MAX);
|
|
|
|
}
|
2019-04-17 18:04:30 -07:00
|
|
|
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds, None).unwrap();
|
2019-04-17 18:04:30 -07:00
|
|
|
|
2019-04-18 21:56:43 -07:00
|
|
|
assert_eq!(
|
|
|
|
blocktree.get_slot_entries(slot, 0, None).unwrap(),
|
|
|
|
original_entries,
|
|
|
|
);
|
2019-04-17 18:04:30 -07:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
let meta = blocktree.meta(slot).unwrap().unwrap();
|
2019-08-20 17:16:06 -07:00
|
|
|
assert_eq!(meta.received, num_shreds);
|
|
|
|
assert_eq!(meta.consumed, num_shreds);
|
2019-04-18 21:56:43 -07:00
|
|
|
assert_eq!(meta.parent_slot, parent_slot);
|
2019-08-20 17:16:06 -07:00
|
|
|
assert_eq!(meta.last_index, num_shreds - 1);
|
2019-04-18 21:56:43 -07:00
|
|
|
}
|
2018-12-20 12:12:04 -08:00
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2018-12-20 12:12:04 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
2019-08-20 17:16:06 -07:00
|
|
|
pub fn test_insert_data_shreds_duplicate() {
|
2018-12-20 12:12:04 -08:00
|
|
|
// Create RocksDb ledger
|
2019-09-03 21:32:51 -07:00
|
|
|
let blocktree_path = get_tmp_ledger_path("test_insert_data_shreds_duplicate");
|
2018-12-20 12:12:04 -08:00
|
|
|
{
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2018-12-20 12:12:04 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Make duplicate entries and shreds
|
2019-02-12 19:54:18 -08:00
|
|
|
let num_unique_entries = 10;
|
2019-08-20 17:16:06 -07:00
|
|
|
let (mut original_shreds, original_entries) =
|
2019-09-03 21:32:51 -07:00
|
|
|
make_slot_entries(0, 0, num_unique_entries);
|
2018-12-20 12:12:04 -08:00
|
|
|
|
2019-08-20 17:16:06 -07:00
|
|
|
// Discard first shred
|
|
|
|
original_shreds.remove(0);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(original_shreds, None).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-08-20 17:16:06 -07:00
|
|
|
assert_eq!(blocktree.get_slot_entries(0, 0, None).unwrap(), vec![]);
|
2018-12-20 12:12:04 -08:00
|
|
|
|
2019-08-20 17:16:06 -07:00
|
|
|
let duplicate_shreds = entries_to_test_shreds(original_entries.clone(), 0, 0, true);
|
2019-08-26 18:27:45 -07:00
|
|
|
let num_shreds = duplicate_shreds.len() as u64;
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(duplicate_shreds, None).unwrap();
|
2018-12-20 12:12:04 -08:00
|
|
|
|
2019-08-20 17:16:06 -07:00
|
|
|
assert_eq!(
|
|
|
|
blocktree.get_slot_entries(0, 0, None).unwrap(),
|
|
|
|
original_entries
|
|
|
|
);
|
2018-12-20 12:12:04 -08:00
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
let meta = blocktree.meta(0).unwrap().unwrap();
|
2019-08-20 17:16:06 -07:00
|
|
|
assert_eq!(meta.consumed, num_shreds);
|
|
|
|
assert_eq!(meta.received, num_shreds);
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(meta.parent_slot, 0);
|
2019-08-20 17:16:06 -07:00
|
|
|
assert_eq!(meta.last_index, num_shreds - 1);
|
2018-12-19 16:11:47 -08:00
|
|
|
}
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2018-12-19 16:11:47 -08:00
|
|
|
}
|
|
|
|
|
2019-02-07 15:10:54 -08:00
|
|
|
#[test]
|
2019-09-03 21:32:51 -07:00
|
|
|
pub fn test_new_shreds_signal() {
|
2019-02-07 15:10:54 -08:00
|
|
|
// Initialize ledger
|
2019-09-03 21:32:51 -07:00
|
|
|
let ledger_path = get_tmp_ledger_path("test_new_shreds_signal");
|
2019-05-09 14:10:04 -07:00
|
|
|
let (ledger, recvr, _) = Blocktree::open_with_signal(&ledger_path).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
let ledger = Arc::new(ledger);
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let entries_per_slot = 50;
|
2019-02-12 19:54:18 -08:00
|
|
|
// Create entries for slot 0
|
2019-09-03 21:32:51 -07:00
|
|
|
let (mut shreds, _) = make_slot_entries(0, 0, entries_per_slot);
|
|
|
|
let shreds_per_slot = shreds.len() as u64;
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Insert second shred, but we're missing the first shred, so no consecutive
|
|
|
|
// shreds starting from slot 0, index 0 should exist.
|
2019-09-05 18:20:30 -07:00
|
|
|
ledger.insert_shreds(vec![shreds.remove(1)], None).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
let timer = Duration::new(1, 0);
|
|
|
|
assert!(recvr.recv_timeout(timer).is_err());
|
2019-09-03 21:32:51 -07:00
|
|
|
// Insert first shred, now we've made a consecutive block
|
2019-09-05 18:20:30 -07:00
|
|
|
ledger.insert_shreds(vec![shreds.remove(0)], None).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
// Wait to get notified of update, should only be one update
|
|
|
|
assert!(recvr.recv_timeout(timer).is_ok());
|
|
|
|
assert!(recvr.try_recv().is_err());
|
|
|
|
// Insert the rest of the ticks
|
2019-09-05 18:20:30 -07:00
|
|
|
ledger.insert_shreds(shreds, None).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
// Wait to get notified of update, should only be one update
|
|
|
|
assert!(recvr.recv_timeout(timer).is_ok());
|
|
|
|
assert!(recvr.try_recv().is_err());
|
|
|
|
|
|
|
|
// Create some other slots, and send batches of ticks for each slot such that each slot
|
2019-09-03 21:32:51 -07:00
|
|
|
// is missing the tick at shred index == slot index - 1. Thus, no consecutive blocks
|
2019-02-07 15:10:54 -08:00
|
|
|
// will be formed
|
2019-09-03 21:32:51 -07:00
|
|
|
let num_slots = shreds_per_slot;
|
|
|
|
let mut shreds: Vec<Shred> = vec![];
|
|
|
|
let mut missing_shreds = vec![];
|
2019-03-05 14:18:29 -08:00
|
|
|
for slot in 1..num_slots + 1 {
|
2019-09-03 21:32:51 -07:00
|
|
|
let (mut slot_shreds, _) = make_slot_entries(slot, slot - 1, entries_per_slot);
|
|
|
|
let missing_shred = slot_shreds.remove(slot as usize - 1);
|
|
|
|
shreds.extend(slot_shreds);
|
|
|
|
missing_shreds.push(missing_shred);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Should be no updates, since no new chains from block 0 were formed
|
2019-09-05 18:20:30 -07:00
|
|
|
ledger.insert_shreds(shreds, None).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
assert!(recvr.recv_timeout(timer).is_err());
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Insert a shred for each slot that doesn't make a consecutive block, we
|
2019-02-07 15:10:54 -08:00
|
|
|
// should get no updates
|
2019-09-03 21:32:51 -07:00
|
|
|
let shreds: Vec<_> = (1..num_slots + 1)
|
2019-03-05 14:18:29 -08:00
|
|
|
.flat_map(|slot| {
|
2019-09-03 21:32:51 -07:00
|
|
|
let (mut shred, _) = make_slot_entries(slot, slot - 1, 1);
|
|
|
|
shred[0].set_index(2 * num_slots as u32);
|
|
|
|
shred
|
2019-02-07 15:10:54 -08:00
|
|
|
})
|
|
|
|
.collect();
|
|
|
|
|
2019-09-05 18:20:30 -07:00
|
|
|
ledger.insert_shreds(shreds, None).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
assert!(recvr.recv_timeout(timer).is_err());
|
|
|
|
|
|
|
|
// For slots 1..num_slots/2, fill in the holes in one batch insertion,
|
|
|
|
// so we should only get one signal
|
2019-09-03 21:32:51 -07:00
|
|
|
let missing_shreds2 = missing_shreds
|
|
|
|
.drain((num_slots / 2) as usize..)
|
|
|
|
.collect_vec();
|
2019-09-05 18:20:30 -07:00
|
|
|
ledger.insert_shreds(missing_shreds, None).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
assert!(recvr.recv_timeout(timer).is_ok());
|
|
|
|
assert!(recvr.try_recv().is_err());
|
|
|
|
|
|
|
|
// Fill in the holes for each of the remaining slots, we should get a single update
|
|
|
|
// for each
|
2019-09-05 18:20:30 -07:00
|
|
|
ledger.insert_shreds(missing_shreds2, None).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
// Destroying database without closing it first is undefined behavior
|
|
|
|
drop(ledger);
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-05-09 14:10:04 -07:00
|
|
|
#[test]
|
2019-09-03 21:32:51 -07:00
|
|
|
pub fn test_completed_shreds_signal() {
|
2019-05-09 14:10:04 -07:00
|
|
|
// Initialize ledger
|
2019-09-03 21:32:51 -07:00
|
|
|
let ledger_path = get_tmp_ledger_path("test_completed_shreds_signal");
|
2019-05-09 14:10:04 -07:00
|
|
|
let (ledger, _, recvr) = Blocktree::open_with_signal(&ledger_path).unwrap();
|
|
|
|
let ledger = Arc::new(ledger);
|
|
|
|
|
|
|
|
let entries_per_slot = 10;
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Create shreds for slot 0
|
|
|
|
let (mut shreds, _) = make_slot_entries(0, 0, entries_per_slot);
|
2019-05-09 14:10:04 -07:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let shred0 = shreds.remove(0);
|
|
|
|
// Insert all but the first shred in the slot, should not be considered complete
|
2019-09-05 18:20:30 -07:00
|
|
|
ledger.insert_shreds(shreds, None).unwrap();
|
2019-05-09 14:10:04 -07:00
|
|
|
assert!(recvr.try_recv().is_err());
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Insert first shred, slot should now be considered complete
|
2019-09-05 18:20:30 -07:00
|
|
|
ledger.insert_shreds(vec![shred0], None).unwrap();
|
2019-05-09 14:10:04 -07:00
|
|
|
assert_eq!(recvr.try_recv().unwrap(), vec![0]);
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
2019-09-03 21:32:51 -07:00
|
|
|
pub fn test_completed_shreds_signal_orphans() {
|
2019-05-09 14:10:04 -07:00
|
|
|
// Initialize ledger
|
2019-09-03 21:32:51 -07:00
|
|
|
let ledger_path = get_tmp_ledger_path("test_completed_shreds_signal_orphans");
|
2019-05-09 14:10:04 -07:00
|
|
|
let (ledger, _, recvr) = Blocktree::open_with_signal(&ledger_path).unwrap();
|
|
|
|
let ledger = Arc::new(ledger);
|
|
|
|
|
|
|
|
let entries_per_slot = 10;
|
|
|
|
let slots = vec![2, 5, 10];
|
2019-09-03 21:32:51 -07:00
|
|
|
let mut all_shreds = make_chaining_slot_entries(&slots[..], entries_per_slot);
|
2019-05-09 14:10:04 -07:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Get the shreds for slot 10, chaining to slot 5
|
|
|
|
let (mut orphan_child, _) = all_shreds.remove(2);
|
2019-05-09 14:10:04 -07:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Get the shreds for slot 5 chaining to slot 2
|
|
|
|
let (mut orphan_shreds, _) = all_shreds.remove(1);
|
2019-05-09 14:10:04 -07:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Insert all but the first shred in the slot, should not be considered complete
|
|
|
|
let orphan_child0 = orphan_child.remove(0);
|
2019-09-05 18:20:30 -07:00
|
|
|
ledger.insert_shreds(orphan_child, None).unwrap();
|
2019-05-09 14:10:04 -07:00
|
|
|
assert!(recvr.try_recv().is_err());
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Insert first shred, slot should now be considered complete
|
2019-09-05 18:20:30 -07:00
|
|
|
ledger.insert_shreds(vec![orphan_child0], None).unwrap();
|
2019-05-09 14:10:04 -07:00
|
|
|
assert_eq!(recvr.try_recv().unwrap(), vec![slots[2]]);
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Insert the shreds for the orphan_slot
|
|
|
|
let orphan_shred0 = orphan_shreds.remove(0);
|
2019-09-05 18:20:30 -07:00
|
|
|
ledger.insert_shreds(orphan_shreds, None).unwrap();
|
2019-05-09 14:10:04 -07:00
|
|
|
assert!(recvr.try_recv().is_err());
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Insert first shred, slot should now be considered complete
|
2019-09-05 18:20:30 -07:00
|
|
|
ledger.insert_shreds(vec![orphan_shred0], None).unwrap();
|
2019-05-09 14:10:04 -07:00
|
|
|
assert_eq!(recvr.try_recv().unwrap(), vec![slots[1]]);
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
2019-09-03 21:32:51 -07:00
|
|
|
pub fn test_completed_shreds_signal_many() {
|
2019-05-09 14:10:04 -07:00
|
|
|
// Initialize ledger
|
2019-09-03 21:32:51 -07:00
|
|
|
let ledger_path = get_tmp_ledger_path("test_completed_shreds_signal_many");
|
2019-05-09 14:10:04 -07:00
|
|
|
let (ledger, _, recvr) = Blocktree::open_with_signal(&ledger_path).unwrap();
|
|
|
|
let ledger = Arc::new(ledger);
|
|
|
|
|
|
|
|
let entries_per_slot = 10;
|
|
|
|
let mut slots = vec![2, 5, 10];
|
2019-09-03 21:32:51 -07:00
|
|
|
let mut all_shreds = make_chaining_slot_entries(&slots[..], entries_per_slot);
|
2019-05-09 14:10:04 -07:00
|
|
|
let disconnected_slot = 4;
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let (shreds0, _) = all_shreds.remove(0);
|
|
|
|
let (shreds1, _) = all_shreds.remove(0);
|
|
|
|
let (shreds2, _) = all_shreds.remove(0);
|
|
|
|
let (shreds3, _) = make_slot_entries(disconnected_slot, 1, entries_per_slot);
|
2019-05-09 14:10:04 -07:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let mut all_shreds: Vec<_> = vec![shreds0, shreds1, shreds2, shreds3]
|
2019-05-09 14:10:04 -07:00
|
|
|
.into_iter()
|
|
|
|
.flatten()
|
|
|
|
.collect();
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
all_shreds.shuffle(&mut thread_rng());
|
2019-09-05 18:20:30 -07:00
|
|
|
ledger.insert_shreds(all_shreds, None).unwrap();
|
2019-05-09 14:10:04 -07:00
|
|
|
let mut result = recvr.try_recv().unwrap();
|
|
|
|
result.sort();
|
|
|
|
slots.push(disconnected_slot);
|
|
|
|
slots.sort();
|
|
|
|
assert_eq!(result, slots);
|
|
|
|
}
|
|
|
|
|
2019-02-07 15:10:54 -08:00
|
|
|
#[test]
|
|
|
|
pub fn test_handle_chaining_basic() {
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree_path = get_tmp_ledger_path("test_handle_chaining_basic");
|
2019-02-07 15:10:54 -08:00
|
|
|
{
|
2019-02-12 19:54:18 -08:00
|
|
|
let entries_per_slot = 2;
|
|
|
|
let num_slots = 3;
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Construct the shreds
|
|
|
|
let (mut shreds, _) = make_many_slot_entries(0, num_slots, entries_per_slot);
|
|
|
|
let shreds_per_slot = shreds.len() / num_slots as usize;
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
// 1) Write to the first slot
|
2019-09-03 21:32:51 -07:00
|
|
|
let shreds1 = shreds
|
|
|
|
.drain(shreds_per_slot..2 * shreds_per_slot)
|
|
|
|
.collect_vec();
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds1, None).unwrap();
|
2019-02-13 15:01:56 -08:00
|
|
|
let s1 = blocktree.meta(1).unwrap().unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
assert!(s1.next_slots.is_empty());
|
|
|
|
// Slot 1 is not trunk because slot 0 hasn't been inserted yet
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(!s1.is_connected);
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(s1.parent_slot, 0);
|
|
|
|
assert_eq!(s1.last_index, entries_per_slot - 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
// 2) Write to the second slot
|
2019-09-03 21:32:51 -07:00
|
|
|
let shreds2 = shreds
|
|
|
|
.drain(shreds_per_slot..2 * shreds_per_slot)
|
|
|
|
.collect_vec();
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds2, None).unwrap();
|
2019-02-13 15:01:56 -08:00
|
|
|
let s2 = blocktree.meta(2).unwrap().unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
assert!(s2.next_slots.is_empty());
|
|
|
|
// Slot 2 is not trunk because slot 0 hasn't been inserted yet
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(!s2.is_connected);
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(s2.parent_slot, 1);
|
|
|
|
assert_eq!(s2.last_index, entries_per_slot - 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
// Check the first slot again, it should chain to the second slot,
|
|
|
|
// but still isn't part of the trunk
|
2019-02-13 15:01:56 -08:00
|
|
|
let s1 = blocktree.meta(1).unwrap().unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
assert_eq!(s1.next_slots, vec![2]);
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(!s1.is_connected);
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(s1.parent_slot, 0);
|
|
|
|
assert_eq!(s1.last_index, entries_per_slot - 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
// 3) Write to the zeroth slot, check that every slot
|
|
|
|
// is now part of the trunk
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds, None).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
for i in 0..3 {
|
2019-02-13 15:01:56 -08:00
|
|
|
let s = blocktree.meta(i).unwrap().unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
// The last slot will not chain to any other slots
|
|
|
|
if i != 2 {
|
|
|
|
assert_eq!(s.next_slots, vec![i + 1]);
|
|
|
|
}
|
2019-02-12 19:54:18 -08:00
|
|
|
if i == 0 {
|
|
|
|
assert_eq!(s.parent_slot, 0);
|
|
|
|
} else {
|
|
|
|
assert_eq!(s.parent_slot, i - 1);
|
|
|
|
}
|
|
|
|
assert_eq!(s.last_index, entries_per_slot - 1);
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(s.is_connected);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
pub fn test_handle_chaining_missing_slots() {
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree_path = get_tmp_ledger_path("test_handle_chaining_missing_slots");
|
2019-02-07 15:10:54 -08:00
|
|
|
{
|
2019-02-12 19:54:18 -08:00
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
let num_slots = 30;
|
2019-02-12 19:54:18 -08:00
|
|
|
let entries_per_slot = 2;
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-02-12 19:54:18 -08:00
|
|
|
// Separate every other slot into two separate vectors
|
|
|
|
let mut slots = vec![];
|
|
|
|
let mut missing_slots = vec![];
|
2019-03-05 14:18:29 -08:00
|
|
|
for slot in 0..num_slots {
|
2019-02-12 19:54:18 -08:00
|
|
|
let parent_slot = {
|
2019-03-05 14:18:29 -08:00
|
|
|
if slot == 0 {
|
2019-02-12 19:54:18 -08:00
|
|
|
0
|
|
|
|
} else {
|
2019-03-05 14:18:29 -08:00
|
|
|
slot - 1
|
2019-02-12 19:54:18 -08:00
|
|
|
}
|
|
|
|
};
|
2019-09-03 21:32:51 -07:00
|
|
|
let (slot_shreds, _) = make_slot_entries(slot, parent_slot, entries_per_slot);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-03-05 14:18:29 -08:00
|
|
|
if slot % 2 == 1 {
|
2019-09-03 21:32:51 -07:00
|
|
|
slots.extend(slot_shreds);
|
2019-02-12 19:54:18 -08:00
|
|
|
} else {
|
2019-09-03 21:32:51 -07:00
|
|
|
missing_slots.extend(slot_shreds);
|
2019-02-12 19:54:18 -08:00
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Write the shreds for every other slot
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(slots, None).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
// Check metadata
|
|
|
|
for i in 0..num_slots {
|
2019-02-12 19:54:18 -08:00
|
|
|
// If "i" is the index of a slot we just inserted, then next_slots should be empty
|
|
|
|
// for slot "i" because no slots chain to that slot, because slot i + 1 is missing.
|
2019-04-06 19:41:22 -07:00
|
|
|
// However, if it's a slot we haven't inserted, aka one of the gaps, then one of the
|
|
|
|
// slots we just inserted will chain to that gap, so next_slots for that orphan slot
|
|
|
|
// won't be empty, but the parent slot is unknown so should equal std::u64::MAX.
|
2019-02-13 15:01:56 -08:00
|
|
|
let s = blocktree.meta(i as u64).unwrap().unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
if i % 2 == 0 {
|
|
|
|
assert_eq!(s.next_slots, vec![i as u64 + 1]);
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(s.parent_slot, std::u64::MAX);
|
2019-02-07 15:10:54 -08:00
|
|
|
} else {
|
|
|
|
assert!(s.next_slots.is_empty());
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(s.parent_slot, i - 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
if i == 0 {
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(s.is_connected);
|
2019-02-07 15:10:54 -08:00
|
|
|
} else {
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(!s.is_connected);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Write the shreds for the other half of the slots that we didn't insert earlier
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(missing_slots, None).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
for i in 0..num_slots {
|
|
|
|
// Check that all the slots chain correctly once the missing slots
|
|
|
|
// have been filled
|
2019-02-13 15:01:56 -08:00
|
|
|
let s = blocktree.meta(i as u64).unwrap().unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
if i != num_slots - 1 {
|
|
|
|
assert_eq!(s.next_slots, vec![i as u64 + 1]);
|
|
|
|
} else {
|
|
|
|
assert!(s.next_slots.is_empty());
|
|
|
|
}
|
2019-02-12 19:54:18 -08:00
|
|
|
|
|
|
|
if i == 0 {
|
|
|
|
assert_eq!(s.parent_slot, 0);
|
|
|
|
} else {
|
|
|
|
assert_eq!(s.parent_slot, i - 1);
|
|
|
|
}
|
|
|
|
assert_eq!(s.last_index, entries_per_slot - 1);
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(s.is_connected);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
2019-03-20 11:19:37 -07:00
|
|
|
pub fn test_forward_chaining_is_connected() {
|
|
|
|
let blocktree_path = get_tmp_ledger_path("test_forward_chaining_is_connected");
|
2019-02-07 15:10:54 -08:00
|
|
|
{
|
2019-02-12 19:54:18 -08:00
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
let num_slots = 15;
|
2019-02-12 19:54:18 -08:00
|
|
|
let entries_per_slot = 2;
|
|
|
|
assert!(entries_per_slot > 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let (mut shreds, _) = make_many_slot_entries(0, num_slots, entries_per_slot);
|
|
|
|
let shreds_per_slot = shreds.len() / num_slots as usize;
|
2019-02-12 19:54:18 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Write the shreds such that every 3rd slot has a gap in the beginning
|
|
|
|
let mut missing_shreds = vec![];
|
|
|
|
for slot in 0..num_slots {
|
|
|
|
let mut shreds_for_slot = shreds.drain(..shreds_per_slot).collect_vec();
|
2019-03-05 14:18:29 -08:00
|
|
|
if slot % 3 == 0 {
|
2019-09-03 21:32:51 -07:00
|
|
|
let shred0 = shreds_for_slot.remove(0);
|
|
|
|
missing_shreds.push(shred0);
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds_for_slot, None).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
} else {
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds_for_slot, None).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check metadata
|
|
|
|
for i in 0..num_slots {
|
2019-02-13 15:01:56 -08:00
|
|
|
let s = blocktree.meta(i as u64).unwrap().unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
// The last slot will not chain to any other slots
|
|
|
|
if i as u64 != num_slots - 1 {
|
|
|
|
assert_eq!(s.next_slots, vec![i as u64 + 1]);
|
|
|
|
} else {
|
|
|
|
assert!(s.next_slots.is_empty());
|
|
|
|
}
|
2019-02-12 19:54:18 -08:00
|
|
|
|
|
|
|
if i == 0 {
|
|
|
|
assert_eq!(s.parent_slot, 0);
|
2019-02-07 15:10:54 -08:00
|
|
|
} else {
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(s.parent_slot, i - 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(s.last_index, entries_per_slot - 1);
|
|
|
|
|
2019-02-07 15:10:54 -08:00
|
|
|
// Other than slot 0, no slots should be part of the trunk
|
|
|
|
if i != 0 {
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(!s.is_connected);
|
2019-02-07 15:10:54 -08:00
|
|
|
} else {
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(s.is_connected);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Iteratively finish every 3rd slot, and check that all slots up to and including
|
|
|
|
// slot_index + 3 become part of the trunk
|
2019-09-03 21:32:51 -07:00
|
|
|
for slot_index in 0..num_slots {
|
2019-02-07 15:10:54 -08:00
|
|
|
if slot_index % 3 == 0 {
|
2019-09-03 21:32:51 -07:00
|
|
|
let shred = missing_shreds.remove(0);
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(vec![shred], None).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
for i in 0..num_slots {
|
2019-02-13 15:01:56 -08:00
|
|
|
let s = blocktree.meta(i as u64).unwrap().unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
if i != num_slots - 1 {
|
|
|
|
assert_eq!(s.next_slots, vec![i as u64 + 1]);
|
|
|
|
} else {
|
|
|
|
assert!(s.next_slots.is_empty());
|
|
|
|
}
|
|
|
|
if i <= slot_index as u64 + 3 {
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(s.is_connected);
|
2019-02-07 15:10:54 -08:00
|
|
|
} else {
|
2019-03-20 11:19:37 -07:00
|
|
|
assert!(!s.is_connected);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
2019-02-12 19:54:18 -08:00
|
|
|
|
|
|
|
if i == 0 {
|
|
|
|
assert_eq!(s.parent_slot, 0);
|
|
|
|
} else {
|
|
|
|
assert_eq!(s.parent_slot, i - 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
assert_eq!(s.last_index, entries_per_slot - 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
2019-09-03 21:32:51 -07:00
|
|
|
/*
|
|
|
|
#[test]
|
|
|
|
pub fn test_chaining_tree() {
|
|
|
|
let blocktree_path = get_tmp_ledger_path("test_chaining_tree");
|
|
|
|
{
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
let num_tree_levels = 6;
|
|
|
|
assert!(num_tree_levels > 1);
|
|
|
|
let branching_factor: u64 = 4;
|
|
|
|
// Number of slots that will be in the tree
|
|
|
|
let num_slots = (branching_factor.pow(num_tree_levels) - 1) / (branching_factor - 1);
|
|
|
|
let erasure_config = ErasureConfig::default();
|
|
|
|
let entries_per_slot = erasure_config.num_data() as u64;
|
|
|
|
assert!(entries_per_slot > 1);
|
|
|
|
|
|
|
|
let (mut shreds, _) = make_many_slot_entries(0, num_slots, entries_per_slot);
|
|
|
|
|
|
|
|
// Insert tree one slot at a time in a random order
|
|
|
|
let mut slots: Vec<_> = (0..num_slots).collect();
|
|
|
|
|
|
|
|
// Get shreds for the slot
|
|
|
|
slots.shuffle(&mut thread_rng());
|
|
|
|
for slot in slots {
|
|
|
|
// Get shreds for the slot "slot"
|
|
|
|
let slot_shreds = &mut shreds
|
|
|
|
[(slot * entries_per_slot) as usize..((slot + 1) * entries_per_slot) as usize];
|
|
|
|
for shred in slot_shreds.iter_mut() {
|
|
|
|
// Get the parent slot of the slot in the tree
|
|
|
|
let slot_parent = {
|
|
|
|
if slot == 0 {
|
|
|
|
0
|
|
|
|
} else {
|
|
|
|
(slot - 1) / branching_factor
|
|
|
|
}
|
|
|
|
};
|
|
|
|
shred.set_parent(slot_parent);
|
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let shared_shreds: Vec<_> = slot_shreds
|
|
|
|
.iter()
|
|
|
|
.cloned()
|
|
|
|
.map(|shred| Arc::new(RwLock::new(shred)))
|
|
|
|
.collect();
|
|
|
|
let mut coding_generator = CodingGenerator::new_from_config(&erasure_config);
|
|
|
|
let coding_shreds = coding_generator.next(&shared_shreds);
|
|
|
|
assert_eq!(coding_shreds.len(), erasure_config.num_coding());
|
2019-02-13 15:01:56 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let mut rng = thread_rng();
|
2019-02-13 15:01:56 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Randomly pick whether to insert erasure or coding shreds first
|
|
|
|
if rng.gen_bool(0.5) {
|
|
|
|
blocktree.write_shreds(slot_shreds).unwrap();
|
|
|
|
blocktree.put_shared_coding_shreds(&coding_shreds).unwrap();
|
|
|
|
} else {
|
|
|
|
blocktree.put_shared_coding_shreds(&coding_shreds).unwrap();
|
|
|
|
blocktree.write_shreds(slot_shreds).unwrap();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Make sure everything chains correctly
|
|
|
|
let last_level =
|
|
|
|
(branching_factor.pow(num_tree_levels - 1) - 1) / (branching_factor - 1);
|
|
|
|
for slot in 0..num_slots {
|
|
|
|
let slot_meta = blocktree.meta(slot).unwrap().unwrap();
|
|
|
|
assert_eq!(slot_meta.consumed, entries_per_slot);
|
|
|
|
assert_eq!(slot_meta.received, entries_per_slot);
|
|
|
|
assert!(slot_meta.is_connected);
|
2019-02-13 15:01:56 -08:00
|
|
|
let slot_parent = {
|
2019-03-05 14:18:29 -08:00
|
|
|
if slot == 0 {
|
2019-02-13 15:01:56 -08:00
|
|
|
0
|
|
|
|
} else {
|
2019-03-05 14:18:29 -08:00
|
|
|
(slot - 1) / branching_factor
|
2019-02-13 15:01:56 -08:00
|
|
|
}
|
|
|
|
};
|
2019-09-03 21:32:51 -07:00
|
|
|
assert_eq!(slot_meta.parent_slot, slot_parent);
|
2019-02-13 15:01:56 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let expected_children: HashSet<_> = {
|
|
|
|
if slot >= last_level {
|
|
|
|
HashSet::new()
|
|
|
|
} else {
|
|
|
|
let first_child_slot = min(num_slots - 1, slot * branching_factor + 1);
|
|
|
|
let last_child_slot = min(num_slots - 1, (slot + 1) * branching_factor);
|
|
|
|
(first_child_slot..last_child_slot + 1).collect()
|
|
|
|
}
|
|
|
|
};
|
2019-02-13 15:01:56 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let result: HashSet<_> = slot_meta.next_slots.iter().cloned().collect();
|
|
|
|
if expected_children.len() != 0 {
|
|
|
|
assert_eq!(slot_meta.next_slots.len(), branching_factor as usize);
|
2019-02-13 15:01:56 -08:00
|
|
|
} else {
|
2019-09-03 21:32:51 -07:00
|
|
|
assert_eq!(slot_meta.next_slots.len(), 0);
|
2019-02-13 15:01:56 -08:00
|
|
|
}
|
2019-09-03 21:32:51 -07:00
|
|
|
assert_eq!(expected_children, result);
|
2019-02-13 15:01:56 -08:00
|
|
|
}
|
2019-09-03 21:32:51 -07:00
|
|
|
|
|
|
|
// No orphan slots should exist
|
|
|
|
assert!(blocktree.orphans_cf.is_empty().unwrap())
|
2019-02-13 15:01:56 -08:00
|
|
|
}
|
2019-03-29 16:07:24 -07:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2019-02-13 15:01:56 -08:00
|
|
|
}
|
2019-09-03 21:32:51 -07:00
|
|
|
*/
|
2019-02-07 15:10:54 -08:00
|
|
|
#[test]
|
|
|
|
pub fn test_get_slots_since() {
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree_path = get_tmp_ledger_path("test_get_slots_since");
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
{
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
// Slot doesn't exist
|
2019-02-07 20:52:39 -08:00
|
|
|
assert!(blocktree.get_slots_since(&vec![0]).unwrap().is_empty());
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-02-28 19:49:22 -08:00
|
|
|
let mut meta0 = SlotMeta::new(0, 0);
|
2019-05-03 14:46:02 -07:00
|
|
|
blocktree.meta_cf.put(0, &meta0).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
// Slot exists, chains to nothing
|
2019-02-28 19:49:22 -08:00
|
|
|
let expected: HashMap<u64, Vec<u64>> =
|
|
|
|
HashMap::from_iter(vec![(0, vec![])].into_iter());
|
|
|
|
assert_eq!(blocktree.get_slots_since(&vec![0]).unwrap(), expected);
|
2019-02-07 15:10:54 -08:00
|
|
|
meta0.next_slots = vec![1, 2];
|
2019-05-03 14:46:02 -07:00
|
|
|
blocktree.meta_cf.put(0, &meta0).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
// Slot exists, chains to some other slots
|
2019-02-28 19:49:22 -08:00
|
|
|
let expected: HashMap<u64, Vec<u64>> =
|
|
|
|
HashMap::from_iter(vec![(0, vec![1, 2])].into_iter());
|
|
|
|
assert_eq!(blocktree.get_slots_since(&vec![0]).unwrap(), expected);
|
|
|
|
assert_eq!(blocktree.get_slots_since(&vec![0, 1]).unwrap(), expected);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
let mut meta3 = SlotMeta::new(3, 1);
|
|
|
|
meta3.next_slots = vec![10, 5];
|
2019-05-03 14:46:02 -07:00
|
|
|
blocktree.meta_cf.put(3, &meta3).unwrap();
|
2019-02-28 19:49:22 -08:00
|
|
|
let expected: HashMap<u64, Vec<u64>> =
|
|
|
|
HashMap::from_iter(vec![(0, vec![1, 2]), (3, vec![10, 5])].into_iter());
|
|
|
|
assert_eq!(blocktree.get_slots_since(&vec![0, 1, 3]).unwrap(), expected);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-03-29 16:07:24 -07:00
|
|
|
#[test]
|
2019-04-06 19:41:22 -07:00
|
|
|
fn test_orphans() {
|
|
|
|
let blocktree_path = get_tmp_ledger_path("test_orphans");
|
2019-03-29 16:07:24 -07:00
|
|
|
{
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Create shreds and entries
|
2019-03-29 16:07:24 -07:00
|
|
|
let entries_per_slot = 1;
|
2019-09-03 21:32:51 -07:00
|
|
|
let (mut shreds, _) = make_many_slot_entries(0, 3, entries_per_slot);
|
|
|
|
let shreds_per_slot = shreds.len() / 3;
|
2019-03-29 16:07:24 -07:00
|
|
|
|
|
|
|
// Write slot 2, which chains to slot 1. We're missing slot 0,
|
2019-04-06 19:41:22 -07:00
|
|
|
// so slot 1 is the orphan
|
2019-09-03 21:32:51 -07:00
|
|
|
let shreds_for_slot = shreds.drain((shreds_per_slot * 2)..).collect_vec();
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds_for_slot, None).unwrap();
|
2019-03-29 16:07:24 -07:00
|
|
|
let meta = blocktree
|
|
|
|
.meta(1)
|
|
|
|
.expect("Expect database get to succeed")
|
|
|
|
.unwrap();
|
2019-04-26 08:52:10 -07:00
|
|
|
assert!(is_orphan(&meta));
|
2019-04-06 19:41:22 -07:00
|
|
|
assert_eq!(blocktree.get_orphans(None), vec![1]);
|
2019-03-29 16:07:24 -07:00
|
|
|
|
|
|
|
// Write slot 1 which chains to slot 0, so now slot 0 is the
|
2019-04-06 19:41:22 -07:00
|
|
|
// orphan, and slot 1 is no longer the orphan.
|
2019-09-03 21:32:51 -07:00
|
|
|
let shreds_for_slot = shreds.drain(shreds_per_slot..).collect_vec();
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds_for_slot, None).unwrap();
|
2019-03-29 16:07:24 -07:00
|
|
|
let meta = blocktree
|
|
|
|
.meta(1)
|
|
|
|
.expect("Expect database get to succeed")
|
|
|
|
.unwrap();
|
2019-04-26 08:52:10 -07:00
|
|
|
assert!(!is_orphan(&meta));
|
2019-03-29 16:07:24 -07:00
|
|
|
let meta = blocktree
|
|
|
|
.meta(0)
|
|
|
|
.expect("Expect database get to succeed")
|
|
|
|
.unwrap();
|
2019-04-26 08:52:10 -07:00
|
|
|
assert!(is_orphan(&meta));
|
2019-04-06 19:41:22 -07:00
|
|
|
assert_eq!(blocktree.get_orphans(None), vec![0]);
|
2019-03-29 16:07:24 -07:00
|
|
|
|
2019-04-06 19:41:22 -07:00
|
|
|
// Write some slot that also chains to existing slots and orphan,
|
2019-03-29 16:07:24 -07:00
|
|
|
// nothing should change
|
2019-09-03 21:32:51 -07:00
|
|
|
let (shred4, _) = make_slot_entries(4, 0, 1);
|
|
|
|
let (shred5, _) = make_slot_entries(5, 1, 1);
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shred4, None).unwrap();
|
|
|
|
blocktree.insert_shreds(shred5, None).unwrap();
|
2019-04-06 19:41:22 -07:00
|
|
|
assert_eq!(blocktree.get_orphans(None), vec![0]);
|
2019-03-29 16:07:24 -07:00
|
|
|
|
2019-04-06 19:41:22 -07:00
|
|
|
// Write zeroth slot, no more orphans
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds, None).unwrap();
|
2019-03-29 16:07:24 -07:00
|
|
|
for i in 0..3 {
|
|
|
|
let meta = blocktree
|
|
|
|
.meta(i)
|
|
|
|
.expect("Expect database get to succeed")
|
|
|
|
.unwrap();
|
2019-04-26 08:52:10 -07:00
|
|
|
assert!(!is_orphan(&meta));
|
2019-03-29 16:07:24 -07:00
|
|
|
}
|
2019-04-06 19:41:22 -07:00
|
|
|
// Orphans cf is empty
|
2019-05-03 14:46:02 -07:00
|
|
|
assert!(blocktree.orphans_cf.is_empty().unwrap())
|
2019-03-29 16:07:24 -07:00
|
|
|
}
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
2019-08-20 17:16:06 -07:00
|
|
|
fn test_insert_data_shreds_slots(name: &str, should_bulk_write: bool) {
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree_path = get_tmp_ledger_path(name);
|
2019-02-07 15:10:54 -08:00
|
|
|
{
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Create shreds and entries
|
2019-02-07 15:10:54 -08:00
|
|
|
let num_entries = 20 as u64;
|
2019-02-12 19:54:18 -08:00
|
|
|
let mut entries = vec![];
|
2019-08-20 17:16:06 -07:00
|
|
|
let mut shreds = vec![];
|
|
|
|
let mut num_shreds_per_slot = 0;
|
2019-03-05 14:18:29 -08:00
|
|
|
for slot in 0..num_entries {
|
2019-02-12 19:54:18 -08:00
|
|
|
let parent_slot = {
|
2019-03-05 14:18:29 -08:00
|
|
|
if slot == 0 {
|
2019-02-12 19:54:18 -08:00
|
|
|
0
|
|
|
|
} else {
|
2019-03-05 14:18:29 -08:00
|
|
|
slot - 1
|
2019-02-12 19:54:18 -08:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let (mut shred, entry) = make_slot_entries(slot, parent_slot, 1);
|
2019-08-20 17:16:06 -07:00
|
|
|
num_shreds_per_slot = shred.len() as u64;
|
|
|
|
shred
|
|
|
|
.iter_mut()
|
|
|
|
.enumerate()
|
|
|
|
.for_each(|(i, shred)| shred.set_index(slot as u32 + i as u32));
|
|
|
|
shreds.extend(shred);
|
2019-02-12 19:54:18 -08:00
|
|
|
entries.extend(entry);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
2019-08-20 17:16:06 -07:00
|
|
|
let num_shreds = shreds.len();
|
2019-09-03 21:32:51 -07:00
|
|
|
// Write shreds to the database
|
2019-02-07 15:10:54 -08:00
|
|
|
if should_bulk_write {
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds, None).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
} else {
|
2019-08-20 17:16:06 -07:00
|
|
|
for _ in 0..num_shreds {
|
|
|
|
let shred = shreds.remove(0);
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(vec![shred], None).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for i in 0..num_entries - 1 {
|
|
|
|
assert_eq!(
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree.get_slot_entries(i, i, None).unwrap()[0],
|
2019-02-12 19:54:18 -08:00
|
|
|
entries[i as usize]
|
2019-02-07 15:10:54 -08:00
|
|
|
);
|
|
|
|
|
2019-04-26 08:52:10 -07:00
|
|
|
let meta = blocktree.meta(i).unwrap().unwrap();
|
2019-08-20 17:16:06 -07:00
|
|
|
assert_eq!(meta.received, i + num_shreds_per_slot);
|
|
|
|
assert_eq!(meta.last_index, i + num_shreds_per_slot - 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
if i != 0 {
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(meta.parent_slot, i - 1);
|
2019-08-20 17:16:06 -07:00
|
|
|
assert_eq!(meta.consumed, 0);
|
2019-02-07 15:10:54 -08:00
|
|
|
} else {
|
2019-02-12 19:54:18 -08:00
|
|
|
assert_eq!(meta.parent_slot, 0);
|
2019-08-20 17:16:06 -07:00
|
|
|
assert_eq!(meta.consumed, num_shreds_per_slot);
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
2019-02-12 19:54:18 -08:00
|
|
|
|
2019-03-27 23:55:51 -07:00
|
|
|
#[test]
|
|
|
|
fn test_find_missing_data_indexes() {
|
|
|
|
let slot = 0;
|
|
|
|
let blocktree_path = get_tmp_ledger_path!();
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
|
|
|
|
// Write entries
|
2019-08-21 15:27:42 -07:00
|
|
|
let gap: u64 = 10;
|
2019-03-27 23:55:51 -07:00
|
|
|
assert!(gap > 3);
|
|
|
|
let num_entries = 10;
|
2019-08-21 15:27:42 -07:00
|
|
|
let entries = make_tiny_test_entries(num_entries);
|
|
|
|
let mut shreds = entries_to_test_shreds(entries, slot, 0, true);
|
|
|
|
let num_shreds = shreds.len();
|
|
|
|
for (i, b) in shreds.iter_mut().enumerate() {
|
|
|
|
b.set_index(i as u32 * gap as u32);
|
2019-03-27 23:55:51 -07:00
|
|
|
b.set_slot(slot);
|
|
|
|
}
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds, None).unwrap();
|
2019-03-27 23:55:51 -07:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Index of the first shred is 0
|
|
|
|
// Index of the second shred is "gap"
|
2019-03-27 23:55:51 -07:00
|
|
|
// Thus, the missing indexes should then be [1, gap - 1] for the input index
|
|
|
|
// range of [0, gap)
|
|
|
|
let expected: Vec<u64> = (1..gap).collect();
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(slot, 0, gap, gap as usize),
|
|
|
|
expected
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(slot, 1, gap, (gap - 1) as usize),
|
|
|
|
expected,
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(slot, 0, gap - 1, (gap - 1) as usize),
|
|
|
|
&expected[..expected.len() - 1],
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(slot, gap - 2, gap, gap as usize),
|
|
|
|
vec![gap - 2, gap - 1],
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(slot, gap - 2, gap, 1),
|
|
|
|
vec![gap - 2],
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(slot, 0, gap, 1),
|
|
|
|
vec![1],
|
|
|
|
);
|
|
|
|
|
|
|
|
// Test with end indexes that are greater than the last item in the ledger
|
|
|
|
let mut expected: Vec<u64> = (1..gap).collect();
|
|
|
|
expected.push(gap + 1);
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(slot, 0, gap + 2, (gap + 2) as usize),
|
|
|
|
expected,
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(slot, 0, gap + 2, (gap - 1) as usize),
|
|
|
|
&expected[..expected.len() - 1],
|
|
|
|
);
|
|
|
|
|
2019-08-21 15:27:42 -07:00
|
|
|
for i in 0..num_shreds as u64 {
|
2019-03-27 23:55:51 -07:00
|
|
|
for j in 0..i {
|
|
|
|
let expected: Vec<u64> = (j..i)
|
|
|
|
.flat_map(|k| {
|
|
|
|
let begin = k * gap + 1;
|
|
|
|
let end = (k + 1) * gap;
|
|
|
|
(begin..end)
|
|
|
|
})
|
|
|
|
.collect();
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(
|
|
|
|
slot,
|
|
|
|
j * gap,
|
|
|
|
i * gap,
|
|
|
|
((i - j) * gap) as usize
|
|
|
|
),
|
|
|
|
expected,
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
drop(blocktree);
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_find_missing_data_indexes_sanity() {
|
|
|
|
let slot = 0;
|
|
|
|
|
|
|
|
let blocktree_path = get_tmp_ledger_path!();
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
|
|
|
|
// Early exit conditions
|
|
|
|
let empty: Vec<u64> = vec![];
|
|
|
|
assert_eq!(blocktree.find_missing_data_indexes(slot, 0, 0, 1), empty);
|
|
|
|
assert_eq!(blocktree.find_missing_data_indexes(slot, 5, 5, 1), empty);
|
|
|
|
assert_eq!(blocktree.find_missing_data_indexes(slot, 4, 3, 1), empty);
|
|
|
|
assert_eq!(blocktree.find_missing_data_indexes(slot, 1, 2, 0), empty);
|
|
|
|
|
2019-08-21 15:27:42 -07:00
|
|
|
let entries = make_tiny_test_entries(20);
|
|
|
|
let mut shreds = entries_to_test_shreds(entries, slot, 0, true);
|
2019-08-26 18:27:45 -07:00
|
|
|
shreds.drain(2..);
|
2019-03-27 23:55:51 -07:00
|
|
|
|
|
|
|
const ONE: u64 = 1;
|
|
|
|
const OTHER: u64 = 4;
|
|
|
|
|
2019-08-21 15:27:42 -07:00
|
|
|
shreds[0].set_index(ONE as u32);
|
|
|
|
shreds[1].set_index(OTHER as u32);
|
2019-03-27 23:55:51 -07:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Insert one shred at index = first_index
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds, None).unwrap();
|
2019-03-27 23:55:51 -07:00
|
|
|
|
|
|
|
const STARTS: u64 = OTHER * 2;
|
|
|
|
const END: u64 = OTHER * 3;
|
|
|
|
const MAX: usize = 10;
|
2019-09-03 21:32:51 -07:00
|
|
|
// The first shred has index = first_index. Thus, for i < first_index,
|
2019-03-27 23:55:51 -07:00
|
|
|
// given the input range of [i, first_index], the missing indexes should be
|
|
|
|
// [i, first_index - 1]
|
|
|
|
for start in 0..STARTS {
|
|
|
|
let result = blocktree.find_missing_data_indexes(
|
|
|
|
slot, start, // start
|
|
|
|
END, //end
|
|
|
|
MAX, //max
|
|
|
|
);
|
|
|
|
let expected: Vec<u64> = (start..END).filter(|i| *i != ONE && *i != OTHER).collect();
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
}
|
|
|
|
|
|
|
|
drop(blocktree);
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
2019-09-03 21:32:51 -07:00
|
|
|
pub fn test_no_missing_shred_indexes() {
|
2019-03-27 23:55:51 -07:00
|
|
|
let slot = 0;
|
|
|
|
let blocktree_path = get_tmp_ledger_path!();
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
|
|
|
|
// Write entries
|
|
|
|
let num_entries = 10;
|
2019-08-21 15:27:42 -07:00
|
|
|
let entries = make_tiny_test_entries(num_entries);
|
|
|
|
let shreds = entries_to_test_shreds(entries, slot, 0, true);
|
|
|
|
let num_shreds = shreds.len();
|
2019-03-27 23:55:51 -07:00
|
|
|
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds, None).unwrap();
|
2019-03-27 23:55:51 -07:00
|
|
|
|
|
|
|
let empty: Vec<u64> = vec![];
|
2019-08-21 15:27:42 -07:00
|
|
|
for i in 0..num_shreds as u64 {
|
2019-03-27 23:55:51 -07:00
|
|
|
for j in 0..i {
|
|
|
|
assert_eq!(
|
|
|
|
blocktree.find_missing_data_indexes(slot, j, i, (i - j) as usize),
|
|
|
|
empty
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
drop(blocktree);
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
2019-04-25 00:04:49 -07:00
|
|
|
#[test]
|
2019-09-04 17:14:42 -07:00
|
|
|
pub fn test_should_insert_data_shred() {
|
|
|
|
let (shreds, _) = make_slot_entries(0, 0, 100);
|
2019-04-25 00:04:49 -07:00
|
|
|
let blocktree_path = get_tmp_ledger_path!();
|
2019-09-03 21:32:51 -07:00
|
|
|
{
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2019-09-04 17:14:42 -07:00
|
|
|
let index_cf = blocktree.db.column::<cf::Index>();
|
|
|
|
let last_root = RwLock::new(0);
|
2019-04-25 00:04:49 -07:00
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
// Insert the first 5 shreds, we don't have a "is_last" shred yet
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree
|
|
|
|
.insert_shreds(shreds[0..5].to_vec(), None)
|
|
|
|
.unwrap();
|
2019-04-25 00:04:49 -07:00
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
// Trying to insert a shred less than `slot_meta.consumed` should fail
|
2019-09-03 21:32:51 -07:00
|
|
|
let slot_meta = blocktree.meta(0).unwrap().unwrap();
|
2019-09-04 17:14:42 -07:00
|
|
|
let index = index_cf.get(0).unwrap().unwrap();
|
2019-09-03 21:32:51 -07:00
|
|
|
assert_eq!(slot_meta.consumed, 5);
|
2019-09-04 17:14:42 -07:00
|
|
|
assert!(!Blocktree::should_insert_data_shred(
|
|
|
|
&shreds[1],
|
|
|
|
&slot_meta,
|
|
|
|
index.data(),
|
|
|
|
&last_root
|
|
|
|
));
|
2019-09-03 21:32:51 -07:00
|
|
|
|
|
|
|
// Trying to insert the same shred again should fail
|
2019-09-04 17:14:42 -07:00
|
|
|
// skip over shred 5 so the `slot_meta.consumed` doesn't increment
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree
|
|
|
|
.insert_shreds(shreds[6..7].to_vec(), None)
|
|
|
|
.unwrap();
|
2019-09-03 21:32:51 -07:00
|
|
|
let slot_meta = blocktree.meta(0).unwrap().unwrap();
|
2019-09-04 17:14:42 -07:00
|
|
|
let index = index_cf.get(0).unwrap().unwrap();
|
|
|
|
assert!(!Blocktree::should_insert_data_shred(
|
|
|
|
&shreds[6],
|
|
|
|
&slot_meta,
|
|
|
|
index.data(),
|
|
|
|
&last_root
|
|
|
|
));
|
2019-09-03 21:32:51 -07:00
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
// Trying to insert another "is_last" shred with index < the received index should fail
|
|
|
|
// skip over shred 7
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree
|
|
|
|
.insert_shreds(shreds[8..9].to_vec(), None)
|
|
|
|
.unwrap();
|
2019-09-03 21:32:51 -07:00
|
|
|
let slot_meta = blocktree.meta(0).unwrap().unwrap();
|
2019-09-04 17:14:42 -07:00
|
|
|
let index = index_cf.get(0).unwrap().unwrap();
|
2019-09-03 21:32:51 -07:00
|
|
|
assert_eq!(slot_meta.received, 9);
|
2019-09-04 17:14:42 -07:00
|
|
|
let shred7 = {
|
|
|
|
if let Shred::Data(ref s) = shreds[7] {
|
|
|
|
Shred::LastInSlot(s.clone())
|
|
|
|
} else {
|
|
|
|
panic!("Shred in unexpected format")
|
|
|
|
}
|
|
|
|
};
|
|
|
|
assert!(!Blocktree::should_insert_data_shred(
|
|
|
|
&shred7,
|
|
|
|
&slot_meta,
|
|
|
|
index.data(),
|
|
|
|
&last_root
|
|
|
|
));
|
2019-09-03 21:32:51 -07:00
|
|
|
|
|
|
|
// Insert all pending shreds
|
2019-09-04 17:14:42 -07:00
|
|
|
let mut shred8 = shreds[8].clone();
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds, None).unwrap();
|
2019-09-03 21:32:51 -07:00
|
|
|
let slot_meta = blocktree.meta(0).unwrap().unwrap();
|
2019-09-04 17:14:42 -07:00
|
|
|
let index = index_cf.get(0).unwrap().unwrap();
|
2019-09-03 21:32:51 -07:00
|
|
|
|
|
|
|
// Trying to insert a shred with index > the "is_last" shred should fail
|
2019-09-04 17:14:42 -07:00
|
|
|
if let Shred::Data(ref mut s) = shred8 {
|
|
|
|
s.header.common_header.slot = slot_meta.last_index + 1;
|
|
|
|
} else {
|
|
|
|
panic!("Shred in unexpected format")
|
|
|
|
}
|
|
|
|
assert!(!Blocktree::should_insert_data_shred(
|
|
|
|
&shred7,
|
2019-09-03 21:32:51 -07:00
|
|
|
&slot_meta,
|
2019-09-04 17:14:42 -07:00
|
|
|
index.data(),
|
|
|
|
&last_root
|
2019-09-03 21:32:51 -07:00
|
|
|
));
|
|
|
|
}
|
2019-04-25 00:04:49 -07:00
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2019-09-04 17:14:42 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
pub fn test_should_insert_coding_shred() {
|
|
|
|
let blocktree_path = get_tmp_ledger_path!();
|
|
|
|
{
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
let index_cf = blocktree.db.column::<cf::Index>();
|
|
|
|
let last_root = RwLock::new(0);
|
|
|
|
|
|
|
|
let mut shred = CodingShred::default();
|
|
|
|
let slot = 1;
|
|
|
|
shred.header.position = 10;
|
|
|
|
shred.header.common_header.index = 11;
|
|
|
|
shred.header.common_header.slot = 1;
|
|
|
|
shred.header.num_coding_shreds = shred.header.position + 1;
|
|
|
|
let coding_shred = Shred::Coding(shred.clone());
|
|
|
|
|
|
|
|
// Insert a good coding shred
|
|
|
|
assert!(Blocktree::should_insert_coding_shred(
|
|
|
|
&coding_shred,
|
|
|
|
Index::new(slot).coding(),
|
|
|
|
&last_root
|
|
|
|
));
|
|
|
|
|
|
|
|
// Insertion should succeed
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree
|
|
|
|
.insert_shreds(vec![coding_shred.clone()], None)
|
|
|
|
.unwrap();
|
2019-09-04 17:14:42 -07:00
|
|
|
|
|
|
|
// Trying to insert the same shred again should fail
|
|
|
|
{
|
|
|
|
let index = index_cf
|
|
|
|
.get(shred.header.common_header.slot)
|
|
|
|
.unwrap()
|
|
|
|
.unwrap();
|
|
|
|
assert!(!Blocktree::should_insert_coding_shred(
|
|
|
|
&coding_shred,
|
|
|
|
index.coding(),
|
|
|
|
&last_root
|
|
|
|
));
|
|
|
|
}
|
|
|
|
|
|
|
|
shred.header.common_header.index += 1;
|
|
|
|
|
|
|
|
// Establish a baseline that works
|
|
|
|
{
|
|
|
|
let index = index_cf
|
|
|
|
.get(shred.header.common_header.slot)
|
|
|
|
.unwrap()
|
|
|
|
.unwrap();
|
|
|
|
assert!(Blocktree::should_insert_coding_shred(
|
|
|
|
&Shred::Coding(shred.clone()),
|
|
|
|
index.coding(),
|
|
|
|
&last_root
|
|
|
|
));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Trying to insert a shred with index < position should fail
|
|
|
|
{
|
|
|
|
let mut shred_ = shred.clone();
|
|
|
|
shred_.header.common_header.index = (shred_.header.position - 1).into();
|
|
|
|
let index = index_cf
|
|
|
|
.get(shred_.header.common_header.slot)
|
|
|
|
.unwrap()
|
|
|
|
.unwrap();
|
|
|
|
assert!(!Blocktree::should_insert_coding_shred(
|
|
|
|
&Shred::Coding(shred_),
|
|
|
|
index.coding(),
|
|
|
|
&last_root
|
|
|
|
));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Trying to insert shred with num_coding == 0 should fail
|
|
|
|
{
|
|
|
|
let mut shred_ = shred.clone();
|
|
|
|
shred_.header.num_coding_shreds = 0;
|
|
|
|
let index = index_cf
|
|
|
|
.get(shred_.header.common_header.slot)
|
|
|
|
.unwrap()
|
|
|
|
.unwrap();
|
|
|
|
assert!(!Blocktree::should_insert_coding_shred(
|
|
|
|
&Shred::Coding(shred_),
|
|
|
|
index.coding(),
|
|
|
|
&last_root
|
|
|
|
));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Trying to insert shred with pos >= num_coding should fail
|
|
|
|
{
|
|
|
|
let mut shred_ = shred.clone();
|
|
|
|
shred_.header.num_coding_shreds = shred_.header.position;
|
|
|
|
let index = index_cf
|
|
|
|
.get(shred_.header.common_header.slot)
|
|
|
|
.unwrap()
|
|
|
|
.unwrap();
|
|
|
|
assert!(!Blocktree::should_insert_coding_shred(
|
|
|
|
&Shred::Coding(shred_),
|
|
|
|
index.coding(),
|
|
|
|
&last_root
|
|
|
|
));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Trying to insert with set_index with num_coding that would imply the last blob
|
|
|
|
// has index > u32::MAX should fail
|
|
|
|
{
|
|
|
|
let mut shred_ = shred.clone();
|
|
|
|
shred_.header.num_coding_shreds = 3;
|
|
|
|
shred_.header.common_header.index = std::u32::MAX - 1;
|
|
|
|
shred_.header.position = 0;
|
|
|
|
let index = index_cf
|
|
|
|
.get(shred_.header.common_header.slot)
|
|
|
|
.unwrap()
|
|
|
|
.unwrap();
|
|
|
|
assert!(!Blocktree::should_insert_coding_shred(
|
|
|
|
&Shred::Coding(shred_.clone()),
|
|
|
|
index.coding(),
|
|
|
|
&last_root
|
|
|
|
));
|
|
|
|
|
|
|
|
// Decreasing the number of num_coding_shreds will put it within the allowed limit
|
|
|
|
shred_.header.num_coding_shreds = 2;
|
|
|
|
let coding_shred = Shred::Coding(shred_);
|
|
|
|
assert!(Blocktree::should_insert_coding_shred(
|
|
|
|
&coding_shred,
|
|
|
|
index.coding(),
|
|
|
|
&last_root
|
|
|
|
));
|
|
|
|
|
|
|
|
// Insertion should succeed
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(vec![coding_shred], None).unwrap();
|
2019-09-04 17:14:42 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// Trying to insert value into slot <= than last root should fail
|
|
|
|
{
|
|
|
|
let mut shred_ = shred.clone();
|
|
|
|
let index = index_cf
|
|
|
|
.get(shred_.header.common_header.slot)
|
|
|
|
.unwrap()
|
|
|
|
.unwrap();
|
|
|
|
shred_.header.common_header.slot = *last_root.read().unwrap();
|
|
|
|
assert!(!Blocktree::should_insert_coding_shred(
|
|
|
|
&Shred::Coding(shred_),
|
|
|
|
index.coding(),
|
|
|
|
&last_root
|
|
|
|
));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
2019-04-25 00:04:49 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
pub fn test_insert_multiple_is_last() {
|
2019-09-03 21:32:51 -07:00
|
|
|
let (shreds, _) = make_slot_entries(0, 0, 20);
|
|
|
|
let num_shreds = shreds.len() as u64;
|
2019-04-25 00:04:49 -07:00
|
|
|
let blocktree_path = get_tmp_ledger_path!();
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds, None).unwrap();
|
2019-09-03 21:32:51 -07:00
|
|
|
let slot_meta = blocktree.meta(0).unwrap().unwrap();
|
|
|
|
|
|
|
|
assert_eq!(slot_meta.consumed, num_shreds);
|
|
|
|
assert_eq!(slot_meta.received, num_shreds);
|
|
|
|
assert_eq!(slot_meta.last_index, num_shreds - 1);
|
|
|
|
assert!(slot_meta.is_full());
|
2019-04-25 00:04:49 -07:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let (shreds, _) = make_slot_entries(0, 0, 22);
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds, None).unwrap();
|
2019-04-25 00:04:49 -07:00
|
|
|
let slot_meta = blocktree.meta(0).unwrap().unwrap();
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
assert_eq!(slot_meta.consumed, num_shreds);
|
|
|
|
assert_eq!(slot_meta.received, num_shreds);
|
|
|
|
assert_eq!(slot_meta.last_index, num_shreds - 1);
|
2019-04-25 00:04:49 -07:00
|
|
|
assert!(slot_meta.is_full());
|
|
|
|
|
|
|
|
drop(blocktree);
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
2019-05-13 22:04:54 -07:00
|
|
|
#[test]
|
|
|
|
fn test_slot_data_iterator() {
|
2019-09-03 21:32:51 -07:00
|
|
|
// Construct the shreds
|
2019-05-13 22:04:54 -07:00
|
|
|
let blocktree_path = get_tmp_ledger_path!();
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2019-09-03 21:32:51 -07:00
|
|
|
let shreds_per_slot = 10;
|
2019-05-13 22:04:54 -07:00
|
|
|
let slots = vec![2, 4, 8, 12];
|
2019-09-03 21:32:51 -07:00
|
|
|
let all_shreds = make_chaining_slot_entries(&slots, shreds_per_slot);
|
|
|
|
let slot_8_shreds = bincode::serialize(&all_shreds[2].0).unwrap();
|
|
|
|
for (slot_shreds, _) in all_shreds {
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(slot_shreds, None).unwrap();
|
2019-05-13 22:04:54 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// Slot doesnt exist, iterator should be empty
|
2019-09-03 21:32:51 -07:00
|
|
|
let shred_iter = blocktree.slot_data_iterator(5).unwrap();
|
|
|
|
let result: Vec<_> = shred_iter.collect();
|
2019-05-13 22:04:54 -07:00
|
|
|
assert_eq!(result, vec![]);
|
|
|
|
|
|
|
|
// Test that the iterator for slot 8 contains what was inserted earlier
|
2019-09-03 21:32:51 -07:00
|
|
|
let shred_iter = blocktree.slot_data_iterator(8).unwrap();
|
|
|
|
let result: Vec<Shred> = shred_iter
|
|
|
|
.map(|(_, bytes)| {
|
|
|
|
let shred: Shred = bincode::deserialize(&bytes).unwrap();
|
|
|
|
shred
|
|
|
|
})
|
|
|
|
.collect();
|
|
|
|
let result_serialized = bincode::serialize(&result).unwrap();
|
|
|
|
assert_eq!(result_serialized.len(), slot_8_shreds.len());
|
|
|
|
assert_eq!(result_serialized, slot_8_shreds);
|
2019-05-13 22:04:54 -07:00
|
|
|
|
|
|
|
drop(blocktree);
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
2019-05-20 19:04:18 -07:00
|
|
|
#[test]
|
2019-05-29 09:43:22 -07:00
|
|
|
fn test_set_roots() {
|
2019-05-20 19:04:18 -07:00
|
|
|
let blocktree_path = get_tmp_ledger_path!();
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
let chained_slots = vec![0, 2, 4, 7, 12, 15];
|
2019-08-27 15:09:41 -07:00
|
|
|
assert_eq!(blocktree.last_root(), 0);
|
2019-05-20 19:04:18 -07:00
|
|
|
|
2019-05-29 09:43:22 -07:00
|
|
|
blocktree.set_roots(&chained_slots).unwrap();
|
2019-05-20 19:04:18 -07:00
|
|
|
|
2019-08-27 15:09:41 -07:00
|
|
|
assert_eq!(blocktree.last_root(), 15);
|
|
|
|
|
2019-05-20 19:04:18 -07:00
|
|
|
for i in chained_slots {
|
|
|
|
assert!(blocktree.is_root(i));
|
|
|
|
}
|
|
|
|
|
|
|
|
drop(blocktree);
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
2019-07-17 14:42:29 -07:00
|
|
|
#[test]
|
|
|
|
fn test_prune() {
|
|
|
|
let blocktree_path = get_tmp_ledger_path!();
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2019-09-03 21:32:51 -07:00
|
|
|
let (shreds, _) = make_many_slot_entries(0, 50, 6);
|
|
|
|
let shreds_per_slot = shreds.len() as u64 / 50;
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds, None).unwrap();
|
2019-07-17 14:42:29 -07:00
|
|
|
blocktree
|
|
|
|
.slot_meta_iterator(0)
|
|
|
|
.unwrap()
|
2019-09-03 21:32:51 -07:00
|
|
|
.for_each(|(_, meta)| assert_eq!(meta.last_index, shreds_per_slot - 1));
|
2019-07-17 14:42:29 -07:00
|
|
|
|
|
|
|
blocktree.prune(5);
|
|
|
|
|
|
|
|
blocktree
|
|
|
|
.slot_meta_iterator(0)
|
|
|
|
.unwrap()
|
|
|
|
.for_each(|(slot, meta)| {
|
|
|
|
assert!(slot <= 5);
|
2019-09-03 21:32:51 -07:00
|
|
|
assert_eq!(meta.last_index, shreds_per_slot - 1)
|
2019-07-17 14:42:29 -07:00
|
|
|
});
|
|
|
|
|
2019-08-27 15:09:41 -07:00
|
|
|
let data_iter = blocktree
|
2019-09-03 21:32:51 -07:00
|
|
|
.data_shred_cf
|
2019-08-27 15:09:41 -07:00
|
|
|
.iter(IteratorMode::From((0, 0), IteratorDirection::Forward))
|
|
|
|
.unwrap();
|
2019-07-17 14:42:29 -07:00
|
|
|
for ((slot, _), _) in data_iter {
|
|
|
|
if slot > 5 {
|
|
|
|
assert!(false);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
drop(blocktree);
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
2019-07-20 13:13:55 -07:00
|
|
|
#[test]
|
|
|
|
fn test_purge_slots() {
|
|
|
|
let blocktree_path = get_tmp_ledger_path!();
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2019-09-03 21:32:51 -07:00
|
|
|
let (shreds, _) = make_many_slot_entries(0, 50, 5);
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds, None).unwrap();
|
2019-07-20 13:13:55 -07:00
|
|
|
|
|
|
|
blocktree.purge_slots(0, Some(5));
|
|
|
|
|
|
|
|
blocktree
|
|
|
|
.slot_meta_iterator(0)
|
|
|
|
.unwrap()
|
|
|
|
.for_each(|(slot, _)| {
|
|
|
|
assert!(slot > 5);
|
|
|
|
});
|
|
|
|
|
|
|
|
blocktree.purge_slots(0, None);
|
|
|
|
|
|
|
|
blocktree.slot_meta_iterator(0).unwrap().for_each(|(_, _)| {
|
|
|
|
assert!(false);
|
|
|
|
});
|
|
|
|
|
|
|
|
drop(blocktree);
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
2019-07-24 17:28:08 -07:00
|
|
|
#[test]
|
|
|
|
fn test_purge_huge() {
|
|
|
|
let blocktree_path = get_tmp_ledger_path!();
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
2019-09-03 21:32:51 -07:00
|
|
|
let (shreds, _) = make_many_slot_entries(0, 5000, 10);
|
2019-09-05 18:20:30 -07:00
|
|
|
blocktree.insert_shreds(shreds, None).unwrap();
|
2019-07-24 17:28:08 -07:00
|
|
|
|
|
|
|
blocktree.purge_slots(0, Some(4999));
|
|
|
|
|
|
|
|
blocktree
|
|
|
|
.slot_meta_iterator(0)
|
|
|
|
.unwrap()
|
|
|
|
.for_each(|(slot, _)| {
|
|
|
|
assert_eq!(slot, 5000);
|
|
|
|
});
|
|
|
|
|
|
|
|
drop(blocktree);
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
2019-07-17 14:42:29 -07:00
|
|
|
#[should_panic]
|
|
|
|
#[test]
|
|
|
|
fn test_prune_out_of_bounds() {
|
|
|
|
let blocktree_path = get_tmp_ledger_path!();
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
|
|
|
|
// slot 5 does not exist, prune should panic
|
|
|
|
blocktree.prune(5);
|
|
|
|
|
|
|
|
drop(blocktree);
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_iter_bounds() {
|
|
|
|
let blocktree_path = get_tmp_ledger_path!();
|
|
|
|
let blocktree = Blocktree::open(&blocktree_path).unwrap();
|
|
|
|
|
|
|
|
// slot 5 does not exist, iter should be ok and should be a noop
|
|
|
|
blocktree
|
|
|
|
.slot_meta_iterator(5)
|
|
|
|
.unwrap()
|
|
|
|
.for_each(|_| assert!(false));
|
|
|
|
|
|
|
|
drop(blocktree);
|
|
|
|
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
pub fn make_slot_entries(
|
2019-08-20 17:16:06 -07:00
|
|
|
slot: u64,
|
|
|
|
parent_slot: u64,
|
|
|
|
num_entries: u64,
|
|
|
|
) -> (Vec<Shred>, Vec<Entry>) {
|
|
|
|
let entries = make_tiny_test_entries(num_entries as usize);
|
|
|
|
let shreds = entries_to_test_shreds(entries.clone(), slot, parent_slot, true);
|
|
|
|
(shreds, entries)
|
|
|
|
}
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
pub fn make_many_slot_entries(
|
2019-08-20 17:16:06 -07:00
|
|
|
start_slot: u64,
|
|
|
|
num_slots: u64,
|
|
|
|
entries_per_slot: u64,
|
|
|
|
) -> (Vec<Shred>, Vec<Entry>) {
|
|
|
|
let mut shreds = vec![];
|
|
|
|
let mut entries = vec![];
|
|
|
|
for slot in start_slot..start_slot + num_slots {
|
|
|
|
let parent_slot = if slot == 0 { 0 } else { slot - 1 };
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let (slot_shreds, slot_entries) =
|
|
|
|
make_slot_entries(slot, parent_slot, entries_per_slot);
|
|
|
|
shreds.extend(slot_shreds);
|
2019-08-20 17:16:06 -07:00
|
|
|
entries.extend(slot_entries);
|
|
|
|
}
|
|
|
|
|
|
|
|
(shreds, entries)
|
|
|
|
}
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Create shreds for slots that have a parent-child relationship defined by the input `chain`
|
|
|
|
pub fn make_chaining_slot_entries(
|
2019-08-20 17:16:06 -07:00
|
|
|
chain: &[u64],
|
|
|
|
entries_per_slot: u64,
|
|
|
|
) -> Vec<(Vec<Shred>, Vec<Entry>)> {
|
|
|
|
let mut slots_shreds_and_entries = vec![];
|
|
|
|
for (i, slot) in chain.iter().enumerate() {
|
|
|
|
let parent_slot = {
|
2019-08-27 15:09:41 -07:00
|
|
|
if *slot == 0 || i == 0 {
|
2019-08-20 17:16:06 -07:00
|
|
|
0
|
|
|
|
} else {
|
|
|
|
chain[i - 1]
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let result = make_slot_entries(*slot, parent_slot, entries_per_slot);
|
2019-08-20 17:16:06 -07:00
|
|
|
slots_shreds_and_entries.push(result);
|
|
|
|
}
|
|
|
|
|
|
|
|
slots_shreds_and_entries
|
|
|
|
}
|
2018-11-15 15:53:31 -08:00
|
|
|
}
|