2019-07-11 13:58:33 -07:00
|
|
|
use crate::erasure::ErasureConfig;
|
2019-10-18 09:28:51 -07:00
|
|
|
use serde::{Deserialize, Serialize};
|
2020-04-09 20:21:31 -07:00
|
|
|
use solana_sdk::clock::Slot;
|
2019-10-22 08:20:19 -07:00
|
|
|
use std::{collections::BTreeSet, ops::RangeBounds};
|
2019-04-11 14:14:57 -07:00
|
|
|
|
|
|
|
#[derive(Clone, Debug, Default, Deserialize, Serialize, Eq, PartialEq)]
|
|
|
|
// The Meta column family
|
|
|
|
pub struct SlotMeta {
|
|
|
|
// The number of slots above the root (the genesis block). The first
|
|
|
|
// slot has slot 0.
|
2019-11-02 00:38:30 -07:00
|
|
|
pub slot: Slot,
|
2019-11-14 11:49:31 -08:00
|
|
|
// The total number of consecutive shreds starting from index 0
|
2019-04-11 14:14:57 -07:00
|
|
|
// we have received for this slot.
|
|
|
|
pub consumed: u64,
|
2019-11-07 11:08:09 -08:00
|
|
|
// The index *plus one* of the highest shred received for this slot. Useful
|
|
|
|
// for checking if the slot has received any shreds yet, and to calculate the
|
2019-04-11 14:14:57 -07:00
|
|
|
// range where there is one or more holes: `(consumed..received)`.
|
|
|
|
pub received: u64,
|
2019-11-07 11:08:09 -08:00
|
|
|
// The timestamp of the first time a shred was added for this slot
|
|
|
|
pub first_shred_timestamp: u64,
|
|
|
|
// The index of the shred that is flagged as the last shred for this slot.
|
2019-04-11 14:14:57 -07:00
|
|
|
pub last_index: u64,
|
|
|
|
// The slot height of the block this one derives from.
|
2019-11-02 00:38:30 -07:00
|
|
|
pub parent_slot: Slot,
|
|
|
|
// The list of slots, each of which contains a block that derives
|
2019-04-11 14:14:57 -07:00
|
|
|
// from this one.
|
2019-11-02 00:38:30 -07:00
|
|
|
pub next_slots: Vec<Slot>,
|
2019-04-11 14:14:57 -07:00
|
|
|
// True if this slot is full (consumed == last_index + 1) and if every
|
|
|
|
// slot that is a parent of this slot is also connected.
|
|
|
|
pub is_connected: bool,
|
2019-10-21 16:15:10 -07:00
|
|
|
// List of start indexes for completed data slots
|
|
|
|
pub completed_data_indexes: Vec<u32>,
|
2019-04-11 14:14:57 -07:00
|
|
|
}
|
|
|
|
|
2019-07-10 11:08:17 -07:00
|
|
|
#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)]
|
2019-11-07 11:08:09 -08:00
|
|
|
/// Index recording presence/absence of shreds
|
2019-07-10 11:08:17 -07:00
|
|
|
pub struct Index {
|
2019-11-02 00:38:30 -07:00
|
|
|
pub slot: Slot,
|
2020-01-13 12:03:19 -08:00
|
|
|
data: ShredIndex,
|
|
|
|
coding: ShredIndex,
|
2019-07-10 11:08:17 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)]
|
2020-01-13 12:03:19 -08:00
|
|
|
pub struct ShredIndex {
|
|
|
|
/// Map representing presence/absence of shreds
|
2020-04-24 15:04:23 -07:00
|
|
|
index: BTreeSet<u64>,
|
2019-07-10 11:08:17 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, Eq, PartialEq)]
|
|
|
|
/// Erasure coding information
|
|
|
|
pub struct ErasureMeta {
|
|
|
|
/// Which erasure set in the slot this is
|
|
|
|
pub set_index: u64,
|
2019-12-12 16:50:29 -08:00
|
|
|
/// First coding index in the FEC set
|
|
|
|
pub first_coding_index: u64,
|
2019-07-10 11:08:17 -07:00
|
|
|
/// Size of shards in this erasure set
|
|
|
|
pub size: usize,
|
2019-07-11 13:58:33 -07:00
|
|
|
/// Erasure configuration for this erasure set
|
2019-08-22 16:32:38 -07:00
|
|
|
pub config: ErasureConfig,
|
2019-07-10 11:08:17 -07:00
|
|
|
}
|
|
|
|
|
2020-01-13 17:21:39 -08:00
|
|
|
#[derive(Deserialize, Serialize)]
|
|
|
|
pub struct DuplicateSlotProof {
|
|
|
|
#[serde(with = "serde_bytes")]
|
|
|
|
pub shred1: Vec<u8>,
|
|
|
|
#[serde(with = "serde_bytes")]
|
|
|
|
pub shred2: Vec<u8>,
|
|
|
|
}
|
|
|
|
|
2019-07-10 11:08:17 -07:00
|
|
|
#[derive(Debug, PartialEq)]
|
|
|
|
pub enum ErasureMetaStatus {
|
|
|
|
CanRecover,
|
|
|
|
DataFull,
|
|
|
|
StillNeed(usize),
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Index {
|
2019-11-02 00:38:30 -07:00
|
|
|
pub(crate) fn new(slot: Slot) -> Self {
|
2019-07-10 11:08:17 -07:00
|
|
|
Index {
|
|
|
|
slot,
|
2020-01-13 12:03:19 -08:00
|
|
|
data: ShredIndex::default(),
|
|
|
|
coding: ShredIndex::default(),
|
2019-07-10 11:08:17 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-01-13 12:03:19 -08:00
|
|
|
pub fn data(&self) -> &ShredIndex {
|
2019-07-10 11:08:17 -07:00
|
|
|
&self.data
|
|
|
|
}
|
2020-01-13 12:03:19 -08:00
|
|
|
pub fn coding(&self) -> &ShredIndex {
|
2019-07-10 11:08:17 -07:00
|
|
|
&self.coding
|
|
|
|
}
|
|
|
|
|
2020-01-13 12:03:19 -08:00
|
|
|
pub fn data_mut(&mut self) -> &mut ShredIndex {
|
2019-07-10 11:08:17 -07:00
|
|
|
&mut self.data
|
|
|
|
}
|
2020-01-13 12:03:19 -08:00
|
|
|
pub fn coding_mut(&mut self) -> &mut ShredIndex {
|
2019-07-10 11:08:17 -07:00
|
|
|
&mut self.coding
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-01-13 12:03:19 -08:00
|
|
|
impl ShredIndex {
|
|
|
|
pub fn num_shreds(&self) -> usize {
|
2019-12-09 00:13:36 -08:00
|
|
|
self.index.len()
|
|
|
|
}
|
|
|
|
|
2019-07-10 11:08:17 -07:00
|
|
|
pub fn present_in_bounds(&self, bounds: impl RangeBounds<u64>) -> usize {
|
|
|
|
self.index.range(bounds).count()
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn is_present(&self, index: u64) -> bool {
|
|
|
|
self.index.contains(&index)
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn set_present(&mut self, index: u64, presence: bool) {
|
|
|
|
if presence {
|
|
|
|
self.index.insert(index);
|
|
|
|
} else {
|
|
|
|
self.index.remove(&index);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn set_many_present(&mut self, presence: impl IntoIterator<Item = (u64, bool)>) {
|
|
|
|
for (idx, present) in presence.into_iter() {
|
|
|
|
self.set_present(idx, present);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-04-11 14:14:57 -07:00
|
|
|
impl SlotMeta {
|
|
|
|
pub fn is_full(&self) -> bool {
|
|
|
|
// last_index is std::u64::MAX when it has no information about how
|
2019-11-07 11:08:09 -08:00
|
|
|
// many shreds will fill this slot.
|
|
|
|
// Note: A full slot with zero shreds is not possible.
|
2019-04-11 14:14:57 -07:00
|
|
|
if self.last_index == std::u64::MAX {
|
|
|
|
return false;
|
|
|
|
}
|
2019-04-25 00:04:49 -07:00
|
|
|
|
|
|
|
// Should never happen
|
|
|
|
if self.consumed > self.last_index + 1 {
|
2020-02-14 11:11:55 -08:00
|
|
|
datapoint_error!(
|
2020-01-13 13:13:52 -08:00
|
|
|
"blockstore_error",
|
2019-05-10 08:33:58 -07:00
|
|
|
(
|
|
|
|
"error",
|
|
|
|
format!(
|
|
|
|
"Observed a slot meta with consumed: {} > meta.last_index + 1: {}",
|
|
|
|
self.consumed,
|
|
|
|
self.last_index + 1
|
|
|
|
),
|
|
|
|
String
|
|
|
|
)
|
2019-04-25 00:04:49 -07:00
|
|
|
);
|
|
|
|
}
|
2019-04-11 14:14:57 -07:00
|
|
|
|
|
|
|
self.consumed == self.last_index + 1
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn is_parent_set(&self) -> bool {
|
|
|
|
self.parent_slot != std::u64::MAX
|
|
|
|
}
|
|
|
|
|
2020-05-05 14:07:21 -07:00
|
|
|
pub fn clear_unconfirmed_slot(&mut self) {
|
|
|
|
let mut new_self = SlotMeta::new_orphan(self.slot);
|
|
|
|
std::mem::swap(&mut new_self.next_slots, &mut self.next_slots);
|
|
|
|
std::mem::swap(self, &mut new_self);
|
|
|
|
}
|
|
|
|
|
2019-11-02 00:38:30 -07:00
|
|
|
pub(crate) fn new(slot: Slot, parent_slot: Slot) -> Self {
|
2019-04-11 14:14:57 -07:00
|
|
|
SlotMeta {
|
|
|
|
slot,
|
|
|
|
consumed: 0,
|
|
|
|
received: 0,
|
2019-11-07 11:08:09 -08:00
|
|
|
first_shred_timestamp: 0,
|
2019-04-11 14:14:57 -07:00
|
|
|
parent_slot,
|
|
|
|
next_slots: vec![],
|
|
|
|
is_connected: slot == 0,
|
|
|
|
last_index: std::u64::MAX,
|
2019-10-21 16:15:10 -07:00
|
|
|
completed_data_indexes: vec![],
|
2019-04-11 14:14:57 -07:00
|
|
|
}
|
|
|
|
}
|
2020-05-05 14:07:21 -07:00
|
|
|
|
|
|
|
pub(crate) fn new_orphan(slot: Slot) -> Self {
|
|
|
|
Self::new(slot, std::u64::MAX)
|
|
|
|
}
|
2019-04-11 14:14:57 -07:00
|
|
|
}
|
|
|
|
|
2019-06-20 20:15:33 -07:00
|
|
|
impl ErasureMeta {
|
2019-12-12 16:50:29 -08:00
|
|
|
pub fn new(set_index: u64, first_coding_index: u64, config: &ErasureConfig) -> ErasureMeta {
|
2019-07-11 13:58:33 -07:00
|
|
|
ErasureMeta {
|
|
|
|
set_index,
|
2019-12-12 16:50:29 -08:00
|
|
|
first_coding_index,
|
2019-07-11 13:58:33 -07:00
|
|
|
size: 0,
|
|
|
|
config: *config,
|
|
|
|
}
|
2019-04-11 14:14:57 -07:00
|
|
|
}
|
|
|
|
|
2019-07-10 11:08:17 -07:00
|
|
|
pub fn status(&self, index: &Index) -> ErasureMetaStatus {
|
|
|
|
use ErasureMetaStatus::*;
|
2019-06-20 20:15:33 -07:00
|
|
|
|
2019-12-12 16:50:29 -08:00
|
|
|
let num_coding = index.coding().present_in_bounds(
|
|
|
|
self.first_coding_index..self.first_coding_index + self.config.num_coding() as u64,
|
|
|
|
);
|
|
|
|
let num_data = index
|
|
|
|
.data()
|
|
|
|
.present_in_bounds(self.set_index..self.set_index + self.config.num_data() as u64);
|
2019-07-10 11:08:17 -07:00
|
|
|
|
2020-02-04 15:45:01 -08:00
|
|
|
let (data_missing, num_needed) = (
|
|
|
|
self.config.num_data().saturating_sub(num_data),
|
|
|
|
self.config.num_data().saturating_sub(num_data + num_coding),
|
2019-07-11 13:58:33 -07:00
|
|
|
);
|
2019-06-20 20:15:33 -07:00
|
|
|
|
2020-02-04 15:45:01 -08:00
|
|
|
if data_missing == 0 {
|
2020-01-21 11:48:09 -08:00
|
|
|
DataFull
|
2020-02-04 15:45:01 -08:00
|
|
|
} else if num_needed == 0 {
|
|
|
|
CanRecover
|
2019-06-20 20:15:33 -07:00
|
|
|
} else {
|
2020-02-04 15:45:01 -08:00
|
|
|
StillNeed(num_needed)
|
2019-06-20 20:15:33 -07:00
|
|
|
}
|
2019-04-19 20:22:51 -07:00
|
|
|
}
|
|
|
|
|
2019-06-20 20:15:33 -07:00
|
|
|
pub fn set_size(&mut self, size: usize) {
|
|
|
|
self.size = size;
|
2019-04-19 20:22:51 -07:00
|
|
|
}
|
|
|
|
|
2019-06-20 20:15:33 -07:00
|
|
|
pub fn size(&self) -> usize {
|
|
|
|
self.size
|
2019-04-11 14:14:57 -07:00
|
|
|
}
|
2019-06-20 20:15:33 -07:00
|
|
|
}
|
2019-04-18 21:56:43 -07:00
|
|
|
|
2020-01-13 17:21:39 -08:00
|
|
|
impl DuplicateSlotProof {
|
|
|
|
pub(crate) fn new(shred1: Vec<u8>, shred2: Vec<u8>) -> Self {
|
|
|
|
DuplicateSlotProof { shred1, shred2 }
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-04-04 20:24:06 -07:00
|
|
|
#[derive(Debug, Default, Deserialize, Serialize, PartialEq)]
|
|
|
|
pub struct TransactionStatusIndexMeta {
|
|
|
|
pub max_slot: Slot,
|
|
|
|
pub frozen: bool,
|
|
|
|
}
|
|
|
|
|
2020-04-08 12:50:39 -07:00
|
|
|
#[derive(Debug, Default, Deserialize, Serialize, PartialEq)]
|
|
|
|
pub struct AddressSignatureMeta {
|
|
|
|
pub writeable: bool,
|
|
|
|
}
|
|
|
|
|
2020-09-22 12:26:32 -07:00
|
|
|
#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)]
|
|
|
|
pub struct PerfSample {
|
|
|
|
pub num_transactions: u64,
|
|
|
|
pub num_slots: u64,
|
|
|
|
pub sample_period_secs: u16,
|
|
|
|
}
|
|
|
|
|
2019-07-10 11:08:17 -07:00
|
|
|
#[cfg(test)]
|
|
|
|
mod test {
|
|
|
|
use super::*;
|
2019-06-20 20:15:33 -07:00
|
|
|
use rand::{seq::SliceRandom, thread_rng};
|
2019-07-10 11:08:17 -07:00
|
|
|
use std::iter::repeat;
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-07-10 11:08:17 -07:00
|
|
|
#[test]
|
|
|
|
fn test_erasure_meta_status() {
|
|
|
|
use ErasureMetaStatus::*;
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-07-10 11:08:17 -07:00
|
|
|
let set_index = 0;
|
2019-07-11 13:58:33 -07:00
|
|
|
let erasure_config = ErasureConfig::default();
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-12-12 16:50:29 -08:00
|
|
|
let mut e_meta = ErasureMeta::new(set_index, set_index, &erasure_config);
|
2019-07-10 11:08:17 -07:00
|
|
|
let mut rng = thread_rng();
|
|
|
|
let mut index = Index::new(0);
|
|
|
|
e_meta.size = 1;
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-07-11 13:58:33 -07:00
|
|
|
let data_indexes = 0..erasure_config.num_data() as u64;
|
|
|
|
let coding_indexes = 0..erasure_config.num_coding() as u64;
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-07-11 13:58:33 -07:00
|
|
|
assert_eq!(e_meta.status(&index), StillNeed(erasure_config.num_data()));
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-07-10 11:08:17 -07:00
|
|
|
index
|
|
|
|
.data_mut()
|
|
|
|
.set_many_present(data_indexes.clone().zip(repeat(true)));
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-07-10 11:08:17 -07:00
|
|
|
assert_eq!(e_meta.status(&index), DataFull);
|
2019-04-11 14:14:57 -07:00
|
|
|
|
2019-07-10 11:08:17 -07:00
|
|
|
index
|
|
|
|
.coding_mut()
|
|
|
|
.set_many_present(coding_indexes.clone().zip(repeat(true)));
|
2019-04-18 21:56:43 -07:00
|
|
|
|
2019-07-10 11:08:17 -07:00
|
|
|
for &idx in data_indexes
|
|
|
|
.clone()
|
|
|
|
.collect::<Vec<_>>()
|
2019-07-11 13:58:33 -07:00
|
|
|
.choose_multiple(&mut rng, erasure_config.num_data())
|
2019-07-10 11:08:17 -07:00
|
|
|
{
|
|
|
|
index.data_mut().set_present(idx, false);
|
2019-04-18 21:56:43 -07:00
|
|
|
|
2019-07-10 11:08:17 -07:00
|
|
|
assert_eq!(e_meta.status(&index), CanRecover);
|
|
|
|
}
|
2019-04-18 21:56:43 -07:00
|
|
|
|
2019-07-10 11:08:17 -07:00
|
|
|
index
|
|
|
|
.data_mut()
|
|
|
|
.set_many_present(data_indexes.zip(repeat(true)));
|
2019-06-20 20:15:33 -07:00
|
|
|
|
2019-07-10 11:08:17 -07:00
|
|
|
for &idx in coding_indexes
|
|
|
|
.collect::<Vec<_>>()
|
2019-07-11 13:58:33 -07:00
|
|
|
.choose_multiple(&mut rng, erasure_config.num_coding())
|
2019-07-10 11:08:17 -07:00
|
|
|
{
|
|
|
|
index.coding_mut().set_present(idx, false);
|
2019-06-20 20:15:33 -07:00
|
|
|
|
2019-07-10 11:08:17 -07:00
|
|
|
assert_eq!(e_meta.status(&index), DataFull);
|
|
|
|
}
|
2019-04-18 21:56:43 -07:00
|
|
|
}
|
2020-05-05 14:07:21 -07:00
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_clear_unconfirmed_slot() {
|
|
|
|
let mut slot_meta = SlotMeta::new_orphan(5);
|
|
|
|
slot_meta.consumed = 5;
|
|
|
|
slot_meta.received = 5;
|
|
|
|
slot_meta.next_slots = vec![6, 7];
|
|
|
|
slot_meta.clear_unconfirmed_slot();
|
|
|
|
|
|
|
|
let mut expected = SlotMeta::new_orphan(5);
|
|
|
|
expected.next_slots = vec![6, 7];
|
|
|
|
assert_eq!(slot_meta, expected);
|
|
|
|
}
|
2019-04-11 14:14:57 -07:00
|
|
|
}
|