2021-04-16 12:04:46 -07:00
|
|
|
//! The `shred` module defines data structures and methods to pull MTU sized data frames from the
|
|
|
|
//! network. There are two types of shreds: data and coding. Data shreds contain entry information
|
|
|
|
//! while coding shreds provide redundancy to protect against dropped network packets (erasures).
|
|
|
|
//!
|
|
|
|
//! +---------------------------------------------------------------------------------------------+
|
|
|
|
//! | Data Shred |
|
|
|
|
//! +---------------------------------------------------------------------------------------------+
|
|
|
|
//! | common | data | payload |
|
|
|
|
//! | header | header | |
|
|
|
|
//! |+---+---+--- |+---+---+---|+----------------------------------------------------------+----+|
|
|
|
|
//! || s | s | . || p | f | s || data (ie ledger entries) | r ||
|
|
|
|
//! || i | h | . || a | l | i || | e ||
|
|
|
|
//! || g | r | . || r | a | z || See notes immediately after shred diagrams for an | s ||
|
|
|
|
//! || n | e | || e | g | e || explanation of the "restricted" section in this payload | t ||
|
|
|
|
//! || a | d | || n | s | || | r ||
|
|
|
|
//! || t | | || t | | || | i ||
|
|
|
|
//! || u | t | || | | || | c ||
|
|
|
|
//! || r | y | || o | | || | t ||
|
|
|
|
//! || e | p | || f | | || | e ||
|
|
|
|
//! || | e | || f | | || | d ||
|
|
|
|
//! |+---+---+--- |+---+---+---+|----------------------------------------------------------+----+|
|
|
|
|
//! +---------------------------------------------------------------------------------------------+
|
|
|
|
//!
|
|
|
|
//! +---------------------------------------------------------------------------------------------+
|
|
|
|
//! | Coding Shred |
|
|
|
|
//! +---------------------------------------------------------------------------------------------+
|
|
|
|
//! | common | coding | payload |
|
|
|
|
//! | header | header | |
|
|
|
|
//! |+---+---+--- |+---+---+---+----------------------------------------------------------------+|
|
|
|
|
//! || s | s | . || n | n | p || data (encoded data shred data) ||
|
|
|
|
//! || i | h | . || u | u | o || ||
|
|
|
|
//! || g | r | . || m | m | s || ||
|
|
|
|
//! || n | e | || | | i || ||
|
|
|
|
//! || a | d | || d | c | t || ||
|
|
|
|
//! || t | | || | | i || ||
|
|
|
|
//! || u | t | || s | s | o || ||
|
|
|
|
//! || r | y | || h | h | n || ||
|
|
|
|
//! || e | p | || r | r | || ||
|
|
|
|
//! || | e | || e | e | || ||
|
|
|
|
//! || | | || d | d | || ||
|
|
|
|
//! |+---+---+--- |+---+---+---+|+--------------------------------------------------------------+|
|
|
|
|
//! +---------------------------------------------------------------------------------------------+
|
|
|
|
//!
|
|
|
|
//! Notes:
|
|
|
|
//! a) Coding shreds encode entire data shreds: both of the headers AND the payload.
|
|
|
|
//! b) Coding shreds require their own headers for identification and etc.
|
|
|
|
//! c) The erasure algorithm requires data shred and coding shred bytestreams to be equal in length.
|
|
|
|
//!
|
|
|
|
//! So, given a) - c), we must restrict data shred's payload length such that the entire coding
|
|
|
|
//! payload can fit into one coding shred / packet.
|
|
|
|
|
2021-09-01 08:44:26 -07:00
|
|
|
use {
|
2022-05-05 16:48:00 -07:00
|
|
|
self::traits::{Shred as _, ShredCode as _, ShredData as _},
|
2022-04-27 15:00:04 -07:00
|
|
|
crate::blockstore::MAX_DATA_SHREDS_PER_SLOT,
|
2022-05-01 12:25:15 -07:00
|
|
|
bitflags::bitflags,
|
2022-04-23 06:33:59 -07:00
|
|
|
num_enum::{IntoPrimitive, TryFromPrimitive},
|
|
|
|
serde::{Deserialize, Serialize},
|
2021-09-01 08:44:26 -07:00
|
|
|
solana_entry::entry::{create_ticks, Entry},
|
2022-05-05 16:48:00 -07:00
|
|
|
solana_perf::packet::{limited_deserialize, Packet},
|
2021-09-01 08:44:26 -07:00
|
|
|
solana_sdk::{
|
|
|
|
clock::Slot,
|
|
|
|
hash::{hashv, Hash},
|
|
|
|
packet::PACKET_DATA_SIZE,
|
|
|
|
pubkey::Pubkey,
|
|
|
|
signature::{Keypair, Signature, Signer},
|
|
|
|
},
|
2022-05-17 15:44:35 -07:00
|
|
|
static_assertions::const_assert_eq,
|
2022-05-05 16:48:00 -07:00
|
|
|
std::fmt::Debug,
|
2021-09-01 08:44:26 -07:00
|
|
|
thiserror::Error,
|
2019-11-02 00:38:30 -07:00
|
|
|
};
|
2022-05-05 16:48:00 -07:00
|
|
|
pub use {
|
|
|
|
self::{
|
|
|
|
legacy::{ShredCode, ShredData},
|
|
|
|
stats::{ProcessShredsStats, ShredFetchStats},
|
|
|
|
},
|
|
|
|
crate::shredder::Shredder,
|
|
|
|
};
|
|
|
|
|
|
|
|
mod legacy;
|
|
|
|
mod stats;
|
|
|
|
mod traits;
|
2019-08-02 15:53:42 -07:00
|
|
|
|
2020-05-19 12:38:18 -07:00
|
|
|
pub type Nonce = u32;
|
|
|
|
|
2019-10-21 12:46:16 -07:00
|
|
|
/// The following constants are computed by hand, and hardcoded.
|
|
|
|
/// `test_shred_constants` ensures that the values are correct.
|
|
|
|
/// Constants are used over lazy_static for performance reasons.
|
2022-04-25 05:43:22 -07:00
|
|
|
const SIZE_OF_COMMON_SHRED_HEADER: usize = 83;
|
|
|
|
const SIZE_OF_DATA_SHRED_HEADER: usize = 5;
|
|
|
|
const SIZE_OF_CODING_SHRED_HEADER: usize = 6;
|
|
|
|
const SIZE_OF_SIGNATURE: usize = 64;
|
|
|
|
const SIZE_OF_SHRED_TYPE: usize = 1;
|
|
|
|
const SIZE_OF_SHRED_SLOT: usize = 8;
|
|
|
|
const SIZE_OF_SHRED_INDEX: usize = 4;
|
2020-05-19 12:38:18 -07:00
|
|
|
pub const SIZE_OF_NONCE: usize = 4;
|
2022-05-17 15:44:35 -07:00
|
|
|
const_assert_eq!(SIZE_OF_CODING_SHRED_HEADERS, 89);
|
2022-04-25 05:43:22 -07:00
|
|
|
const SIZE_OF_CODING_SHRED_HEADERS: usize =
|
2019-10-21 12:46:16 -07:00
|
|
|
SIZE_OF_COMMON_SHRED_HEADER + SIZE_OF_CODING_SHRED_HEADER;
|
2022-05-17 15:44:35 -07:00
|
|
|
// Maximum size of data that a data-shred may contain (excluding headers).
|
|
|
|
const_assert_eq!(SIZE_OF_DATA_SHRED_PAYLOAD, 1051);
|
2019-10-21 12:46:16 -07:00
|
|
|
pub const SIZE_OF_DATA_SHRED_PAYLOAD: usize = PACKET_DATA_SIZE
|
|
|
|
- SIZE_OF_COMMON_SHRED_HEADER
|
|
|
|
- SIZE_OF_DATA_SHRED_HEADER
|
2021-04-16 12:04:46 -07:00
|
|
|
- SIZE_OF_CODING_SHRED_HEADERS
|
2020-05-19 12:38:18 -07:00
|
|
|
- SIZE_OF_NONCE;
|
2022-05-17 15:44:35 -07:00
|
|
|
const_assert_eq!(SHRED_DATA_OFFSET, 88);
|
2022-04-25 05:43:22 -07:00
|
|
|
const SHRED_DATA_OFFSET: usize = SIZE_OF_COMMON_SHRED_HEADER + SIZE_OF_DATA_SHRED_HEADER;
|
|
|
|
|
|
|
|
const OFFSET_OF_SHRED_TYPE: usize = SIZE_OF_SIGNATURE;
|
|
|
|
const OFFSET_OF_SHRED_SLOT: usize = SIZE_OF_SIGNATURE + SIZE_OF_SHRED_TYPE;
|
|
|
|
const OFFSET_OF_SHRED_INDEX: usize = OFFSET_OF_SHRED_SLOT + SIZE_OF_SHRED_SLOT;
|
2022-05-17 15:44:35 -07:00
|
|
|
const_assert_eq!(SHRED_PAYLOAD_SIZE, 1228);
|
2022-04-28 16:42:37 -07:00
|
|
|
const SHRED_PAYLOAD_SIZE: usize = PACKET_DATA_SIZE - SIZE_OF_NONCE;
|
2019-09-18 18:00:07 -07:00
|
|
|
|
2019-11-07 16:38:06 -08:00
|
|
|
pub const MAX_DATA_SHREDS_PER_FEC_BLOCK: u32 = 32;
|
2019-09-19 16:29:52 -07:00
|
|
|
|
2022-05-02 16:33:53 -07:00
|
|
|
// LAST_SHRED_IN_SLOT also implies DATA_COMPLETE_SHRED.
|
|
|
|
// So it cannot be LAST_SHRED_IN_SLOT if not also DATA_COMPLETE_SHRED.
|
2022-05-01 12:25:15 -07:00
|
|
|
bitflags! {
|
|
|
|
#[derive(Default, Serialize, Deserialize)]
|
|
|
|
pub struct ShredFlags:u8 {
|
|
|
|
const SHRED_TICK_REFERENCE_MASK = 0b0011_1111;
|
2022-05-02 16:33:53 -07:00
|
|
|
const DATA_COMPLETE_SHRED = 0b0100_0000;
|
|
|
|
const LAST_SHRED_IN_SLOT = 0b1100_0000;
|
2022-05-01 12:25:15 -07:00
|
|
|
}
|
|
|
|
}
|
2019-09-19 16:29:52 -07:00
|
|
|
|
2022-04-25 16:19:37 -07:00
|
|
|
#[derive(Debug, Error)]
|
|
|
|
pub enum Error {
|
|
|
|
#[error(transparent)]
|
|
|
|
BincodeError(#[from] bincode::Error),
|
2022-04-27 15:00:04 -07:00
|
|
|
#[error(transparent)]
|
|
|
|
ErasureError(#[from] reed_solomon_erasure::Error),
|
2022-04-29 12:42:15 -07:00
|
|
|
#[error("Invalid data shred index: {0}")]
|
|
|
|
InvalidDataShredIndex(/*shred index:*/ u32),
|
2022-04-25 16:19:37 -07:00
|
|
|
#[error("Invalid data size: {size}, payload: {payload}")]
|
|
|
|
InvalidDataSize { size: u16, payload: usize },
|
2022-04-29 12:42:15 -07:00
|
|
|
#[error("Invalid erasure shard index: {0:?}")]
|
|
|
|
InvalidErasureShardIndex(/*headers:*/ Box<dyn Debug>),
|
2022-04-25 16:19:37 -07:00
|
|
|
#[error("Invalid num coding shreds: {0}")]
|
|
|
|
InvalidNumCodingShreds(u16),
|
|
|
|
#[error("Invalid parent_offset: {parent_offset}, slot: {slot}")]
|
2020-04-19 21:15:09 -07:00
|
|
|
InvalidParentOffset { slot: Slot, parent_offset: u16 },
|
2022-04-25 16:19:37 -07:00
|
|
|
#[error("Invalid parent slot: {parent_slot}, slot: {slot}")]
|
|
|
|
InvalidParentSlot { slot: Slot, parent_slot: Slot },
|
2022-04-29 12:42:15 -07:00
|
|
|
#[error("Invalid payload size: {0}")]
|
|
|
|
InvalidPayloadSize(/*payload size:*/ usize),
|
2022-05-02 16:33:53 -07:00
|
|
|
#[error("Invalid shred flags: {0}")]
|
|
|
|
InvalidShredFlags(u8),
|
2022-04-28 16:42:37 -07:00
|
|
|
#[error("Invalid shred type")]
|
|
|
|
InvalidShredType,
|
2019-10-17 15:44:15 -07:00
|
|
|
}
|
|
|
|
|
2021-11-16 09:50:56 -08:00
|
|
|
#[repr(u8)]
|
2022-04-23 06:33:59 -07:00
|
|
|
#[derive(
|
|
|
|
Clone,
|
|
|
|
Copy,
|
|
|
|
Debug,
|
|
|
|
Eq,
|
|
|
|
Hash,
|
|
|
|
PartialEq,
|
|
|
|
AbiEnumVisitor,
|
|
|
|
AbiExample,
|
|
|
|
Deserialize,
|
|
|
|
IntoPrimitive,
|
|
|
|
Serialize,
|
|
|
|
TryFromPrimitive,
|
|
|
|
)]
|
|
|
|
#[serde(into = "u8", try_from = "u8")]
|
2021-11-16 09:50:56 -08:00
|
|
|
pub enum ShredType {
|
|
|
|
Data = 0b1010_0101,
|
|
|
|
Code = 0b0101_1010,
|
|
|
|
}
|
|
|
|
|
2019-10-15 20:48:45 -07:00
|
|
|
/// A common header that is present in data and code shred headers
|
2022-04-29 12:42:15 -07:00
|
|
|
#[derive(Clone, Copy, Debug, PartialEq, Deserialize, Serialize)]
|
2022-04-25 05:43:22 -07:00
|
|
|
struct ShredCommonHeader {
|
|
|
|
signature: Signature,
|
|
|
|
shred_type: ShredType,
|
|
|
|
slot: Slot,
|
|
|
|
index: u32,
|
|
|
|
version: u16,
|
|
|
|
fec_set_index: u32,
|
2019-09-19 16:29:52 -07:00
|
|
|
}
|
|
|
|
|
2019-10-15 20:48:45 -07:00
|
|
|
/// The data shred header has parent offset and flags
|
2022-05-05 16:48:00 -07:00
|
|
|
#[derive(Clone, Copy, Debug, PartialEq, Deserialize, Serialize)]
|
2022-04-25 05:43:22 -07:00
|
|
|
struct DataShredHeader {
|
|
|
|
parent_offset: u16,
|
2022-05-01 12:25:15 -07:00
|
|
|
flags: ShredFlags,
|
2022-04-25 05:43:22 -07:00
|
|
|
size: u16, // common shred header + data shred header + data
|
2019-09-19 16:29:52 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/// The coding shred header has FEC information
|
2022-05-05 16:48:00 -07:00
|
|
|
#[derive(Clone, Copy, Debug, PartialEq, Deserialize, Serialize)]
|
2022-04-25 05:43:22 -07:00
|
|
|
struct CodingShredHeader {
|
|
|
|
num_data_shreds: u16,
|
|
|
|
num_coding_shreds: u16,
|
|
|
|
position: u16,
|
2019-09-19 16:29:52 -07:00
|
|
|
}
|
|
|
|
|
2019-09-17 18:22:46 -07:00
|
|
|
#[derive(Clone, Debug, PartialEq)]
|
2022-05-05 16:48:00 -07:00
|
|
|
pub enum Shred {
|
|
|
|
ShredCode(ShredCode),
|
|
|
|
ShredData(ShredData),
|
2019-09-16 20:28:54 -07:00
|
|
|
}
|
|
|
|
|
2021-12-16 06:18:55 -08:00
|
|
|
/// Tuple which uniquely identifies a shred should it exists.
|
2021-12-14 09:34:02 -08:00
|
|
|
#[derive(Clone, Copy, Eq, Hash, PartialEq)]
|
|
|
|
pub struct ShredId(Slot, /*shred index:*/ u32, ShredType);
|
|
|
|
|
|
|
|
impl ShredId {
|
|
|
|
pub(crate) fn new(slot: Slot, index: u32, shred_type: ShredType) -> ShredId {
|
|
|
|
ShredId(slot, index, shred_type)
|
|
|
|
}
|
|
|
|
|
|
|
|
pub(crate) fn unwrap(&self) -> (Slot, /*shred index:*/ u32, ShredType) {
|
|
|
|
(self.0, self.1, self.2)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-12-16 06:18:55 -08:00
|
|
|
/// Tuple which identifies erasure coding set that the shred belongs to.
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
|
|
|
pub(crate) struct ErasureSetId(Slot, /*fec_set_index:*/ u32);
|
|
|
|
|
|
|
|
impl ErasureSetId {
|
|
|
|
pub(crate) fn slot(&self) -> Slot {
|
|
|
|
self.0
|
|
|
|
}
|
|
|
|
|
|
|
|
// Storage key for ErasureMeta in blockstore db.
|
|
|
|
pub(crate) fn store_key(&self) -> (Slot, /*fec_set_index:*/ u64) {
|
|
|
|
(self.0, u64::from(self.1))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-05 16:48:00 -07:00
|
|
|
macro_rules! dispatch {
|
|
|
|
($vis:vis fn $name:ident(&self $(, $arg:ident : $ty:ty)?) $(-> $out:ty)?) => {
|
|
|
|
#[inline]
|
|
|
|
$vis fn $name(&self $(, $arg:$ty)?) $(-> $out)? {
|
|
|
|
match self {
|
|
|
|
Self::ShredCode(shred) => shred.$name($($arg, )?),
|
|
|
|
Self::ShredData(shred) => shred.$name($($arg, )?),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
};
|
|
|
|
($vis:vis fn $name:ident(self $(, $arg:ident : $ty:ty)?) $(-> $out:ty)?) => {
|
|
|
|
#[inline]
|
|
|
|
$vis fn $name(self $(, $arg:$ty)?) $(-> $out)? {
|
|
|
|
match self {
|
|
|
|
Self::ShredCode(shred) => shred.$name($($arg, )?),
|
|
|
|
Self::ShredData(shred) => shred.$name($($arg, )?),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
};
|
|
|
|
($vis:vis fn $name:ident(&mut self $(, $arg:ident : $ty:ty)?) $(-> $out:ty)?) => {
|
|
|
|
#[inline]
|
|
|
|
$vis fn $name(&mut self $(, $arg:$ty)?) $(-> $out)? {
|
|
|
|
match self {
|
|
|
|
Self::ShredCode(shred) => shred.$name($($arg, )?),
|
|
|
|
Self::ShredData(shred) => shred.$name($($arg, )?),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-09-18 16:24:30 -07:00
|
|
|
impl Shred {
|
2022-05-05 16:48:00 -07:00
|
|
|
dispatch!(fn common_header(&self) -> &ShredCommonHeader);
|
|
|
|
dispatch!(fn set_signature(&mut self, signature: Signature));
|
|
|
|
dispatch!(fn signed_payload(&self) -> &[u8]);
|
|
|
|
|
|
|
|
// Returns the portion of the shred's payload which is erasure coded.
|
|
|
|
dispatch!(pub(crate) fn erasure_shard(self) -> Result<Vec<u8>, Error>);
|
|
|
|
// Like Shred::erasure_shard but returning a slice.
|
|
|
|
dispatch!(pub(crate) fn erasure_shard_as_slice(&self) -> Result<&[u8], Error>);
|
|
|
|
// Returns the shard index within the erasure coding set.
|
|
|
|
dispatch!(pub(crate) fn erasure_shard_index(&self) -> Option<usize>);
|
|
|
|
|
|
|
|
dispatch!(pub fn into_payload(self) -> Vec<u8>);
|
|
|
|
dispatch!(pub fn payload(&self) -> &Vec<u8>);
|
|
|
|
dispatch!(pub fn sanitize(&self) -> Result<(), Error>);
|
|
|
|
|
|
|
|
// Only for tests.
|
|
|
|
dispatch!(pub fn set_index(&mut self, index: u32));
|
|
|
|
dispatch!(pub fn set_slot(&mut self, slot: Slot));
|
|
|
|
|
2019-12-12 13:27:33 -08:00
|
|
|
pub fn copy_to_packet(&self, packet: &mut Packet) {
|
2022-05-05 16:48:00 -07:00
|
|
|
let payload = self.payload();
|
|
|
|
let size = payload.len();
|
|
|
|
packet.data[..size].copy_from_slice(&payload[..]);
|
|
|
|
packet.meta.size = size;
|
2019-12-12 13:27:33 -08:00
|
|
|
}
|
|
|
|
|
2022-04-25 05:43:22 -07:00
|
|
|
// TODO: Should this sanitize output?
|
2019-10-08 00:42:51 -07:00
|
|
|
pub fn new_from_data(
|
2019-11-02 00:38:30 -07:00
|
|
|
slot: Slot,
|
2019-10-08 00:42:51 -07:00
|
|
|
index: u32,
|
|
|
|
parent_offset: u16,
|
2022-04-27 11:04:10 -07:00
|
|
|
data: &[u8],
|
2022-05-02 16:33:53 -07:00
|
|
|
flags: ShredFlags,
|
2019-11-06 13:27:58 -08:00
|
|
|
reference_tick: u8,
|
2019-11-18 18:05:02 -08:00
|
|
|
version: u16,
|
2019-12-12 16:50:29 -08:00
|
|
|
fec_set_index: u32,
|
2019-10-08 00:42:51 -07:00
|
|
|
) -> Self {
|
2022-05-05 16:48:00 -07:00
|
|
|
Self::from(ShredData::new_from_data(
|
2019-11-18 18:05:02 -08:00
|
|
|
slot,
|
|
|
|
index,
|
2022-05-02 16:33:53 -07:00
|
|
|
parent_offset,
|
2022-05-05 16:48:00 -07:00
|
|
|
data,
|
2022-05-02 16:33:53 -07:00
|
|
|
flags,
|
2022-05-05 16:48:00 -07:00
|
|
|
reference_tick,
|
|
|
|
version,
|
|
|
|
fec_set_index,
|
|
|
|
))
|
2019-10-18 22:55:59 -07:00
|
|
|
}
|
2019-10-08 00:42:51 -07:00
|
|
|
|
2022-05-05 16:48:00 -07:00
|
|
|
pub fn new_from_serialized_shred(shred: Vec<u8>) -> Result<Self, Error> {
|
|
|
|
Ok(match Self::shred_type_from_payload(&shred)? {
|
|
|
|
ShredType::Code => Self::from(ShredCode::from_payload(shred)?),
|
|
|
|
ShredType::Data => Self::from(ShredData::from_payload(shred)?),
|
|
|
|
})
|
2019-09-17 18:22:46 -07:00
|
|
|
}
|
|
|
|
|
2022-04-27 11:04:10 -07:00
|
|
|
pub fn new_from_parity_shard(
|
2022-04-19 13:00:05 -07:00
|
|
|
slot: Slot,
|
|
|
|
index: u32,
|
2022-04-27 11:04:10 -07:00
|
|
|
parity_shard: &[u8],
|
2022-04-19 13:00:05 -07:00
|
|
|
fec_set_index: u32,
|
|
|
|
num_data_shreds: u16,
|
|
|
|
num_coding_shreds: u16,
|
|
|
|
position: u16,
|
|
|
|
version: u16,
|
2022-04-25 05:43:22 -07:00
|
|
|
) -> Self {
|
2022-05-05 16:48:00 -07:00
|
|
|
Self::from(ShredCode::new_from_parity_shard(
|
2022-04-19 13:00:05 -07:00
|
|
|
slot,
|
2022-05-05 16:48:00 -07:00
|
|
|
index,
|
|
|
|
parity_shard,
|
2022-04-19 13:00:05 -07:00
|
|
|
fec_set_index,
|
2022-04-25 05:43:22 -07:00
|
|
|
num_data_shreds,
|
|
|
|
num_coding_shreds,
|
2021-12-05 06:42:09 -08:00
|
|
|
position,
|
2022-05-05 16:48:00 -07:00
|
|
|
version,
|
|
|
|
))
|
2019-09-16 20:28:54 -07:00
|
|
|
}
|
|
|
|
|
2021-12-14 09:34:02 -08:00
|
|
|
/// Unique identifier for each shred.
|
|
|
|
pub fn id(&self) -> ShredId {
|
|
|
|
ShredId(self.slot(), self.index(), self.shred_type())
|
|
|
|
}
|
|
|
|
|
2019-11-02 00:38:30 -07:00
|
|
|
pub fn slot(&self) -> Slot {
|
2022-05-05 16:48:00 -07:00
|
|
|
self.common_header().slot
|
2019-09-16 20:28:54 -07:00
|
|
|
}
|
|
|
|
|
2022-04-25 16:19:37 -07:00
|
|
|
pub fn parent(&self) -> Result<Slot, Error> {
|
2022-05-05 16:48:00 -07:00
|
|
|
match self {
|
|
|
|
Self::ShredCode(_) => Err(Error::InvalidShredType),
|
|
|
|
Self::ShredData(shred) => shred.parent(),
|
2019-09-16 20:28:54 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn index(&self) -> u32 {
|
2022-05-05 16:48:00 -07:00
|
|
|
self.common_header().index
|
2019-09-16 20:28:54 -07:00
|
|
|
}
|
|
|
|
|
2022-04-27 15:00:04 -07:00
|
|
|
pub(crate) fn data(&self) -> Result<&[u8], Error> {
|
2022-05-05 16:48:00 -07:00
|
|
|
match self {
|
|
|
|
Self::ShredCode(_) => Err(Error::InvalidShredType),
|
|
|
|
Self::ShredData(shred) => shred.data(),
|
2022-04-27 15:00:04 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-04-25 05:43:22 -07:00
|
|
|
// Possibly trimmed payload;
|
|
|
|
// Should only be used when storing shreds to blockstore.
|
|
|
|
pub(crate) fn bytes_to_store(&self) -> &[u8] {
|
2022-05-05 16:48:00 -07:00
|
|
|
match self {
|
|
|
|
Self::ShredCode(shred) => shred.payload(),
|
|
|
|
Self::ShredData(shred) => shred.bytes_to_store(),
|
2022-04-25 05:43:22 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-04-28 16:42:37 -07:00
|
|
|
// Possibly zero pads bytes stored in blockstore.
|
2022-05-05 16:48:00 -07:00
|
|
|
pub(crate) fn resize_stored_shred(shred: Vec<u8>) -> Result<Vec<u8>, Error> {
|
|
|
|
match Self::shred_type_from_payload(&shred)? {
|
|
|
|
ShredType::Code => ShredCode::resize_stored_shred(shred),
|
|
|
|
ShredType::Data => ShredData::resize_stored_shred(shred),
|
2022-04-28 16:42:37 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-04-25 05:43:22 -07:00
|
|
|
pub fn fec_set_index(&self) -> u32 {
|
2022-05-05 16:48:00 -07:00
|
|
|
self.common_header().fec_set_index
|
2021-12-09 08:43:57 -08:00
|
|
|
}
|
|
|
|
|
2021-12-10 12:08:04 -08:00
|
|
|
pub(crate) fn first_coding_index(&self) -> Option<u32> {
|
2022-05-05 16:48:00 -07:00
|
|
|
match self {
|
|
|
|
Self::ShredCode(shred) => shred.first_coding_index(),
|
|
|
|
Self::ShredData(_) => None,
|
2022-04-25 16:19:37 -07:00
|
|
|
}
|
2021-12-09 08:43:57 -08:00
|
|
|
}
|
|
|
|
|
2019-11-18 18:05:02 -08:00
|
|
|
pub fn version(&self) -> u16 {
|
2022-05-05 16:48:00 -07:00
|
|
|
self.common_header().version
|
2019-11-18 18:05:02 -08:00
|
|
|
}
|
|
|
|
|
2021-12-16 06:18:55 -08:00
|
|
|
// Identifier for the erasure coding set that the shred belongs to.
|
|
|
|
pub(crate) fn erasure_set(&self) -> ErasureSetId {
|
|
|
|
ErasureSetId(self.slot(), self.fec_set_index())
|
|
|
|
}
|
|
|
|
|
2019-09-16 20:28:54 -07:00
|
|
|
pub fn signature(&self) -> Signature {
|
2022-05-05 16:48:00 -07:00
|
|
|
self.common_header().signature
|
2019-09-16 20:28:54 -07:00
|
|
|
}
|
|
|
|
|
2022-04-19 13:00:05 -07:00
|
|
|
pub fn sign(&mut self, keypair: &Keypair) {
|
2022-05-05 16:48:00 -07:00
|
|
|
let signature = keypair.sign_message(self.signed_payload());
|
|
|
|
self.set_signature(signature);
|
2022-04-19 13:00:05 -07:00
|
|
|
}
|
|
|
|
|
2022-03-30 08:23:29 -07:00
|
|
|
pub fn seed(&self, leader_pubkey: Pubkey) -> [u8; 32] {
|
|
|
|
hashv(&[
|
|
|
|
&self.slot().to_le_bytes(),
|
|
|
|
&self.index().to_le_bytes(),
|
|
|
|
&leader_pubkey.to_bytes(),
|
|
|
|
])
|
|
|
|
.to_bytes()
|
2019-09-16 20:28:54 -07:00
|
|
|
}
|
|
|
|
|
2021-11-16 09:50:56 -08:00
|
|
|
#[inline]
|
|
|
|
pub fn shred_type(&self) -> ShredType {
|
2022-05-05 16:48:00 -07:00
|
|
|
self.common_header().shred_type
|
|
|
|
}
|
|
|
|
|
|
|
|
fn shred_type_from_payload(shred: &[u8]) -> Result<ShredType, Error> {
|
|
|
|
match shred.get(OFFSET_OF_SHRED_TYPE) {
|
|
|
|
None => Err(Error::InvalidPayloadSize(shred.len())),
|
|
|
|
Some(shred_type) => match ShredType::try_from(*shred_type) {
|
|
|
|
Err(_) => Err(Error::InvalidShredType),
|
|
|
|
Ok(shred_type) => Ok(shred_type),
|
|
|
|
},
|
|
|
|
}
|
2021-11-16 09:50:56 -08:00
|
|
|
}
|
|
|
|
|
2019-09-16 20:28:54 -07:00
|
|
|
pub fn is_data(&self) -> bool {
|
2021-11-16 09:50:56 -08:00
|
|
|
self.shred_type() == ShredType::Data
|
2019-09-16 20:28:54 -07:00
|
|
|
}
|
2019-10-16 15:41:43 -07:00
|
|
|
pub fn is_code(&self) -> bool {
|
2021-11-16 09:50:56 -08:00
|
|
|
self.shred_type() == ShredType::Code
|
2019-10-16 15:41:43 -07:00
|
|
|
}
|
2019-09-16 20:28:54 -07:00
|
|
|
|
|
|
|
pub fn last_in_slot(&self) -> bool {
|
2022-05-05 16:48:00 -07:00
|
|
|
match self {
|
|
|
|
Self::ShredCode(_) => false,
|
|
|
|
Self::ShredData(shred) => shred.last_in_slot(),
|
2019-09-16 20:28:54 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-09-17 18:22:46 -07:00
|
|
|
/// This is not a safe function. It only changes the meta information.
|
|
|
|
/// Use this only for test code which doesn't care about actual shred
|
|
|
|
pub fn set_last_in_slot(&mut self) {
|
2022-05-05 16:48:00 -07:00
|
|
|
match self {
|
|
|
|
Self::ShredCode(_) => (),
|
|
|
|
Self::ShredData(shred) => shred.set_last_in_slot(),
|
2019-09-17 18:22:46 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-09-16 20:28:54 -07:00
|
|
|
pub fn data_complete(&self) -> bool {
|
2022-05-05 16:48:00 -07:00
|
|
|
match self {
|
|
|
|
Self::ShredCode(_) => false,
|
|
|
|
Self::ShredData(shred) => shred.data_complete(),
|
2019-09-16 20:28:54 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-04-25 05:43:22 -07:00
|
|
|
pub(crate) fn reference_tick(&self) -> u8 {
|
2022-05-05 16:48:00 -07:00
|
|
|
match self {
|
|
|
|
Self::ShredCode(_) => ShredFlags::SHRED_TICK_REFERENCE_MASK.bits(),
|
|
|
|
Self::ShredData(shred) => shred.reference_tick(),
|
2019-11-06 13:27:58 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-03-12 05:44:06 -08:00
|
|
|
// Get slot from a shred packet with partial deserialize
|
2022-05-05 16:48:00 -07:00
|
|
|
pub fn get_slot_from_packet(packet: &Packet) -> Option<Slot> {
|
|
|
|
let buffer = packet.data.get(OFFSET_OF_SHRED_SLOT..)?;
|
|
|
|
limited_deserialize(buffer).ok()
|
2021-03-12 05:44:06 -08:00
|
|
|
}
|
|
|
|
|
2022-04-25 05:43:22 -07:00
|
|
|
pub(crate) fn reference_tick_from_data(data: &[u8]) -> u8 {
|
2022-05-05 16:48:00 -07:00
|
|
|
// TODO: should check the type bit as well!
|
|
|
|
const SHRED_FLAGS_OFFSET: usize = SIZE_OF_COMMON_SHRED_HEADER + std::mem::size_of::<u16>();
|
|
|
|
data[SHRED_FLAGS_OFFSET] & ShredFlags::SHRED_TICK_REFERENCE_MASK.bits()
|
2019-11-07 11:08:09 -08:00
|
|
|
}
|
|
|
|
|
2019-09-17 18:22:46 -07:00
|
|
|
pub fn verify(&self, pubkey: &Pubkey) -> bool {
|
2022-05-05 16:48:00 -07:00
|
|
|
let message = self.signed_payload();
|
|
|
|
self.signature().verify(pubkey.as_ref(), message)
|
2019-09-17 18:22:46 -07:00
|
|
|
}
|
2022-04-25 05:43:22 -07:00
|
|
|
|
|
|
|
// Returns true if the erasure coding of the two shreds mismatch.
|
2022-05-05 16:48:00 -07:00
|
|
|
pub(crate) fn erasure_mismatch(&self, other: &Self) -> Result<bool, Error> {
|
|
|
|
match (self, other) {
|
|
|
|
(Self::ShredCode(shred), Self::ShredCode(other)) => Ok(shred.erasure_mismatch(other)),
|
2022-04-28 16:42:37 -07:00
|
|
|
_ => Err(Error::InvalidShredType),
|
2022-04-25 05:43:22 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-05 16:48:00 -07:00
|
|
|
pub(crate) fn num_data_shreds(&self) -> Result<u16, Error> {
|
|
|
|
match self {
|
|
|
|
Self::ShredCode(shred) => Ok(shred.num_data_shreds()),
|
|
|
|
Self::ShredData(_) => Err(Error::InvalidShredType),
|
2022-04-25 05:43:22 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-05 16:48:00 -07:00
|
|
|
pub(crate) fn num_coding_shreds(&self) -> Result<u16, Error> {
|
|
|
|
match self {
|
|
|
|
Self::ShredCode(shred) => Ok(shred.num_coding_shreds()),
|
|
|
|
Self::ShredData(_) => Err(Error::InvalidShredType),
|
2022-04-25 05:43:22 -07:00
|
|
|
}
|
|
|
|
}
|
2019-09-12 21:52:13 -07:00
|
|
|
}
|
|
|
|
|
2022-05-05 16:48:00 -07:00
|
|
|
impl From<ShredCode> for Shred {
|
|
|
|
fn from(shred: ShredCode) -> Self {
|
|
|
|
Self::ShredCode(shred)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl From<ShredData> for Shred {
|
|
|
|
fn from(shred: ShredData) -> Self {
|
|
|
|
Self::ShredData(shred)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-12-15 16:50:40 -08:00
|
|
|
// Get slot, index, and type from a packet with partial deserialize
|
|
|
|
pub fn get_shred_slot_index_type(
|
|
|
|
p: &Packet,
|
|
|
|
stats: &mut ShredFetchStats,
|
2021-11-16 09:50:56 -08:00
|
|
|
) -> Option<(Slot, u32, ShredType)> {
|
2020-12-15 16:50:40 -08:00
|
|
|
let index_start = OFFSET_OF_SHRED_INDEX;
|
|
|
|
let index_end = index_start + SIZE_OF_SHRED_INDEX;
|
|
|
|
let slot_start = OFFSET_OF_SHRED_SLOT;
|
|
|
|
let slot_end = slot_start + SIZE_OF_SHRED_SLOT;
|
|
|
|
|
|
|
|
debug_assert!(index_end > slot_end);
|
|
|
|
debug_assert!(index_end > OFFSET_OF_SHRED_TYPE);
|
|
|
|
|
|
|
|
if index_end > p.meta.size {
|
|
|
|
stats.index_overrun += 1;
|
|
|
|
return None;
|
|
|
|
}
|
|
|
|
|
2022-01-21 16:01:22 -08:00
|
|
|
let index = match limited_deserialize::<u32>(&p.data[index_start..index_end]) {
|
|
|
|
Ok(x) => x,
|
2020-12-15 16:50:40 -08:00
|
|
|
Err(_e) => {
|
|
|
|
stats.index_bad_deserialize += 1;
|
|
|
|
return None;
|
|
|
|
}
|
2022-01-21 16:01:22 -08:00
|
|
|
};
|
2020-12-15 16:50:40 -08:00
|
|
|
|
|
|
|
if index >= MAX_DATA_SHREDS_PER_SLOT as u32 {
|
|
|
|
stats.index_out_of_bounds += 1;
|
|
|
|
return None;
|
|
|
|
}
|
|
|
|
|
2022-01-21 16:01:22 -08:00
|
|
|
let slot = match limited_deserialize::<Slot>(&p.data[slot_start..slot_end]) {
|
|
|
|
Ok(x) => x,
|
2020-12-15 16:50:40 -08:00
|
|
|
Err(_e) => {
|
|
|
|
stats.slot_bad_deserialize += 1;
|
|
|
|
return None;
|
|
|
|
}
|
2022-01-21 16:01:22 -08:00
|
|
|
};
|
2020-12-15 16:50:40 -08:00
|
|
|
|
2022-04-23 06:33:59 -07:00
|
|
|
let shred_type = match ShredType::try_from(p.data[OFFSET_OF_SHRED_TYPE]) {
|
|
|
|
Err(_) => {
|
2021-11-16 09:50:56 -08:00
|
|
|
stats.bad_shred_type += 1;
|
|
|
|
return None;
|
|
|
|
}
|
2022-04-23 06:33:59 -07:00
|
|
|
Ok(shred_type) => shred_type,
|
2021-11-16 09:50:56 -08:00
|
|
|
};
|
|
|
|
Some((slot, index, shred_type))
|
2020-12-15 16:50:40 -08:00
|
|
|
}
|
|
|
|
|
2020-05-19 12:38:18 -07:00
|
|
|
pub fn max_ticks_per_n_shreds(num_shreds: u64, shred_data_size: Option<usize>) -> u64 {
|
2019-10-31 13:38:50 -07:00
|
|
|
let ticks = create_ticks(1, 0, Hash::default());
|
2020-05-19 12:38:18 -07:00
|
|
|
max_entries_per_n_shred(&ticks[0], num_shreds, shred_data_size)
|
2019-10-08 00:42:51 -07:00
|
|
|
}
|
|
|
|
|
2020-05-19 12:38:18 -07:00
|
|
|
pub fn max_entries_per_n_shred(
|
|
|
|
entry: &Entry,
|
|
|
|
num_shreds: u64,
|
|
|
|
shred_data_size: Option<usize>,
|
|
|
|
) -> u64 {
|
|
|
|
let shred_data_size = shred_data_size.unwrap_or(SIZE_OF_DATA_SHRED_PAYLOAD) as u64;
|
2019-10-08 00:42:51 -07:00
|
|
|
let vec_size = bincode::serialized_size(&vec![entry]).unwrap();
|
|
|
|
let entry_size = bincode::serialized_size(entry).unwrap();
|
|
|
|
let count_size = vec_size - entry_size;
|
|
|
|
|
|
|
|
(shred_data_size * num_shreds - count_size) / entry_size
|
|
|
|
}
|
|
|
|
|
2019-12-11 11:10:21 -08:00
|
|
|
pub fn verify_test_data_shred(
|
|
|
|
shred: &Shred,
|
|
|
|
index: u32,
|
|
|
|
slot: Slot,
|
|
|
|
parent: Slot,
|
|
|
|
pk: &Pubkey,
|
|
|
|
verify: bool,
|
|
|
|
is_last_in_slot: bool,
|
2021-03-16 03:09:16 -07:00
|
|
|
is_last_data: bool,
|
2019-12-11 11:10:21 -08:00
|
|
|
) {
|
2022-04-28 16:42:37 -07:00
|
|
|
shred.sanitize().unwrap();
|
2022-05-05 16:48:00 -07:00
|
|
|
assert_eq!(shred.payload().len(), SHRED_PAYLOAD_SIZE);
|
2019-12-11 11:10:21 -08:00
|
|
|
assert!(shred.is_data());
|
|
|
|
assert_eq!(shred.index(), index);
|
|
|
|
assert_eq!(shred.slot(), slot);
|
2021-12-09 08:43:57 -08:00
|
|
|
assert_eq!(shred.parent().unwrap(), parent);
|
2019-12-11 11:10:21 -08:00
|
|
|
assert_eq!(verify, shred.verify(pk));
|
|
|
|
if is_last_in_slot {
|
|
|
|
assert!(shred.last_in_slot());
|
|
|
|
} else {
|
|
|
|
assert!(!shred.last_in_slot());
|
|
|
|
}
|
2021-03-16 03:09:16 -07:00
|
|
|
if is_last_data {
|
2019-12-11 11:10:21 -08:00
|
|
|
assert!(shred.data_complete());
|
|
|
|
} else {
|
|
|
|
assert!(!shred.data_complete());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-08-02 15:53:42 -07:00
|
|
|
#[cfg(test)]
|
2022-04-25 05:43:22 -07:00
|
|
|
mod tests {
|
2022-05-01 06:11:45 -07:00
|
|
|
use {
|
|
|
|
super::*,
|
|
|
|
bincode::serialized_size,
|
|
|
|
matches::assert_matches,
|
|
|
|
rand::Rng,
|
|
|
|
rand_chacha::{rand_core::SeedableRng, ChaChaRng},
|
2022-05-05 16:48:00 -07:00
|
|
|
solana_sdk::{shred_version, signature::Signer},
|
2022-05-01 06:11:45 -07:00
|
|
|
};
|
|
|
|
|
|
|
|
fn bs58_decode<T: AsRef<[u8]>>(data: T) -> Vec<u8> {
|
|
|
|
bs58::decode(data).into_vec().unwrap()
|
|
|
|
}
|
2019-08-02 15:53:42 -07:00
|
|
|
|
2019-10-21 12:46:16 -07:00
|
|
|
#[test]
|
|
|
|
fn test_shred_constants() {
|
2022-04-29 12:42:15 -07:00
|
|
|
let common_header = ShredCommonHeader {
|
|
|
|
signature: Signature::default(),
|
|
|
|
shred_type: ShredType::Code,
|
|
|
|
slot: Slot::MAX,
|
|
|
|
index: u32::MAX,
|
|
|
|
version: u16::MAX,
|
|
|
|
fec_set_index: u32::MAX,
|
|
|
|
};
|
2022-05-05 16:48:00 -07:00
|
|
|
let data_shred_header = DataShredHeader {
|
|
|
|
parent_offset: u16::MAX,
|
|
|
|
flags: ShredFlags::all(),
|
|
|
|
size: u16::MAX,
|
|
|
|
};
|
|
|
|
let coding_shred_header = CodingShredHeader {
|
|
|
|
num_data_shreds: u16::MAX,
|
|
|
|
num_coding_shreds: u16::MAX,
|
|
|
|
position: u16::MAX,
|
|
|
|
};
|
2019-10-21 12:46:16 -07:00
|
|
|
assert_eq!(
|
|
|
|
SIZE_OF_COMMON_SHRED_HEADER,
|
2022-04-29 12:42:15 -07:00
|
|
|
serialized_size(&common_header).unwrap() as usize
|
2019-10-21 12:46:16 -07:00
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
SIZE_OF_CODING_SHRED_HEADER,
|
2022-05-05 16:48:00 -07:00
|
|
|
serialized_size(&coding_shred_header).unwrap() as usize
|
2019-10-21 12:46:16 -07:00
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
SIZE_OF_DATA_SHRED_HEADER,
|
2022-05-05 16:48:00 -07:00
|
|
|
serialized_size(&data_shred_header).unwrap() as usize
|
2019-10-21 12:46:16 -07:00
|
|
|
);
|
2020-05-19 12:38:18 -07:00
|
|
|
let data_shred_header_with_size = DataShredHeader {
|
|
|
|
size: 1000,
|
2022-05-05 16:48:00 -07:00
|
|
|
..data_shred_header
|
2020-05-19 12:38:18 -07:00
|
|
|
};
|
|
|
|
assert_eq!(
|
|
|
|
SIZE_OF_DATA_SHRED_HEADER,
|
|
|
|
serialized_size(&data_shred_header_with_size).unwrap() as usize
|
|
|
|
);
|
2019-10-21 12:46:16 -07:00
|
|
|
assert_eq!(
|
|
|
|
SIZE_OF_SIGNATURE,
|
|
|
|
bincode::serialized_size(&Signature::default()).unwrap() as usize
|
|
|
|
);
|
2019-12-30 07:42:09 -08:00
|
|
|
assert_eq!(
|
|
|
|
SIZE_OF_SHRED_TYPE,
|
2022-04-29 12:42:15 -07:00
|
|
|
bincode::serialized_size(&ShredType::Code).unwrap() as usize
|
2019-12-30 07:42:09 -08:00
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
SIZE_OF_SHRED_SLOT,
|
|
|
|
bincode::serialized_size(&Slot::default()).unwrap() as usize
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
SIZE_OF_SHRED_INDEX,
|
2022-04-29 12:42:15 -07:00
|
|
|
bincode::serialized_size(&common_header.index).unwrap() as usize
|
2019-12-30 07:42:09 -08:00
|
|
|
);
|
2019-10-21 12:46:16 -07:00
|
|
|
}
|
|
|
|
|
2019-11-18 18:05:02 -08:00
|
|
|
#[test]
|
|
|
|
fn test_version_from_hash() {
|
|
|
|
let hash = [
|
|
|
|
0xa5u8, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5,
|
|
|
|
0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a,
|
|
|
|
0xa5, 0xa5, 0x5a, 0x5a,
|
|
|
|
];
|
2020-02-24 09:18:08 -08:00
|
|
|
let version = shred_version::version_from_hash(&Hash::new(&hash));
|
2020-01-24 08:14:27 -08:00
|
|
|
assert_eq!(version, 1);
|
2019-11-18 18:05:02 -08:00
|
|
|
let hash = [
|
|
|
|
0xa5u8, 0xa5, 0x5a, 0x5a, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
|
|
0, 0, 0, 0, 0, 0, 0, 0,
|
|
|
|
];
|
2020-02-24 09:18:08 -08:00
|
|
|
let version = shred_version::version_from_hash(&Hash::new(&hash));
|
2019-11-18 18:05:02 -08:00
|
|
|
assert_eq!(version, 0xffff);
|
|
|
|
let hash = [
|
|
|
|
0xa5u8, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
|
|
];
|
2020-02-24 09:18:08 -08:00
|
|
|
let version = shred_version::version_from_hash(&Hash::new(&hash));
|
2020-01-24 08:14:27 -08:00
|
|
|
assert_eq!(version, 0x5a5b);
|
2019-11-18 18:05:02 -08:00
|
|
|
}
|
2019-12-12 16:50:29 -08:00
|
|
|
|
2020-04-19 21:15:09 -07:00
|
|
|
#[test]
|
|
|
|
fn test_invalid_parent_offset() {
|
2022-05-02 16:33:53 -07:00
|
|
|
let shred = Shred::new_from_data(10, 0, 1000, &[1, 2, 3], ShredFlags::empty(), 0, 1, 0);
|
2020-04-19 21:15:09 -07:00
|
|
|
let mut packet = Packet::default();
|
|
|
|
shred.copy_to_packet(&mut packet);
|
|
|
|
let shred_res = Shred::new_from_serialized_shred(packet.data.to_vec());
|
|
|
|
assert_matches!(
|
2021-12-09 08:43:57 -08:00
|
|
|
shred.parent(),
|
2022-04-25 16:19:37 -07:00
|
|
|
Err(Error::InvalidParentOffset {
|
|
|
|
slot: 10,
|
|
|
|
parent_offset: 1000
|
|
|
|
})
|
|
|
|
);
|
|
|
|
assert_matches!(
|
|
|
|
shred_res,
|
|
|
|
Err(Error::InvalidParentOffset {
|
2020-04-19 21:15:09 -07:00
|
|
|
slot: 10,
|
|
|
|
parent_offset: 1000
|
|
|
|
})
|
|
|
|
);
|
|
|
|
}
|
2020-12-15 16:50:40 -08:00
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_shred_offsets() {
|
|
|
|
solana_logger::setup();
|
|
|
|
let mut packet = Packet::default();
|
2022-05-02 16:33:53 -07:00
|
|
|
let shred = Shred::new_from_data(1, 3, 0, &[], ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 0);
|
2020-12-15 16:50:40 -08:00
|
|
|
shred.copy_to_packet(&mut packet);
|
|
|
|
let mut stats = ShredFetchStats::default();
|
|
|
|
let ret = get_shred_slot_index_type(&packet, &mut stats);
|
2021-11-16 09:50:56 -08:00
|
|
|
assert_eq!(Some((1, 3, ShredType::Data)), ret);
|
2020-12-15 16:50:40 -08:00
|
|
|
assert_eq!(stats, ShredFetchStats::default());
|
|
|
|
|
|
|
|
packet.meta.size = OFFSET_OF_SHRED_TYPE;
|
|
|
|
assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
|
|
|
|
assert_eq!(stats.index_overrun, 1);
|
|
|
|
|
|
|
|
packet.meta.size = OFFSET_OF_SHRED_INDEX;
|
|
|
|
assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
|
|
|
|
assert_eq!(stats.index_overrun, 2);
|
|
|
|
|
|
|
|
packet.meta.size = OFFSET_OF_SHRED_INDEX + 1;
|
|
|
|
assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
|
|
|
|
assert_eq!(stats.index_overrun, 3);
|
|
|
|
|
|
|
|
packet.meta.size = OFFSET_OF_SHRED_INDEX + SIZE_OF_SHRED_INDEX - 1;
|
|
|
|
assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
|
|
|
|
assert_eq!(stats.index_overrun, 4);
|
|
|
|
|
|
|
|
packet.meta.size = OFFSET_OF_SHRED_INDEX + SIZE_OF_SHRED_INDEX;
|
|
|
|
assert_eq!(
|
2021-11-16 09:50:56 -08:00
|
|
|
Some((1, 3, ShredType::Data)),
|
2020-12-15 16:50:40 -08:00
|
|
|
get_shred_slot_index_type(&packet, &mut stats)
|
|
|
|
);
|
|
|
|
assert_eq!(stats.index_overrun, 4);
|
|
|
|
|
2022-04-27 11:04:10 -07:00
|
|
|
let shred = Shred::new_from_parity_shard(
|
2021-12-05 06:42:09 -08:00
|
|
|
8, // slot
|
|
|
|
2, // index
|
2022-04-27 11:04:10 -07:00
|
|
|
&[], // parity_shard
|
2021-12-05 06:42:09 -08:00
|
|
|
10, // fec_set_index
|
|
|
|
30, // num_data
|
|
|
|
4, // num_code
|
|
|
|
1, // position
|
|
|
|
200, // version
|
|
|
|
);
|
2020-12-15 16:50:40 -08:00
|
|
|
shred.copy_to_packet(&mut packet);
|
|
|
|
assert_eq!(
|
2021-11-16 09:50:56 -08:00
|
|
|
Some((8, 2, ShredType::Code)),
|
2020-12-15 16:50:40 -08:00
|
|
|
get_shred_slot_index_type(&packet, &mut stats)
|
|
|
|
);
|
|
|
|
|
2022-05-02 16:33:53 -07:00
|
|
|
let shred = Shred::new_from_data(
|
|
|
|
1,
|
|
|
|
std::u32::MAX - 10,
|
|
|
|
0,
|
|
|
|
&[],
|
|
|
|
ShredFlags::LAST_SHRED_IN_SLOT,
|
|
|
|
0,
|
|
|
|
0,
|
|
|
|
0,
|
|
|
|
);
|
2020-12-15 16:50:40 -08:00
|
|
|
shred.copy_to_packet(&mut packet);
|
|
|
|
assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
|
|
|
|
assert_eq!(1, stats.index_out_of_bounds);
|
|
|
|
|
2022-04-27 11:04:10 -07:00
|
|
|
let shred = Shred::new_from_parity_shard(
|
2021-12-05 06:42:09 -08:00
|
|
|
8, // slot
|
|
|
|
2, // index
|
2022-04-27 11:04:10 -07:00
|
|
|
&[], // parity_shard
|
2021-12-05 06:42:09 -08:00
|
|
|
10, // fec_set_index
|
|
|
|
30, // num_data_shreds
|
|
|
|
4, // num_coding_shreds
|
|
|
|
3, // position
|
|
|
|
200, // version
|
|
|
|
);
|
2020-12-15 16:50:40 -08:00
|
|
|
shred.copy_to_packet(&mut packet);
|
2021-11-16 09:50:56 -08:00
|
|
|
packet.data[OFFSET_OF_SHRED_TYPE] = u8::MAX;
|
2020-12-15 16:50:40 -08:00
|
|
|
|
|
|
|
assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
|
|
|
|
assert_eq!(1, stats.bad_shred_type);
|
|
|
|
}
|
2021-11-16 09:50:56 -08:00
|
|
|
|
|
|
|
// Asserts that ShredType is backward compatible with u8.
|
|
|
|
#[test]
|
|
|
|
fn test_shred_type_compat() {
|
|
|
|
assert_eq!(std::mem::size_of::<ShredType>(), std::mem::size_of::<u8>());
|
2022-04-23 06:33:59 -07:00
|
|
|
assert_matches!(ShredType::try_from(0u8), Err(_));
|
|
|
|
assert_matches!(ShredType::try_from(1u8), Err(_));
|
2021-11-16 09:50:56 -08:00
|
|
|
assert_matches!(bincode::deserialize::<ShredType>(&[0u8]), Err(_));
|
2022-04-23 06:33:59 -07:00
|
|
|
assert_matches!(bincode::deserialize::<ShredType>(&[1u8]), Err(_));
|
2021-11-16 09:50:56 -08:00
|
|
|
// data shred
|
|
|
|
assert_eq!(ShredType::Data as u8, 0b1010_0101);
|
2022-04-23 06:33:59 -07:00
|
|
|
assert_eq!(ShredType::try_from(0b1010_0101), Ok(ShredType::Data));
|
2021-11-16 09:50:56 -08:00
|
|
|
let buf = bincode::serialize(&ShredType::Data).unwrap();
|
|
|
|
assert_eq!(buf, vec![0b1010_0101]);
|
|
|
|
assert_matches!(
|
|
|
|
bincode::deserialize::<ShredType>(&[0b1010_0101]),
|
|
|
|
Ok(ShredType::Data)
|
|
|
|
);
|
|
|
|
// coding shred
|
|
|
|
assert_eq!(ShredType::Code as u8, 0b0101_1010);
|
2022-04-23 06:33:59 -07:00
|
|
|
assert_eq!(ShredType::try_from(0b0101_1010), Ok(ShredType::Code));
|
2021-11-16 09:50:56 -08:00
|
|
|
let buf = bincode::serialize(&ShredType::Code).unwrap();
|
|
|
|
assert_eq!(buf, vec![0b0101_1010]);
|
|
|
|
assert_matches!(
|
|
|
|
bincode::deserialize::<ShredType>(&[0b0101_1010]),
|
|
|
|
Ok(ShredType::Code)
|
|
|
|
);
|
|
|
|
}
|
2022-04-25 05:43:22 -07:00
|
|
|
|
2022-05-01 06:11:45 -07:00
|
|
|
#[test]
|
|
|
|
fn test_serde_compat_shred_data() {
|
|
|
|
const SEED: &str = "6qG9NGWEtoTugS4Zgs46u8zTccEJuRHtrNMiUayLHCxt";
|
2022-05-02 16:33:53 -07:00
|
|
|
const PAYLOAD: &str = "hNX8YgJCQwSFGJkZ6qZLiepwPjpctC9UCsMD1SNNQurBXv\
|
|
|
|
rm7KKfLmPRMM9CpWHt6MsJuEWpDXLGwH9qdziJzGKhBMfYH63avcchjdaUiMqzVip7cUD\
|
|
|
|
kqZ9zZJMrHCCUDnxxKMupsJWKroUSjKeo7hrug2KfHah85VckXpRna4R9QpH7tf2WVBTD\
|
|
|
|
M4m3EerctsEQs8eZaTRxzTVkhtJYdNf74KZbH58dc3Yn2qUxF1mexWoPS6L5oZBatx";
|
2022-05-01 06:11:45 -07:00
|
|
|
let mut rng = {
|
|
|
|
let seed = <[u8; 32]>::try_from(bs58_decode(SEED)).unwrap();
|
|
|
|
ChaChaRng::from_seed(seed)
|
|
|
|
};
|
|
|
|
let mut data = [0u8; SIZE_OF_DATA_SHRED_PAYLOAD];
|
|
|
|
rng.fill(&mut data[..]);
|
|
|
|
let keypair = Keypair::generate(&mut rng);
|
|
|
|
let mut shred = Shred::new_from_data(
|
|
|
|
141939602, // slot
|
|
|
|
28685, // index
|
|
|
|
36390, // parent_offset
|
|
|
|
&data, // data
|
2022-05-02 16:33:53 -07:00
|
|
|
ShredFlags::LAST_SHRED_IN_SLOT,
|
|
|
|
37, // reference_tick
|
|
|
|
45189, // version
|
|
|
|
28657, // fec_set_index
|
2022-05-01 06:11:45 -07:00
|
|
|
);
|
|
|
|
shred.sign(&keypair);
|
|
|
|
assert!(shred.verify(&keypair.pubkey()));
|
|
|
|
assert_matches!(shred.sanitize(), Ok(()));
|
|
|
|
let mut payload = bs58_decode(PAYLOAD);
|
|
|
|
payload.extend({
|
|
|
|
let skip = payload.len() - SHRED_DATA_OFFSET;
|
|
|
|
data.iter().skip(skip).copied()
|
|
|
|
});
|
|
|
|
let mut packet = Packet::default();
|
|
|
|
packet.data[..payload.len()].copy_from_slice(&payload);
|
|
|
|
packet.meta.size = payload.len();
|
|
|
|
assert_eq!(shred.bytes_to_store(), payload);
|
2022-05-02 08:02:06 -07:00
|
|
|
assert_eq!(shred, Shred::new_from_serialized_shred(payload).unwrap());
|
|
|
|
assert_eq!(
|
|
|
|
shred.reference_tick(),
|
|
|
|
Shred::reference_tick_from_data(&packet.data)
|
|
|
|
);
|
|
|
|
assert_eq!(Shred::get_slot_from_packet(&packet), Some(shred.slot()));
|
|
|
|
assert_eq!(
|
|
|
|
get_shred_slot_index_type(&packet, &mut ShredFetchStats::default()),
|
|
|
|
Some((shred.slot(), shred.index(), shred.shred_type()))
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_serde_compat_shred_data_empty() {
|
|
|
|
const SEED: &str = "E3M5hm8yAEB7iPhQxFypAkLqxNeZCTuGBDMa8Jdrghoo";
|
|
|
|
const PAYLOAD: &str = "nRNFVBEsV9FEM5KfmsCXJsgELRSkCV55drTavdy5aZPnsp\
|
|
|
|
B8WvsgY99ZuNHDnwkrqe6Lx7ARVmercwugR5HwDcLA9ivKMypk9PNucDPLs67TXWy6k9R\
|
|
|
|
ozKmy";
|
|
|
|
let mut rng = {
|
|
|
|
let seed = <[u8; 32]>::try_from(bs58_decode(SEED)).unwrap();
|
|
|
|
ChaChaRng::from_seed(seed)
|
|
|
|
};
|
|
|
|
let keypair = Keypair::generate(&mut rng);
|
|
|
|
let mut shred = Shred::new_from_data(
|
|
|
|
142076266, // slot
|
|
|
|
21443, // index
|
|
|
|
51279, // parent_offset
|
|
|
|
&[], // data
|
2022-05-02 16:33:53 -07:00
|
|
|
ShredFlags::DATA_COMPLETE_SHRED,
|
|
|
|
49, // reference_tick
|
|
|
|
59445, // version
|
|
|
|
21414, // fec_set_index
|
2022-05-02 08:02:06 -07:00
|
|
|
);
|
|
|
|
shred.sign(&keypair);
|
|
|
|
assert!(shred.verify(&keypair.pubkey()));
|
|
|
|
assert_matches!(shred.sanitize(), Ok(()));
|
|
|
|
let payload = bs58_decode(PAYLOAD);
|
|
|
|
let mut packet = Packet::default();
|
|
|
|
packet.data[..payload.len()].copy_from_slice(&payload);
|
|
|
|
packet.meta.size = payload.len();
|
|
|
|
assert_eq!(shred.bytes_to_store(), payload);
|
2022-05-01 06:11:45 -07:00
|
|
|
assert_eq!(shred, Shred::new_from_serialized_shred(payload).unwrap());
|
|
|
|
assert_eq!(
|
|
|
|
shred.reference_tick(),
|
|
|
|
Shred::reference_tick_from_data(&packet.data)
|
|
|
|
);
|
|
|
|
assert_eq!(Shred::get_slot_from_packet(&packet), Some(shred.slot()));
|
|
|
|
assert_eq!(
|
|
|
|
get_shred_slot_index_type(&packet, &mut ShredFetchStats::default()),
|
|
|
|
Some((shred.slot(), shred.index(), shred.shred_type()))
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_serde_compat_shred_code() {
|
|
|
|
const SEED: &str = "4jfjh3UZVyaEgvyG9oQmNyFY9yHDmbeH9eUhnBKkrcrN";
|
|
|
|
const PAYLOAD: &str = "3xGsXwzkPpLFuKwbbfKMUxt1B6VqQPzbvvAkxRNCX9kNEP\
|
|
|
|
sa2VifwGBtFuNm3CWXdmQizDz5vJjDHu6ZqqaBCSfrHurag87qAXwTtjNPhZzKEew5pLc\
|
|
|
|
aY6cooiAch2vpfixNYSDjnirozje5cmUtGuYs1asXwsAKSN3QdWHz3XGParWkZeUMAzRV\
|
|
|
|
1UPEDZ7vETKbxeNixKbzZzo47Lakh3C35hS74ocfj23CWoW1JpkETkXjUpXcfcv6cS";
|
|
|
|
let mut rng = {
|
|
|
|
let seed = <[u8; 32]>::try_from(bs58_decode(SEED)).unwrap();
|
|
|
|
ChaChaRng::from_seed(seed)
|
|
|
|
};
|
2022-05-05 16:48:00 -07:00
|
|
|
let mut parity_shard = vec![0u8; /*ENCODED_PAYLOAD_SIZE:*/ 1139];
|
2022-05-01 06:11:45 -07:00
|
|
|
rng.fill(&mut parity_shard[..]);
|
|
|
|
let keypair = Keypair::generate(&mut rng);
|
|
|
|
let mut shred = Shred::new_from_parity_shard(
|
|
|
|
141945197, // slot
|
|
|
|
23418, // index
|
|
|
|
&parity_shard,
|
|
|
|
21259, // fec_set_index
|
|
|
|
32, // num_data_shreds
|
|
|
|
58, // num_coding_shreds
|
|
|
|
43, // position
|
|
|
|
47298, // version
|
|
|
|
);
|
|
|
|
shred.sign(&keypair);
|
|
|
|
assert!(shred.verify(&keypair.pubkey()));
|
|
|
|
assert_matches!(shred.sanitize(), Ok(()));
|
|
|
|
let mut payload = bs58_decode(PAYLOAD);
|
|
|
|
payload.extend({
|
|
|
|
let skip = payload.len() - SIZE_OF_CODING_SHRED_HEADERS;
|
|
|
|
parity_shard.iter().skip(skip).copied()
|
|
|
|
});
|
|
|
|
let mut packet = Packet::default();
|
|
|
|
packet.data[..payload.len()].copy_from_slice(&payload);
|
|
|
|
packet.meta.size = payload.len();
|
|
|
|
assert_eq!(shred.bytes_to_store(), payload);
|
|
|
|
assert_eq!(shred, Shred::new_from_serialized_shred(payload).unwrap());
|
|
|
|
assert_eq!(Shred::get_slot_from_packet(&packet), Some(shred.slot()));
|
|
|
|
assert_eq!(
|
|
|
|
get_shred_slot_index_type(&packet, &mut ShredFetchStats::default()),
|
|
|
|
Some((shred.slot(), shred.index(), shred.shred_type()))
|
|
|
|
);
|
|
|
|
}
|
2022-05-01 12:25:15 -07:00
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_shred_flags() {
|
|
|
|
fn make_shred(is_last_data: bool, is_last_in_slot: bool, reference_tick: u8) -> Shred {
|
2022-05-02 16:33:53 -07:00
|
|
|
let flags = if is_last_in_slot {
|
|
|
|
assert!(is_last_data);
|
|
|
|
ShredFlags::LAST_SHRED_IN_SLOT
|
|
|
|
} else if is_last_data {
|
|
|
|
ShredFlags::DATA_COMPLETE_SHRED
|
|
|
|
} else {
|
|
|
|
ShredFlags::empty()
|
|
|
|
};
|
2022-05-01 12:25:15 -07:00
|
|
|
Shred::new_from_data(
|
|
|
|
0, // slot
|
|
|
|
0, // index
|
|
|
|
0, // parent_offset
|
|
|
|
&[], // data
|
2022-05-02 16:33:53 -07:00
|
|
|
flags,
|
2022-05-01 12:25:15 -07:00
|
|
|
reference_tick,
|
|
|
|
0, // version
|
|
|
|
0, // fec_set_index
|
|
|
|
)
|
|
|
|
}
|
|
|
|
fn check_shred_flags(
|
|
|
|
shred: &Shred,
|
|
|
|
is_last_data: bool,
|
|
|
|
is_last_in_slot: bool,
|
|
|
|
reference_tick: u8,
|
|
|
|
) {
|
|
|
|
assert_eq!(shred.data_complete(), is_last_data);
|
|
|
|
assert_eq!(shred.last_in_slot(), is_last_in_slot);
|
|
|
|
assert_eq!(shred.reference_tick(), reference_tick.min(63u8));
|
|
|
|
assert_eq!(
|
2022-05-05 16:48:00 -07:00
|
|
|
Shred::reference_tick_from_data(shred.payload()),
|
2022-05-01 12:25:15 -07:00
|
|
|
reference_tick.min(63u8),
|
|
|
|
);
|
|
|
|
}
|
|
|
|
for is_last_data in [false, true] {
|
|
|
|
for is_last_in_slot in [false, true] {
|
2022-05-02 16:33:53 -07:00
|
|
|
// LAST_SHRED_IN_SLOT also implies DATA_COMPLETE_SHRED. So it
|
|
|
|
// cannot be LAST_SHRED_IN_SLOT if not DATA_COMPLETE_SHRED.
|
|
|
|
let is_last_in_slot = is_last_in_slot && is_last_data;
|
2022-05-01 12:25:15 -07:00
|
|
|
for reference_tick in [0, 37, 63, 64, 80, 128, 255] {
|
|
|
|
let mut shred = make_shred(is_last_data, is_last_in_slot, reference_tick);
|
|
|
|
check_shred_flags(&shred, is_last_data, is_last_in_slot, reference_tick);
|
|
|
|
shred.set_last_in_slot();
|
2022-05-02 16:33:53 -07:00
|
|
|
check_shred_flags(&shred, true, true, reference_tick);
|
2022-05-01 12:25:15 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2022-05-02 16:33:53 -07:00
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_shred_flags_serde() {
|
|
|
|
let flags: ShredFlags = bincode::deserialize(&[0b0111_0001]).unwrap();
|
|
|
|
assert!(flags.contains(ShredFlags::DATA_COMPLETE_SHRED));
|
|
|
|
assert!(!flags.contains(ShredFlags::LAST_SHRED_IN_SLOT));
|
|
|
|
assert_eq!((flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits(), 49u8);
|
|
|
|
assert_eq!(bincode::serialize(&flags).unwrap(), [0b0111_0001]);
|
|
|
|
|
|
|
|
let flags: ShredFlags = bincode::deserialize(&[0b1110_0101]).unwrap();
|
|
|
|
assert!(flags.contains(ShredFlags::DATA_COMPLETE_SHRED));
|
|
|
|
assert!(flags.contains(ShredFlags::LAST_SHRED_IN_SLOT));
|
|
|
|
assert_eq!((flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits(), 37u8);
|
|
|
|
assert_eq!(bincode::serialize(&flags).unwrap(), [0b1110_0101]);
|
|
|
|
|
|
|
|
let flags: ShredFlags = bincode::deserialize(&[0b1011_1101]).unwrap();
|
|
|
|
assert!(!flags.contains(ShredFlags::DATA_COMPLETE_SHRED));
|
|
|
|
assert!(!flags.contains(ShredFlags::LAST_SHRED_IN_SLOT));
|
|
|
|
assert_eq!((flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits(), 61u8);
|
|
|
|
assert_eq!(bincode::serialize(&flags).unwrap(), [0b1011_1101]);
|
|
|
|
}
|
2019-08-02 15:53:42 -07:00
|
|
|
}
|