2021-04-16 12:04:46 -07:00
|
|
|
//! The `shred` module defines data structures and methods to pull MTU sized data frames from the
|
|
|
|
//! network. There are two types of shreds: data and coding. Data shreds contain entry information
|
|
|
|
//! while coding shreds provide redundancy to protect against dropped network packets (erasures).
|
|
|
|
//!
|
|
|
|
//! +---------------------------------------------------------------------------------------------+
|
|
|
|
//! | Data Shred |
|
|
|
|
//! +---------------------------------------------------------------------------------------------+
|
|
|
|
//! | common | data | payload |
|
|
|
|
//! | header | header | |
|
|
|
|
//! |+---+---+--- |+---+---+---|+----------------------------------------------------------+----+|
|
|
|
|
//! || s | s | . || p | f | s || data (ie ledger entries) | r ||
|
|
|
|
//! || i | h | . || a | l | i || | e ||
|
|
|
|
//! || g | r | . || r | a | z || See notes immediately after shred diagrams for an | s ||
|
|
|
|
//! || n | e | || e | g | e || explanation of the "restricted" section in this payload | t ||
|
|
|
|
//! || a | d | || n | s | || | r ||
|
|
|
|
//! || t | | || t | | || | i ||
|
|
|
|
//! || u | t | || | | || | c ||
|
|
|
|
//! || r | y | || o | | || | t ||
|
|
|
|
//! || e | p | || f | | || | e ||
|
|
|
|
//! || | e | || f | | || | d ||
|
|
|
|
//! |+---+---+--- |+---+---+---+|----------------------------------------------------------+----+|
|
|
|
|
//! +---------------------------------------------------------------------------------------------+
|
|
|
|
//!
|
|
|
|
//! +---------------------------------------------------------------------------------------------+
|
|
|
|
//! | Coding Shred |
|
|
|
|
//! +---------------------------------------------------------------------------------------------+
|
|
|
|
//! | common | coding | payload |
|
|
|
|
//! | header | header | |
|
|
|
|
//! |+---+---+--- |+---+---+---+----------------------------------------------------------------+|
|
|
|
|
//! || s | s | . || n | n | p || data (encoded data shred data) ||
|
|
|
|
//! || i | h | . || u | u | o || ||
|
|
|
|
//! || g | r | . || m | m | s || ||
|
|
|
|
//! || n | e | || | | i || ||
|
|
|
|
//! || a | d | || d | c | t || ||
|
|
|
|
//! || t | | || | | i || ||
|
|
|
|
//! || u | t | || s | s | o || ||
|
|
|
|
//! || r | y | || h | h | n || ||
|
|
|
|
//! || e | p | || r | r | || ||
|
|
|
|
//! || | e | || e | e | || ||
|
|
|
|
//! || | | || d | d | || ||
|
|
|
|
//! |+---+---+--- |+---+---+---+|+--------------------------------------------------------------+|
|
|
|
|
//! +---------------------------------------------------------------------------------------------+
|
|
|
|
//!
|
|
|
|
//! Notes:
|
|
|
|
//! a) Coding shreds encode entire data shreds: both of the headers AND the payload.
|
|
|
|
//! b) Coding shreds require their own headers for identification and etc.
|
|
|
|
//! c) The erasure algorithm requires data shred and coding shred bytestreams to be equal in length.
|
|
|
|
//!
|
|
|
|
//! So, given a) - c), we must restrict data shred's payload length such that the entire coding
|
|
|
|
//! payload can fit into one coding shred / packet.
|
|
|
|
|
2022-08-12 11:02:01 -07:00
|
|
|
#[cfg(test)]
|
|
|
|
pub(crate) use shred_code::MAX_CODE_SHREDS_PER_SLOT;
|
2021-09-01 08:44:26 -07:00
|
|
|
use {
|
2022-12-31 09:08:25 -08:00
|
|
|
self::{merkle::MerkleRoot, shred_code::ShredCode, traits::Shred as _},
|
2022-06-28 05:45:50 -07:00
|
|
|
crate::blockstore::{self, MAX_DATA_SHREDS_PER_SLOT},
|
2022-05-01 12:25:15 -07:00
|
|
|
bitflags::bitflags,
|
2022-04-23 06:33:59 -07:00
|
|
|
num_enum::{IntoPrimitive, TryFromPrimitive},
|
2022-09-15 14:50:22 -07:00
|
|
|
rayon::ThreadPool,
|
2022-08-19 14:07:32 -07:00
|
|
|
reed_solomon_erasure::Error::TooFewShardsPresent,
|
2022-04-23 06:33:59 -07:00
|
|
|
serde::{Deserialize, Serialize},
|
2021-09-01 08:44:26 -07:00
|
|
|
solana_entry::entry::{create_ticks, Entry},
|
2022-06-19 08:30:18 -07:00
|
|
|
solana_perf::packet::Packet,
|
2021-09-01 08:44:26 -07:00
|
|
|
solana_sdk::{
|
|
|
|
clock::Slot,
|
|
|
|
hash::{hashv, Hash},
|
|
|
|
pubkey::Pubkey,
|
2022-08-16 05:30:38 -07:00
|
|
|
signature::{Keypair, Signature, Signer, SIGNATURE_BYTES},
|
2021-09-01 08:44:26 -07:00
|
|
|
},
|
2022-05-17 15:44:35 -07:00
|
|
|
static_assertions::const_assert_eq,
|
2022-09-15 14:50:22 -07:00
|
|
|
std::{fmt::Debug, time::Instant},
|
2021-09-01 08:44:26 -07:00
|
|
|
thiserror::Error,
|
2019-11-02 00:38:30 -07:00
|
|
|
};
|
2022-08-25 13:51:55 -07:00
|
|
|
pub use {
|
|
|
|
self::{
|
|
|
|
shred_data::ShredData,
|
|
|
|
stats::{ProcessShredsStats, ShredFetchStats},
|
|
|
|
},
|
2022-09-25 11:09:47 -07:00
|
|
|
crate::shredder::{ReedSolomonCache, Shredder},
|
2022-08-25 13:51:55 -07:00
|
|
|
};
|
2022-05-05 16:48:00 -07:00
|
|
|
|
2022-05-30 05:51:19 -07:00
|
|
|
mod common;
|
2022-05-05 16:48:00 -07:00
|
|
|
mod legacy;
|
2022-06-07 15:41:03 -07:00
|
|
|
mod merkle;
|
2022-05-30 05:51:19 -07:00
|
|
|
mod shred_code;
|
|
|
|
mod shred_data;
|
2022-05-05 16:48:00 -07:00
|
|
|
mod stats;
|
|
|
|
mod traits;
|
2019-08-02 15:53:42 -07:00
|
|
|
|
2020-05-19 12:38:18 -07:00
|
|
|
pub type Nonce = u32;
|
2022-12-16 10:56:21 -08:00
|
|
|
const_assert_eq!(SIZE_OF_NONCE, 4);
|
|
|
|
pub const SIZE_OF_NONCE: usize = std::mem::size_of::<Nonce>();
|
2020-05-19 12:38:18 -07:00
|
|
|
|
2019-10-21 12:46:16 -07:00
|
|
|
/// The following constants are computed by hand, and hardcoded.
|
|
|
|
/// `test_shred_constants` ensures that the values are correct.
|
|
|
|
/// Constants are used over lazy_static for performance reasons.
|
2022-04-25 05:43:22 -07:00
|
|
|
const SIZE_OF_COMMON_SHRED_HEADER: usize = 83;
|
2022-05-30 05:51:19 -07:00
|
|
|
const SIZE_OF_DATA_SHRED_HEADERS: usize = 88;
|
|
|
|
const SIZE_OF_CODING_SHRED_HEADERS: usize = 89;
|
2022-08-16 05:30:38 -07:00
|
|
|
const SIZE_OF_SIGNATURE: usize = SIGNATURE_BYTES;
|
2022-05-30 05:51:00 -07:00
|
|
|
const SIZE_OF_SHRED_VARIANT: usize = 1;
|
2022-04-25 05:43:22 -07:00
|
|
|
const SIZE_OF_SHRED_SLOT: usize = 8;
|
|
|
|
|
2022-05-30 05:51:00 -07:00
|
|
|
const OFFSET_OF_SHRED_VARIANT: usize = SIZE_OF_SIGNATURE;
|
|
|
|
const OFFSET_OF_SHRED_SLOT: usize = SIZE_OF_SIGNATURE + SIZE_OF_SHRED_VARIANT;
|
2022-04-25 05:43:22 -07:00
|
|
|
const OFFSET_OF_SHRED_INDEX: usize = OFFSET_OF_SHRED_SLOT + SIZE_OF_SHRED_SLOT;
|
2019-09-18 18:00:07 -07:00
|
|
|
|
removes buffering when generating coding shreds in broadcast (#25807)
Given the 32:32 erasure recovery schema, current implementation requires
exactly 32 data shreds to generate coding shreds for the batch (except
for the final erasure batch in each slot).
As a result, when serializing ledger entries to data shreds, if the
number of data shreds is not a multiple of 32, the coding shreds for the
last batch cannot be generated until there are more data shreds to
complete the batch to 32 data shreds. This adds latency in generating
and broadcasting coding shreds.
In addition, with Merkle variants for shreds, data shreds cannot be
signed and broadcasted until coding shreds are also generated. As a
result *both* code and data shreds will be delayed before broadcast if
we still require exactly 32 data shreds for each batch.
This commit instead always generates and broadcast coding shreds as soon
as there any number of data shreds available. When serializing entries
to shreds:
* if the number of resulting data shreds is less than 32, then more
coding shreds will be generated so that the resulting erasure batch
has the same recovery probabilities as a 32:32 batch.
* if the number of data shreds is more than 32, then the data shreds are
split uniformly into erasure batches with _at least_ 32 data shreds in
each batch. Each erasure batch will have the same number of code and
data shreds.
For example:
* If there are 19 data shreds, 27 coding shreds are generated. The
resulting 19(data):27(code) erasure batch has the same recovery
probabilities as a 32:32 batch.
* If there are 107 data shreds, they are split into 3 batches of 36:36,
36:36 and 35:35 data:code shreds each.
A consequence of this change is that code and data shreds indices will
no longer align as there will be more coding shreds than data shreds
(not only in the last batch in each slot but also in the intermediate
ones);
2022-08-11 05:44:27 -07:00
|
|
|
// Shreds are uniformly split into erasure batches with a "target" number of
|
|
|
|
// data shreds per each batch as below. The actual number of data shreds in
|
|
|
|
// each erasure batch depends on the number of shreds obtained from serializing
|
|
|
|
// a &[Entry].
|
|
|
|
pub const DATA_SHREDS_PER_FEC_BLOCK: usize = 32;
|
2019-09-19 16:29:52 -07:00
|
|
|
|
2022-05-30 05:51:19 -07:00
|
|
|
// For legacy tests and benchmarks.
|
|
|
|
const_assert_eq!(LEGACY_SHRED_DATA_CAPACITY, 1051);
|
|
|
|
pub const LEGACY_SHRED_DATA_CAPACITY: usize = legacy::ShredData::CAPACITY;
|
|
|
|
|
2022-05-02 16:33:53 -07:00
|
|
|
// LAST_SHRED_IN_SLOT also implies DATA_COMPLETE_SHRED.
|
|
|
|
// So it cannot be LAST_SHRED_IN_SLOT if not also DATA_COMPLETE_SHRED.
|
2022-05-01 12:25:15 -07:00
|
|
|
bitflags! {
|
|
|
|
#[derive(Default, Serialize, Deserialize)]
|
|
|
|
pub struct ShredFlags:u8 {
|
|
|
|
const SHRED_TICK_REFERENCE_MASK = 0b0011_1111;
|
2022-05-02 16:33:53 -07:00
|
|
|
const DATA_COMPLETE_SHRED = 0b0100_0000;
|
|
|
|
const LAST_SHRED_IN_SLOT = 0b1100_0000;
|
2022-05-01 12:25:15 -07:00
|
|
|
}
|
|
|
|
}
|
2019-09-19 16:29:52 -07:00
|
|
|
|
2022-04-25 16:19:37 -07:00
|
|
|
#[derive(Debug, Error)]
|
|
|
|
pub enum Error {
|
|
|
|
#[error(transparent)]
|
|
|
|
BincodeError(#[from] bincode::Error),
|
2022-04-27 15:00:04 -07:00
|
|
|
#[error(transparent)]
|
|
|
|
ErasureError(#[from] reed_solomon_erasure::Error),
|
2022-04-25 16:19:37 -07:00
|
|
|
#[error("Invalid data size: {size}, payload: {payload}")]
|
|
|
|
InvalidDataSize { size: u16, payload: usize },
|
2022-04-29 12:42:15 -07:00
|
|
|
#[error("Invalid erasure shard index: {0:?}")]
|
2022-09-15 14:50:22 -07:00
|
|
|
InvalidErasureShardIndex(/*headers:*/ Box<dyn Debug + Send>),
|
2022-06-07 15:41:03 -07:00
|
|
|
#[error("Invalid merkle proof")]
|
|
|
|
InvalidMerkleProof,
|
2022-04-25 16:19:37 -07:00
|
|
|
#[error("Invalid num coding shreds: {0}")]
|
|
|
|
InvalidNumCodingShreds(u16),
|
|
|
|
#[error("Invalid parent_offset: {parent_offset}, slot: {slot}")]
|
2020-04-19 21:15:09 -07:00
|
|
|
InvalidParentOffset { slot: Slot, parent_offset: u16 },
|
2022-04-25 16:19:37 -07:00
|
|
|
#[error("Invalid parent slot: {parent_slot}, slot: {slot}")]
|
|
|
|
InvalidParentSlot { slot: Slot, parent_slot: Slot },
|
2022-04-29 12:42:15 -07:00
|
|
|
#[error("Invalid payload size: {0}")]
|
|
|
|
InvalidPayloadSize(/*payload size:*/ usize),
|
2022-06-07 15:41:03 -07:00
|
|
|
#[error("Invalid proof size: {0}")]
|
|
|
|
InvalidProofSize(/*proof_size:*/ u8),
|
2022-08-19 14:07:32 -07:00
|
|
|
#[error("Invalid recovered shred")]
|
|
|
|
InvalidRecoveredShred,
|
|
|
|
#[error("Invalid shard size: {0}")]
|
|
|
|
InvalidShardSize(/*shard_size:*/ usize),
|
2022-05-02 16:33:53 -07:00
|
|
|
#[error("Invalid shred flags: {0}")]
|
|
|
|
InvalidShredFlags(u8),
|
2022-06-19 08:30:18 -07:00
|
|
|
#[error("Invalid {0:?} shred index: {1}")]
|
|
|
|
InvalidShredIndex(ShredType, /*shred index:*/ u32),
|
2022-04-28 16:42:37 -07:00
|
|
|
#[error("Invalid shred type")]
|
|
|
|
InvalidShredType,
|
2022-05-30 05:51:00 -07:00
|
|
|
#[error("Invalid shred variant")]
|
|
|
|
InvalidShredVariant,
|
2022-06-07 15:41:03 -07:00
|
|
|
#[error(transparent)]
|
|
|
|
IoError(#[from] std::io::Error),
|
2022-09-15 14:50:22 -07:00
|
|
|
#[error("Unknown proof size")]
|
|
|
|
UnknownProofSize,
|
2019-10-17 15:44:15 -07:00
|
|
|
}
|
|
|
|
|
2021-11-16 09:50:56 -08:00
|
|
|
#[repr(u8)]
|
2022-04-23 06:33:59 -07:00
|
|
|
#[derive(
|
|
|
|
Clone,
|
|
|
|
Copy,
|
|
|
|
Debug,
|
|
|
|
Eq,
|
|
|
|
Hash,
|
|
|
|
PartialEq,
|
|
|
|
AbiEnumVisitor,
|
|
|
|
AbiExample,
|
|
|
|
Deserialize,
|
|
|
|
IntoPrimitive,
|
|
|
|
Serialize,
|
|
|
|
TryFromPrimitive,
|
|
|
|
)]
|
|
|
|
#[serde(into = "u8", try_from = "u8")]
|
2021-11-16 09:50:56 -08:00
|
|
|
pub enum ShredType {
|
|
|
|
Data = 0b1010_0101,
|
|
|
|
Code = 0b0101_1010,
|
|
|
|
}
|
|
|
|
|
2022-05-30 05:51:00 -07:00
|
|
|
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Deserialize, Serialize)]
|
|
|
|
#[serde(into = "u8", try_from = "u8")]
|
|
|
|
enum ShredVariant {
|
|
|
|
LegacyCode, // 0b0101_1010
|
|
|
|
LegacyData, // 0b1010_0101
|
2022-06-07 15:41:03 -07:00
|
|
|
// proof_size is the number of proof entries in the merkle tree branch.
|
|
|
|
MerkleCode(/*proof_size:*/ u8), // 0b0100_????
|
|
|
|
MerkleData(/*proof_size:*/ u8), // 0b1000_????
|
2022-05-30 05:51:00 -07:00
|
|
|
}
|
|
|
|
|
2019-10-15 20:48:45 -07:00
|
|
|
/// A common header that is present in data and code shred headers
|
2022-05-22 18:00:42 -07:00
|
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)]
|
2022-04-25 05:43:22 -07:00
|
|
|
struct ShredCommonHeader {
|
|
|
|
signature: Signature,
|
2022-05-30 05:51:00 -07:00
|
|
|
shred_variant: ShredVariant,
|
2022-04-25 05:43:22 -07:00
|
|
|
slot: Slot,
|
|
|
|
index: u32,
|
|
|
|
version: u16,
|
|
|
|
fec_set_index: u32,
|
2019-09-19 16:29:52 -07:00
|
|
|
}
|
|
|
|
|
2019-10-15 20:48:45 -07:00
|
|
|
/// The data shred header has parent offset and flags
|
2022-05-22 18:00:42 -07:00
|
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)]
|
2022-04-25 05:43:22 -07:00
|
|
|
struct DataShredHeader {
|
|
|
|
parent_offset: u16,
|
2022-05-01 12:25:15 -07:00
|
|
|
flags: ShredFlags,
|
2022-04-25 05:43:22 -07:00
|
|
|
size: u16, // common shred header + data shred header + data
|
2019-09-19 16:29:52 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/// The coding shred header has FEC information
|
2022-05-22 18:00:42 -07:00
|
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)]
|
2022-04-25 05:43:22 -07:00
|
|
|
struct CodingShredHeader {
|
|
|
|
num_data_shreds: u16,
|
|
|
|
num_coding_shreds: u16,
|
2022-08-19 14:07:32 -07:00
|
|
|
position: u16, // [0..num_coding_shreds)
|
2019-09-19 16:29:52 -07:00
|
|
|
}
|
|
|
|
|
2022-05-22 18:00:42 -07:00
|
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
2022-05-05 16:48:00 -07:00
|
|
|
pub enum Shred {
|
|
|
|
ShredCode(ShredCode),
|
|
|
|
ShredData(ShredData),
|
2019-09-16 20:28:54 -07:00
|
|
|
}
|
|
|
|
|
2022-12-31 09:08:25 -08:00
|
|
|
pub(crate) enum SignedData<'a> {
|
|
|
|
Chunk(&'a [u8]), // Chunk of payload past signature.
|
|
|
|
MerkleRoot(MerkleRoot),
|
|
|
|
}
|
|
|
|
|
|
|
|
impl<'a> AsRef<[u8]> for SignedData<'a> {
|
|
|
|
fn as_ref(&self) -> &[u8] {
|
|
|
|
match self {
|
|
|
|
Self::Chunk(chunk) => chunk,
|
|
|
|
Self::MerkleRoot(root) => root,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-12-16 06:18:55 -08:00
|
|
|
/// Tuple which uniquely identifies a shred should it exists.
|
2022-06-30 05:13:00 -07:00
|
|
|
#[derive(Clone, Copy, Eq, Debug, Hash, PartialEq)]
|
2021-12-14 09:34:02 -08:00
|
|
|
pub struct ShredId(Slot, /*shred index:*/ u32, ShredType);
|
|
|
|
|
|
|
|
impl ShredId {
|
|
|
|
pub(crate) fn new(slot: Slot, index: u32, shred_type: ShredType) -> ShredId {
|
|
|
|
ShredId(slot, index, shred_type)
|
|
|
|
}
|
|
|
|
|
2022-06-30 05:13:00 -07:00
|
|
|
pub fn slot(&self) -> Slot {
|
|
|
|
self.0
|
|
|
|
}
|
|
|
|
|
2021-12-14 09:34:02 -08:00
|
|
|
pub(crate) fn unwrap(&self) -> (Slot, /*shred index:*/ u32, ShredType) {
|
|
|
|
(self.0, self.1, self.2)
|
|
|
|
}
|
2022-06-27 10:58:43 -07:00
|
|
|
|
|
|
|
pub fn seed(&self, leader: &Pubkey) -> [u8; 32] {
|
|
|
|
let ShredId(slot, index, shred_type) = self;
|
|
|
|
hashv(&[
|
|
|
|
&slot.to_le_bytes(),
|
|
|
|
&u8::from(*shred_type).to_le_bytes(),
|
|
|
|
&index.to_le_bytes(),
|
|
|
|
AsRef::<[u8]>::as_ref(leader),
|
|
|
|
])
|
|
|
|
.to_bytes()
|
|
|
|
}
|
2021-12-14 09:34:02 -08:00
|
|
|
}
|
|
|
|
|
2021-12-16 06:18:55 -08:00
|
|
|
/// Tuple which identifies erasure coding set that the shred belongs to.
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
|
|
|
pub(crate) struct ErasureSetId(Slot, /*fec_set_index:*/ u32);
|
|
|
|
|
|
|
|
impl ErasureSetId {
|
|
|
|
pub(crate) fn slot(&self) -> Slot {
|
|
|
|
self.0
|
|
|
|
}
|
|
|
|
|
|
|
|
// Storage key for ErasureMeta in blockstore db.
|
|
|
|
pub(crate) fn store_key(&self) -> (Slot, /*fec_set_index:*/ u64) {
|
|
|
|
(self.0, u64::from(self.1))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-05 16:48:00 -07:00
|
|
|
macro_rules! dispatch {
|
|
|
|
($vis:vis fn $name:ident(&self $(, $arg:ident : $ty:ty)?) $(-> $out:ty)?) => {
|
|
|
|
#[inline]
|
|
|
|
$vis fn $name(&self $(, $arg:$ty)?) $(-> $out)? {
|
|
|
|
match self {
|
|
|
|
Self::ShredCode(shred) => shred.$name($($arg, )?),
|
|
|
|
Self::ShredData(shred) => shred.$name($($arg, )?),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
};
|
|
|
|
($vis:vis fn $name:ident(self $(, $arg:ident : $ty:ty)?) $(-> $out:ty)?) => {
|
|
|
|
#[inline]
|
|
|
|
$vis fn $name(self $(, $arg:$ty)?) $(-> $out)? {
|
|
|
|
match self {
|
|
|
|
Self::ShredCode(shred) => shred.$name($($arg, )?),
|
|
|
|
Self::ShredData(shred) => shred.$name($($arg, )?),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
};
|
|
|
|
($vis:vis fn $name:ident(&mut self $(, $arg:ident : $ty:ty)?) $(-> $out:ty)?) => {
|
|
|
|
#[inline]
|
|
|
|
$vis fn $name(&mut self $(, $arg:$ty)?) $(-> $out)? {
|
|
|
|
match self {
|
|
|
|
Self::ShredCode(shred) => shred.$name($($arg, )?),
|
|
|
|
Self::ShredData(shred) => shred.$name($($arg, )?),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-08-19 14:07:32 -07:00
|
|
|
use dispatch;
|
|
|
|
|
2019-09-18 16:24:30 -07:00
|
|
|
impl Shred {
|
2022-05-05 16:48:00 -07:00
|
|
|
dispatch!(fn common_header(&self) -> &ShredCommonHeader);
|
|
|
|
dispatch!(fn set_signature(&mut self, signature: Signature));
|
2022-12-31 09:08:25 -08:00
|
|
|
dispatch!(fn signed_data(&self) -> Result<SignedData, Error>);
|
2022-05-05 16:48:00 -07:00
|
|
|
|
|
|
|
// Returns the portion of the shred's payload which is erasure coded.
|
|
|
|
dispatch!(pub(crate) fn erasure_shard(self) -> Result<Vec<u8>, Error>);
|
|
|
|
// Like Shred::erasure_shard but returning a slice.
|
|
|
|
dispatch!(pub(crate) fn erasure_shard_as_slice(&self) -> Result<&[u8], Error>);
|
|
|
|
// Returns the shard index within the erasure coding set.
|
2022-05-30 05:51:19 -07:00
|
|
|
dispatch!(pub(crate) fn erasure_shard_index(&self) -> Result<usize, Error>);
|
2022-05-05 16:48:00 -07:00
|
|
|
|
|
|
|
dispatch!(pub fn into_payload(self) -> Vec<u8>);
|
|
|
|
dispatch!(pub fn payload(&self) -> &Vec<u8>);
|
|
|
|
dispatch!(pub fn sanitize(&self) -> Result<(), Error>);
|
|
|
|
|
|
|
|
// Only for tests.
|
|
|
|
dispatch!(pub fn set_index(&mut self, index: u32));
|
|
|
|
dispatch!(pub fn set_slot(&mut self, slot: Slot));
|
|
|
|
|
2019-12-12 13:27:33 -08:00
|
|
|
pub fn copy_to_packet(&self, packet: &mut Packet) {
|
2022-05-05 16:48:00 -07:00
|
|
|
let payload = self.payload();
|
|
|
|
let size = payload.len();
|
2022-05-25 09:52:54 -07:00
|
|
|
packet.buffer_mut()[..size].copy_from_slice(&payload[..]);
|
2022-12-06 03:54:49 -08:00
|
|
|
packet.meta_mut().size = size;
|
2019-12-12 13:27:33 -08:00
|
|
|
}
|
|
|
|
|
2022-04-25 05:43:22 -07:00
|
|
|
// TODO: Should this sanitize output?
|
2019-10-08 00:42:51 -07:00
|
|
|
pub fn new_from_data(
|
2019-11-02 00:38:30 -07:00
|
|
|
slot: Slot,
|
2019-10-08 00:42:51 -07:00
|
|
|
index: u32,
|
|
|
|
parent_offset: u16,
|
2022-04-27 11:04:10 -07:00
|
|
|
data: &[u8],
|
2022-05-02 16:33:53 -07:00
|
|
|
flags: ShredFlags,
|
2019-11-06 13:27:58 -08:00
|
|
|
reference_tick: u8,
|
2019-11-18 18:05:02 -08:00
|
|
|
version: u16,
|
2019-12-12 16:50:29 -08:00
|
|
|
fec_set_index: u32,
|
2019-10-08 00:42:51 -07:00
|
|
|
) -> Self {
|
2022-05-05 16:48:00 -07:00
|
|
|
Self::from(ShredData::new_from_data(
|
2019-11-18 18:05:02 -08:00
|
|
|
slot,
|
|
|
|
index,
|
2022-05-02 16:33:53 -07:00
|
|
|
parent_offset,
|
2022-05-05 16:48:00 -07:00
|
|
|
data,
|
2022-05-02 16:33:53 -07:00
|
|
|
flags,
|
2022-05-05 16:48:00 -07:00
|
|
|
reference_tick,
|
|
|
|
version,
|
|
|
|
fec_set_index,
|
|
|
|
))
|
2019-10-18 22:55:59 -07:00
|
|
|
}
|
2019-10-08 00:42:51 -07:00
|
|
|
|
2022-05-05 16:48:00 -07:00
|
|
|
pub fn new_from_serialized_shred(shred: Vec<u8>) -> Result<Self, Error> {
|
2022-05-30 05:51:00 -07:00
|
|
|
Ok(match layout::get_shred_variant(&shred)? {
|
|
|
|
ShredVariant::LegacyCode => {
|
|
|
|
let shred = legacy::ShredCode::from_payload(shred)?;
|
2022-05-30 05:51:19 -07:00
|
|
|
Self::from(ShredCode::from(shred))
|
2022-05-30 05:51:00 -07:00
|
|
|
}
|
|
|
|
ShredVariant::LegacyData => {
|
|
|
|
let shred = legacy::ShredData::from_payload(shred)?;
|
2022-05-30 05:51:19 -07:00
|
|
|
Self::from(ShredData::from(shred))
|
2022-05-30 05:51:00 -07:00
|
|
|
}
|
2022-06-07 15:41:03 -07:00
|
|
|
ShredVariant::MerkleCode(_) => {
|
|
|
|
let shred = merkle::ShredCode::from_payload(shred)?;
|
|
|
|
Self::from(ShredCode::from(shred))
|
|
|
|
}
|
|
|
|
ShredVariant::MerkleData(_) => {
|
|
|
|
let shred = merkle::ShredData::from_payload(shred)?;
|
|
|
|
Self::from(ShredData::from(shred))
|
|
|
|
}
|
2022-05-05 16:48:00 -07:00
|
|
|
})
|
2019-09-17 18:22:46 -07:00
|
|
|
}
|
|
|
|
|
2022-04-27 11:04:10 -07:00
|
|
|
pub fn new_from_parity_shard(
|
2022-04-19 13:00:05 -07:00
|
|
|
slot: Slot,
|
|
|
|
index: u32,
|
2022-04-27 11:04:10 -07:00
|
|
|
parity_shard: &[u8],
|
2022-04-19 13:00:05 -07:00
|
|
|
fec_set_index: u32,
|
|
|
|
num_data_shreds: u16,
|
|
|
|
num_coding_shreds: u16,
|
|
|
|
position: u16,
|
|
|
|
version: u16,
|
2022-04-25 05:43:22 -07:00
|
|
|
) -> Self {
|
2022-05-05 16:48:00 -07:00
|
|
|
Self::from(ShredCode::new_from_parity_shard(
|
2022-04-19 13:00:05 -07:00
|
|
|
slot,
|
2022-05-05 16:48:00 -07:00
|
|
|
index,
|
|
|
|
parity_shard,
|
2022-04-19 13:00:05 -07:00
|
|
|
fec_set_index,
|
2022-04-25 05:43:22 -07:00
|
|
|
num_data_shreds,
|
|
|
|
num_coding_shreds,
|
2021-12-05 06:42:09 -08:00
|
|
|
position,
|
2022-05-05 16:48:00 -07:00
|
|
|
version,
|
|
|
|
))
|
2019-09-16 20:28:54 -07:00
|
|
|
}
|
|
|
|
|
2021-12-14 09:34:02 -08:00
|
|
|
/// Unique identifier for each shred.
|
|
|
|
pub fn id(&self) -> ShredId {
|
|
|
|
ShredId(self.slot(), self.index(), self.shred_type())
|
|
|
|
}
|
|
|
|
|
2019-11-02 00:38:30 -07:00
|
|
|
pub fn slot(&self) -> Slot {
|
2022-05-05 16:48:00 -07:00
|
|
|
self.common_header().slot
|
2019-09-16 20:28:54 -07:00
|
|
|
}
|
|
|
|
|
2022-04-25 16:19:37 -07:00
|
|
|
pub fn parent(&self) -> Result<Slot, Error> {
|
2022-05-05 16:48:00 -07:00
|
|
|
match self {
|
|
|
|
Self::ShredCode(_) => Err(Error::InvalidShredType),
|
|
|
|
Self::ShredData(shred) => shred.parent(),
|
2019-09-16 20:28:54 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn index(&self) -> u32 {
|
2022-05-05 16:48:00 -07:00
|
|
|
self.common_header().index
|
2019-09-16 20:28:54 -07:00
|
|
|
}
|
|
|
|
|
2022-04-27 15:00:04 -07:00
|
|
|
pub(crate) fn data(&self) -> Result<&[u8], Error> {
|
2022-05-05 16:48:00 -07:00
|
|
|
match self {
|
|
|
|
Self::ShredCode(_) => Err(Error::InvalidShredType),
|
|
|
|
Self::ShredData(shred) => shred.data(),
|
2022-04-27 15:00:04 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-04-25 05:43:22 -07:00
|
|
|
// Possibly trimmed payload;
|
|
|
|
// Should only be used when storing shreds to blockstore.
|
|
|
|
pub(crate) fn bytes_to_store(&self) -> &[u8] {
|
2022-05-05 16:48:00 -07:00
|
|
|
match self {
|
|
|
|
Self::ShredCode(shred) => shred.payload(),
|
|
|
|
Self::ShredData(shred) => shred.bytes_to_store(),
|
2022-04-25 05:43:22 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn fec_set_index(&self) -> u32 {
|
2022-05-05 16:48:00 -07:00
|
|
|
self.common_header().fec_set_index
|
2021-12-09 08:43:57 -08:00
|
|
|
}
|
|
|
|
|
2021-12-10 12:08:04 -08:00
|
|
|
pub(crate) fn first_coding_index(&self) -> Option<u32> {
|
2022-05-05 16:48:00 -07:00
|
|
|
match self {
|
|
|
|
Self::ShredCode(shred) => shred.first_coding_index(),
|
|
|
|
Self::ShredData(_) => None,
|
2022-04-25 16:19:37 -07:00
|
|
|
}
|
2021-12-09 08:43:57 -08:00
|
|
|
}
|
|
|
|
|
2019-11-18 18:05:02 -08:00
|
|
|
pub fn version(&self) -> u16 {
|
2022-05-05 16:48:00 -07:00
|
|
|
self.common_header().version
|
2019-11-18 18:05:02 -08:00
|
|
|
}
|
|
|
|
|
2021-12-16 06:18:55 -08:00
|
|
|
// Identifier for the erasure coding set that the shred belongs to.
|
|
|
|
pub(crate) fn erasure_set(&self) -> ErasureSetId {
|
|
|
|
ErasureSetId(self.slot(), self.fec_set_index())
|
|
|
|
}
|
|
|
|
|
2022-12-31 09:08:25 -08:00
|
|
|
pub fn signature(&self) -> &Signature {
|
|
|
|
&self.common_header().signature
|
2019-09-16 20:28:54 -07:00
|
|
|
}
|
|
|
|
|
2022-04-19 13:00:05 -07:00
|
|
|
pub fn sign(&mut self, keypair: &Keypair) {
|
2022-12-31 09:08:25 -08:00
|
|
|
let data = self.signed_data().unwrap();
|
|
|
|
let signature = keypair.sign_message(data.as_ref());
|
2022-05-05 16:48:00 -07:00
|
|
|
self.set_signature(signature);
|
2022-04-19 13:00:05 -07:00
|
|
|
}
|
|
|
|
|
2021-11-16 09:50:56 -08:00
|
|
|
#[inline]
|
|
|
|
pub fn shred_type(&self) -> ShredType {
|
2022-05-30 05:51:00 -07:00
|
|
|
ShredType::from(self.common_header().shred_variant)
|
2022-05-05 16:48:00 -07:00
|
|
|
}
|
|
|
|
|
2019-09-16 20:28:54 -07:00
|
|
|
pub fn is_data(&self) -> bool {
|
2021-11-16 09:50:56 -08:00
|
|
|
self.shred_type() == ShredType::Data
|
2019-09-16 20:28:54 -07:00
|
|
|
}
|
2019-10-16 15:41:43 -07:00
|
|
|
pub fn is_code(&self) -> bool {
|
2021-11-16 09:50:56 -08:00
|
|
|
self.shred_type() == ShredType::Code
|
2019-10-16 15:41:43 -07:00
|
|
|
}
|
2019-09-16 20:28:54 -07:00
|
|
|
|
|
|
|
pub fn last_in_slot(&self) -> bool {
|
2022-05-05 16:48:00 -07:00
|
|
|
match self {
|
|
|
|
Self::ShredCode(_) => false,
|
|
|
|
Self::ShredData(shred) => shred.last_in_slot(),
|
2019-09-16 20:28:54 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-09-17 18:22:46 -07:00
|
|
|
/// This is not a safe function. It only changes the meta information.
|
|
|
|
/// Use this only for test code which doesn't care about actual shred
|
|
|
|
pub fn set_last_in_slot(&mut self) {
|
2022-05-05 16:48:00 -07:00
|
|
|
match self {
|
|
|
|
Self::ShredCode(_) => (),
|
|
|
|
Self::ShredData(shred) => shred.set_last_in_slot(),
|
2019-09-17 18:22:46 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-09-16 20:28:54 -07:00
|
|
|
pub fn data_complete(&self) -> bool {
|
2022-05-05 16:48:00 -07:00
|
|
|
match self {
|
|
|
|
Self::ShredCode(_) => false,
|
|
|
|
Self::ShredData(shred) => shred.data_complete(),
|
2019-09-16 20:28:54 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-04-25 05:43:22 -07:00
|
|
|
pub(crate) fn reference_tick(&self) -> u8 {
|
2022-05-05 16:48:00 -07:00
|
|
|
match self {
|
|
|
|
Self::ShredCode(_) => ShredFlags::SHRED_TICK_REFERENCE_MASK.bits(),
|
|
|
|
Self::ShredData(shred) => shred.reference_tick(),
|
2019-11-06 13:27:58 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-08-19 14:07:32 -07:00
|
|
|
#[must_use]
|
2019-09-17 18:22:46 -07:00
|
|
|
pub fn verify(&self, pubkey: &Pubkey) -> bool {
|
2022-12-31 09:08:25 -08:00
|
|
|
match self.signed_data() {
|
|
|
|
Ok(data) => self.signature().verify(pubkey.as_ref(), data.as_ref()),
|
|
|
|
Err(_) => false,
|
|
|
|
}
|
2019-09-17 18:22:46 -07:00
|
|
|
}
|
2022-04-25 05:43:22 -07:00
|
|
|
|
|
|
|
// Returns true if the erasure coding of the two shreds mismatch.
|
2022-05-05 16:48:00 -07:00
|
|
|
pub(crate) fn erasure_mismatch(&self, other: &Self) -> Result<bool, Error> {
|
|
|
|
match (self, other) {
|
|
|
|
(Self::ShredCode(shred), Self::ShredCode(other)) => Ok(shred.erasure_mismatch(other)),
|
2022-04-28 16:42:37 -07:00
|
|
|
_ => Err(Error::InvalidShredType),
|
2022-04-25 05:43:22 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-05 16:48:00 -07:00
|
|
|
pub(crate) fn num_data_shreds(&self) -> Result<u16, Error> {
|
|
|
|
match self {
|
|
|
|
Self::ShredCode(shred) => Ok(shred.num_data_shreds()),
|
|
|
|
Self::ShredData(_) => Err(Error::InvalidShredType),
|
2022-04-25 05:43:22 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-05 16:48:00 -07:00
|
|
|
pub(crate) fn num_coding_shreds(&self) -> Result<u16, Error> {
|
|
|
|
match self {
|
|
|
|
Self::ShredCode(shred) => Ok(shred.num_coding_shreds()),
|
|
|
|
Self::ShredData(_) => Err(Error::InvalidShredType),
|
2022-04-25 05:43:22 -07:00
|
|
|
}
|
|
|
|
}
|
2019-09-12 21:52:13 -07:00
|
|
|
}
|
|
|
|
|
2022-05-26 06:06:27 -07:00
|
|
|
// Helper methods to extract pieces of the shred from the payload
|
|
|
|
// without deserializing the entire payload.
|
|
|
|
pub mod layout {
|
2022-12-31 09:08:25 -08:00
|
|
|
use {super::*, std::ops::Range};
|
2022-12-29 15:32:58 -08:00
|
|
|
#[cfg(test)]
|
2022-12-30 10:52:10 -08:00
|
|
|
use {
|
|
|
|
rand::{seq::SliceRandom, Rng},
|
|
|
|
std::collections::HashMap,
|
|
|
|
};
|
|
|
|
|
2022-05-30 05:51:19 -07:00
|
|
|
fn get_shred_size(packet: &Packet) -> Option<usize> {
|
2022-06-02 18:05:06 -07:00
|
|
|
let size = packet.data(..)?.len();
|
2022-12-06 03:54:49 -08:00
|
|
|
if packet.meta().repair() {
|
2022-05-30 05:51:19 -07:00
|
|
|
size.checked_sub(SIZE_OF_NONCE)
|
2022-05-26 06:06:27 -07:00
|
|
|
} else {
|
2022-05-30 05:51:19 -07:00
|
|
|
Some(size)
|
2022-05-26 06:06:27 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-30 05:51:19 -07:00
|
|
|
pub fn get_shred(packet: &Packet) -> Option<&[u8]> {
|
|
|
|
let size = get_shred_size(packet)?;
|
2022-07-05 14:41:19 -07:00
|
|
|
packet.data(..size)
|
2022-05-26 06:06:27 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
pub(crate) fn get_signature(shred: &[u8]) -> Option<Signature> {
|
|
|
|
Some(Signature::new(shred.get(..SIZE_OF_SIGNATURE)?))
|
|
|
|
}
|
|
|
|
|
|
|
|
pub(crate) const fn get_signature_range() -> Range<usize> {
|
|
|
|
0..SIZE_OF_SIGNATURE
|
|
|
|
}
|
|
|
|
|
2022-05-30 05:51:00 -07:00
|
|
|
pub(super) fn get_shred_variant(shred: &[u8]) -> Result<ShredVariant, Error> {
|
|
|
|
let shred_variant = match shred.get(OFFSET_OF_SHRED_VARIANT) {
|
|
|
|
None => return Err(Error::InvalidPayloadSize(shred.len())),
|
|
|
|
Some(shred_variant) => *shred_variant,
|
|
|
|
};
|
|
|
|
ShredVariant::try_from(shred_variant).map_err(|_| Error::InvalidShredVariant)
|
|
|
|
}
|
|
|
|
|
2022-06-30 05:13:00 -07:00
|
|
|
#[inline]
|
2022-05-26 06:06:27 -07:00
|
|
|
pub(super) fn get_shred_type(shred: &[u8]) -> Result<ShredType, Error> {
|
2022-05-30 05:51:00 -07:00
|
|
|
let shred_variant = get_shred_variant(shred)?;
|
|
|
|
Ok(ShredType::from(shred_variant))
|
2022-05-26 06:06:27 -07:00
|
|
|
}
|
|
|
|
|
2022-06-30 05:13:00 -07:00
|
|
|
#[inline]
|
2022-05-26 06:06:27 -07:00
|
|
|
pub fn get_slot(shred: &[u8]) -> Option<Slot> {
|
2022-06-19 08:30:18 -07:00
|
|
|
<[u8; 8]>::try_from(shred.get(OFFSET_OF_SHRED_SLOT..)?.get(..8)?)
|
|
|
|
.map(Slot::from_le_bytes)
|
|
|
|
.ok()
|
2022-05-26 06:06:27 -07:00
|
|
|
}
|
|
|
|
|
2022-06-30 05:13:00 -07:00
|
|
|
#[inline]
|
2022-05-26 06:06:27 -07:00
|
|
|
pub(super) fn get_index(shred: &[u8]) -> Option<u32> {
|
2022-06-19 08:30:18 -07:00
|
|
|
<[u8; 4]>::try_from(shred.get(OFFSET_OF_SHRED_INDEX..)?.get(..4)?)
|
|
|
|
.map(u32::from_le_bytes)
|
|
|
|
.ok()
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn get_version(shred: &[u8]) -> Option<u16> {
|
2022-12-29 15:32:58 -08:00
|
|
|
<[u8; 2]>::try_from(shred.get(77..79)?)
|
2022-06-19 08:30:18 -07:00
|
|
|
.map(u16::from_le_bytes)
|
|
|
|
.ok()
|
2022-05-26 06:06:27 -07:00
|
|
|
}
|
|
|
|
|
2022-06-28 05:45:50 -07:00
|
|
|
// The caller should verify first that the shred is data and not code!
|
|
|
|
pub(super) fn get_parent_offset(shred: &[u8]) -> Option<u16> {
|
|
|
|
debug_assert_eq!(get_shred_type(shred).unwrap(), ShredType::Data);
|
2022-12-29 15:32:58 -08:00
|
|
|
<[u8; 2]>::try_from(shred.get(83..85)?)
|
2022-06-28 05:45:50 -07:00
|
|
|
.map(u16::from_le_bytes)
|
|
|
|
.ok()
|
|
|
|
}
|
|
|
|
|
2022-06-30 05:13:00 -07:00
|
|
|
#[inline]
|
|
|
|
pub fn get_shred_id(shred: &[u8]) -> Option<ShredId> {
|
|
|
|
Some(ShredId(
|
|
|
|
get_slot(shred)?,
|
|
|
|
get_index(shred)?,
|
|
|
|
get_shred_type(shred).ok()?,
|
|
|
|
))
|
|
|
|
}
|
|
|
|
|
2022-12-30 10:52:10 -08:00
|
|
|
pub(crate) fn get_signed_data(shred: &[u8]) -> Option<SignedData> {
|
|
|
|
let data = match get_shred_variant(shred).ok()? {
|
|
|
|
ShredVariant::LegacyCode | ShredVariant::LegacyData => {
|
|
|
|
let chunk = shred.get(self::legacy::SIGNED_MESSAGE_OFFSETS)?;
|
|
|
|
SignedData::Chunk(chunk)
|
|
|
|
}
|
2022-06-07 15:41:03 -07:00
|
|
|
ShredVariant::MerkleCode(proof_size) => {
|
2022-12-30 10:52:10 -08:00
|
|
|
let merkle_root = self::merkle::ShredCode::get_merkle_root(shred, proof_size)?;
|
|
|
|
SignedData::MerkleRoot(merkle_root)
|
2022-06-07 15:41:03 -07:00
|
|
|
}
|
|
|
|
ShredVariant::MerkleData(proof_size) => {
|
2022-12-30 10:52:10 -08:00
|
|
|
let merkle_root = self::merkle::ShredData::get_merkle_root(shred, proof_size)?;
|
|
|
|
SignedData::MerkleRoot(merkle_root)
|
2022-06-07 15:41:03 -07:00
|
|
|
}
|
2022-05-30 05:51:19 -07:00
|
|
|
};
|
2022-12-30 10:52:10 -08:00
|
|
|
Some(data)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Returns offsets within the shred payload which is signed.
|
|
|
|
pub(crate) fn get_signed_data_offsets(shred: &[u8]) -> Option<Range<usize>> {
|
|
|
|
let offsets = match get_shred_variant(shred).ok()? {
|
|
|
|
ShredVariant::LegacyCode | ShredVariant::LegacyData => legacy::SIGNED_MESSAGE_OFFSETS,
|
|
|
|
ShredVariant::MerkleCode(proof_size) => {
|
|
|
|
merkle::ShredCode::get_signed_data_offsets(proof_size)?
|
|
|
|
}
|
|
|
|
ShredVariant::MerkleData(proof_size) => {
|
|
|
|
merkle::ShredData::get_signed_data_offsets(proof_size)?
|
|
|
|
}
|
|
|
|
};
|
|
|
|
(offsets.end <= shred.len()).then_some(offsets)
|
2022-05-26 06:06:27 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
pub(crate) fn get_reference_tick(shred: &[u8]) -> Result<u8, Error> {
|
|
|
|
if get_shred_type(shred)? != ShredType::Data {
|
|
|
|
return Err(Error::InvalidShredType);
|
|
|
|
}
|
2022-12-29 15:32:58 -08:00
|
|
|
let flags = match shred.get(85) {
|
2022-05-26 06:06:27 -07:00
|
|
|
None => return Err(Error::InvalidPayloadSize(shred.len())),
|
|
|
|
Some(flags) => flags,
|
|
|
|
};
|
|
|
|
Ok(flags & ShredFlags::SHRED_TICK_REFERENCE_MASK.bits())
|
|
|
|
}
|
2022-12-29 15:32:58 -08:00
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
pub(crate) fn get_merkle_root(shred: &[u8]) -> Option<MerkleRoot> {
|
|
|
|
match get_shred_variant(shred).ok()? {
|
|
|
|
ShredVariant::LegacyCode | ShredVariant::LegacyData => None,
|
|
|
|
ShredVariant::MerkleCode(proof_size) => {
|
|
|
|
merkle::ShredCode::get_merkle_root(shred, proof_size)
|
|
|
|
}
|
|
|
|
ShredVariant::MerkleData(proof_size) => {
|
|
|
|
merkle::ShredData::get_merkle_root(shred, proof_size)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2022-12-30 10:52:10 -08:00
|
|
|
|
|
|
|
// Minimally corrupts the packet so that the signature no longer verifies.
|
|
|
|
#[cfg(test)]
|
|
|
|
pub(crate) fn corrupt_packet<R: Rng>(
|
|
|
|
rng: &mut R,
|
|
|
|
packet: &mut Packet,
|
|
|
|
keypairs: &HashMap<Slot, Keypair>,
|
|
|
|
) {
|
|
|
|
fn modify_packet<R: Rng>(rng: &mut R, packet: &mut Packet, offsets: Range<usize>) {
|
|
|
|
let buffer = packet.buffer_mut();
|
|
|
|
let byte = buffer[offsets].choose_mut(rng).unwrap();
|
|
|
|
*byte = rng.gen::<u8>().max(1u8).wrapping_add(*byte);
|
|
|
|
}
|
|
|
|
let shred = get_shred(packet).unwrap();
|
|
|
|
let merkle_proof_size = match get_shred_variant(shred).unwrap() {
|
|
|
|
ShredVariant::LegacyCode | ShredVariant::LegacyData => None,
|
|
|
|
ShredVariant::MerkleCode(proof_size) | ShredVariant::MerkleData(proof_size) => {
|
|
|
|
Some(proof_size)
|
|
|
|
}
|
|
|
|
};
|
|
|
|
let coin_flip: bool = rng.gen();
|
|
|
|
if coin_flip {
|
|
|
|
// Corrupt one byte within the signature offsets.
|
|
|
|
modify_packet(rng, packet, 0..SIGNATURE_BYTES);
|
|
|
|
} else {
|
|
|
|
// Corrupt one byte within the signed data offsets.
|
|
|
|
let size = shred.len();
|
|
|
|
let offsets = get_signed_data_offsets(shred).unwrap();
|
|
|
|
modify_packet(rng, packet, offsets);
|
|
|
|
if let Some(proof_size) = merkle_proof_size {
|
|
|
|
// Also need to corrupt the merkle proof.
|
|
|
|
// Proof entries are each 20 bytes at the end of shreds.
|
|
|
|
let offset = usize::from(proof_size) * 20;
|
|
|
|
modify_packet(rng, packet, size - offset..size);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Assert that the signature no longer verifies.
|
|
|
|
let shred = get_shred(packet).unwrap();
|
|
|
|
let slot = get_slot(shred).unwrap();
|
|
|
|
let signature = get_signature(shred).unwrap();
|
|
|
|
if coin_flip {
|
|
|
|
let pubkey = keypairs[&slot].pubkey();
|
|
|
|
let data = get_signed_data(shred).unwrap();
|
|
|
|
assert!(!signature.verify(pubkey.as_ref(), data.as_ref()));
|
|
|
|
let offsets = get_signed_data_offsets(shred).unwrap();
|
|
|
|
assert!(!signature.verify(pubkey.as_ref(), &shred[offsets]));
|
|
|
|
} else {
|
|
|
|
// Slot may have been corrupted and no longer mapping to a keypair.
|
|
|
|
let pubkey = keypairs.get(&slot).map(Keypair::pubkey).unwrap_or_default();
|
|
|
|
if let Some(data) = get_signed_data(shred) {
|
|
|
|
assert!(!signature.verify(pubkey.as_ref(), data.as_ref()));
|
|
|
|
}
|
|
|
|
let offsets = get_signed_data_offsets(shred).unwrap_or_default();
|
|
|
|
assert!(!signature.verify(pubkey.as_ref(), &shred[offsets]));
|
|
|
|
}
|
|
|
|
}
|
2022-05-26 06:06:27 -07:00
|
|
|
}
|
|
|
|
|
2022-05-05 16:48:00 -07:00
|
|
|
impl From<ShredCode> for Shred {
|
|
|
|
fn from(shred: ShredCode) -> Self {
|
|
|
|
Self::ShredCode(shred)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl From<ShredData> for Shred {
|
|
|
|
fn from(shred: ShredData) -> Self {
|
|
|
|
Self::ShredData(shred)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-08-19 14:07:32 -07:00
|
|
|
impl From<merkle::Shred> for Shred {
|
|
|
|
fn from(shred: merkle::Shred) -> Self {
|
|
|
|
match shred {
|
|
|
|
merkle::Shred::ShredCode(shred) => Self::ShredCode(ShredCode::Merkle(shred)),
|
|
|
|
merkle::Shred::ShredData(shred) => Self::ShredData(ShredData::Merkle(shred)),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl TryFrom<Shred> for merkle::Shred {
|
|
|
|
type Error = Error;
|
|
|
|
|
|
|
|
fn try_from(shred: Shred) -> Result<Self, Self::Error> {
|
|
|
|
match shred {
|
|
|
|
Shred::ShredCode(ShredCode::Legacy(_)) => Err(Error::InvalidShredVariant),
|
|
|
|
Shred::ShredCode(ShredCode::Merkle(shred)) => Ok(Self::ShredCode(shred)),
|
|
|
|
Shred::ShredData(ShredData::Legacy(_)) => Err(Error::InvalidShredVariant),
|
|
|
|
Shred::ShredData(ShredData::Merkle(shred)) => Ok(Self::ShredData(shred)),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-30 05:51:00 -07:00
|
|
|
impl From<ShredVariant> for ShredType {
|
|
|
|
#[inline]
|
|
|
|
fn from(shred_variant: ShredVariant) -> Self {
|
|
|
|
match shred_variant {
|
|
|
|
ShredVariant::LegacyCode => ShredType::Code,
|
|
|
|
ShredVariant::LegacyData => ShredType::Data,
|
2022-06-07 15:41:03 -07:00
|
|
|
ShredVariant::MerkleCode(_) => ShredType::Code,
|
|
|
|
ShredVariant::MerkleData(_) => ShredType::Data,
|
2022-05-30 05:51:00 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl From<ShredVariant> for u8 {
|
|
|
|
fn from(shred_variant: ShredVariant) -> u8 {
|
|
|
|
match shred_variant {
|
|
|
|
ShredVariant::LegacyCode => u8::from(ShredType::Code),
|
|
|
|
ShredVariant::LegacyData => u8::from(ShredType::Data),
|
2022-06-07 15:41:03 -07:00
|
|
|
ShredVariant::MerkleCode(proof_size) => proof_size | 0x40,
|
|
|
|
ShredVariant::MerkleData(proof_size) => proof_size | 0x80,
|
2022-05-30 05:51:00 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl TryFrom<u8> for ShredVariant {
|
|
|
|
type Error = Error;
|
|
|
|
fn try_from(shred_variant: u8) -> Result<Self, Self::Error> {
|
|
|
|
if shred_variant == u8::from(ShredType::Code) {
|
|
|
|
Ok(ShredVariant::LegacyCode)
|
|
|
|
} else if shred_variant == u8::from(ShredType::Data) {
|
|
|
|
Ok(ShredVariant::LegacyData)
|
|
|
|
} else {
|
2022-06-07 15:41:03 -07:00
|
|
|
match shred_variant & 0xF0 {
|
|
|
|
0x40 => Ok(ShredVariant::MerkleCode(shred_variant & 0x0F)),
|
|
|
|
0x80 => Ok(ShredVariant::MerkleData(shred_variant & 0x0F)),
|
|
|
|
_ => Err(Error::InvalidShredVariant),
|
|
|
|
}
|
2022-05-30 05:51:00 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-09-25 11:09:47 -07:00
|
|
|
pub(crate) fn recover(
|
|
|
|
shreds: Vec<Shred>,
|
|
|
|
reed_solomon_cache: &ReedSolomonCache,
|
|
|
|
) -> Result<Vec<Shred>, Error> {
|
2022-08-19 14:07:32 -07:00
|
|
|
match shreds
|
|
|
|
.first()
|
|
|
|
.ok_or(TooFewShardsPresent)?
|
|
|
|
.common_header()
|
|
|
|
.shred_variant
|
|
|
|
{
|
2022-09-25 11:09:47 -07:00
|
|
|
ShredVariant::LegacyData | ShredVariant::LegacyCode => {
|
|
|
|
Shredder::try_recovery(shreds, reed_solomon_cache)
|
|
|
|
}
|
2022-08-19 14:07:32 -07:00
|
|
|
ShredVariant::MerkleCode(_) | ShredVariant::MerkleData(_) => {
|
|
|
|
let shreds = shreds
|
|
|
|
.into_iter()
|
|
|
|
.map(merkle::Shred::try_from)
|
|
|
|
.collect::<Result<_, _>>()?;
|
2022-09-25 11:09:47 -07:00
|
|
|
Ok(merkle::recover(shreds, reed_solomon_cache)?
|
2022-08-19 14:07:32 -07:00
|
|
|
.into_iter()
|
|
|
|
.map(Shred::from)
|
|
|
|
.collect())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-09-15 14:50:22 -07:00
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
|
|
pub(crate) fn make_merkle_shreds_from_entries(
|
|
|
|
thread_pool: &ThreadPool,
|
|
|
|
keypair: &Keypair,
|
|
|
|
entries: &[Entry],
|
|
|
|
slot: Slot,
|
|
|
|
parent_slot: Slot,
|
|
|
|
shred_version: u16,
|
|
|
|
reference_tick: u8,
|
|
|
|
is_last_in_slot: bool,
|
|
|
|
next_shred_index: u32,
|
|
|
|
next_code_index: u32,
|
2022-09-25 11:09:47 -07:00
|
|
|
reed_solomon_cache: &ReedSolomonCache,
|
2022-09-15 14:50:22 -07:00
|
|
|
stats: &mut ProcessShredsStats,
|
|
|
|
) -> Result<Vec<Shred>, Error> {
|
|
|
|
let now = Instant::now();
|
|
|
|
let entries = bincode::serialize(entries)?;
|
|
|
|
stats.serialize_elapsed += now.elapsed().as_micros() as u64;
|
|
|
|
let shreds = merkle::make_shreds_from_data(
|
|
|
|
thread_pool,
|
|
|
|
keypair,
|
|
|
|
&entries[..],
|
|
|
|
slot,
|
|
|
|
parent_slot,
|
|
|
|
shred_version,
|
|
|
|
reference_tick,
|
|
|
|
is_last_in_slot,
|
|
|
|
next_shred_index,
|
|
|
|
next_code_index,
|
2022-09-25 11:09:47 -07:00
|
|
|
reed_solomon_cache,
|
2022-09-15 14:50:22 -07:00
|
|
|
stats,
|
|
|
|
)?;
|
|
|
|
Ok(shreds.into_iter().flatten().map(Shred::from).collect())
|
|
|
|
}
|
|
|
|
|
2022-07-05 14:41:19 -07:00
|
|
|
// Accepts shreds in the slot range [root + 1, max_slot].
|
2022-06-28 05:45:50 -07:00
|
|
|
#[must_use]
|
|
|
|
pub fn should_discard_shred(
|
2022-05-26 06:06:27 -07:00
|
|
|
packet: &Packet,
|
2022-06-28 05:45:50 -07:00
|
|
|
root: Slot,
|
2022-07-05 14:41:19 -07:00
|
|
|
max_slot: Slot,
|
2022-06-28 05:45:50 -07:00
|
|
|
shred_version: u16,
|
2020-12-15 16:50:40 -08:00
|
|
|
stats: &mut ShredFetchStats,
|
2022-06-28 05:45:50 -07:00
|
|
|
) -> bool {
|
2022-07-05 14:41:19 -07:00
|
|
|
debug_assert!(root < max_slot);
|
2022-05-30 05:51:19 -07:00
|
|
|
let shred = match layout::get_shred(packet) {
|
|
|
|
None => {
|
|
|
|
stats.index_overrun += 1;
|
2022-06-28 05:45:50 -07:00
|
|
|
return true;
|
2022-05-30 05:51:19 -07:00
|
|
|
}
|
|
|
|
Some(shred) => shred,
|
|
|
|
};
|
2022-07-05 14:41:19 -07:00
|
|
|
match layout::get_version(shred) {
|
|
|
|
None => {
|
|
|
|
stats.index_overrun += 1;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
Some(version) => {
|
|
|
|
if version != shred_version {
|
|
|
|
stats.shred_version_mismatch += 1;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
2020-12-15 16:50:40 -08:00
|
|
|
}
|
2022-05-26 06:06:27 -07:00
|
|
|
let shred_type = match layout::get_shred_type(shred) {
|
|
|
|
Ok(shred_type) => shred_type,
|
|
|
|
Err(_) => {
|
|
|
|
stats.bad_shred_type += 1;
|
2022-06-28 05:45:50 -07:00
|
|
|
return true;
|
2020-12-15 16:50:40 -08:00
|
|
|
}
|
2022-01-21 16:01:22 -08:00
|
|
|
};
|
2022-05-26 06:06:27 -07:00
|
|
|
let slot = match layout::get_slot(shred) {
|
2022-06-28 05:45:50 -07:00
|
|
|
Some(slot) => {
|
2022-07-05 14:41:19 -07:00
|
|
|
if slot > max_slot {
|
2022-06-28 05:45:50 -07:00
|
|
|
stats.slot_out_of_range += 1;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
slot
|
|
|
|
}
|
2022-05-26 06:06:27 -07:00
|
|
|
None => {
|
2020-12-15 16:50:40 -08:00
|
|
|
stats.slot_bad_deserialize += 1;
|
2022-06-28 05:45:50 -07:00
|
|
|
return true;
|
2020-12-15 16:50:40 -08:00
|
|
|
}
|
2022-01-21 16:01:22 -08:00
|
|
|
};
|
2022-05-26 06:06:27 -07:00
|
|
|
let index = match layout::get_index(shred) {
|
|
|
|
Some(index) => index,
|
|
|
|
None => {
|
|
|
|
stats.index_bad_deserialize += 1;
|
2022-06-28 05:45:50 -07:00
|
|
|
return true;
|
2021-11-16 09:50:56 -08:00
|
|
|
}
|
|
|
|
};
|
2022-06-28 05:45:50 -07:00
|
|
|
match shred_type {
|
|
|
|
ShredType::Code => {
|
|
|
|
if index >= shred_code::MAX_CODE_SHREDS_PER_SLOT as u32 {
|
|
|
|
stats.index_out_of_bounds += 1;
|
|
|
|
return true;
|
|
|
|
}
|
2022-06-30 12:45:10 -07:00
|
|
|
if slot <= root {
|
|
|
|
stats.slot_out_of_range += 1;
|
|
|
|
return true;
|
|
|
|
}
|
2022-06-28 05:45:50 -07:00
|
|
|
}
|
|
|
|
ShredType::Data => {
|
|
|
|
if index >= MAX_DATA_SHREDS_PER_SLOT as u32 {
|
|
|
|
stats.index_out_of_bounds += 1;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
let parent_offset = match layout::get_parent_offset(shred) {
|
|
|
|
Some(parent_offset) => parent_offset,
|
|
|
|
None => {
|
|
|
|
stats.bad_parent_offset += 1;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
let parent = match slot.checked_sub(Slot::from(parent_offset)) {
|
|
|
|
Some(parent) => parent,
|
|
|
|
None => {
|
|
|
|
stats.bad_parent_offset += 1;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
if !blockstore::verify_shred_slots(slot, parent, root) {
|
|
|
|
stats.slot_out_of_range += 1;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
2022-05-26 06:06:27 -07:00
|
|
|
}
|
2022-07-05 14:41:19 -07:00
|
|
|
false
|
2020-12-15 16:50:40 -08:00
|
|
|
}
|
|
|
|
|
2020-05-19 12:38:18 -07:00
|
|
|
pub fn max_ticks_per_n_shreds(num_shreds: u64, shred_data_size: Option<usize>) -> u64 {
|
2019-10-31 13:38:50 -07:00
|
|
|
let ticks = create_ticks(1, 0, Hash::default());
|
2020-05-19 12:38:18 -07:00
|
|
|
max_entries_per_n_shred(&ticks[0], num_shreds, shred_data_size)
|
2019-10-08 00:42:51 -07:00
|
|
|
}
|
|
|
|
|
2020-05-19 12:38:18 -07:00
|
|
|
pub fn max_entries_per_n_shred(
|
|
|
|
entry: &Entry,
|
|
|
|
num_shreds: u64,
|
|
|
|
shred_data_size: Option<usize>,
|
|
|
|
) -> u64 {
|
2022-06-07 15:41:03 -07:00
|
|
|
let data_buffer_size = ShredData::capacity(/*merkle_proof_size:*/ None).unwrap();
|
2022-05-30 05:51:19 -07:00
|
|
|
let shred_data_size = shred_data_size.unwrap_or(data_buffer_size) as u64;
|
2019-10-08 00:42:51 -07:00
|
|
|
let vec_size = bincode::serialized_size(&vec![entry]).unwrap();
|
|
|
|
let entry_size = bincode::serialized_size(entry).unwrap();
|
|
|
|
let count_size = vec_size - entry_size;
|
|
|
|
|
|
|
|
(shred_data_size * num_shreds - count_size) / entry_size
|
|
|
|
}
|
|
|
|
|
2019-12-11 11:10:21 -08:00
|
|
|
pub fn verify_test_data_shred(
|
|
|
|
shred: &Shred,
|
|
|
|
index: u32,
|
|
|
|
slot: Slot,
|
|
|
|
parent: Slot,
|
|
|
|
pk: &Pubkey,
|
|
|
|
verify: bool,
|
|
|
|
is_last_in_slot: bool,
|
2021-03-16 03:09:16 -07:00
|
|
|
is_last_data: bool,
|
2019-12-11 11:10:21 -08:00
|
|
|
) {
|
2022-04-28 16:42:37 -07:00
|
|
|
shred.sanitize().unwrap();
|
2019-12-11 11:10:21 -08:00
|
|
|
assert!(shred.is_data());
|
|
|
|
assert_eq!(shred.index(), index);
|
|
|
|
assert_eq!(shred.slot(), slot);
|
2021-12-09 08:43:57 -08:00
|
|
|
assert_eq!(shred.parent().unwrap(), parent);
|
2019-12-11 11:10:21 -08:00
|
|
|
assert_eq!(verify, shred.verify(pk));
|
|
|
|
if is_last_in_slot {
|
|
|
|
assert!(shred.last_in_slot());
|
|
|
|
} else {
|
|
|
|
assert!(!shred.last_in_slot());
|
|
|
|
}
|
2021-03-16 03:09:16 -07:00
|
|
|
if is_last_data {
|
2019-12-11 11:10:21 -08:00
|
|
|
assert!(shred.data_complete());
|
|
|
|
} else {
|
|
|
|
assert!(!shred.data_complete());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-08-02 15:53:42 -07:00
|
|
|
#[cfg(test)]
|
2022-04-25 05:43:22 -07:00
|
|
|
mod tests {
|
2022-05-01 06:11:45 -07:00
|
|
|
use {
|
|
|
|
super::*,
|
|
|
|
bincode::serialized_size,
|
|
|
|
matches::assert_matches,
|
|
|
|
rand::Rng,
|
|
|
|
rand_chacha::{rand_core::SeedableRng, ChaChaRng},
|
2022-05-05 16:48:00 -07:00
|
|
|
solana_sdk::{shred_version, signature::Signer},
|
2022-05-01 06:11:45 -07:00
|
|
|
};
|
|
|
|
|
2022-12-29 15:32:58 -08:00
|
|
|
const SIZE_OF_SHRED_INDEX: usize = 4;
|
|
|
|
|
2022-05-01 06:11:45 -07:00
|
|
|
fn bs58_decode<T: AsRef<[u8]>>(data: T) -> Vec<u8> {
|
|
|
|
bs58::decode(data).into_vec().unwrap()
|
|
|
|
}
|
2019-08-02 15:53:42 -07:00
|
|
|
|
2019-10-21 12:46:16 -07:00
|
|
|
#[test]
|
|
|
|
fn test_shred_constants() {
|
2022-04-29 12:42:15 -07:00
|
|
|
let common_header = ShredCommonHeader {
|
|
|
|
signature: Signature::default(),
|
2022-05-30 05:51:00 -07:00
|
|
|
shred_variant: ShredVariant::LegacyCode,
|
2022-04-29 12:42:15 -07:00
|
|
|
slot: Slot::MAX,
|
|
|
|
index: u32::MAX,
|
|
|
|
version: u16::MAX,
|
|
|
|
fec_set_index: u32::MAX,
|
|
|
|
};
|
2022-05-05 16:48:00 -07:00
|
|
|
let data_shred_header = DataShredHeader {
|
|
|
|
parent_offset: u16::MAX,
|
|
|
|
flags: ShredFlags::all(),
|
|
|
|
size: u16::MAX,
|
|
|
|
};
|
|
|
|
let coding_shred_header = CodingShredHeader {
|
|
|
|
num_data_shreds: u16::MAX,
|
|
|
|
num_coding_shreds: u16::MAX,
|
|
|
|
position: u16::MAX,
|
|
|
|
};
|
2019-10-21 12:46:16 -07:00
|
|
|
assert_eq!(
|
|
|
|
SIZE_OF_COMMON_SHRED_HEADER,
|
2022-04-29 12:42:15 -07:00
|
|
|
serialized_size(&common_header).unwrap() as usize
|
2019-10-21 12:46:16 -07:00
|
|
|
);
|
|
|
|
assert_eq!(
|
2022-05-30 05:51:19 -07:00
|
|
|
SIZE_OF_CODING_SHRED_HEADERS - SIZE_OF_COMMON_SHRED_HEADER,
|
2022-05-05 16:48:00 -07:00
|
|
|
serialized_size(&coding_shred_header).unwrap() as usize
|
2019-10-21 12:46:16 -07:00
|
|
|
);
|
|
|
|
assert_eq!(
|
2022-05-30 05:51:19 -07:00
|
|
|
SIZE_OF_DATA_SHRED_HEADERS - SIZE_OF_COMMON_SHRED_HEADER,
|
2022-05-05 16:48:00 -07:00
|
|
|
serialized_size(&data_shred_header).unwrap() as usize
|
2019-10-21 12:46:16 -07:00
|
|
|
);
|
2020-05-19 12:38:18 -07:00
|
|
|
let data_shred_header_with_size = DataShredHeader {
|
|
|
|
size: 1000,
|
2022-05-05 16:48:00 -07:00
|
|
|
..data_shred_header
|
2020-05-19 12:38:18 -07:00
|
|
|
};
|
|
|
|
assert_eq!(
|
2022-05-30 05:51:19 -07:00
|
|
|
SIZE_OF_DATA_SHRED_HEADERS - SIZE_OF_COMMON_SHRED_HEADER,
|
2020-05-19 12:38:18 -07:00
|
|
|
serialized_size(&data_shred_header_with_size).unwrap() as usize
|
|
|
|
);
|
2019-10-21 12:46:16 -07:00
|
|
|
assert_eq!(
|
|
|
|
SIZE_OF_SIGNATURE,
|
|
|
|
bincode::serialized_size(&Signature::default()).unwrap() as usize
|
|
|
|
);
|
2019-12-30 07:42:09 -08:00
|
|
|
assert_eq!(
|
2022-05-30 05:51:00 -07:00
|
|
|
SIZE_OF_SHRED_VARIANT,
|
2022-06-07 15:41:03 -07:00
|
|
|
bincode::serialized_size(&ShredVariant::MerkleCode(15)).unwrap() as usize
|
2019-12-30 07:42:09 -08:00
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
SIZE_OF_SHRED_SLOT,
|
|
|
|
bincode::serialized_size(&Slot::default()).unwrap() as usize
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
SIZE_OF_SHRED_INDEX,
|
2022-04-29 12:42:15 -07:00
|
|
|
bincode::serialized_size(&common_header.index).unwrap() as usize
|
2019-12-30 07:42:09 -08:00
|
|
|
);
|
2019-10-21 12:46:16 -07:00
|
|
|
}
|
|
|
|
|
2019-11-18 18:05:02 -08:00
|
|
|
#[test]
|
|
|
|
fn test_version_from_hash() {
|
|
|
|
let hash = [
|
|
|
|
0xa5u8, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5,
|
|
|
|
0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a,
|
|
|
|
0xa5, 0xa5, 0x5a, 0x5a,
|
|
|
|
];
|
2020-02-24 09:18:08 -08:00
|
|
|
let version = shred_version::version_from_hash(&Hash::new(&hash));
|
2020-01-24 08:14:27 -08:00
|
|
|
assert_eq!(version, 1);
|
2019-11-18 18:05:02 -08:00
|
|
|
let hash = [
|
|
|
|
0xa5u8, 0xa5, 0x5a, 0x5a, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
|
|
0, 0, 0, 0, 0, 0, 0, 0,
|
|
|
|
];
|
2020-02-24 09:18:08 -08:00
|
|
|
let version = shred_version::version_from_hash(&Hash::new(&hash));
|
2019-11-18 18:05:02 -08:00
|
|
|
assert_eq!(version, 0xffff);
|
|
|
|
let hash = [
|
|
|
|
0xa5u8, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
|
|
];
|
2020-02-24 09:18:08 -08:00
|
|
|
let version = shred_version::version_from_hash(&Hash::new(&hash));
|
2020-01-24 08:14:27 -08:00
|
|
|
assert_eq!(version, 0x5a5b);
|
2019-11-18 18:05:02 -08:00
|
|
|
}
|
2019-12-12 16:50:29 -08:00
|
|
|
|
2020-04-19 21:15:09 -07:00
|
|
|
#[test]
|
|
|
|
fn test_invalid_parent_offset() {
|
2022-05-02 16:33:53 -07:00
|
|
|
let shred = Shred::new_from_data(10, 0, 1000, &[1, 2, 3], ShredFlags::empty(), 0, 1, 0);
|
2020-04-19 21:15:09 -07:00
|
|
|
let mut packet = Packet::default();
|
|
|
|
shred.copy_to_packet(&mut packet);
|
2022-06-02 18:05:06 -07:00
|
|
|
let shred_res = Shred::new_from_serialized_shred(packet.data(..).unwrap().to_vec());
|
2020-04-19 21:15:09 -07:00
|
|
|
assert_matches!(
|
2021-12-09 08:43:57 -08:00
|
|
|
shred.parent(),
|
2022-04-25 16:19:37 -07:00
|
|
|
Err(Error::InvalidParentOffset {
|
|
|
|
slot: 10,
|
|
|
|
parent_offset: 1000
|
|
|
|
})
|
|
|
|
);
|
|
|
|
assert_matches!(
|
|
|
|
shred_res,
|
|
|
|
Err(Error::InvalidParentOffset {
|
2020-04-19 21:15:09 -07:00
|
|
|
slot: 10,
|
|
|
|
parent_offset: 1000
|
|
|
|
})
|
|
|
|
);
|
|
|
|
}
|
2020-12-15 16:50:40 -08:00
|
|
|
|
|
|
|
#[test]
|
2022-06-28 05:45:50 -07:00
|
|
|
fn test_should_discard_shred() {
|
2020-12-15 16:50:40 -08:00
|
|
|
solana_logger::setup();
|
|
|
|
let mut packet = Packet::default();
|
2022-06-28 05:45:50 -07:00
|
|
|
let root = 1;
|
|
|
|
let shred_version = 798;
|
2022-07-05 14:41:19 -07:00
|
|
|
let max_slot = 16;
|
2022-06-28 05:45:50 -07:00
|
|
|
let shred = Shred::new_from_data(
|
|
|
|
2, // slot
|
|
|
|
3, // index
|
|
|
|
1, // parent_offset
|
|
|
|
&[], // data
|
|
|
|
ShredFlags::LAST_SHRED_IN_SLOT,
|
|
|
|
0, // reference_tick
|
|
|
|
shred_version,
|
|
|
|
0, // fec_set_index
|
|
|
|
);
|
2020-12-15 16:50:40 -08:00
|
|
|
shred.copy_to_packet(&mut packet);
|
|
|
|
let mut stats = ShredFetchStats::default();
|
2022-06-28 05:45:50 -07:00
|
|
|
assert!(!should_discard_shred(
|
|
|
|
&packet,
|
|
|
|
root,
|
2022-07-05 14:41:19 -07:00
|
|
|
max_slot,
|
2022-06-28 05:45:50 -07:00
|
|
|
shred_version,
|
|
|
|
&mut stats
|
|
|
|
));
|
2020-12-15 16:50:40 -08:00
|
|
|
assert_eq!(stats, ShredFetchStats::default());
|
|
|
|
|
2022-12-06 03:54:49 -08:00
|
|
|
packet.meta_mut().size = OFFSET_OF_SHRED_VARIANT;
|
2022-06-28 05:45:50 -07:00
|
|
|
assert!(should_discard_shred(
|
|
|
|
&packet,
|
|
|
|
root,
|
2022-07-05 14:41:19 -07:00
|
|
|
max_slot,
|
2022-06-28 05:45:50 -07:00
|
|
|
shred_version,
|
|
|
|
&mut stats
|
|
|
|
));
|
2020-12-15 16:50:40 -08:00
|
|
|
assert_eq!(stats.index_overrun, 1);
|
|
|
|
|
2022-12-06 03:54:49 -08:00
|
|
|
packet.meta_mut().size = OFFSET_OF_SHRED_INDEX;
|
2022-06-28 05:45:50 -07:00
|
|
|
assert!(should_discard_shred(
|
|
|
|
&packet,
|
|
|
|
root,
|
2022-07-05 14:41:19 -07:00
|
|
|
max_slot,
|
2022-06-28 05:45:50 -07:00
|
|
|
shred_version,
|
|
|
|
&mut stats
|
|
|
|
));
|
2020-12-15 16:50:40 -08:00
|
|
|
assert_eq!(stats.index_overrun, 2);
|
|
|
|
|
2022-12-06 03:54:49 -08:00
|
|
|
packet.meta_mut().size = OFFSET_OF_SHRED_INDEX + 1;
|
2022-06-28 05:45:50 -07:00
|
|
|
assert!(should_discard_shred(
|
|
|
|
&packet,
|
|
|
|
root,
|
2022-07-05 14:41:19 -07:00
|
|
|
max_slot,
|
2022-06-28 05:45:50 -07:00
|
|
|
shred_version,
|
|
|
|
&mut stats
|
|
|
|
));
|
2020-12-15 16:50:40 -08:00
|
|
|
assert_eq!(stats.index_overrun, 3);
|
|
|
|
|
2022-12-06 03:54:49 -08:00
|
|
|
packet.meta_mut().size = OFFSET_OF_SHRED_INDEX + SIZE_OF_SHRED_INDEX - 1;
|
2022-06-28 05:45:50 -07:00
|
|
|
assert!(should_discard_shred(
|
|
|
|
&packet,
|
|
|
|
root,
|
2022-07-05 14:41:19 -07:00
|
|
|
max_slot,
|
2022-06-28 05:45:50 -07:00
|
|
|
shred_version,
|
|
|
|
&mut stats
|
|
|
|
));
|
2020-12-15 16:50:40 -08:00
|
|
|
assert_eq!(stats.index_overrun, 4);
|
|
|
|
|
2022-12-06 03:54:49 -08:00
|
|
|
packet.meta_mut().size = OFFSET_OF_SHRED_INDEX + SIZE_OF_SHRED_INDEX + 2;
|
2022-06-28 05:45:50 -07:00
|
|
|
assert!(should_discard_shred(
|
|
|
|
&packet,
|
|
|
|
root,
|
2022-07-05 14:41:19 -07:00
|
|
|
max_slot,
|
2022-06-28 05:45:50 -07:00
|
|
|
shred_version,
|
|
|
|
&mut stats
|
|
|
|
));
|
|
|
|
assert_eq!(stats.bad_parent_offset, 1);
|
2020-12-15 16:50:40 -08:00
|
|
|
|
2022-04-27 11:04:10 -07:00
|
|
|
let shred = Shred::new_from_parity_shard(
|
2021-12-05 06:42:09 -08:00
|
|
|
8, // slot
|
|
|
|
2, // index
|
2022-04-27 11:04:10 -07:00
|
|
|
&[], // parity_shard
|
2021-12-05 06:42:09 -08:00
|
|
|
10, // fec_set_index
|
|
|
|
30, // num_data
|
|
|
|
4, // num_code
|
|
|
|
1, // position
|
2022-06-28 05:45:50 -07:00
|
|
|
shred_version,
|
2021-12-05 06:42:09 -08:00
|
|
|
);
|
2020-12-15 16:50:40 -08:00
|
|
|
shred.copy_to_packet(&mut packet);
|
2022-06-28 05:45:50 -07:00
|
|
|
assert!(!should_discard_shred(
|
|
|
|
&packet,
|
|
|
|
root,
|
2022-07-05 14:41:19 -07:00
|
|
|
max_slot,
|
2022-06-28 05:45:50 -07:00
|
|
|
shred_version,
|
|
|
|
&mut stats
|
|
|
|
));
|
2020-12-15 16:50:40 -08:00
|
|
|
|
2022-05-02 16:33:53 -07:00
|
|
|
let shred = Shred::new_from_data(
|
2022-06-28 05:45:50 -07:00
|
|
|
2, // slot
|
|
|
|
std::u32::MAX - 10, // index
|
|
|
|
1, // parent_offset
|
|
|
|
&[], // data
|
2022-05-02 16:33:53 -07:00
|
|
|
ShredFlags::LAST_SHRED_IN_SLOT,
|
2022-06-28 05:45:50 -07:00
|
|
|
0, // reference_tick
|
|
|
|
shred_version,
|
|
|
|
0, // fec_set_index
|
2022-05-02 16:33:53 -07:00
|
|
|
);
|
2020-12-15 16:50:40 -08:00
|
|
|
shred.copy_to_packet(&mut packet);
|
2022-06-28 05:45:50 -07:00
|
|
|
assert!(should_discard_shred(
|
|
|
|
&packet,
|
|
|
|
root,
|
2022-07-05 14:41:19 -07:00
|
|
|
max_slot,
|
2022-06-28 05:45:50 -07:00
|
|
|
shred_version,
|
|
|
|
&mut stats
|
|
|
|
));
|
2020-12-15 16:50:40 -08:00
|
|
|
assert_eq!(1, stats.index_out_of_bounds);
|
|
|
|
|
2022-04-27 11:04:10 -07:00
|
|
|
let shred = Shred::new_from_parity_shard(
|
2021-12-05 06:42:09 -08:00
|
|
|
8, // slot
|
|
|
|
2, // index
|
2022-04-27 11:04:10 -07:00
|
|
|
&[], // parity_shard
|
2021-12-05 06:42:09 -08:00
|
|
|
10, // fec_set_index
|
|
|
|
30, // num_data_shreds
|
|
|
|
4, // num_coding_shreds
|
|
|
|
3, // position
|
2022-07-05 14:41:19 -07:00
|
|
|
shred_version,
|
2021-12-05 06:42:09 -08:00
|
|
|
);
|
2020-12-15 16:50:40 -08:00
|
|
|
shred.copy_to_packet(&mut packet);
|
2022-07-05 14:41:19 -07:00
|
|
|
assert!(!should_discard_shred(
|
|
|
|
&packet,
|
|
|
|
root,
|
|
|
|
max_slot,
|
|
|
|
shred_version,
|
|
|
|
&mut stats
|
|
|
|
));
|
2022-05-30 05:51:00 -07:00
|
|
|
packet.buffer_mut()[OFFSET_OF_SHRED_VARIANT] = u8::MAX;
|
2020-12-15 16:50:40 -08:00
|
|
|
|
2022-06-28 05:45:50 -07:00
|
|
|
assert!(should_discard_shred(
|
|
|
|
&packet,
|
|
|
|
root,
|
2022-07-05 14:41:19 -07:00
|
|
|
max_slot,
|
|
|
|
shred_version,
|
|
|
|
&mut stats
|
|
|
|
));
|
|
|
|
assert_eq!(1, stats.bad_shred_type);
|
|
|
|
assert_eq!(stats.shred_version_mismatch, 0);
|
|
|
|
|
|
|
|
packet.buffer_mut()[OFFSET_OF_SHRED_INDEX + SIZE_OF_SHRED_INDEX + 1] = u8::MAX;
|
|
|
|
assert!(should_discard_shred(
|
|
|
|
&packet,
|
|
|
|
root,
|
|
|
|
max_slot,
|
2022-06-28 05:45:50 -07:00
|
|
|
shred_version,
|
|
|
|
&mut stats
|
|
|
|
));
|
2020-12-15 16:50:40 -08:00
|
|
|
assert_eq!(1, stats.bad_shred_type);
|
2022-07-05 14:41:19 -07:00
|
|
|
assert_eq!(stats.shred_version_mismatch, 1);
|
2020-12-15 16:50:40 -08:00
|
|
|
}
|
2021-11-16 09:50:56 -08:00
|
|
|
|
|
|
|
// Asserts that ShredType is backward compatible with u8.
|
|
|
|
#[test]
|
|
|
|
fn test_shred_type_compat() {
|
|
|
|
assert_eq!(std::mem::size_of::<ShredType>(), std::mem::size_of::<u8>());
|
2022-04-23 06:33:59 -07:00
|
|
|
assert_matches!(ShredType::try_from(0u8), Err(_));
|
|
|
|
assert_matches!(ShredType::try_from(1u8), Err(_));
|
2021-11-16 09:50:56 -08:00
|
|
|
assert_matches!(bincode::deserialize::<ShredType>(&[0u8]), Err(_));
|
2022-04-23 06:33:59 -07:00
|
|
|
assert_matches!(bincode::deserialize::<ShredType>(&[1u8]), Err(_));
|
2021-11-16 09:50:56 -08:00
|
|
|
// data shred
|
|
|
|
assert_eq!(ShredType::Data as u8, 0b1010_0101);
|
2022-05-30 05:51:00 -07:00
|
|
|
assert_eq!(u8::from(ShredType::Data), 0b1010_0101);
|
2022-04-23 06:33:59 -07:00
|
|
|
assert_eq!(ShredType::try_from(0b1010_0101), Ok(ShredType::Data));
|
2021-11-16 09:50:56 -08:00
|
|
|
let buf = bincode::serialize(&ShredType::Data).unwrap();
|
|
|
|
assert_eq!(buf, vec![0b1010_0101]);
|
|
|
|
assert_matches!(
|
|
|
|
bincode::deserialize::<ShredType>(&[0b1010_0101]),
|
|
|
|
Ok(ShredType::Data)
|
|
|
|
);
|
|
|
|
// coding shred
|
|
|
|
assert_eq!(ShredType::Code as u8, 0b0101_1010);
|
2022-05-30 05:51:00 -07:00
|
|
|
assert_eq!(u8::from(ShredType::Code), 0b0101_1010);
|
2022-04-23 06:33:59 -07:00
|
|
|
assert_eq!(ShredType::try_from(0b0101_1010), Ok(ShredType::Code));
|
2021-11-16 09:50:56 -08:00
|
|
|
let buf = bincode::serialize(&ShredType::Code).unwrap();
|
|
|
|
assert_eq!(buf, vec![0b0101_1010]);
|
|
|
|
assert_matches!(
|
|
|
|
bincode::deserialize::<ShredType>(&[0b0101_1010]),
|
|
|
|
Ok(ShredType::Code)
|
|
|
|
);
|
|
|
|
}
|
2022-04-25 05:43:22 -07:00
|
|
|
|
2022-05-30 05:51:00 -07:00
|
|
|
#[test]
|
|
|
|
fn test_shred_variant_compat() {
|
|
|
|
assert_matches!(ShredVariant::try_from(0u8), Err(_));
|
|
|
|
assert_matches!(ShredVariant::try_from(1u8), Err(_));
|
|
|
|
assert_matches!(ShredVariant::try_from(0b0101_0000), Err(_));
|
|
|
|
assert_matches!(ShredVariant::try_from(0b1010_0000), Err(_));
|
|
|
|
assert_matches!(bincode::deserialize::<ShredVariant>(&[0b0101_0000]), Err(_));
|
|
|
|
assert_matches!(bincode::deserialize::<ShredVariant>(&[0b1010_0000]), Err(_));
|
|
|
|
// Legacy coding shred.
|
|
|
|
assert_eq!(u8::from(ShredVariant::LegacyCode), 0b0101_1010);
|
|
|
|
assert_eq!(ShredType::from(ShredVariant::LegacyCode), ShredType::Code);
|
|
|
|
assert_matches!(
|
|
|
|
ShredVariant::try_from(0b0101_1010),
|
|
|
|
Ok(ShredVariant::LegacyCode)
|
|
|
|
);
|
|
|
|
let buf = bincode::serialize(&ShredVariant::LegacyCode).unwrap();
|
|
|
|
assert_eq!(buf, vec![0b0101_1010]);
|
|
|
|
assert_matches!(
|
|
|
|
bincode::deserialize::<ShredVariant>(&[0b0101_1010]),
|
|
|
|
Ok(ShredVariant::LegacyCode)
|
|
|
|
);
|
|
|
|
// Legacy data shred.
|
|
|
|
assert_eq!(u8::from(ShredVariant::LegacyData), 0b1010_0101);
|
|
|
|
assert_eq!(ShredType::from(ShredVariant::LegacyData), ShredType::Data);
|
|
|
|
assert_matches!(
|
|
|
|
ShredVariant::try_from(0b1010_0101),
|
|
|
|
Ok(ShredVariant::LegacyData)
|
|
|
|
);
|
|
|
|
let buf = bincode::serialize(&ShredVariant::LegacyData).unwrap();
|
|
|
|
assert_eq!(buf, vec![0b1010_0101]);
|
|
|
|
assert_matches!(
|
|
|
|
bincode::deserialize::<ShredVariant>(&[0b1010_0101]),
|
|
|
|
Ok(ShredVariant::LegacyData)
|
|
|
|
);
|
2022-06-07 15:41:03 -07:00
|
|
|
// Merkle coding shred.
|
|
|
|
assert_eq!(u8::from(ShredVariant::MerkleCode(5)), 0b0100_0101);
|
|
|
|
assert_eq!(
|
|
|
|
ShredType::from(ShredVariant::MerkleCode(5)),
|
|
|
|
ShredType::Code
|
|
|
|
);
|
|
|
|
assert_matches!(
|
|
|
|
ShredVariant::try_from(0b0100_0101),
|
|
|
|
Ok(ShredVariant::MerkleCode(5))
|
|
|
|
);
|
|
|
|
let buf = bincode::serialize(&ShredVariant::MerkleCode(5)).unwrap();
|
|
|
|
assert_eq!(buf, vec![0b0100_0101]);
|
|
|
|
assert_matches!(
|
|
|
|
bincode::deserialize::<ShredVariant>(&[0b0100_0101]),
|
|
|
|
Ok(ShredVariant::MerkleCode(5))
|
|
|
|
);
|
|
|
|
for proof_size in 0..=15u8 {
|
|
|
|
let byte = proof_size | 0b0100_0000;
|
|
|
|
assert_eq!(u8::from(ShredVariant::MerkleCode(proof_size)), byte);
|
|
|
|
assert_eq!(
|
|
|
|
ShredType::from(ShredVariant::MerkleCode(proof_size)),
|
|
|
|
ShredType::Code
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
ShredVariant::try_from(byte).unwrap(),
|
|
|
|
ShredVariant::MerkleCode(proof_size)
|
|
|
|
);
|
|
|
|
let buf = bincode::serialize(&ShredVariant::MerkleCode(proof_size)).unwrap();
|
|
|
|
assert_eq!(buf, vec![byte]);
|
|
|
|
assert_eq!(
|
|
|
|
bincode::deserialize::<ShredVariant>(&[byte]).unwrap(),
|
|
|
|
ShredVariant::MerkleCode(proof_size)
|
|
|
|
);
|
|
|
|
}
|
|
|
|
// Merkle data shred.
|
|
|
|
assert_eq!(u8::from(ShredVariant::MerkleData(10)), 0b1000_1010);
|
|
|
|
assert_eq!(
|
|
|
|
ShredType::from(ShredVariant::MerkleData(10)),
|
|
|
|
ShredType::Data
|
|
|
|
);
|
|
|
|
assert_matches!(
|
|
|
|
ShredVariant::try_from(0b1000_1010),
|
|
|
|
Ok(ShredVariant::MerkleData(10))
|
|
|
|
);
|
|
|
|
let buf = bincode::serialize(&ShredVariant::MerkleData(10)).unwrap();
|
|
|
|
assert_eq!(buf, vec![0b1000_1010]);
|
|
|
|
assert_matches!(
|
|
|
|
bincode::deserialize::<ShredVariant>(&[0b1000_1010]),
|
|
|
|
Ok(ShredVariant::MerkleData(10))
|
|
|
|
);
|
|
|
|
for proof_size in 0..=15u8 {
|
|
|
|
let byte = proof_size | 0b1000_0000;
|
|
|
|
assert_eq!(u8::from(ShredVariant::MerkleData(proof_size)), byte);
|
|
|
|
assert_eq!(
|
|
|
|
ShredType::from(ShredVariant::MerkleData(proof_size)),
|
|
|
|
ShredType::Data
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
ShredVariant::try_from(byte).unwrap(),
|
|
|
|
ShredVariant::MerkleData(proof_size)
|
|
|
|
);
|
|
|
|
let buf = bincode::serialize(&ShredVariant::MerkleData(proof_size)).unwrap();
|
|
|
|
assert_eq!(buf, vec![byte]);
|
|
|
|
assert_eq!(
|
|
|
|
bincode::deserialize::<ShredVariant>(&[byte]).unwrap(),
|
|
|
|
ShredVariant::MerkleData(proof_size)
|
|
|
|
);
|
|
|
|
}
|
2022-05-30 05:51:00 -07:00
|
|
|
}
|
|
|
|
|
2022-06-27 10:58:43 -07:00
|
|
|
#[test]
|
|
|
|
fn test_shred_seed() {
|
|
|
|
let mut rng = ChaChaRng::from_seed([147u8; 32]);
|
|
|
|
let leader = Pubkey::new_from_array(rng.gen());
|
|
|
|
let key = ShredId(
|
|
|
|
141939602, // slot
|
|
|
|
28685, // index
|
|
|
|
ShredType::Data,
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
bs58::encode(key.seed(&leader)).into_string(),
|
|
|
|
"Gp4kUM4ZpWGQN5XSCyM9YHYWEBCAZLa94ZQuSgDE4r56"
|
|
|
|
);
|
|
|
|
let leader = Pubkey::new_from_array(rng.gen());
|
|
|
|
let key = ShredId(
|
|
|
|
141945197, // slot
|
|
|
|
23418, // index
|
|
|
|
ShredType::Code,
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
bs58::encode(key.seed(&leader)).into_string(),
|
|
|
|
"G1gmFe1QUM8nhDApk6BqvPgw3TQV2Qc5bpKppa96qbVb"
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
2022-06-19 08:30:18 -07:00
|
|
|
fn verify_shred_layout(shred: &Shred, packet: &Packet) {
|
|
|
|
let data = layout::get_shred(packet).unwrap();
|
2022-06-30 05:13:00 -07:00
|
|
|
assert_eq!(data, packet.data(..).unwrap());
|
2022-06-19 08:30:18 -07:00
|
|
|
assert_eq!(layout::get_slot(data), Some(shred.slot()));
|
|
|
|
assert_eq!(layout::get_index(data), Some(shred.index()));
|
|
|
|
assert_eq!(layout::get_version(data), Some(shred.version()));
|
2022-06-30 05:13:00 -07:00
|
|
|
assert_eq!(layout::get_shred_id(data), Some(shred.id()));
|
2022-12-31 09:08:25 -08:00
|
|
|
assert_eq!(layout::get_signature(data), Some(*shred.signature()));
|
2022-06-30 05:13:00 -07:00
|
|
|
assert_eq!(layout::get_shred_type(data).unwrap(), shred.shred_type());
|
2022-06-19 08:30:18 -07:00
|
|
|
match shred.shred_type() {
|
|
|
|
ShredType::Code => {
|
|
|
|
assert_matches!(
|
|
|
|
layout::get_reference_tick(data),
|
|
|
|
Err(Error::InvalidShredType)
|
|
|
|
);
|
|
|
|
}
|
|
|
|
ShredType::Data => {
|
|
|
|
assert_eq!(
|
|
|
|
layout::get_reference_tick(data).unwrap(),
|
|
|
|
shred.reference_tick()
|
|
|
|
);
|
2022-06-28 05:45:50 -07:00
|
|
|
let parent_offset = layout::get_parent_offset(data).unwrap();
|
|
|
|
let slot = layout::get_slot(data).unwrap();
|
|
|
|
let parent = slot.checked_sub(Slot::from(parent_offset)).unwrap();
|
|
|
|
assert_eq!(parent, shred.parent().unwrap());
|
2022-06-19 08:30:18 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-01 06:11:45 -07:00
|
|
|
#[test]
|
|
|
|
fn test_serde_compat_shred_data() {
|
|
|
|
const SEED: &str = "6qG9NGWEtoTugS4Zgs46u8zTccEJuRHtrNMiUayLHCxt";
|
2022-05-02 16:33:53 -07:00
|
|
|
const PAYLOAD: &str = "hNX8YgJCQwSFGJkZ6qZLiepwPjpctC9UCsMD1SNNQurBXv\
|
|
|
|
rm7KKfLmPRMM9CpWHt6MsJuEWpDXLGwH9qdziJzGKhBMfYH63avcchjdaUiMqzVip7cUD\
|
|
|
|
kqZ9zZJMrHCCUDnxxKMupsJWKroUSjKeo7hrug2KfHah85VckXpRna4R9QpH7tf2WVBTD\
|
|
|
|
M4m3EerctsEQs8eZaTRxzTVkhtJYdNf74KZbH58dc3Yn2qUxF1mexWoPS6L5oZBatx";
|
2022-05-01 06:11:45 -07:00
|
|
|
let mut rng = {
|
|
|
|
let seed = <[u8; 32]>::try_from(bs58_decode(SEED)).unwrap();
|
|
|
|
ChaChaRng::from_seed(seed)
|
|
|
|
};
|
2022-05-30 05:51:19 -07:00
|
|
|
let mut data = [0u8; legacy::ShredData::CAPACITY];
|
2022-05-01 06:11:45 -07:00
|
|
|
rng.fill(&mut data[..]);
|
|
|
|
let keypair = Keypair::generate(&mut rng);
|
|
|
|
let mut shred = Shred::new_from_data(
|
|
|
|
141939602, // slot
|
|
|
|
28685, // index
|
|
|
|
36390, // parent_offset
|
|
|
|
&data, // data
|
2022-05-02 16:33:53 -07:00
|
|
|
ShredFlags::LAST_SHRED_IN_SLOT,
|
|
|
|
37, // reference_tick
|
|
|
|
45189, // version
|
|
|
|
28657, // fec_set_index
|
2022-05-01 06:11:45 -07:00
|
|
|
);
|
|
|
|
shred.sign(&keypair);
|
|
|
|
assert!(shred.verify(&keypair.pubkey()));
|
|
|
|
assert_matches!(shred.sanitize(), Ok(()));
|
|
|
|
let mut payload = bs58_decode(PAYLOAD);
|
|
|
|
payload.extend({
|
2022-05-30 05:51:19 -07:00
|
|
|
let skip = payload.len() - SIZE_OF_DATA_SHRED_HEADERS;
|
2022-05-01 06:11:45 -07:00
|
|
|
data.iter().skip(skip).copied()
|
|
|
|
});
|
|
|
|
let mut packet = Packet::default();
|
2022-05-25 09:52:54 -07:00
|
|
|
packet.buffer_mut()[..payload.len()].copy_from_slice(&payload);
|
2022-12-06 03:54:49 -08:00
|
|
|
packet.meta_mut().size = payload.len();
|
2022-05-01 06:11:45 -07:00
|
|
|
assert_eq!(shred.bytes_to_store(), payload);
|
2022-05-02 08:02:06 -07:00
|
|
|
assert_eq!(shred, Shred::new_from_serialized_shred(payload).unwrap());
|
2022-06-19 08:30:18 -07:00
|
|
|
verify_shred_layout(&shred, &packet);
|
2022-05-02 08:02:06 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_serde_compat_shred_data_empty() {
|
|
|
|
const SEED: &str = "E3M5hm8yAEB7iPhQxFypAkLqxNeZCTuGBDMa8Jdrghoo";
|
|
|
|
const PAYLOAD: &str = "nRNFVBEsV9FEM5KfmsCXJsgELRSkCV55drTavdy5aZPnsp\
|
|
|
|
B8WvsgY99ZuNHDnwkrqe6Lx7ARVmercwugR5HwDcLA9ivKMypk9PNucDPLs67TXWy6k9R\
|
|
|
|
ozKmy";
|
|
|
|
let mut rng = {
|
|
|
|
let seed = <[u8; 32]>::try_from(bs58_decode(SEED)).unwrap();
|
|
|
|
ChaChaRng::from_seed(seed)
|
|
|
|
};
|
|
|
|
let keypair = Keypair::generate(&mut rng);
|
|
|
|
let mut shred = Shred::new_from_data(
|
|
|
|
142076266, // slot
|
|
|
|
21443, // index
|
|
|
|
51279, // parent_offset
|
|
|
|
&[], // data
|
2022-05-02 16:33:53 -07:00
|
|
|
ShredFlags::DATA_COMPLETE_SHRED,
|
|
|
|
49, // reference_tick
|
|
|
|
59445, // version
|
|
|
|
21414, // fec_set_index
|
2022-05-02 08:02:06 -07:00
|
|
|
);
|
|
|
|
shred.sign(&keypair);
|
|
|
|
assert!(shred.verify(&keypair.pubkey()));
|
|
|
|
assert_matches!(shred.sanitize(), Ok(()));
|
|
|
|
let payload = bs58_decode(PAYLOAD);
|
|
|
|
let mut packet = Packet::default();
|
2022-05-25 09:52:54 -07:00
|
|
|
packet.buffer_mut()[..payload.len()].copy_from_slice(&payload);
|
2022-12-06 03:54:49 -08:00
|
|
|
packet.meta_mut().size = payload.len();
|
2022-05-02 08:02:06 -07:00
|
|
|
assert_eq!(shred.bytes_to_store(), payload);
|
2022-05-01 06:11:45 -07:00
|
|
|
assert_eq!(shred, Shred::new_from_serialized_shred(payload).unwrap());
|
2022-06-19 08:30:18 -07:00
|
|
|
verify_shred_layout(&shred, &packet);
|
2022-05-01 06:11:45 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_serde_compat_shred_code() {
|
|
|
|
const SEED: &str = "4jfjh3UZVyaEgvyG9oQmNyFY9yHDmbeH9eUhnBKkrcrN";
|
|
|
|
const PAYLOAD: &str = "3xGsXwzkPpLFuKwbbfKMUxt1B6VqQPzbvvAkxRNCX9kNEP\
|
|
|
|
sa2VifwGBtFuNm3CWXdmQizDz5vJjDHu6ZqqaBCSfrHurag87qAXwTtjNPhZzKEew5pLc\
|
|
|
|
aY6cooiAch2vpfixNYSDjnirozje5cmUtGuYs1asXwsAKSN3QdWHz3XGParWkZeUMAzRV\
|
|
|
|
1UPEDZ7vETKbxeNixKbzZzo47Lakh3C35hS74ocfj23CWoW1JpkETkXjUpXcfcv6cS";
|
|
|
|
let mut rng = {
|
|
|
|
let seed = <[u8; 32]>::try_from(bs58_decode(SEED)).unwrap();
|
|
|
|
ChaChaRng::from_seed(seed)
|
|
|
|
};
|
2022-05-30 05:51:19 -07:00
|
|
|
let mut parity_shard = vec![0u8; legacy::SIZE_OF_ERASURE_ENCODED_SLICE];
|
2022-05-01 06:11:45 -07:00
|
|
|
rng.fill(&mut parity_shard[..]);
|
|
|
|
let keypair = Keypair::generate(&mut rng);
|
|
|
|
let mut shred = Shred::new_from_parity_shard(
|
|
|
|
141945197, // slot
|
|
|
|
23418, // index
|
|
|
|
&parity_shard,
|
|
|
|
21259, // fec_set_index
|
|
|
|
32, // num_data_shreds
|
|
|
|
58, // num_coding_shreds
|
|
|
|
43, // position
|
|
|
|
47298, // version
|
|
|
|
);
|
|
|
|
shred.sign(&keypair);
|
|
|
|
assert!(shred.verify(&keypair.pubkey()));
|
|
|
|
assert_matches!(shred.sanitize(), Ok(()));
|
|
|
|
let mut payload = bs58_decode(PAYLOAD);
|
|
|
|
payload.extend({
|
|
|
|
let skip = payload.len() - SIZE_OF_CODING_SHRED_HEADERS;
|
|
|
|
parity_shard.iter().skip(skip).copied()
|
|
|
|
});
|
|
|
|
let mut packet = Packet::default();
|
2022-05-25 09:52:54 -07:00
|
|
|
packet.buffer_mut()[..payload.len()].copy_from_slice(&payload);
|
2022-12-06 03:54:49 -08:00
|
|
|
packet.meta_mut().size = payload.len();
|
2022-05-01 06:11:45 -07:00
|
|
|
assert_eq!(shred.bytes_to_store(), payload);
|
|
|
|
assert_eq!(shred, Shred::new_from_serialized_shred(payload).unwrap());
|
2022-06-19 08:30:18 -07:00
|
|
|
verify_shred_layout(&shred, &packet);
|
2022-05-01 06:11:45 -07:00
|
|
|
}
|
2022-05-01 12:25:15 -07:00
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_shred_flags() {
|
|
|
|
fn make_shred(is_last_data: bool, is_last_in_slot: bool, reference_tick: u8) -> Shred {
|
2022-05-02 16:33:53 -07:00
|
|
|
let flags = if is_last_in_slot {
|
|
|
|
assert!(is_last_data);
|
|
|
|
ShredFlags::LAST_SHRED_IN_SLOT
|
|
|
|
} else if is_last_data {
|
|
|
|
ShredFlags::DATA_COMPLETE_SHRED
|
|
|
|
} else {
|
|
|
|
ShredFlags::empty()
|
|
|
|
};
|
2022-05-01 12:25:15 -07:00
|
|
|
Shred::new_from_data(
|
|
|
|
0, // slot
|
|
|
|
0, // index
|
|
|
|
0, // parent_offset
|
|
|
|
&[], // data
|
2022-05-02 16:33:53 -07:00
|
|
|
flags,
|
2022-05-01 12:25:15 -07:00
|
|
|
reference_tick,
|
|
|
|
0, // version
|
|
|
|
0, // fec_set_index
|
|
|
|
)
|
|
|
|
}
|
|
|
|
fn check_shred_flags(
|
|
|
|
shred: &Shred,
|
|
|
|
is_last_data: bool,
|
|
|
|
is_last_in_slot: bool,
|
|
|
|
reference_tick: u8,
|
|
|
|
) {
|
|
|
|
assert_eq!(shred.data_complete(), is_last_data);
|
|
|
|
assert_eq!(shred.last_in_slot(), is_last_in_slot);
|
|
|
|
assert_eq!(shred.reference_tick(), reference_tick.min(63u8));
|
|
|
|
assert_eq!(
|
2022-05-26 06:06:27 -07:00
|
|
|
layout::get_reference_tick(shred.payload()).unwrap(),
|
2022-05-01 12:25:15 -07:00
|
|
|
reference_tick.min(63u8),
|
|
|
|
);
|
|
|
|
}
|
|
|
|
for is_last_data in [false, true] {
|
|
|
|
for is_last_in_slot in [false, true] {
|
2022-05-02 16:33:53 -07:00
|
|
|
// LAST_SHRED_IN_SLOT also implies DATA_COMPLETE_SHRED. So it
|
|
|
|
// cannot be LAST_SHRED_IN_SLOT if not DATA_COMPLETE_SHRED.
|
|
|
|
let is_last_in_slot = is_last_in_slot && is_last_data;
|
2022-05-01 12:25:15 -07:00
|
|
|
for reference_tick in [0, 37, 63, 64, 80, 128, 255] {
|
|
|
|
let mut shred = make_shred(is_last_data, is_last_in_slot, reference_tick);
|
|
|
|
check_shred_flags(&shred, is_last_data, is_last_in_slot, reference_tick);
|
|
|
|
shred.set_last_in_slot();
|
2022-05-02 16:33:53 -07:00
|
|
|
check_shred_flags(&shred, true, true, reference_tick);
|
2022-05-01 12:25:15 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2022-05-02 16:33:53 -07:00
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_shred_flags_serde() {
|
|
|
|
let flags: ShredFlags = bincode::deserialize(&[0b0111_0001]).unwrap();
|
|
|
|
assert!(flags.contains(ShredFlags::DATA_COMPLETE_SHRED));
|
|
|
|
assert!(!flags.contains(ShredFlags::LAST_SHRED_IN_SLOT));
|
|
|
|
assert_eq!((flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits(), 49u8);
|
|
|
|
assert_eq!(bincode::serialize(&flags).unwrap(), [0b0111_0001]);
|
|
|
|
|
|
|
|
let flags: ShredFlags = bincode::deserialize(&[0b1110_0101]).unwrap();
|
|
|
|
assert!(flags.contains(ShredFlags::DATA_COMPLETE_SHRED));
|
|
|
|
assert!(flags.contains(ShredFlags::LAST_SHRED_IN_SLOT));
|
|
|
|
assert_eq!((flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits(), 37u8);
|
|
|
|
assert_eq!(bincode::serialize(&flags).unwrap(), [0b1110_0101]);
|
|
|
|
|
|
|
|
let flags: ShredFlags = bincode::deserialize(&[0b1011_1101]).unwrap();
|
|
|
|
assert!(!flags.contains(ShredFlags::DATA_COMPLETE_SHRED));
|
|
|
|
assert!(!flags.contains(ShredFlags::LAST_SHRED_IN_SLOT));
|
|
|
|
assert_eq!((flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits(), 61u8);
|
|
|
|
assert_eq!(bincode::serialize(&flags).unwrap(), [0b1011_1101]);
|
|
|
|
}
|
2019-08-02 15:53:42 -07:00
|
|
|
}
|