solana/ledger/src/shred/shred_data.rs

160 lines
5.4 KiB
Rust
Raw Normal View History

use {
crate::shred::{
self,
common::dispatch,
adds a new shred variant embedding merkle tree hashes of the erasure batch (#25237) Coding shreds can only be signed once erasure codings are already generated. Therefore coding shreds recovered from erasure codings lack slot leader's signature and so cannot be retransmitted to the rest of the cluster. shred/merkle.rs implements a new shred variant where we generate merkle tree for each erasure encoded batch and each shred includes: * root of the merkle tree (Hash truncated to 20 bytes). * slot leader's signature of the root of the merkle tree. * merkle tree nodes along the branch the shred belongs to, where hashes are trimmed to 20 bytes during tree construction. This schema results in the same signature for all shreds within an erasure batch. When recovering shreds from erasure codes, we can reconstruct merkle tree for the batch and for each recovered shred also recover respective merkle tree branch; then snap the slot leader's signature from any of the shreds received from turbine and retransmit all recovered code or data shreds. Backward compatibility is achieved by encoding shred variant at byte 65 of payload (previously shred-type at this position): * 0b0101_1010 indicates a legacy coding shred, which is also equal to ShredType::Code for backward compatibility. * 0b1010_0101 indicates a legacy data shred, which is also equal to ShredType::Data for backward compatibility. * 0b0100_???? indicates a merkle coding shred with merkle branch size indicated by the last 4 bits. * 0b1000_???? indicates a merkle data shred with merkle branch size indicated by the last 4 bits. Merkle root and branch are encoded at the end of the shred payload.
2022-06-07 15:41:03 -07:00
legacy, merkle,
traits::{Shred as _, ShredData as ShredDataTrait},
DataShredHeader, Error, ShredCommonHeader, ShredFlags, ShredType, ShredVariant,
MAX_DATA_SHREDS_PER_SLOT,
},
solana_sdk::{clock::Slot, signature::Signature},
};
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum ShredData {
Legacy(legacy::ShredData),
adds a new shred variant embedding merkle tree hashes of the erasure batch (#25237) Coding shreds can only be signed once erasure codings are already generated. Therefore coding shreds recovered from erasure codings lack slot leader's signature and so cannot be retransmitted to the rest of the cluster. shred/merkle.rs implements a new shred variant where we generate merkle tree for each erasure encoded batch and each shred includes: * root of the merkle tree (Hash truncated to 20 bytes). * slot leader's signature of the root of the merkle tree. * merkle tree nodes along the branch the shred belongs to, where hashes are trimmed to 20 bytes during tree construction. This schema results in the same signature for all shreds within an erasure batch. When recovering shreds from erasure codes, we can reconstruct merkle tree for the batch and for each recovered shred also recover respective merkle tree branch; then snap the slot leader's signature from any of the shreds received from turbine and retransmit all recovered code or data shreds. Backward compatibility is achieved by encoding shred variant at byte 65 of payload (previously shred-type at this position): * 0b0101_1010 indicates a legacy coding shred, which is also equal to ShredType::Code for backward compatibility. * 0b1010_0101 indicates a legacy data shred, which is also equal to ShredType::Data for backward compatibility. * 0b0100_???? indicates a merkle coding shred with merkle branch size indicated by the last 4 bits. * 0b1000_???? indicates a merkle data shred with merkle branch size indicated by the last 4 bits. Merkle root and branch are encoded at the end of the shred payload.
2022-06-07 15:41:03 -07:00
Merkle(merkle::ShredData),
}
impl ShredData {
dispatch!(fn data_header(&self) -> &DataShredHeader);
dispatch!(pub(super) fn common_header(&self) -> &ShredCommonHeader);
dispatch!(pub(super) fn data(&self) -> Result<&[u8], Error>);
dispatch!(pub(super) fn erasure_shard(self) -> Result<Vec<u8>, Error>);
dispatch!(pub(super) fn erasure_shard_as_slice(&self) -> Result<&[u8], Error>);
dispatch!(pub(super) fn erasure_shard_index(&self) -> Result<usize, Error>);
dispatch!(pub(super) fn into_payload(self) -> Vec<u8>);
dispatch!(pub(super) fn parent(&self) -> Result<Slot, Error>);
dispatch!(pub(super) fn payload(&self) -> &Vec<u8>);
dispatch!(pub(super) fn sanitize(&self) -> Result<(), Error>);
dispatch!(pub(super) fn set_signature(&mut self, signature: Signature));
dispatch!(pub(super) fn signed_message(&self) -> &[u8]);
// Only for tests.
dispatch!(pub(super) fn set_index(&mut self, index: u32));
dispatch!(pub(super) fn set_slot(&mut self, slot: Slot));
pub(super) fn new_from_data(
slot: Slot,
index: u32,
parent_offset: u16,
data: &[u8],
flags: ShredFlags,
reference_tick: u8,
version: u16,
fec_set_index: u32,
) -> Self {
Self::from(legacy::ShredData::new_from_data(
slot,
index,
parent_offset,
data,
flags,
reference_tick,
version,
fec_set_index,
))
}
pub(super) fn last_in_slot(&self) -> bool {
let flags = self.data_header().flags;
flags.contains(ShredFlags::LAST_SHRED_IN_SLOT)
}
pub(super) fn data_complete(&self) -> bool {
let flags = self.data_header().flags;
flags.contains(ShredFlags::DATA_COMPLETE_SHRED)
}
pub(super) fn reference_tick(&self) -> u8 {
let flags = self.data_header().flags;
(flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits()
}
// Possibly trimmed payload;
// Should only be used when storing shreds to blockstore.
pub(super) fn bytes_to_store(&self) -> &[u8] {
match self {
Self::Legacy(shred) => shred.bytes_to_store(),
adds a new shred variant embedding merkle tree hashes of the erasure batch (#25237) Coding shreds can only be signed once erasure codings are already generated. Therefore coding shreds recovered from erasure codings lack slot leader's signature and so cannot be retransmitted to the rest of the cluster. shred/merkle.rs implements a new shred variant where we generate merkle tree for each erasure encoded batch and each shred includes: * root of the merkle tree (Hash truncated to 20 bytes). * slot leader's signature of the root of the merkle tree. * merkle tree nodes along the branch the shred belongs to, where hashes are trimmed to 20 bytes during tree construction. This schema results in the same signature for all shreds within an erasure batch. When recovering shreds from erasure codes, we can reconstruct merkle tree for the batch and for each recovered shred also recover respective merkle tree branch; then snap the slot leader's signature from any of the shreds received from turbine and retransmit all recovered code or data shreds. Backward compatibility is achieved by encoding shred variant at byte 65 of payload (previously shred-type at this position): * 0b0101_1010 indicates a legacy coding shred, which is also equal to ShredType::Code for backward compatibility. * 0b1010_0101 indicates a legacy data shred, which is also equal to ShredType::Data for backward compatibility. * 0b0100_???? indicates a merkle coding shred with merkle branch size indicated by the last 4 bits. * 0b1000_???? indicates a merkle data shred with merkle branch size indicated by the last 4 bits. Merkle root and branch are encoded at the end of the shred payload.
2022-06-07 15:41:03 -07:00
Self::Merkle(shred) => shred.payload(),
}
}
// Possibly zero pads bytes stored in blockstore.
pub(crate) fn resize_stored_shred(shred: Vec<u8>) -> Result<Vec<u8>, Error> {
match shred::layout::get_shred_variant(&shred)? {
adds a new shred variant embedding merkle tree hashes of the erasure batch (#25237) Coding shreds can only be signed once erasure codings are already generated. Therefore coding shreds recovered from erasure codings lack slot leader's signature and so cannot be retransmitted to the rest of the cluster. shred/merkle.rs implements a new shred variant where we generate merkle tree for each erasure encoded batch and each shred includes: * root of the merkle tree (Hash truncated to 20 bytes). * slot leader's signature of the root of the merkle tree. * merkle tree nodes along the branch the shred belongs to, where hashes are trimmed to 20 bytes during tree construction. This schema results in the same signature for all shreds within an erasure batch. When recovering shreds from erasure codes, we can reconstruct merkle tree for the batch and for each recovered shred also recover respective merkle tree branch; then snap the slot leader's signature from any of the shreds received from turbine and retransmit all recovered code or data shreds. Backward compatibility is achieved by encoding shred variant at byte 65 of payload (previously shred-type at this position): * 0b0101_1010 indicates a legacy coding shred, which is also equal to ShredType::Code for backward compatibility. * 0b1010_0101 indicates a legacy data shred, which is also equal to ShredType::Data for backward compatibility. * 0b0100_???? indicates a merkle coding shred with merkle branch size indicated by the last 4 bits. * 0b1000_???? indicates a merkle data shred with merkle branch size indicated by the last 4 bits. Merkle root and branch are encoded at the end of the shred payload.
2022-06-07 15:41:03 -07:00
ShredVariant::LegacyCode | ShredVariant::MerkleCode(_) => Err(Error::InvalidShredType),
ShredVariant::MerkleData(_) => {
if shred.len() != merkle::ShredData::SIZE_OF_PAYLOAD {
return Err(Error::InvalidPayloadSize(shred.len()));
}
Ok(shred)
}
ShredVariant::LegacyData => legacy::ShredData::resize_stored_shred(shred),
}
}
// Maximum size of ledger data that can be embedded in a data-shred.
adds a new shred variant embedding merkle tree hashes of the erasure batch (#25237) Coding shreds can only be signed once erasure codings are already generated. Therefore coding shreds recovered from erasure codings lack slot leader's signature and so cannot be retransmitted to the rest of the cluster. shred/merkle.rs implements a new shred variant where we generate merkle tree for each erasure encoded batch and each shred includes: * root of the merkle tree (Hash truncated to 20 bytes). * slot leader's signature of the root of the merkle tree. * merkle tree nodes along the branch the shred belongs to, where hashes are trimmed to 20 bytes during tree construction. This schema results in the same signature for all shreds within an erasure batch. When recovering shreds from erasure codes, we can reconstruct merkle tree for the batch and for each recovered shred also recover respective merkle tree branch; then snap the slot leader's signature from any of the shreds received from turbine and retransmit all recovered code or data shreds. Backward compatibility is achieved by encoding shred variant at byte 65 of payload (previously shred-type at this position): * 0b0101_1010 indicates a legacy coding shred, which is also equal to ShredType::Code for backward compatibility. * 0b1010_0101 indicates a legacy data shred, which is also equal to ShredType::Data for backward compatibility. * 0b0100_???? indicates a merkle coding shred with merkle branch size indicated by the last 4 bits. * 0b1000_???? indicates a merkle data shred with merkle branch size indicated by the last 4 bits. Merkle root and branch are encoded at the end of the shred payload.
2022-06-07 15:41:03 -07:00
// merkle_proof_size is the number of proof entries in the merkle tree
// branch. None indicates a legacy data-shred.
pub fn capacity(merkle_proof_size: Option<u8>) -> Result<usize, Error> {
adds a new shred variant embedding merkle tree hashes of the erasure batch (#25237) Coding shreds can only be signed once erasure codings are already generated. Therefore coding shreds recovered from erasure codings lack slot leader's signature and so cannot be retransmitted to the rest of the cluster. shred/merkle.rs implements a new shred variant where we generate merkle tree for each erasure encoded batch and each shred includes: * root of the merkle tree (Hash truncated to 20 bytes). * slot leader's signature of the root of the merkle tree. * merkle tree nodes along the branch the shred belongs to, where hashes are trimmed to 20 bytes during tree construction. This schema results in the same signature for all shreds within an erasure batch. When recovering shreds from erasure codes, we can reconstruct merkle tree for the batch and for each recovered shred also recover respective merkle tree branch; then snap the slot leader's signature from any of the shreds received from turbine and retransmit all recovered code or data shreds. Backward compatibility is achieved by encoding shred variant at byte 65 of payload (previously shred-type at this position): * 0b0101_1010 indicates a legacy coding shred, which is also equal to ShredType::Code for backward compatibility. * 0b1010_0101 indicates a legacy data shred, which is also equal to ShredType::Data for backward compatibility. * 0b0100_???? indicates a merkle coding shred with merkle branch size indicated by the last 4 bits. * 0b1000_???? indicates a merkle data shred with merkle branch size indicated by the last 4 bits. Merkle root and branch are encoded at the end of the shred payload.
2022-06-07 15:41:03 -07:00
match merkle_proof_size {
None => Ok(legacy::ShredData::CAPACITY),
Some(proof_size) => merkle::ShredData::capacity(proof_size),
}
}
// Only for tests.
pub(super) fn set_last_in_slot(&mut self) {
match self {
Self::Legacy(shred) => shred.set_last_in_slot(),
Self::Merkle(_) => panic!("Not Implemented!"),
}
}
}
impl From<legacy::ShredData> for ShredData {
fn from(shred: legacy::ShredData) -> Self {
Self::Legacy(shred)
}
}
adds a new shred variant embedding merkle tree hashes of the erasure batch (#25237) Coding shreds can only be signed once erasure codings are already generated. Therefore coding shreds recovered from erasure codings lack slot leader's signature and so cannot be retransmitted to the rest of the cluster. shred/merkle.rs implements a new shred variant where we generate merkle tree for each erasure encoded batch and each shred includes: * root of the merkle tree (Hash truncated to 20 bytes). * slot leader's signature of the root of the merkle tree. * merkle tree nodes along the branch the shred belongs to, where hashes are trimmed to 20 bytes during tree construction. This schema results in the same signature for all shreds within an erasure batch. When recovering shreds from erasure codes, we can reconstruct merkle tree for the batch and for each recovered shred also recover respective merkle tree branch; then snap the slot leader's signature from any of the shreds received from turbine and retransmit all recovered code or data shreds. Backward compatibility is achieved by encoding shred variant at byte 65 of payload (previously shred-type at this position): * 0b0101_1010 indicates a legacy coding shred, which is also equal to ShredType::Code for backward compatibility. * 0b1010_0101 indicates a legacy data shred, which is also equal to ShredType::Data for backward compatibility. * 0b0100_???? indicates a merkle coding shred with merkle branch size indicated by the last 4 bits. * 0b1000_???? indicates a merkle data shred with merkle branch size indicated by the last 4 bits. Merkle root and branch are encoded at the end of the shred payload.
2022-06-07 15:41:03 -07:00
impl From<merkle::ShredData> for ShredData {
fn from(shred: merkle::ShredData) -> Self {
Self::Merkle(shred)
}
}
#[inline]
pub(super) fn erasure_shard_index<T: ShredDataTrait>(shred: &T) -> Option<usize> {
let fec_set_index = shred.common_header().fec_set_index;
let index = shred.common_header().index.checked_sub(fec_set_index)?;
usize::try_from(index).ok()
}
pub(super) fn sanitize<T: ShredDataTrait>(shred: &T) -> Result<(), Error> {
if shred.payload().len() != T::SIZE_OF_PAYLOAD {
return Err(Error::InvalidPayloadSize(shred.payload().len()));
}
let common_header = shred.common_header();
let data_header = shred.data_header();
if common_header.index as usize >= MAX_DATA_SHREDS_PER_SLOT {
return Err(Error::InvalidShredIndex(
ShredType::Data,
common_header.index,
));
}
let flags = data_header.flags;
if flags.intersects(ShredFlags::LAST_SHRED_IN_SLOT)
&& !flags.contains(ShredFlags::DATA_COMPLETE_SHRED)
{
return Err(Error::InvalidShredFlags(data_header.flags.bits()));
}
let _data = shred.data()?;
let _parent = shred.parent()?;
let _shard_index = shred.erasure_shard_index()?;
let _erasure_shard = shred.erasure_shard_as_slice()?;
Ok(())
}