(de-)serializes shred headers through a Cursor (#24876)
Current serialize/de-serialize code for shreds manually tracks offsets into the payload; This can instead be done with std::io::Cursor. https://github.com/solana-labs/solana/blob/e812430e2/ledger/src/shred.rs#L232-L258
This commit is contained in:
parent
74b586ae71
commit
4f947a0db3
|
@ -55,7 +55,6 @@ pub use crate::{
|
||||||
};
|
};
|
||||||
use {
|
use {
|
||||||
crate::blockstore::MAX_DATA_SHREDS_PER_SLOT,
|
crate::blockstore::MAX_DATA_SHREDS_PER_SLOT,
|
||||||
bincode::config::Options,
|
|
||||||
bitflags::bitflags,
|
bitflags::bitflags,
|
||||||
num_enum::{IntoPrimitive, TryFromPrimitive},
|
num_enum::{IntoPrimitive, TryFromPrimitive},
|
||||||
serde::{Deserialize, Serialize},
|
serde::{Deserialize, Serialize},
|
||||||
|
@ -68,7 +67,7 @@ use {
|
||||||
pubkey::Pubkey,
|
pubkey::Pubkey,
|
||||||
signature::{Keypair, Signature, Signer},
|
signature::{Keypair, Signature, Signer},
|
||||||
},
|
},
|
||||||
std::{fmt::Debug, mem::size_of, ops::RangeInclusive},
|
std::{fmt::Debug, io::Cursor, mem::size_of, ops::RangeInclusive},
|
||||||
thiserror::Error,
|
thiserror::Error,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -229,34 +228,6 @@ impl ErasureSetId {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Shred {
|
impl Shred {
|
||||||
fn deserialize_obj<'de, T>(index: &mut usize, size: usize, buf: &'de [u8]) -> bincode::Result<T>
|
|
||||||
where
|
|
||||||
T: Deserialize<'de>,
|
|
||||||
{
|
|
||||||
let end = std::cmp::min(*index + size, buf.len());
|
|
||||||
let ret = bincode::options()
|
|
||||||
.with_limit(PACKET_DATA_SIZE as u64)
|
|
||||||
.with_fixint_encoding()
|
|
||||||
.allow_trailing_bytes()
|
|
||||||
.deserialize(&buf[*index..end])?;
|
|
||||||
*index += size;
|
|
||||||
Ok(ret)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn serialize_obj_into<'de, T>(
|
|
||||||
index: &mut usize,
|
|
||||||
size: usize,
|
|
||||||
buf: &'de mut [u8],
|
|
||||||
obj: &T,
|
|
||||||
) -> bincode::Result<()>
|
|
||||||
where
|
|
||||||
T: Serialize,
|
|
||||||
{
|
|
||||||
bincode::serialize_into(&mut buf[*index..*index + size], obj)?;
|
|
||||||
*index += size;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn copy_to_packet(&self, packet: &mut Packet) {
|
pub fn copy_to_packet(&self, packet: &mut Packet) {
|
||||||
let len = self.payload.len();
|
let len = self.payload.len();
|
||||||
packet.data[..len].copy_from_slice(&self.payload[..]);
|
packet.data[..len].copy_from_slice(&self.payload[..]);
|
||||||
|
@ -306,24 +277,13 @@ impl Shred {
|
||||||
data_header.flags |= ShredFlags::LAST_SHRED_IN_SLOT
|
data_header.flags |= ShredFlags::LAST_SHRED_IN_SLOT
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut start = 0;
|
let mut cursor = Cursor::new(&mut payload[..]);
|
||||||
Self::serialize_obj_into(
|
bincode::serialize_into(&mut cursor, &common_header).unwrap();
|
||||||
&mut start,
|
bincode::serialize_into(&mut cursor, &data_header).unwrap();
|
||||||
SIZE_OF_COMMON_SHRED_HEADER,
|
|
||||||
&mut payload,
|
|
||||||
&common_header,
|
|
||||||
)
|
|
||||||
.expect("Failed to write common header into shred buffer");
|
|
||||||
|
|
||||||
Self::serialize_obj_into(
|
|
||||||
&mut start,
|
|
||||||
SIZE_OF_DATA_SHRED_HEADER,
|
|
||||||
&mut payload,
|
|
||||||
&data_header,
|
|
||||||
)
|
|
||||||
.expect("Failed to write data header into shred buffer");
|
|
||||||
// TODO: Need to check if data is too large!
|
// TODO: Need to check if data is too large!
|
||||||
payload[start..start + data.len()].copy_from_slice(data);
|
let offset = cursor.position() as usize;
|
||||||
|
debug_assert_eq!(offset, SHRED_DATA_OFFSET);
|
||||||
|
payload[offset..offset + data.len()].copy_from_slice(data);
|
||||||
Self {
|
Self {
|
||||||
common_header,
|
common_header,
|
||||||
data_header,
|
data_header,
|
||||||
|
@ -333,23 +293,19 @@ impl Shred {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn new_from_serialized_shred(mut payload: Vec<u8>) -> Result<Self, Error> {
|
pub fn new_from_serialized_shred(mut payload: Vec<u8>) -> Result<Self, Error> {
|
||||||
let mut start = 0;
|
let mut cursor = Cursor::new(&payload[..]);
|
||||||
let common_header: ShredCommonHeader =
|
let common_header: ShredCommonHeader = bincode::deserialize_from(&mut cursor)?;
|
||||||
Self::deserialize_obj(&mut start, SIZE_OF_COMMON_SHRED_HEADER, &payload)?;
|
|
||||||
|
|
||||||
// Shreds should be padded out to SHRED_PAYLOAD_SIZE
|
|
||||||
// so that erasure generation/recovery works correctly
|
|
||||||
// But only the data_header.size is stored in blockstore.
|
|
||||||
payload.resize(SHRED_PAYLOAD_SIZE, 0);
|
|
||||||
let (data_header, coding_header) = match common_header.shred_type {
|
let (data_header, coding_header) = match common_header.shred_type {
|
||||||
ShredType::Code => {
|
ShredType::Code => {
|
||||||
let coding_header: CodingShredHeader =
|
let coding_header = bincode::deserialize_from(&mut cursor)?;
|
||||||
Self::deserialize_obj(&mut start, SIZE_OF_CODING_SHRED_HEADER, &payload)?;
|
// see: https://github.com/solana-labs/solana/pull/10109
|
||||||
|
payload.truncate(SHRED_PAYLOAD_SIZE);
|
||||||
(DataShredHeader::default(), coding_header)
|
(DataShredHeader::default(), coding_header)
|
||||||
}
|
}
|
||||||
ShredType::Data => {
|
ShredType::Data => {
|
||||||
let data_header: DataShredHeader =
|
let data_header = bincode::deserialize_from(&mut cursor)?;
|
||||||
Self::deserialize_obj(&mut start, SIZE_OF_DATA_SHRED_HEADER, &payload)?;
|
// see: https://github.com/solana-labs/solana/pull/16602
|
||||||
|
payload.resize(SHRED_PAYLOAD_SIZE, 0u8);
|
||||||
(data_header, CodingShredHeader::default())
|
(data_header, CodingShredHeader::default())
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -386,24 +342,14 @@ impl Shred {
|
||||||
position,
|
position,
|
||||||
};
|
};
|
||||||
let mut payload = vec![0; SHRED_PAYLOAD_SIZE];
|
let mut payload = vec![0; SHRED_PAYLOAD_SIZE];
|
||||||
let mut start = 0;
|
let mut cursor = Cursor::new(&mut payload[..]);
|
||||||
Self::serialize_obj_into(
|
bincode::serialize_into(&mut cursor, &common_header).unwrap();
|
||||||
&mut start,
|
bincode::serialize_into(&mut cursor, &coding_header).unwrap();
|
||||||
SIZE_OF_COMMON_SHRED_HEADER,
|
|
||||||
&mut payload,
|
|
||||||
&common_header,
|
|
||||||
)
|
|
||||||
.expect("Failed to write header into shred buffer");
|
|
||||||
Self::serialize_obj_into(
|
|
||||||
&mut start,
|
|
||||||
SIZE_OF_CODING_SHRED_HEADER,
|
|
||||||
&mut payload,
|
|
||||||
&coding_header,
|
|
||||||
)
|
|
||||||
.expect("Failed to write coding header into shred buffer");
|
|
||||||
// Tests may have an empty parity_shard.
|
// Tests may have an empty parity_shard.
|
||||||
if !parity_shard.is_empty() {
|
if !parity_shard.is_empty() {
|
||||||
payload[SIZE_OF_CODING_SHRED_HEADERS..].copy_from_slice(parity_shard);
|
let offset = cursor.position() as usize;
|
||||||
|
debug_assert_eq!(offset, SIZE_OF_CODING_SHRED_HEADERS);
|
||||||
|
payload[offset..].copy_from_slice(parity_shard);
|
||||||
}
|
}
|
||||||
Shred {
|
Shred {
|
||||||
common_header,
|
common_header,
|
||||||
|
@ -491,9 +437,14 @@ impl Shred {
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
match shred_type {
|
match shred_type {
|
||||||
ShredType::Code => Ok(shred),
|
ShredType::Code => {
|
||||||
|
if shred.len() != SHRED_PAYLOAD_SIZE {
|
||||||
|
return Err(Error::InvalidPayloadSize(shred.len()));
|
||||||
|
}
|
||||||
|
Ok(shred)
|
||||||
|
}
|
||||||
ShredType::Data => {
|
ShredType::Data => {
|
||||||
if shred.len() > SHRED_PAYLOAD_SIZE {
|
if !(SHRED_DATA_OFFSET..SHRED_PAYLOAD_SIZE).contains(&shred.len()) {
|
||||||
return Err(Error::InvalidPayloadSize(shred.len()));
|
return Err(Error::InvalidPayloadSize(shred.len()));
|
||||||
}
|
}
|
||||||
shred.resize(SHRED_PAYLOAD_SIZE, 0u8);
|
shred.resize(SHRED_PAYLOAD_SIZE, 0u8);
|
||||||
|
@ -627,24 +578,12 @@ impl Shred {
|
||||||
|
|
||||||
pub fn set_index(&mut self, index: u32) {
|
pub fn set_index(&mut self, index: u32) {
|
||||||
self.common_header.index = index;
|
self.common_header.index = index;
|
||||||
Self::serialize_obj_into(
|
bincode::serialize_into(&mut self.payload[..], &self.common_header).unwrap();
|
||||||
&mut 0,
|
|
||||||
SIZE_OF_COMMON_SHRED_HEADER,
|
|
||||||
&mut self.payload,
|
|
||||||
&self.common_header,
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn set_slot(&mut self, slot: Slot) {
|
pub fn set_slot(&mut self, slot: Slot) {
|
||||||
self.common_header.slot = slot;
|
self.common_header.slot = slot;
|
||||||
Self::serialize_obj_into(
|
bincode::serialize_into(&mut self.payload[..], &self.common_header).unwrap();
|
||||||
&mut 0,
|
|
||||||
SIZE_OF_COMMON_SHRED_HEADER,
|
|
||||||
&mut self.payload,
|
|
||||||
&self.common_header,
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn signature(&self) -> Signature {
|
pub fn signature(&self) -> Signature {
|
||||||
|
@ -695,6 +634,8 @@ impl Shred {
|
||||||
if self.is_data() {
|
if self.is_data() {
|
||||||
self.data_header.flags |= ShredFlags::LAST_SHRED_IN_SLOT
|
self.data_header.flags |= ShredFlags::LAST_SHRED_IN_SLOT
|
||||||
}
|
}
|
||||||
|
let buffer = &mut self.payload[SIZE_OF_COMMON_SHRED_HEADER..];
|
||||||
|
bincode::serialize_into(buffer, &self.data_header).unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
@ -704,17 +645,9 @@ impl Shred {
|
||||||
.flags
|
.flags
|
||||||
.remove(ShredFlags::DATA_COMPLETE_SHRED);
|
.remove(ShredFlags::DATA_COMPLETE_SHRED);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Data header starts after the shred common header
|
// Data header starts after the shred common header
|
||||||
let mut start = SIZE_OF_COMMON_SHRED_HEADER;
|
let buffer = &mut self.payload[SIZE_OF_COMMON_SHRED_HEADER..];
|
||||||
let size_of_data_shred_header = SIZE_OF_DATA_SHRED_HEADER;
|
bincode::serialize_into(buffer, &self.data_header).unwrap();
|
||||||
Self::serialize_obj_into(
|
|
||||||
&mut start,
|
|
||||||
size_of_data_shred_header,
|
|
||||||
&mut self.payload,
|
|
||||||
&self.data_header,
|
|
||||||
)
|
|
||||||
.expect("Failed to write data header into shred buffer");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn data_complete(&self) -> bool {
|
pub fn data_complete(&self) -> bool {
|
||||||
|
@ -1276,6 +1209,48 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_serde_compat_shred_data_empty() {
|
||||||
|
const SEED: &str = "E3M5hm8yAEB7iPhQxFypAkLqxNeZCTuGBDMa8Jdrghoo";
|
||||||
|
const PAYLOAD: &str = "nRNFVBEsV9FEM5KfmsCXJsgELRSkCV55drTavdy5aZPnsp\
|
||||||
|
B8WvsgY99ZuNHDnwkrqe6Lx7ARVmercwugR5HwDcLA9ivKMypk9PNucDPLs67TXWy6k9R\
|
||||||
|
ozKmy";
|
||||||
|
let mut rng = {
|
||||||
|
let seed = <[u8; 32]>::try_from(bs58_decode(SEED)).unwrap();
|
||||||
|
ChaChaRng::from_seed(seed)
|
||||||
|
};
|
||||||
|
let keypair = Keypair::generate(&mut rng);
|
||||||
|
let mut shred = Shred::new_from_data(
|
||||||
|
142076266, // slot
|
||||||
|
21443, // index
|
||||||
|
51279, // parent_offset
|
||||||
|
&[], // data
|
||||||
|
true, // is_last_data
|
||||||
|
false, // is_last_in_slot
|
||||||
|
49, // reference_tick
|
||||||
|
59445, // version
|
||||||
|
21414, // fec_set_index
|
||||||
|
);
|
||||||
|
shred.sign(&keypair);
|
||||||
|
assert!(shred.verify(&keypair.pubkey()));
|
||||||
|
assert_matches!(shred.sanitize(), Ok(()));
|
||||||
|
let payload = bs58_decode(PAYLOAD);
|
||||||
|
let mut packet = Packet::default();
|
||||||
|
packet.data[..payload.len()].copy_from_slice(&payload);
|
||||||
|
packet.meta.size = payload.len();
|
||||||
|
assert_eq!(shred.bytes_to_store(), payload);
|
||||||
|
assert_eq!(shred, Shred::new_from_serialized_shred(payload).unwrap());
|
||||||
|
assert_eq!(
|
||||||
|
shred.reference_tick(),
|
||||||
|
Shred::reference_tick_from_data(&packet.data)
|
||||||
|
);
|
||||||
|
assert_eq!(Shred::get_slot_from_packet(&packet), Some(shred.slot()));
|
||||||
|
assert_eq!(
|
||||||
|
get_shred_slot_index_type(&packet, &mut ShredFetchStats::default()),
|
||||||
|
Some((shred.slot(), shred.index(), shred.shred_type()))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_serde_compat_shred_code() {
|
fn test_serde_compat_shred_code() {
|
||||||
const SEED: &str = "4jfjh3UZVyaEgvyG9oQmNyFY9yHDmbeH9eUhnBKkrcrN";
|
const SEED: &str = "4jfjh3UZVyaEgvyG9oQmNyFY9yHDmbeH9eUhnBKkrcrN";
|
||||||
|
|
Loading…
Reference in New Issue