From 0e306093940610e656ce8b933674f7010ed4d96f Mon Sep 17 00:00:00 2001 From: behzad nouri Date: Mon, 15 Aug 2022 19:02:32 +0000 Subject: [PATCH] adds Shred{Code,Data}::SIZE_OF_HEADERS trait constants (#27144) --- ledger/src/shred/legacy.rs | 37 +++++++++++++++++++------------------ ledger/src/shred/merkle.rs | 36 +++++++++++++++++++----------------- ledger/src/shred/traits.rs | 2 ++ 3 files changed, 40 insertions(+), 35 deletions(-) diff --git a/ledger/src/shred/legacy.rs b/ledger/src/shred/legacy.rs index fdf97b8b1c..1096ef1f54 100644 --- a/ledger/src/shred/legacy.rs +++ b/ledger/src/shred/legacy.rs @@ -20,15 +20,15 @@ const_assert_eq!(ShredData::SIZE_OF_PAYLOAD, ShredCode::SIZE_OF_PAYLOAD); const_assert_eq!(ShredData::SIZE_OF_PAYLOAD, 1228); const_assert_eq!(ShredData::CAPACITY, 1051); -// SIZE_OF_CODING_SHRED_HEADERS bytes at the end of data shreds +// ShredCode::SIZE_OF_HEADERS bytes at the end of data shreds // is never used and is not part of erasure coding. const_assert_eq!(SIZE_OF_ERASURE_ENCODED_SLICE, 1139); pub(super) const SIZE_OF_ERASURE_ENCODED_SLICE: usize = - ShredCode::SIZE_OF_PAYLOAD - SIZE_OF_CODING_SHRED_HEADERS; + ShredCode::SIZE_OF_PAYLOAD - ShredCode::SIZE_OF_HEADERS; // Layout: {common, data} headers | data | zero padding -// Everything up to SIZE_OF_CODING_SHRED_HEADERS bytes at the end (which is -// part of zero padding) is erasure coded. +// Everything up to ShredCode::SIZE_OF_HEADERS bytes at the end (which is part +// of zero padding) is erasure coded. // All payload past signature, including the entirety of zero paddings, is // signed. #[derive(Clone, Debug, Eq, PartialEq)] @@ -52,6 +52,7 @@ impl Shred for ShredData { // Legacy data shreds are always zero padded and // the same size as coding shreds. const SIZE_OF_PAYLOAD: usize = shred_code::ShredCode::SIZE_OF_PAYLOAD; + const SIZE_OF_HEADERS: usize = SIZE_OF_DATA_SHRED_HEADERS; fn from_payload(mut payload: Vec) -> Result { let mut cursor = Cursor::new(&payload[..]); @@ -64,7 +65,7 @@ impl Shred for ShredData { // Repair packets have nonce at the end of packet payload; see: // https://github.com/solana-labs/solana/pull/10109 // https://github.com/solana-labs/solana/pull/16602 - if payload.len() < SIZE_OF_DATA_SHRED_HEADERS { + if payload.len() < Self::SIZE_OF_HEADERS { return Err(Error::InvalidPayloadSize(payload.len())); } payload.resize(Self::SIZE_OF_PAYLOAD, 0u8); @@ -116,6 +117,7 @@ impl Shred for ShredData { impl Shred for ShredCode { impl_shred_common!(); const SIZE_OF_PAYLOAD: usize = shred_code::ShredCode::SIZE_OF_PAYLOAD; + const SIZE_OF_HEADERS: usize = SIZE_OF_CODING_SHRED_HEADERS; fn from_payload(mut payload: Vec) -> Result { let mut cursor = Cursor::new(&payload[..]); @@ -147,10 +149,9 @@ impl Shred for ShredCode { return Err(Error::InvalidPayloadSize(self.payload.len())); } let mut shard = self.payload; - // SIZE_OF_CODING_SHRED_HEADERS bytes at the beginning of the - // coding shreds contains the header and is not part of erasure - // coding. - shard.drain(..SIZE_OF_CODING_SHRED_HEADERS); + // ShredCode::SIZE_OF_HEADERS bytes at the beginning of the coding + // shreds contains the header and is not part of erasure coding. + shard.drain(..Self::SIZE_OF_HEADERS); Ok(shard) } @@ -158,7 +159,7 @@ impl Shred for ShredCode { if self.payload.len() != Self::SIZE_OF_PAYLOAD { return Err(Error::InvalidPayloadSize(self.payload.len())); } - Ok(&self.payload[SIZE_OF_CODING_SHRED_HEADERS..]) + Ok(&self.payload[Self::SIZE_OF_HEADERS..]) } fn sanitize(&self) -> Result<(), Error> { @@ -185,15 +186,15 @@ impl ShredDataTrait for ShredData { let size = usize::from(self.data_header.size); #[allow(clippy::manual_range_contains)] if size > self.payload.len() - || size < SIZE_OF_DATA_SHRED_HEADERS - || size > SIZE_OF_DATA_SHRED_HEADERS + Self::CAPACITY + || size < Self::SIZE_OF_HEADERS + || size > Self::SIZE_OF_HEADERS + Self::CAPACITY { return Err(Error::InvalidDataSize { size: self.data_header.size, payload: self.payload.len(), }); } - Ok(&self.payload[SIZE_OF_DATA_SHRED_HEADERS..size]) + Ok(&self.payload[Self::SIZE_OF_HEADERS..size]) } // Only for tests. @@ -214,7 +215,7 @@ impl ShredCodeTrait for ShredCode { impl ShredData { // Maximum size of ledger data that can be embedded in a data-shred. pub(super) const CAPACITY: usize = - Self::SIZE_OF_PAYLOAD - SIZE_OF_DATA_SHRED_HEADERS - SIZE_OF_CODING_SHRED_HEADERS; + Self::SIZE_OF_PAYLOAD - Self::SIZE_OF_HEADERS - ShredCode::SIZE_OF_HEADERS; pub(super) fn new_from_data( slot: Slot, @@ -235,7 +236,7 @@ impl ShredData { version, fec_set_index, }; - let size = (data.len() + SIZE_OF_DATA_SHRED_HEADERS) as u16; + let size = (data.len() + Self::SIZE_OF_HEADERS) as u16; let flags = flags | unsafe { ShredFlags::from_bits_unchecked( @@ -254,7 +255,7 @@ impl ShredData { bincode::serialize_into(&mut cursor, &data_header).unwrap(); // TODO: Need to check if data is too large! let offset = cursor.position() as usize; - debug_assert_eq!(offset, SIZE_OF_DATA_SHRED_HEADERS); + debug_assert_eq!(offset, Self::SIZE_OF_HEADERS); payload[offset..offset + data.len()].copy_from_slice(data); Self { common_header, @@ -271,7 +272,7 @@ impl ShredData { pub(super) fn resize_stored_shred(mut shred: Vec) -> Result, Error> { // Old shreds might have been extra zero padded. - if !(SIZE_OF_DATA_SHRED_HEADERS..=ShredCode::SIZE_OF_PAYLOAD).contains(&shred.len()) { + if !(Self::SIZE_OF_HEADERS..=Self::SIZE_OF_PAYLOAD).contains(&shred.len()) { return Err(Error::InvalidPayloadSize(shred.len())); } shred.resize(Self::SIZE_OF_PAYLOAD, 0u8); @@ -310,7 +311,7 @@ impl ShredCode { // Tests may have an empty parity_shard. if !parity_shard.is_empty() { let offset = cursor.position() as usize; - debug_assert_eq!(offset, SIZE_OF_CODING_SHRED_HEADERS); + debug_assert_eq!(offset, Self::SIZE_OF_HEADERS); payload[offset..].copy_from_slice(parity_shard); } Self { diff --git a/ledger/src/shred/merkle.rs b/ledger/src/shred/merkle.rs index 7eb3bf2d69..5529fcf3a6 100644 --- a/ledger/src/shred/merkle.rs +++ b/ledger/src/shred/merkle.rs @@ -76,12 +76,12 @@ impl ShredData { // Maximum size of ledger data that can be embedded in a data-shred. // Also equal to: // ShredCode::size_of_erasure_encoded_slice(proof_size).unwrap() - // - SIZE_OF_DATA_SHRED_HEADERS + // - ShredData::SIZE_OF_HEADERS // + SIZE_OF_SIGNATURE pub(super) fn capacity(proof_size: u8) -> Result { Self::SIZE_OF_PAYLOAD .checked_sub( - SIZE_OF_DATA_SHRED_HEADERS + Self::SIZE_OF_HEADERS + SIZE_OF_MERKLE_ROOT + usize::from(proof_size) * SIZE_OF_MERKLE_PROOF_ENTRY, ) @@ -90,7 +90,7 @@ impl ShredData { pub(super) fn get_signed_message_range(proof_size: u8) -> Option> { let data_buffer_size = Self::capacity(proof_size).ok()?; - let offset = SIZE_OF_DATA_SHRED_HEADERS + data_buffer_size; + let offset = Self::SIZE_OF_HEADERS + data_buffer_size; Some(offset..offset + SIZE_OF_MERKLE_ROOT) } @@ -121,7 +121,7 @@ impl ShredCode { // generated. Coding shred headers cannot be erasure coded either. Self::SIZE_OF_PAYLOAD .checked_sub( - SIZE_OF_CODING_SHRED_HEADERS + Self::SIZE_OF_HEADERS + SIZE_OF_MERKLE_ROOT + SIZE_OF_MERKLE_PROOF_ENTRY * usize::from(proof_size), ) @@ -133,7 +133,7 @@ impl ShredCode { let shard_size = Self::size_of_erasure_encoded_slice(proof_size)?; let chunk = self .payload - .get(SIZE_OF_SIGNATURE..SIZE_OF_CODING_SHRED_HEADERS + shard_size) + .get(SIZE_OF_SIGNATURE..Self::SIZE_OF_HEADERS + shard_size) .ok_or(Error::InvalidPayloadSize(self.payload.len()))?; Ok(hashv(&[MERKLE_HASH_PREFIX_LEAF, chunk])) } @@ -146,7 +146,7 @@ impl ShredCode { pub(super) fn get_signed_message_range(proof_size: u8) -> Option> { let offset = - SIZE_OF_CODING_SHRED_HEADERS + Self::size_of_erasure_encoded_slice(proof_size).ok()?; + Self::SIZE_OF_HEADERS + Self::size_of_erasure_encoded_slice(proof_size).ok()?; Some(offset..offset + SIZE_OF_MERKLE_ROOT) } @@ -161,12 +161,13 @@ impl Shred for ShredData { impl_shred_common!(); // Also equal to: - // SIZE_OF_DATA_SHRED_HEADERS + // ShredData::SIZE_OF_HEADERS // + ShredData::capacity(proof_size).unwrap() // + SIZE_OF_MERKLE_ROOT // + usize::from(proof_size) * SIZE_OF_MERKLE_PROOF_ENTRY const SIZE_OF_PAYLOAD: usize = - ShredCode::SIZE_OF_PAYLOAD - SIZE_OF_CODING_SHRED_HEADERS + SIZE_OF_SIGNATURE; + ShredCode::SIZE_OF_PAYLOAD - ShredCode::SIZE_OF_HEADERS + SIZE_OF_SIGNATURE; + const SIZE_OF_HEADERS: usize = SIZE_OF_DATA_SHRED_HEADERS; fn from_payload(mut payload: Vec) -> Result { if payload.len() < Self::SIZE_OF_PAYLOAD { @@ -213,7 +214,7 @@ impl Shred for ShredData { let proof_size = self.proof_size()?; let data_buffer_size = Self::capacity(proof_size)?; let mut shard = self.payload; - shard.truncate(SIZE_OF_DATA_SHRED_HEADERS + data_buffer_size); + shard.truncate(Self::SIZE_OF_HEADERS + data_buffer_size); shard.drain(0..SIZE_OF_SIGNATURE); Ok(shard) } @@ -225,7 +226,7 @@ impl Shred for ShredData { let proof_size = self.proof_size()?; let data_buffer_size = Self::capacity(proof_size)?; self.payload - .get(SIZE_OF_SIGNATURE..SIZE_OF_DATA_SHRED_HEADERS + data_buffer_size) + .get(SIZE_OF_SIGNATURE..Self::SIZE_OF_HEADERS + data_buffer_size) .ok_or(Error::InvalidPayloadSize(self.payload.len())) } @@ -252,6 +253,7 @@ impl Shred for ShredData { impl Shred for ShredCode { impl_shred_common!(); const SIZE_OF_PAYLOAD: usize = shred_code::ShredCode::SIZE_OF_PAYLOAD; + const SIZE_OF_HEADERS: usize = SIZE_OF_CODING_SHRED_HEADERS; fn from_payload(mut payload: Vec) -> Result { let mut cursor = Cursor::new(&payload[..]); @@ -296,7 +298,7 @@ impl Shred for ShredCode { let proof_size = self.proof_size()?; let shard_size = Self::size_of_erasure_encoded_slice(proof_size)?; let mut shard = self.payload; - shard.drain(..SIZE_OF_CODING_SHRED_HEADERS); + shard.drain(..Self::SIZE_OF_HEADERS); shard.truncate(shard_size); Ok(shard) } @@ -308,7 +310,7 @@ impl Shred for ShredCode { let proof_size = self.proof_size()?; let shard_size = Self::size_of_erasure_encoded_slice(proof_size)?; self.payload - .get(SIZE_OF_CODING_SHRED_HEADERS..SIZE_OF_CODING_SHRED_HEADERS + shard_size) + .get(Self::SIZE_OF_HEADERS..Self::SIZE_OF_HEADERS + shard_size) .ok_or(Error::InvalidPayloadSize(self.payload.len())) } @@ -343,15 +345,15 @@ impl ShredDataTrait for ShredData { let data_buffer_size = Self::capacity(proof_size)?; let size = usize::from(self.data_header.size); if size > self.payload.len() - || size < SIZE_OF_DATA_SHRED_HEADERS - || size > SIZE_OF_DATA_SHRED_HEADERS + data_buffer_size + || size < Self::SIZE_OF_HEADERS + || size > Self::SIZE_OF_HEADERS + data_buffer_size { return Err(Error::InvalidDataSize { size: self.data_header.size, payload: self.payload.len(), }); } - Ok(&self.payload[SIZE_OF_DATA_SHRED_HEADERS..size]) + Ok(&self.payload[Self::SIZE_OF_HEADERS..size]) } // Only for tests. @@ -439,7 +441,7 @@ mod test { // Total size of a data shred including headers and merkle branch. fn shred_data_size_of_payload(proof_size: u8) -> usize { - SIZE_OF_DATA_SHRED_HEADERS + ShredData::SIZE_OF_HEADERS + ShredData::capacity(proof_size).unwrap() + SIZE_OF_MERKLE_ROOT + usize::from(proof_size) * SIZE_OF_MERKLE_PROOF_ENTRY @@ -451,7 +453,7 @@ mod test { // size of erasure encoded header. fn shred_data_capacity(proof_size: u8) -> usize { const SIZE_OF_ERASURE_ENCODED_HEADER: usize = - SIZE_OF_DATA_SHRED_HEADERS - SIZE_OF_SIGNATURE; + ShredData::SIZE_OF_HEADERS - SIZE_OF_SIGNATURE; ShredCode::size_of_erasure_encoded_slice(proof_size).unwrap() - SIZE_OF_ERASURE_ENCODED_HEADER } diff --git a/ledger/src/shred/traits.rs b/ledger/src/shred/traits.rs index 70e049113d..b5326b62a7 100644 --- a/ledger/src/shred/traits.rs +++ b/ledger/src/shred/traits.rs @@ -7,6 +7,8 @@ pub(super) trait Shred: Sized { // Total size of payload including headers, merkle // branches (if any), zero paddings, etc. const SIZE_OF_PAYLOAD: usize; + // Size of common and code/data headers. + const SIZE_OF_HEADERS: usize; fn from_payload(shred: Vec) -> Result; fn common_header(&self) -> &ShredCommonHeader;