blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove unnecessary serialize of shred data (#5967) 

* Remove unnecessary serialize of shred data

* remove obsolete code

* fix golden hash
This commit is contained in:
Pankaj Garg 2019-09-18 20:08:27 -07:00 committed by GitHub
parent 0d16db2d1b
commit 0dbf7995b5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 93 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
core/src/blocktree.rs
View File

@ -1667,7 +1667,6 @@ pub fn entries_to_test_shreds(
pub mod tests { pub mod tests {
use super::*; use super::*;
use crate::entry::{create_ticks, Entry}; use crate::entry::{create_ticks, Entry};
use crate::shred::{DataShredHeader, CODING_SHRED};
use itertools::Itertools; use itertools::Itertools;
use rand::seq::SliceRandom; use rand::seq::SliceRandom;
use rand::thread_rng; use rand::thread_rng;
@ -3107,13 +3106,8 @@ pub mod tests {
let index_cf = blocktree.db.column::<cf::Index>(); let index_cf = blocktree.db.column::<cf::Index>();
let last_root = RwLock::new(0); let last_root = RwLock::new(0);
let mut shred = DataShredHeader::default();
let slot = 1; let slot = 1;
shred.common_header.header.shred_type = CODING_SHRED; let mut shred = Shredder::new_coding_shred_header(slot, 11, 11, 11, 10);
shred.common_header.header.position = 10;
shred.common_header.header.coding_header.index = 11;
shred.common_header.header.coding_header.slot = 1;
shred.common_header.header.num_coding_shreds = shred.common_header.header.position + 1;
let coding_shred = Shred::new_empty_from_header(shred.clone()); let coding_shred = Shred::new_empty_from_header(shred.clone());
// Insert a good coding shred // Insert a good coding shred
@ -3131,7 +3125,7 @@ pub mod tests {
// Trying to insert the same shred again should fail // Trying to insert the same shred again should fail
{ {
let index = index_cf let index = index_cf
.get(shred.common_header.header.coding_header.slot) .get(shred.common_header.coding_header.slot)
.unwrap() .unwrap()
.unwrap(); .unwrap();
assert!(!Blocktree::should_insert_coding_shred( assert!(!Blocktree::should_insert_coding_shred(
@ -3141,13 +3135,13 @@ pub mod tests {
)); ));
} }
shred.common_header.header.coding_header.index += 1; shred.common_header.coding_header.index += 1;
// Establish a baseline that works // Establish a baseline that works
{ {
let coding_shred = Shred::new_empty_from_header(shred.clone()); let coding_shred = Shred::new_empty_from_header(shred.clone());
let index = index_cf let index = index_cf
.get(shred.common_header.header.coding_header.slot) .get(shred.common_header.coding_header.slot)
.unwrap() .unwrap()
.unwrap(); .unwrap();
assert!(Blocktree::should_insert_coding_shred( assert!(Blocktree::should_insert_coding_shred(
@ -3160,7 +3154,7 @@ pub mod tests {
// Trying to insert a shred with index < position should fail // Trying to insert a shred with index < position should fail
{ {
let mut coding_shred = Shred::new_empty_from_header(shred.clone()); let mut coding_shred = Shred::new_empty_from_header(shred.clone());
let index = coding_shred.headers.common_header.header.position - 1; let index = coding_shred.headers.common_header.position - 1;
coding_shred.set_index(index as u32); coding_shred.set_index(index as u32);
let index = index_cf.get(coding_shred.slot()).unwrap().unwrap(); let index = index_cf.get(coding_shred.slot()).unwrap().unwrap();
@ -3174,7 +3168,7 @@ pub mod tests {
// Trying to insert shred with num_coding == 0 should fail // Trying to insert shred with num_coding == 0 should fail
{ {
let mut coding_shred = Shred::new_empty_from_header(shred.clone()); let mut coding_shred = Shred::new_empty_from_header(shred.clone());
coding_shred.headers.common_header.header.num_coding_shreds = 0; coding_shred.headers.common_header.num_coding_shreds = 0;
let index = index_cf.get(coding_shred.slot()).unwrap().unwrap(); let index = index_cf.get(coding_shred.slot()).unwrap().unwrap();
assert!(!Blocktree::should_insert_coding_shred( assert!(!Blocktree::should_insert_coding_shred(
&coding_shred, &coding_shred,
@ -3186,8 +3180,8 @@ pub mod tests {
// Trying to insert shred with pos >= num_coding should fail // Trying to insert shred with pos >= num_coding should fail
{ {
let mut coding_shred = Shred::new_empty_from_header(shred.clone()); let mut coding_shred = Shred::new_empty_from_header(shred.clone());
coding_shred.headers.common_header.header.num_coding_shreds = coding_shred.headers.common_header.num_coding_shreds =
coding_shred.headers.common_header.header.position; coding_shred.headers.common_header.position;
let index = index_cf.get(coding_shred.slot()).unwrap().unwrap(); let index = index_cf.get(coding_shred.slot()).unwrap().unwrap();
assert!(!Blocktree::should_insert_coding_shred( assert!(!Blocktree::should_insert_coding_shred(
&coding_shred, &coding_shred,
@ -3200,14 +3194,9 @@ pub mod tests {
// has index > u32::MAX should fail // has index > u32::MAX should fail
{ {
let mut coding_shred = Shred::new_empty_from_header(shred.clone()); let mut coding_shred = Shred::new_empty_from_header(shred.clone());
coding_shred.headers.common_header.header.num_coding_shreds = 3; coding_shred.headers.common_header.num_coding_shreds = 3;
coding_shred coding_shred.headers.common_header.coding_header.index = std::u32::MAX - 1;
.headers coding_shred.headers.common_header.position = 0;
.common_header
.header
.coding_header
.index = std::u32::MAX - 1;
coding_shred.headers.common_header.header.position = 0;
let index = index_cf.get(coding_shred.slot()).unwrap().unwrap(); let index = index_cf.get(coding_shred.slot()).unwrap().unwrap();
assert!(!Blocktree::should_insert_coding_shred( assert!(!Blocktree::should_insert_coding_shred(
&coding_shred, &coding_shred,
@ -3216,7 +3205,7 @@ pub mod tests {
)); ));
// Decreasing the number of num_coding_shreds will put it within the allowed limit // Decreasing the number of num_coding_shreds will put it within the allowed limit
coding_shred.headers.common_header.header.num_coding_shreds = 2; coding_shred.headers.common_header.num_coding_shreds = 2;
assert!(Blocktree::should_insert_coding_shred( assert!(Blocktree::should_insert_coding_shred(
&coding_shred, &coding_shred,
index.coding(), index.coding(),

2
core/src/chacha.rs
View File

@ -153,7 +153,7 @@ mod tests {
hasher.hash(&buf[..size]); hasher.hash(&buf[..size]);
// golden needs to be updated if blob stuff changes.... // golden needs to be updated if blob stuff changes....
let golden: Hash = "3LWNjNqC6HncoWUhXbk6cUH8NSM675aZqRPGUC4Zq21H" let golden: Hash = "CLGvEayebjdgnLdttFAweZE9rqVkehXqEStUifG9kiU9"
.parse() .parse()
.unwrap(); .unwrap();

254
core/src/shred.rs
View File

@ -12,19 +12,15 @@ use solana_rayon_threadlimit::get_thread_count;
use solana_sdk::packet::PACKET_DATA_SIZE; use solana_sdk::packet::PACKET_DATA_SIZE;
use solana_sdk::pubkey::Pubkey; use solana_sdk::pubkey::Pubkey;
use solana_sdk::signature::{Keypair, KeypairUtil, Signature}; use solana_sdk::signature::{Keypair, KeypairUtil, Signature};
use std::io;
use std::io::{Error as IOError, ErrorKind, Write}; use std::io::{Error as IOError, ErrorKind, Write};
use std::sync::Arc; use std::sync::Arc;
use std::{cmp, io};
lazy_static! { lazy_static! {
static ref SIZE_OF_CODING_SHRED_HEADER: usize = static ref SIZE_OF_CODING_SHRED_HEADER: usize =
{ serialized_size(&CodingShredHeader::default()).unwrap() as usize }; { serialized_size(&CodingShredHeader::default()).unwrap() as usize };
static ref SIZE_OF_DATA_SHRED_HEADER: usize = static ref SIZE_OF_DATA_SHRED_HEADER: usize =
{ serialized_size(&DataShredHeader::default()).unwrap() as usize }; { serialized_size(&DataShredHeader::default()).unwrap() as usize };
static ref SIZE_OF_EMPTY_CODING_SHRED: usize =
{ serialized_size(&CodingShred::empty_shred()).unwrap() as usize };
static ref SIZE_OF_EMPTY_DATA_SHRED: usize =
{ serialized_size(&DataShred::empty_shred()).unwrap() as usize };
static ref SIZE_OF_SIGNATURE: usize = static ref SIZE_OF_SIGNATURE: usize =
{ bincode::serialized_size(&Signature::default()).unwrap() as usize }; { bincode::serialized_size(&Signature::default()).unwrap() as usize };
static ref SIZE_OF_EMPTY_VEC: usize = static ref SIZE_OF_EMPTY_VEC: usize =
@ -60,7 +56,7 @@ impl Shred {
let header = if shred_type == CODING_SHRED { let header = if shred_type == CODING_SHRED {
let end = *SIZE_OF_CODING_SHRED_HEADER; let end = *SIZE_OF_CODING_SHRED_HEADER;
let mut header = DataShredHeader::default(); let mut header = DataShredHeader::default();
header.common_header.header = bincode::deserialize(&shred_buf[..end])?; header.common_header = bincode::deserialize(&shred_buf[..end])?;
header header
} else { } else {
let end = *SIZE_OF_DATA_SHRED_HEADER; let end = *SIZE_OF_DATA_SHRED_HEADER;
@ -80,7 +76,7 @@ impl Shred {
if self.is_data() { if self.is_data() {
&self.headers.data_header &self.headers.data_header
} else { } else {
&self.headers.common_header.header.coding_header &self.headers.common_header.coding_header
} }
} }
@ -88,7 +84,7 @@ impl Shred {
if self.is_data() { if self.is_data() {
&mut self.headers.data_header &mut self.headers.data_header
} else { } else {
&mut self.headers.common_header.header.coding_header &mut self.headers.common_header.coding_header
} }
} }
@ -133,7 +129,7 @@ impl Shred {
} }
pub fn is_data(&self) -> bool { pub fn is_data(&self) -> bool {
self.headers.common_header.header.shred_type == DATA_SHRED self.headers.common_header.shred_type == DATA_SHRED
} }
pub fn last_in_slot(&self) -> bool { pub fn last_in_slot(&self) -> bool {
@ -162,7 +158,7 @@ impl Shred {
pub fn coding_params(&self) -> Option<(u16, u16, u16)> { pub fn coding_params(&self) -> Option<(u16, u16, u16)> {
if !self.is_data() { if !self.is_data() {
let header = &self.headers.common_header.header; let header = &self.headers.common_header;
Some(( Some((
header.num_data_shreds, header.num_data_shreds,
header.num_coding_shreds, header.num_coding_shreds,
@ -175,7 +171,7 @@ impl Shred {
pub fn verify(&self, pubkey: &Pubkey) -> bool { pub fn verify(&self, pubkey: &Pubkey) -> bool {
let signed_payload_offset = if self.is_data() { let signed_payload_offset = if self.is_data() {
CodingShred::overhead() *SIZE_OF_CODING_SHRED_HEADER
} else { } else {
*SIZE_OF_SHRED_TYPE *SIZE_OF_SHRED_TYPE
} + *SIZE_OF_SIGNATURE; } + *SIZE_OF_SIGNATURE;
@ -205,7 +201,7 @@ pub struct ShredCommonHeader {
/// A common header that is present at start of every data shred /// A common header that is present at start of every data shred
#[derive(Serialize, Clone, Deserialize, PartialEq, Debug)] #[derive(Serialize, Clone, Deserialize, PartialEq, Debug)]
pub struct DataShredHeader { pub struct DataShredHeader {
pub common_header: CodingShred, pub common_header: CodingShredHeader,
pub data_header: ShredCommonHeader, pub data_header: ShredCommonHeader,
pub parent_offset: u16, pub parent_offset: u16,
pub flags: u8, pub flags: u8,
@ -221,27 +217,12 @@ pub struct CodingShredHeader {
pub position: u16, pub position: u16,
} }
#[derive(Serialize, Clone, Deserialize, PartialEq, Debug)]
pub struct DataShred {
pub header: DataShredHeader,
pub payload: Vec<u8>,
}
#[derive(Serialize, Clone, Deserialize, PartialEq, Debug)]
pub struct CodingShred {
pub header: CodingShredHeader,
pub payload: Vec<u8>,
}
impl Default for DataShredHeader { impl Default for DataShredHeader {
fn default() -> Self { fn default() -> Self {
DataShredHeader { DataShredHeader {
common_header: CodingShred { common_header: CodingShredHeader {
header: CodingShredHeader { shred_type: DATA_SHRED,
shred_type: DATA_SHRED, ..CodingShredHeader::default()
..CodingShredHeader::default()
},
payload: vec![],
}, },
data_header: ShredCommonHeader::default(), data_header: ShredCommonHeader::default(),
parent_offset: 0, parent_offset: 0,
@ -262,84 +243,6 @@ impl Default for CodingShredHeader {
} }
} }
/// Default shred is sized correctly to meet MTU/Packet size requirements
impl Default for DataShred {
fn default() -> Self {
let size = PACKET_DATA_SIZE - *SIZE_OF_EMPTY_DATA_SHRED;
DataShred {
header: DataShredHeader::default(),
payload: vec![0; size],
}
}
}
/// Default shred is sized correctly to meet MTU/Packet size requirements
impl Default for CodingShred {
fn default() -> Self {
let size = PACKET_DATA_SIZE - *SIZE_OF_EMPTY_CODING_SHRED;
CodingShred {
header: CodingShredHeader::default(),
payload: vec![0; size],
}
}
}
/// Common trait implemented by all types of shreds
pub trait ShredCommon {
/// Write at a particular offset in the shred. Returns amount written and leftover capacity
fn write_at(&mut self, offset: usize, buf: &[u8]) -> (usize, usize);
/// Overhead of shred enum and headers
fn overhead() -> usize;
/// Utility function to create an empty shred
fn empty_shred() -> Self;
}
impl ShredCommon for DataShred {
fn write_at(&mut self, offset: usize, buf: &[u8]) -> (usize, usize) {
let mut capacity = self.payload.len().saturating_sub(offset);
let slice_len = cmp::min(capacity, buf.len());
capacity -= slice_len;
if slice_len > 0 {
self.payload[offset..offset + slice_len].copy_from_slice(&buf[..slice_len]);
}
(slice_len, capacity)
}
fn overhead() -> usize {
*SIZE_OF_EMPTY_DATA_SHRED - *SIZE_OF_EMPTY_VEC
}
fn empty_shred() -> Self {
DataShred {
header: DataShredHeader::default(),
payload: vec![],
}
}
}
impl ShredCommon for CodingShred {
fn write_at(&mut self, offset: usize, buf: &[u8]) -> (usize, usize) {
let mut capacity = self.payload.len().saturating_sub(offset);
let slice_len = cmp::min(capacity, buf.len());
capacity -= slice_len;
if slice_len > 0 {
self.payload[offset..offset + slice_len].copy_from_slice(&buf[..slice_len]);
}
(slice_len, capacity)
}
fn overhead() -> usize {
*SIZE_OF_EMPTY_CODING_SHRED
}
fn empty_shred() -> Self {
CodingShred {
header: CodingShredHeader::default(),
payload: vec![],
}
}
}
#[derive(Debug)] #[derive(Debug)]
pub struct Shredder { pub struct Shredder {
slot: u64, slot: u64,
@ -350,14 +253,17 @@ pub struct Shredder {
signer: Arc<Keypair>, signer: Arc<Keypair>,
pub shreds: Vec<Shred>, pub shreds: Vec<Shred>,
fec_set_shred_start: usize, fec_set_shred_start: usize,
active_shred: DataShred, active_shred: Vec<u8>,
active_shred_header: DataShredHeader,
active_offset: usize, active_offset: usize,
} }
impl Write for Shredder { impl Write for Shredder {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> { fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
let written = self.active_offset; let offset = self.active_offset + *SIZE_OF_DATA_SHRED_HEADER;
let (slice_len, capacity) = self.active_shred.write_at(written, buf); let slice_len = std::cmp::min(buf.len(), PACKET_DATA_SIZE - offset);
self.active_shred[offset..offset + slice_len].copy_from_slice(&buf[..slice_len]);
let capacity = PACKET_DATA_SIZE - offset - slice_len;
if buf.len() > slice_len || capacity == 0 { if buf.len() > slice_len || capacity == 0 {
self.finalize_data_shred(); self.finalize_data_shred();
@ -408,11 +314,11 @@ impl Shredder {
), ),
))) )))
} else { } else {
let mut data_shred = DataShred::default(); let mut header = DataShredHeader::default();
data_shred.header.data_header.slot = slot; header.data_header.slot = slot;
data_shred.header.data_header.index = index; header.data_header.index = index;
data_shred.header.parent_offset = (slot - parent) as u16; header.parent_offset = (slot - parent) as u16;
let active_shred = data_shred; let active_shred = vec![0; PACKET_DATA_SIZE];
Ok(Shredder { Ok(Shredder {
slot, slot,
index, index,
@ -423,6 +329,7 @@ impl Shredder {
shreds: vec![], shreds: vec![],
fec_set_shred_start: 0, fec_set_shred_start: 0,
active_shred, active_shred,
active_shred_header: header,
active_offset: 0, active_offset: 0,
}) })
} }
@ -439,7 +346,7 @@ impl Shredder {
} }
fn sign_unsigned_shreds_and_generate_codes(&mut self) { fn sign_unsigned_shreds_and_generate_codes(&mut self) {
let signature_offset = CodingShred::overhead(); let signature_offset = *SIZE_OF_CODING_SHRED_HEADER;
let signer = self.signer.clone(); let signer = self.signer.clone();
PAR_THREAD_POOL.with(|thread_pool| { PAR_THREAD_POOL.with(|thread_pool| {
thread_pool.borrow().install(|| { thread_pool.borrow().install(|| {
@ -465,41 +372,43 @@ impl Shredder {
/// Finalize a data shred. Update the shred index for the next shred /// Finalize a data shred. Update the shred index for the next shred
fn finalize_data_shred(&mut self) { fn finalize_data_shred(&mut self) {
let mut data = Vec::with_capacity(PACKET_DATA_SIZE);
bincode::serialize_into(&mut data, &self.active_shred).expect("Failed to serialize shred");
self.active_offset = 0; self.active_offset = 0;
self.index += 1; self.index += 1;
let mut shred = self.new_data_shred(); // Swap header
std::mem::swap(&mut shred, &mut self.active_shred); let mut header = DataShredHeader::default();
let shred_info = Shred::new(shred.header, data); header.data_header.slot = self.slot;
self.shreds.push(shred_info); header.data_header.index = self.index;
header.parent_offset = self.parent_offset;
std::mem::swap(&mut header, &mut self.active_shred_header);
// Swap shred buffer
let mut shred_buf = vec![0; PACKET_DATA_SIZE];
std::mem::swap(&mut shred_buf, &mut self.active_shred);
let mut wr = io::Cursor::new(&mut shred_buf[..*SIZE_OF_DATA_SHRED_HEADER]);
bincode::serialize_into(&mut wr, &header)
.expect("Failed to write header into shred buffer");
let shred = Shred::new(header, shred_buf);
self.shreds.push(shred);
} }
/// Creates a new data shred pub fn new_coding_shred_header(
fn new_data_shred(&self) -> DataShred {
let mut data_shred = DataShred::default();
data_shred.header.data_header.slot = self.slot;
data_shred.header.data_header.index = self.index;
data_shred.header.parent_offset = self.parent_offset;
data_shred
}
pub fn new_coding_shred(
slot: u64, slot: u64,
index: u32, index: u32,
num_data: usize, num_data: usize,
num_code: usize, num_code: usize,
position: usize, position: usize,
) -> CodingShred { ) -> DataShredHeader {
let mut coding_shred = CodingShred::default(); let mut header = DataShredHeader::default();
coding_shred.header.coding_header.slot = slot; header.common_header.shred_type = CODING_SHRED;
coding_shred.header.coding_header.index = index; header.common_header.coding_header.index = index;
coding_shred.header.num_data_shreds = num_data as u16; header.common_header.coding_header.slot = slot;
coding_shred.header.num_coding_shreds = num_code as u16; header.common_header.num_coding_shreds = num_code as u16;
coding_shred.header.position = position as u16; header.common_header.num_data_shreds = num_data as u16;
coding_shred header.common_header.position = position as u16;
header
} }
/// Generates coding shreds for the data shreds in the current FEC set /// Generates coding shreds for the data shreds in the current FEC set
@ -513,7 +422,7 @@ impl Shredder {
let start_index = self.index - num_data as u32; let start_index = self.index - num_data as u32;
// All information after coding shred field in a data shred is encoded // All information after coding shred field in a data shred is encoded
let coding_block_offset = CodingShred::overhead(); let coding_block_offset = *SIZE_OF_CODING_SHRED_HEADER;
let data_ptrs: Vec<_> = self.shreds[self.fec_set_shred_start..] let data_ptrs: Vec<_> = self.shreds[self.fec_set_shred_start..]
.iter() .iter()
.map(|data| &data.payload[coding_block_offset..]) .map(|data| &data.payload[coding_block_offset..])
@ -522,15 +431,15 @@ impl Shredder {
// Create empty coding shreds, with correctly populated headers // Create empty coding shreds, with correctly populated headers
let mut coding_shreds = Vec::with_capacity(num_coding); let mut coding_shreds = Vec::with_capacity(num_coding);
(0..num_coding).for_each(|i| { (0..num_coding).for_each(|i| {
let shred = bincode::serialize(&Self::new_coding_shred( let header = Self::new_coding_shred_header(
self.slot, self.slot,
start_index + i as u32, start_index + i as u32,
num_data, num_data,
num_coding, num_coding,
i, i,
)) );
.unwrap(); let shred = Shred::new_empty_from_header(header);
coding_shreds.push(shred); coding_shreds.push(shred.payload);
}); });
// Grab pointers for the coding blocks // Grab pointers for the coding blocks
@ -546,15 +455,14 @@ impl Shredder {
// append to the shred list // append to the shred list
coding_shreds.into_iter().enumerate().for_each(|(i, code)| { coding_shreds.into_iter().enumerate().for_each(|(i, code)| {
let mut header = DataShredHeader::default(); let header = Self::new_coding_shred_header(
header.common_header.header.shred_type = CODING_SHRED; self.slot,
header.common_header.header.coding_header.index = start_index + i as u32; start_index + i as u32,
header.common_header.header.coding_header.slot = self.slot; num_data,
header.common_header.header.num_coding_shreds = num_coding as u16; num_coding,
header.common_header.header.num_data_shreds = num_data as u16; i,
header.common_header.header.position = i as u16; );
let shred_info = Shred::new(header, code); self.shreds.push(Shred::new(header, code));
self.shreds.push(shred_info);
}); });
self.fec_set_index = self.index; self.fec_set_index = self.index;
} }
@ -564,12 +472,12 @@ impl Shredder {
/// If there's an active data shred, morph it into the final shred /// If there's an active data shred, morph it into the final shred
/// If the current active data shred is first in slot, finalize it and create a new shred /// If the current active data shred is first in slot, finalize it and create a new shred
fn make_final_data_shred(&mut self, last_in_slot: u8) { fn make_final_data_shred(&mut self, last_in_slot: u8) {
if self.active_shred.header.data_header.index == 0 { if self.active_shred_header.data_header.index == 0 {
self.finalize_data_shred(); self.finalize_data_shred();
} }
self.active_shred.header.flags |= DATA_COMPLETE_SHRED; self.active_shred_header.flags |= DATA_COMPLETE_SHRED;
if last_in_slot == LAST_SHRED_IN_SLOT { if last_in_slot == LAST_SHRED_IN_SLOT {
self.active_shred.header.flags |= LAST_SHRED_IN_SLOT; self.active_shred_header.flags |= LAST_SHRED_IN_SLOT;
} }
self.finalize_data_shred(); self.finalize_data_shred();
self.sign_unsigned_shreds_and_generate_codes(); self.sign_unsigned_shreds_and_generate_codes();
@ -618,21 +526,22 @@ impl Shredder {
first_index: usize, first_index: usize,
missing: usize, missing: usize,
) -> Vec<u8> { ) -> Vec<u8> {
if missing < first_index + num_data { let header = if missing < first_index + num_data {
let mut data_shred = DataShred::default(); let mut header = DataShredHeader::default();
data_shred.header.data_header.slot = slot; header.data_header.slot = slot;
data_shred.header.data_header.index = missing as u32; header.data_header.index = missing as u32;
bincode::serialize(&data_shred).unwrap() header
} else { } else {
bincode::serialize(&Self::new_coding_shred( Self::new_coding_shred_header(
slot, slot,
missing.saturating_sub(num_data) as u32, missing.saturating_sub(num_data) as u32,
num_data, num_data,
num_coding, num_coding,
missing - first_index - num_data, missing - first_index - num_data,
)) )
.unwrap() };
} let shred = Shred::new_empty_from_header(header);
shred.payload
} }
pub fn try_recovery( pub fn try_recovery(
@ -646,7 +555,7 @@ impl Shredder {
let mut recovered_code = vec![]; let mut recovered_code = vec![];
let fec_set_size = num_data + num_coding; let fec_set_size = num_data + num_coding;
if num_coding > 0 && shreds.len() < fec_set_size { if num_coding > 0 && shreds.len() < fec_set_size {
let coding_block_offset = CodingShred::overhead(); let coding_block_offset = *SIZE_OF_CODING_SHRED_HEADER;
// Let's try recovering missing shreds using erasure // Let's try recovering missing shreds using erasure
let mut present = &mut vec![true; fec_set_size]; let mut present = &mut vec![true; fec_set_size];
@ -764,7 +673,7 @@ impl Shredder {
data_shred_bufs[..num_data] data_shred_bufs[..num_data]
.iter() .iter()
.flat_map(|data| { .flat_map(|data| {
let offset = *SIZE_OF_EMPTY_DATA_SHRED; let offset = *SIZE_OF_DATA_SHRED_HEADER;
data[offset as usize..].iter() data[offset as usize..].iter()
}) })
.cloned() .cloned()
@ -819,9 +728,6 @@ mod tests {
assert!(shredder.shreds.is_empty()); assert!(shredder.shreds.is_empty());
assert_eq!(shredder.active_offset, 0); assert_eq!(shredder.active_offset, 0);
assert!(DataShred::overhead() < PACKET_DATA_SIZE);
assert!(CodingShred::overhead() < PACKET_DATA_SIZE);
// Test0: Write some data to shred. Not enough to create a signed shred // Test0: Write some data to shred. Not enough to create a signed shred
let data: Vec<u8> = (0..25).collect(); let data: Vec<u8> = (0..25).collect();
assert_eq!(shredder.write(&data).unwrap(), data.len()); assert_eq!(shredder.write(&data).unwrap(), data.len());
@ -1372,7 +1278,7 @@ mod tests {
let mut index = 0; let mut index = 0;
while index < shredder.shreds.len() { while index < shredder.shreds.len() {
let num_data_shreds = cmp::min( let num_data_shreds = std::cmp::min(
MAX_DATA_SHREDS_PER_FEC_BLOCK as usize, MAX_DATA_SHREDS_PER_FEC_BLOCK as usize,
(shredder.shreds.len() - index) / 2, (shredder.shreds.len() - index) / 2,
); );