Allow coding shred index to be different than data shred index (#7438)
* Allow coding shred index to be different than data shred index * move fec_set_index to shred's common header * fix bench
This commit is contained in:
parent
cc5408482e
commit
d94041e98d
|
@ -75,7 +75,7 @@ fn bench_deshredder(bencher: &mut Bencher) {
|
||||||
fn bench_deserialize_hdr(bencher: &mut Bencher) {
|
fn bench_deserialize_hdr(bencher: &mut Bencher) {
|
||||||
let data = vec![0; SIZE_OF_DATA_SHRED_PAYLOAD];
|
let data = vec![0; SIZE_OF_DATA_SHRED_PAYLOAD];
|
||||||
|
|
||||||
let shred = Shred::new_from_data(2, 1, 1, Some(&data), true, true, 0, 0);
|
let shred = Shred::new_from_data(2, 1, 1, Some(&data), true, true, 0, 0, 1);
|
||||||
|
|
||||||
bencher.iter(|| {
|
bencher.iter(|| {
|
||||||
let payload = shred.payload.clone();
|
let payload = shred.payload.clone();
|
||||||
|
|
|
@ -66,6 +66,7 @@ impl StandardBroadcastRun {
|
||||||
true,
|
true,
|
||||||
max_ticks_in_slot & SHRED_TICK_REFERENCE_MASK,
|
max_ticks_in_slot & SHRED_TICK_REFERENCE_MASK,
|
||||||
self.shred_version,
|
self.shred_version,
|
||||||
|
last_unfinished_slot.next_shred_index,
|
||||||
))
|
))
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
|
|
|
@ -166,7 +166,7 @@ mod tests {
|
||||||
hasher.hash(&buf[..size]);
|
hasher.hash(&buf[..size]);
|
||||||
|
|
||||||
// golden needs to be updated if shred structure changes....
|
// golden needs to be updated if shred structure changes....
|
||||||
let golden: Hash = "9K6NR4cazo7Jzk2CpyXmNaZMGqvfXG83JzyJipkoHare"
|
let golden: Hash = "2rq8nR6rns2T5zsbQAGBDZb41NVtacneLgkCH17CVxZm"
|
||||||
.parse()
|
.parse()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
|
|
|
@ -92,6 +92,7 @@ pub mod tests {
|
||||||
true,
|
true,
|
||||||
0,
|
0,
|
||||||
0,
|
0,
|
||||||
|
0xc0de,
|
||||||
);
|
);
|
||||||
let mut batch = [Packets::default(), Packets::default()];
|
let mut batch = [Packets::default(), Packets::default()];
|
||||||
|
|
||||||
|
@ -110,6 +111,7 @@ pub mod tests {
|
||||||
true,
|
true,
|
||||||
0,
|
0,
|
||||||
0,
|
0,
|
||||||
|
0xc0de,
|
||||||
);
|
);
|
||||||
Shredder::sign_shred(&keypair, &mut shred);
|
Shredder::sign_shred(&keypair, &mut shred);
|
||||||
batch[1].packets.resize(1, Packet::default());
|
batch[1].packets.resize(1, Packet::default());
|
||||||
|
@ -133,14 +135,32 @@ pub mod tests {
|
||||||
let mut batch = vec![Packets::default()];
|
let mut batch = vec![Packets::default()];
|
||||||
batch[0].packets.resize(2, Packet::default());
|
batch[0].packets.resize(2, Packet::default());
|
||||||
|
|
||||||
let mut shred =
|
let mut shred = Shred::new_from_data(
|
||||||
Shred::new_from_data(0, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true, 0, 0);
|
0,
|
||||||
|
0xc0de,
|
||||||
|
0xdead,
|
||||||
|
Some(&[1, 2, 3, 4]),
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0xc0de,
|
||||||
|
);
|
||||||
Shredder::sign_shred(&leader_keypair, &mut shred);
|
Shredder::sign_shred(&leader_keypair, &mut shred);
|
||||||
batch[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload);
|
batch[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload);
|
||||||
batch[0].packets[0].meta.size = shred.payload.len();
|
batch[0].packets[0].meta.size = shred.payload.len();
|
||||||
|
|
||||||
let mut shred =
|
let mut shred = Shred::new_from_data(
|
||||||
Shred::new_from_data(0, 0xbeef, 0xc0de, Some(&[1, 2, 3, 4]), true, true, 0, 0);
|
0,
|
||||||
|
0xbeef,
|
||||||
|
0xc0de,
|
||||||
|
Some(&[1, 2, 3, 4]),
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0xc0de,
|
||||||
|
);
|
||||||
let wrong_keypair = Keypair::new();
|
let wrong_keypair = Keypair::new();
|
||||||
Shredder::sign_shred(&wrong_keypair, &mut shred);
|
Shredder::sign_shred(&wrong_keypair, &mut shred);
|
||||||
batch[0].packets[1].data[0..shred.payload.len()].copy_from_slice(&shred.payload);
|
batch[0].packets[1].data[0..shred.payload.len()].copy_from_slice(&shred.payload);
|
||||||
|
|
|
@ -363,7 +363,7 @@ mod test {
|
||||||
);
|
);
|
||||||
|
|
||||||
// If it's a coding shred, test that slot >= root
|
// If it's a coding shred, test that slot >= root
|
||||||
let (common, coding) = Shredder::new_coding_shred_header(5, 5, 6, 6, 0, 0);
|
let (common, coding) = Shredder::new_coding_shred_header(5, 5, 5, 6, 6, 0, 0);
|
||||||
let mut coding_shred =
|
let mut coding_shred =
|
||||||
Shred::new_empty_from_header(common, DataShredHeader::default(), coding);
|
Shred::new_empty_from_header(common, DataShredHeader::default(), coding);
|
||||||
Shredder::sign_shred(&leader_keypair, &mut coding_shred);
|
Shredder::sign_shred(&leader_keypair, &mut coding_shred);
|
||||||
|
|
|
@ -391,7 +391,11 @@ impl Blocktree {
|
||||||
"blocktree-erasure",
|
"blocktree-erasure",
|
||||||
("slot", slot as i64, i64),
|
("slot", slot as i64, i64),
|
||||||
("start_index", set_index as i64, i64),
|
("start_index", set_index as i64, i64),
|
||||||
("end_index", erasure_meta.end_indexes().0 as i64, i64),
|
(
|
||||||
|
"end_index",
|
||||||
|
(erasure_meta.set_index + erasure_meta.config.num_data() as u64) as i64,
|
||||||
|
i64
|
||||||
|
),
|
||||||
("recovery_attempted", attempted, bool),
|
("recovery_attempted", attempted, bool),
|
||||||
("recovery_status", status, String),
|
("recovery_status", status, String),
|
||||||
("recovered", recovered as i64, i64),
|
("recovered", recovered as i64, i64),
|
||||||
|
@ -424,8 +428,10 @@ impl Blocktree {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
(set_index..set_index + erasure_meta.config.num_coding() as u64).for_each(
|
(erasure_meta.first_coding_index
|
||||||
|i| {
|
..erasure_meta.first_coding_index
|
||||||
|
+ erasure_meta.config.num_coding() as u64)
|
||||||
|
.for_each(|i| {
|
||||||
if let Some(shred) = prev_inserted_codes
|
if let Some(shred) = prev_inserted_codes
|
||||||
.remove(&(slot, i))
|
.remove(&(slot, i))
|
||||||
.map(|s| {
|
.map(|s| {
|
||||||
|
@ -454,13 +460,13 @@ impl Blocktree {
|
||||||
{
|
{
|
||||||
available_shreds.push(shred);
|
available_shreds.push(shred);
|
||||||
}
|
}
|
||||||
},
|
});
|
||||||
);
|
|
||||||
if let Ok(mut result) = Shredder::try_recovery(
|
if let Ok(mut result) = Shredder::try_recovery(
|
||||||
available_shreds,
|
available_shreds,
|
||||||
erasure_meta.config.num_data(),
|
erasure_meta.config.num_data(),
|
||||||
erasure_meta.config.num_coding(),
|
erasure_meta.config.num_coding(),
|
||||||
set_index as usize,
|
set_index as usize,
|
||||||
|
erasure_meta.first_coding_index as usize,
|
||||||
slot,
|
slot,
|
||||||
) {
|
) {
|
||||||
submit_metrics(true, "complete".into(), result.len());
|
submit_metrics(true, "complete".into(), result.len());
|
||||||
|
@ -687,17 +693,21 @@ impl Blocktree {
|
||||||
if is_trusted
|
if is_trusted
|
||||||
|| Blocktree::should_insert_coding_shred(&shred, index_meta.coding(), &self.last_root)
|
|| Blocktree::should_insert_coding_shred(&shred, index_meta.coding(), &self.last_root)
|
||||||
{
|
{
|
||||||
let set_index = shred_index - u64::from(shred.coding_header.position);
|
let set_index = u64::from(shred.common_header.fec_set_index);
|
||||||
let erasure_config = ErasureConfig::new(
|
let erasure_config = ErasureConfig::new(
|
||||||
shred.coding_header.num_data_shreds as usize,
|
shred.coding_header.num_data_shreds as usize,
|
||||||
shred.coding_header.num_coding_shreds as usize,
|
shred.coding_header.num_coding_shreds as usize,
|
||||||
);
|
);
|
||||||
|
|
||||||
let erasure_meta = erasure_metas.entry((slot, set_index)).or_insert_with(|| {
|
let erasure_meta = erasure_metas.entry((slot, set_index)).or_insert_with(|| {
|
||||||
|
let first_coding_index =
|
||||||
|
u64::from(shred.index()) - u64::from(shred.coding_header.position);
|
||||||
self.erasure_meta_cf
|
self.erasure_meta_cf
|
||||||
.get((slot, set_index))
|
.get((slot, set_index))
|
||||||
.expect("Expect database get to succeed")
|
.expect("Expect database get to succeed")
|
||||||
.unwrap_or_else(|| ErasureMeta::new(set_index, &erasure_config))
|
.unwrap_or_else(|| {
|
||||||
|
ErasureMeta::new(set_index, first_coding_index, &erasure_config)
|
||||||
|
})
|
||||||
});
|
});
|
||||||
|
|
||||||
if erasure_config != erasure_meta.config {
|
if erasure_config != erasure_meta.config {
|
||||||
|
@ -3535,7 +3545,17 @@ pub mod tests {
|
||||||
let gap: u64 = 10;
|
let gap: u64 = 10;
|
||||||
let shreds: Vec<_> = (0..64)
|
let shreds: Vec<_> = (0..64)
|
||||||
.map(|i| {
|
.map(|i| {
|
||||||
Shred::new_from_data(slot, (i * gap) as u32, 0, None, false, false, i as u8, 0)
|
Shred::new_from_data(
|
||||||
|
slot,
|
||||||
|
(i * gap) as u32,
|
||||||
|
0,
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
i as u8,
|
||||||
|
0,
|
||||||
|
(i * gap) as u32,
|
||||||
|
)
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
blocktree.insert_shreds(shreds, None, false).unwrap();
|
blocktree.insert_shreds(shreds, None, false).unwrap();
|
||||||
|
@ -3726,7 +3746,8 @@ pub mod tests {
|
||||||
let last_root = RwLock::new(0);
|
let last_root = RwLock::new(0);
|
||||||
|
|
||||||
let slot = 1;
|
let slot = 1;
|
||||||
let (mut shred, coding) = Shredder::new_coding_shred_header(slot, 11, 11, 11, 10, 0);
|
let (mut shred, coding) =
|
||||||
|
Shredder::new_coding_shred_header(slot, 11, 11, 11, 11, 10, 0);
|
||||||
let coding_shred = Shred::new_empty_from_header(
|
let coding_shred = Shred::new_empty_from_header(
|
||||||
shred.clone(),
|
shred.clone(),
|
||||||
DataShredHeader::default(),
|
DataShredHeader::default(),
|
||||||
|
@ -4149,6 +4170,7 @@ pub mod tests {
|
||||||
true,
|
true,
|
||||||
0,
|
0,
|
||||||
0,
|
0,
|
||||||
|
next_shred_index as u32,
|
||||||
)];
|
)];
|
||||||
|
|
||||||
// With the corruption, nothing should be returned, even though an
|
// With the corruption, nothing should be returned, even though an
|
||||||
|
|
|
@ -59,6 +59,8 @@ pub struct CodingIndex {
|
||||||
pub struct ErasureMeta {
|
pub struct ErasureMeta {
|
||||||
/// Which erasure set in the slot this is
|
/// Which erasure set in the slot this is
|
||||||
pub set_index: u64,
|
pub set_index: u64,
|
||||||
|
/// First coding index in the FEC set
|
||||||
|
pub first_coding_index: u64,
|
||||||
/// Size of shards in this erasure set
|
/// Size of shards in this erasure set
|
||||||
pub size: usize,
|
pub size: usize,
|
||||||
/// Erasure configuration for this erasure set
|
/// Erasure configuration for this erasure set
|
||||||
|
@ -200,9 +202,10 @@ impl SlotMeta {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ErasureMeta {
|
impl ErasureMeta {
|
||||||
pub fn new(set_index: u64, config: &ErasureConfig) -> ErasureMeta {
|
pub fn new(set_index: u64, first_coding_index: u64, config: &ErasureConfig) -> ErasureMeta {
|
||||||
ErasureMeta {
|
ErasureMeta {
|
||||||
set_index,
|
set_index,
|
||||||
|
first_coding_index,
|
||||||
size: 0,
|
size: 0,
|
||||||
config: *config,
|
config: *config,
|
||||||
}
|
}
|
||||||
|
@ -211,11 +214,12 @@ impl ErasureMeta {
|
||||||
pub fn status(&self, index: &Index) -> ErasureMetaStatus {
|
pub fn status(&self, index: &Index) -> ErasureMetaStatus {
|
||||||
use ErasureMetaStatus::*;
|
use ErasureMetaStatus::*;
|
||||||
|
|
||||||
let start_idx = self.start_index();
|
let num_coding = index.coding().present_in_bounds(
|
||||||
let (data_end_idx, coding_end_idx) = self.end_indexes();
|
self.first_coding_index..self.first_coding_index + self.config.num_coding() as u64,
|
||||||
|
);
|
||||||
let num_coding = index.coding().present_in_bounds(start_idx..coding_end_idx);
|
let num_data = index
|
||||||
let num_data = index.data().present_in_bounds(start_idx..data_end_idx);
|
.data()
|
||||||
|
.present_in_bounds(self.set_index..self.set_index + self.config.num_data() as u64);
|
||||||
|
|
||||||
let (data_missing, coding_missing) = (
|
let (data_missing, coding_missing) = (
|
||||||
self.config.num_data() - num_data,
|
self.config.num_data() - num_data,
|
||||||
|
@ -240,23 +244,6 @@ impl ErasureMeta {
|
||||||
pub fn size(&self) -> usize {
|
pub fn size(&self) -> usize {
|
||||||
self.size
|
self.size
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn set_index_for(index: u64, num_data: usize) -> u64 {
|
|
||||||
index / num_data as u64
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn start_index(&self) -> u64 {
|
|
||||||
self.set_index
|
|
||||||
}
|
|
||||||
|
|
||||||
/// returns a tuple of (data_end, coding_end)
|
|
||||||
pub fn end_indexes(&self) -> (u64, u64) {
|
|
||||||
let start = self.start_index();
|
|
||||||
(
|
|
||||||
start + self.config.num_data() as u64,
|
|
||||||
start + self.config.num_coding() as u64,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
@ -272,7 +259,7 @@ mod test {
|
||||||
let set_index = 0;
|
let set_index = 0;
|
||||||
let erasure_config = ErasureConfig::default();
|
let erasure_config = ErasureConfig::default();
|
||||||
|
|
||||||
let mut e_meta = ErasureMeta::new(set_index, &erasure_config);
|
let mut e_meta = ErasureMeta::new(set_index, set_index, &erasure_config);
|
||||||
let mut rng = thread_rng();
|
let mut rng = thread_rng();
|
||||||
let mut index = Index::new(0);
|
let mut index = Index::new(0);
|
||||||
e_meta.size = 1;
|
e_meta.size = 1;
|
||||||
|
|
|
@ -27,7 +27,7 @@ use thiserror::Error;
|
||||||
/// The following constants are computed by hand, and hardcoded.
|
/// The following constants are computed by hand, and hardcoded.
|
||||||
/// `test_shred_constants` ensures that the values are correct.
|
/// `test_shred_constants` ensures that the values are correct.
|
||||||
/// Constants are used over lazy_static for performance reasons.
|
/// Constants are used over lazy_static for performance reasons.
|
||||||
pub const SIZE_OF_COMMON_SHRED_HEADER: usize = 79;
|
pub const SIZE_OF_COMMON_SHRED_HEADER: usize = 83;
|
||||||
pub const SIZE_OF_DATA_SHRED_HEADER: usize = 3;
|
pub const SIZE_OF_DATA_SHRED_HEADER: usize = 3;
|
||||||
pub const SIZE_OF_CODING_SHRED_HEADER: usize = 6;
|
pub const SIZE_OF_CODING_SHRED_HEADER: usize = 6;
|
||||||
pub const SIZE_OF_SIGNATURE: usize = 64;
|
pub const SIZE_OF_SIGNATURE: usize = 64;
|
||||||
|
@ -87,6 +87,7 @@ pub struct ShredCommonHeader {
|
||||||
pub slot: Slot,
|
pub slot: Slot,
|
||||||
pub index: u32,
|
pub index: u32,
|
||||||
pub version: u16,
|
pub version: u16,
|
||||||
|
pub fec_set_index: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The data shred header has parent offset and flags
|
/// The data shred header has parent offset and flags
|
||||||
|
@ -153,12 +154,14 @@ impl Shred {
|
||||||
is_last_in_slot: bool,
|
is_last_in_slot: bool,
|
||||||
reference_tick: u8,
|
reference_tick: u8,
|
||||||
version: u16,
|
version: u16,
|
||||||
|
fec_set_index: u32,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let mut payload = vec![0; PACKET_DATA_SIZE];
|
let mut payload = vec![0; PACKET_DATA_SIZE];
|
||||||
let common_header = ShredCommonHeader {
|
let common_header = ShredCommonHeader {
|
||||||
slot,
|
slot,
|
||||||
index,
|
index,
|
||||||
version,
|
version,
|
||||||
|
fec_set_index,
|
||||||
..ShredCommonHeader::default()
|
..ShredCommonHeader::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -460,6 +463,11 @@ impl Shredder {
|
||||||
.map(|(i, shred_data)| {
|
.map(|(i, shred_data)| {
|
||||||
let shred_index = next_shred_index + i as u32;
|
let shred_index = next_shred_index + i as u32;
|
||||||
|
|
||||||
|
// Each FEC block has maximum MAX_DATA_SHREDS_PER_FEC_BLOCK shreds
|
||||||
|
// "FEC set index" is the index of first data shred in that FEC block
|
||||||
|
let fec_set_index =
|
||||||
|
shred_index - (i % MAX_DATA_SHREDS_PER_FEC_BLOCK as usize) as u32;
|
||||||
|
|
||||||
let (is_last_data, is_last_in_slot) = {
|
let (is_last_data, is_last_in_slot) = {
|
||||||
if shred_index == last_shred_index {
|
if shred_index == last_shred_index {
|
||||||
(true, is_last_in_slot)
|
(true, is_last_in_slot)
|
||||||
|
@ -477,6 +485,7 @@ impl Shredder {
|
||||||
is_last_in_slot,
|
is_last_in_slot,
|
||||||
self.reference_tick,
|
self.reference_tick,
|
||||||
self.version,
|
self.version,
|
||||||
|
fec_set_index,
|
||||||
);
|
);
|
||||||
|
|
||||||
Shredder::sign_shred(&self.keypair, &mut shred);
|
Shredder::sign_shred(&self.keypair, &mut shred);
|
||||||
|
@ -541,6 +550,7 @@ impl Shredder {
|
||||||
pub fn new_coding_shred_header(
|
pub fn new_coding_shred_header(
|
||||||
slot: Slot,
|
slot: Slot,
|
||||||
index: u32,
|
index: u32,
|
||||||
|
fec_set_index: u32,
|
||||||
num_data: usize,
|
num_data: usize,
|
||||||
num_code: usize,
|
num_code: usize,
|
||||||
position: usize,
|
position: usize,
|
||||||
|
@ -551,6 +561,7 @@ impl Shredder {
|
||||||
index,
|
index,
|
||||||
slot,
|
slot,
|
||||||
version,
|
version,
|
||||||
|
fec_set_index,
|
||||||
..ShredCommonHeader::default()
|
..ShredCommonHeader::default()
|
||||||
};
|
};
|
||||||
(
|
(
|
||||||
|
@ -592,6 +603,7 @@ impl Shredder {
|
||||||
let (header, coding_header) = Self::new_coding_shred_header(
|
let (header, coding_header) = Self::new_coding_shred_header(
|
||||||
slot,
|
slot,
|
||||||
start_index + i as u32,
|
start_index + i as u32,
|
||||||
|
start_index,
|
||||||
num_data,
|
num_data,
|
||||||
num_coding,
|
num_coding,
|
||||||
i,
|
i,
|
||||||
|
@ -622,6 +634,7 @@ impl Shredder {
|
||||||
let (common_header, coding_header) = Self::new_coding_shred_header(
|
let (common_header, coding_header) = Self::new_coding_shred_header(
|
||||||
slot,
|
slot,
|
||||||
start_index + i as u32,
|
start_index + i as u32,
|
||||||
|
start_index,
|
||||||
num_data,
|
num_data,
|
||||||
num_coding,
|
num_coding,
|
||||||
i,
|
i,
|
||||||
|
@ -679,6 +692,7 @@ impl Shredder {
|
||||||
num_data: usize,
|
num_data: usize,
|
||||||
num_coding: usize,
|
num_coding: usize,
|
||||||
first_index: usize,
|
first_index: usize,
|
||||||
|
first_code_index: usize,
|
||||||
slot: Slot,
|
slot: Slot,
|
||||||
) -> std::result::Result<Vec<Shred>, reed_solomon_erasure::Error> {
|
) -> std::result::Result<Vec<Shred>, reed_solomon_erasure::Error> {
|
||||||
let mut recovered_data = vec![];
|
let mut recovered_data = vec![];
|
||||||
|
@ -691,7 +705,8 @@ impl Shredder {
|
||||||
let mut shred_bufs: Vec<Vec<u8>> = shreds
|
let mut shred_bufs: Vec<Vec<u8>> = shreds
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.flat_map(|shred| {
|
.flat_map(|shred| {
|
||||||
let index = Self::get_shred_index(&shred, num_data);
|
let index =
|
||||||
|
Self::get_shred_index(&shred, num_data, first_index, first_code_index);
|
||||||
let mut blocks = Self::fill_in_missing_shreds(
|
let mut blocks = Self::fill_in_missing_shreds(
|
||||||
num_data,
|
num_data,
|
||||||
num_coding,
|
num_coding,
|
||||||
|
@ -789,11 +804,16 @@ impl Shredder {
|
||||||
Ok(Self::reassemble_payload(num_data, data_shred_bufs))
|
Ok(Self::reassemble_payload(num_data, data_shred_bufs))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_shred_index(shred: &Shred, num_data: usize) -> usize {
|
fn get_shred_index(
|
||||||
|
shred: &Shred,
|
||||||
|
num_data: usize,
|
||||||
|
first_data_index: usize,
|
||||||
|
first_code_index: usize,
|
||||||
|
) -> usize {
|
||||||
if shred.is_data() {
|
if shred.is_data() {
|
||||||
shred.index() as usize
|
shred.index() as usize
|
||||||
} else {
|
} else {
|
||||||
shred.index() as usize + num_data
|
shred.index() as usize + num_data + first_data_index - first_code_index
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1168,6 +1188,7 @@ pub mod tests {
|
||||||
num_data_shreds,
|
num_data_shreds,
|
||||||
num_data_shreds,
|
num_data_shreds,
|
||||||
0,
|
0,
|
||||||
|
0,
|
||||||
slot
|
slot
|
||||||
),
|
),
|
||||||
Err(reed_solomon_erasure::Error::TooFewShardsPresent)
|
Err(reed_solomon_erasure::Error::TooFewShardsPresent)
|
||||||
|
@ -1179,6 +1200,7 @@ pub mod tests {
|
||||||
num_data_shreds,
|
num_data_shreds,
|
||||||
num_data_shreds,
|
num_data_shreds,
|
||||||
0,
|
0,
|
||||||
|
0,
|
||||||
slot,
|
slot,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
@ -1196,6 +1218,7 @@ pub mod tests {
|
||||||
num_data_shreds,
|
num_data_shreds,
|
||||||
num_data_shreds,
|
num_data_shreds,
|
||||||
0,
|
0,
|
||||||
|
0,
|
||||||
slot,
|
slot,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
@ -1243,6 +1266,7 @@ pub mod tests {
|
||||||
num_data_shreds,
|
num_data_shreds,
|
||||||
num_data_shreds,
|
num_data_shreds,
|
||||||
0,
|
0,
|
||||||
|
0,
|
||||||
slot,
|
slot,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
@ -1315,6 +1339,7 @@ pub mod tests {
|
||||||
num_data_shreds,
|
num_data_shreds,
|
||||||
num_data_shreds,
|
num_data_shreds,
|
||||||
25,
|
25,
|
||||||
|
25,
|
||||||
slot,
|
slot,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
@ -1346,6 +1371,7 @@ pub mod tests {
|
||||||
num_data_shreds,
|
num_data_shreds,
|
||||||
num_data_shreds,
|
num_data_shreds,
|
||||||
25,
|
25,
|
||||||
|
25,
|
||||||
slot + 1,
|
slot + 1,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
@ -1358,6 +1384,7 @@ pub mod tests {
|
||||||
num_data_shreds,
|
num_data_shreds,
|
||||||
num_data_shreds,
|
num_data_shreds,
|
||||||
15,
|
15,
|
||||||
|
15,
|
||||||
slot,
|
slot,
|
||||||
),
|
),
|
||||||
Err(reed_solomon_erasure::Error::TooFewShardsPresent)
|
Err(reed_solomon_erasure::Error::TooFewShardsPresent)
|
||||||
|
@ -1365,7 +1392,7 @@ pub mod tests {
|
||||||
|
|
||||||
// Test8: Try recovery/reassembly with incorrect index. Hint: does not recover any shreds
|
// Test8: Try recovery/reassembly with incorrect index. Hint: does not recover any shreds
|
||||||
assert_matches!(
|
assert_matches!(
|
||||||
Shredder::try_recovery(shred_info, num_data_shreds, num_data_shreds, 35, slot,),
|
Shredder::try_recovery(shred_info, num_data_shreds, num_data_shreds, 35, 35, slot,),
|
||||||
Err(reed_solomon_erasure::Error::TooFewShardsPresent)
|
Err(reed_solomon_erasure::Error::TooFewShardsPresent)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
@ -1419,4 +1446,42 @@ pub mod tests {
|
||||||
let version = Shred::version_from_hash(&Hash::new(&hash));
|
let version = Shred::version_from_hash(&Hash::new(&hash));
|
||||||
assert_eq!(version, 0x5a5a);
|
assert_eq!(version, 0x5a5a);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_shred_fec_set_index() {
|
||||||
|
let keypair = Arc::new(Keypair::new());
|
||||||
|
let hash = hash(Hash::default().as_ref());
|
||||||
|
let version = Shred::version_from_hash(&hash);
|
||||||
|
assert_ne!(version, 0);
|
||||||
|
let shredder =
|
||||||
|
Shredder::new(0, 0, 0.5, keypair, 0, version).expect("Failed in creating shredder");
|
||||||
|
|
||||||
|
let entries: Vec<_> = (0..500)
|
||||||
|
.map(|_| {
|
||||||
|
let keypair0 = Keypair::new();
|
||||||
|
let keypair1 = Keypair::new();
|
||||||
|
let tx0 =
|
||||||
|
system_transaction::transfer(&keypair0, &keypair1.pubkey(), 1, Hash::default());
|
||||||
|
Entry::new(&Hash::default(), 1, vec![tx0])
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let start_index = 0x12;
|
||||||
|
let (data_shreds, coding_shreds, _next_index) =
|
||||||
|
shredder.entries_to_shreds(&entries, true, start_index);
|
||||||
|
|
||||||
|
let max_per_block = MAX_DATA_SHREDS_PER_FEC_BLOCK as usize;
|
||||||
|
data_shreds.iter().enumerate().for_each(|(i, s)| {
|
||||||
|
let expected_fec_set_index = start_index + ((i / max_per_block) * max_per_block) as u32;
|
||||||
|
assert_eq!(s.common_header.fec_set_index, expected_fec_set_index);
|
||||||
|
});
|
||||||
|
|
||||||
|
coding_shreds.iter().enumerate().for_each(|(i, s)| {
|
||||||
|
// There'll be half the number of coding shreds, as FEC rate is 0.5
|
||||||
|
// So multiply i with 2
|
||||||
|
let expected_fec_set_index =
|
||||||
|
start_index + ((i * 2 / max_per_block) * max_per_block) as u32;
|
||||||
|
assert_eq!(s.common_header.fec_set_index, expected_fec_set_index);
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -456,8 +456,17 @@ pub mod tests {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let mut packet = Packet::default();
|
let mut packet = Packet::default();
|
||||||
let slot = 0xdeadc0de;
|
let slot = 0xdeadc0de;
|
||||||
let mut shred =
|
let mut shred = Shred::new_from_data(
|
||||||
Shred::new_from_data(slot, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true, 0, 0);
|
slot,
|
||||||
|
0xc0de,
|
||||||
|
0xdead,
|
||||||
|
Some(&[1, 2, 3, 4]),
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0xc0de,
|
||||||
|
);
|
||||||
assert_eq!(shred.slot(), slot);
|
assert_eq!(shred.slot(), slot);
|
||||||
let keypair = Keypair::new();
|
let keypair = Keypair::new();
|
||||||
Shredder::sign_shred(&keypair, &mut shred);
|
Shredder::sign_shred(&keypair, &mut shred);
|
||||||
|
@ -490,8 +499,17 @@ pub mod tests {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let mut batch = [Packets::default()];
|
let mut batch = [Packets::default()];
|
||||||
let slot = 0xdeadc0de;
|
let slot = 0xdeadc0de;
|
||||||
let mut shred =
|
let mut shred = Shred::new_from_data(
|
||||||
Shred::new_from_data(slot, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true, 0, 0);
|
slot,
|
||||||
|
0xc0de,
|
||||||
|
0xdead,
|
||||||
|
Some(&[1, 2, 3, 4]),
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0xc0de,
|
||||||
|
);
|
||||||
let keypair = Keypair::new();
|
let keypair = Keypair::new();
|
||||||
Shredder::sign_shred(&keypair, &mut shred);
|
Shredder::sign_shred(&keypair, &mut shred);
|
||||||
batch[0].packets.resize(1, Packet::default());
|
batch[0].packets.resize(1, Packet::default());
|
||||||
|
@ -533,8 +551,17 @@ pub mod tests {
|
||||||
|
|
||||||
let mut batch = [Packets::default()];
|
let mut batch = [Packets::default()];
|
||||||
let slot = 0xdeadc0de;
|
let slot = 0xdeadc0de;
|
||||||
let mut shred =
|
let mut shred = Shred::new_from_data(
|
||||||
Shred::new_from_data(slot, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true, 0, 0);
|
slot,
|
||||||
|
0xc0de,
|
||||||
|
0xdead,
|
||||||
|
Some(&[1, 2, 3, 4]),
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0xc0de,
|
||||||
|
);
|
||||||
let keypair = Keypair::new();
|
let keypair = Keypair::new();
|
||||||
Shredder::sign_shred(&keypair, &mut shred);
|
Shredder::sign_shred(&keypair, &mut shred);
|
||||||
batch[0].packets.resize(1, Packet::default());
|
batch[0].packets.resize(1, Packet::default());
|
||||||
|
@ -598,6 +625,7 @@ pub mod tests {
|
||||||
true,
|
true,
|
||||||
1,
|
1,
|
||||||
2,
|
2,
|
||||||
|
0xc0de,
|
||||||
);
|
);
|
||||||
shred.copy_to_packet(p);
|
shred.copy_to_packet(p);
|
||||||
}
|
}
|
||||||
|
@ -631,8 +659,17 @@ pub mod tests {
|
||||||
let mut batch = [Packets::default()];
|
let mut batch = [Packets::default()];
|
||||||
let slot = 0xdeadc0de;
|
let slot = 0xdeadc0de;
|
||||||
let keypair = Keypair::new();
|
let keypair = Keypair::new();
|
||||||
let shred =
|
let shred = Shred::new_from_data(
|
||||||
Shred::new_from_data(slot, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true, 0, 0);
|
slot,
|
||||||
|
0xc0de,
|
||||||
|
0xdead,
|
||||||
|
Some(&[1, 2, 3, 4]),
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0xc0de,
|
||||||
|
);
|
||||||
batch[0].packets.resize(1, Packet::default());
|
batch[0].packets.resize(1, Packet::default());
|
||||||
batch[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload);
|
batch[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload);
|
||||||
batch[0].packets[0].meta.size = shred.payload.len();
|
batch[0].packets[0].meta.size = shred.payload.len();
|
||||||
|
|
|
@ -63,6 +63,7 @@ fn test_multi_fec_block_coding() {
|
||||||
MAX_DATA_SHREDS_PER_FEC_BLOCK as usize,
|
MAX_DATA_SHREDS_PER_FEC_BLOCK as usize,
|
||||||
MAX_DATA_SHREDS_PER_FEC_BLOCK as usize,
|
MAX_DATA_SHREDS_PER_FEC_BLOCK as usize,
|
||||||
shred_start_index,
|
shred_start_index,
|
||||||
|
shred_start_index,
|
||||||
slot,
|
slot,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
Loading…
Reference in New Issue