diff --git a/core/benches/cluster_info.rs b/core/benches/cluster_info.rs index f7660b3a0..644e86bf6 100644 --- a/core/benches/cluster_info.rs +++ b/core/benches/cluster_info.rs @@ -51,7 +51,7 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) { let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); const NUM_SHREDS: usize = 32; - let shred = Shred::new_from_data(0, 0, 0, None, false, false, 0, 0, 0); + let shred = Shred::new_from_data(0, 0, 0, &[], false, false, 0, 0, 0); let shreds = vec![shred; NUM_SHREDS]; let mut stakes = HashMap::new(); const NUM_PEERS: usize = 200; diff --git a/core/benches/cluster_nodes.rs b/core/benches/cluster_nodes.rs index f51303112..d566703d7 100644 --- a/core/benches/cluster_nodes.rs +++ b/core/benches/cluster_nodes.rs @@ -39,7 +39,7 @@ fn get_retransmit_peers_deterministic( let parent_offset = if slot == 0 { 0 } else { 1 }; for i in 0..num_simulated_shreds { let index = i as u32; - let shred = Shred::new_from_data(slot, index, parent_offset, None, false, false, 0, 0, 0); + let shred = Shred::new_from_data(slot, index, parent_offset, &[], false, false, 0, 0, 0); let (_neighbors, _children) = cluster_nodes.get_retransmit_peers( *slot_leader, &shred, diff --git a/core/benches/shredder.rs b/core/benches/shredder.rs index 89571b9c9..04b891b8c 100644 --- a/core/benches/shredder.rs +++ b/core/benches/shredder.rs @@ -123,7 +123,7 @@ fn bench_deshredder(bencher: &mut Bencher) { fn bench_deserialize_hdr(bencher: &mut Bencher) { let data = vec![0; SIZE_OF_DATA_SHRED_PAYLOAD]; - let shred = Shred::new_from_data(2, 1, 1, Some(&data), true, true, 0, 0, 1); + let shred = Shred::new_from_data(2, 1, 1, &data, true, true, 0, 0, 1); bencher.iter(|| { let payload = shred.payload().clone(); diff --git a/core/src/broadcast_stage/standard_broadcast_run.rs b/core/src/broadcast_stage/standard_broadcast_run.rs index efae4d320..4136174a8 100644 --- a/core/src/broadcast_stage/standard_broadcast_run.rs +++ b/core/src/broadcast_stage/standard_broadcast_run.rs @@ -76,7 +76,7 @@ impl StandardBroadcastRun { state.slot, state.next_shred_index, parent_offset as u16, - None, // data + &[], // data true, // is_last_in_fec_set true, // is_last_in_slot reference_tick, diff --git a/core/src/outstanding_requests.rs b/core/src/outstanding_requests.rs index d3ef1d5d0..d521a242a 100644 --- a/core/src/outstanding_requests.rs +++ b/core/src/outstanding_requests.rs @@ -107,7 +107,7 @@ pub(crate) mod tests { let repair_type = ShredRepairType::Orphan(9); let mut outstanding_requests = OutstandingRequests::default(); let nonce = outstanding_requests.add_request(repair_type, timestamp()); - let shred = Shred::new_from_data(0, 0, 0, None, false, false, 0, 0, 0); + let shred = Shred::new_from_data(0, 0, 0, &[], false, false, 0, 0, 0); let expire_timestamp = outstanding_requests .requests @@ -127,7 +127,7 @@ pub(crate) mod tests { let mut outstanding_requests = OutstandingRequests::default(); let nonce = outstanding_requests.add_request(repair_type, timestamp()); - let shred = Shred::new_from_data(0, 0, 0, None, false, false, 0, 0, 0); + let shred = Shred::new_from_data(0, 0, 0, &[], false, false, 0, 0, 0); let mut expire_timestamp = outstanding_requests .requests .get(&nonce) diff --git a/core/src/repair_response.rs b/core/src/repair_response.rs index b2235c087..262dc4d03 100644 --- a/core/src/repair_response.rs +++ b/core/src/repair_response.rs @@ -69,7 +69,7 @@ mod test { slot, 0xc0de, 0xdead, - Some(&[1, 2, 3, 4]), + &[1, 2, 3, 4], true, true, 0, diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs index 1a99a1e52..ac3091060 100644 --- a/core/src/replay_stage.rs +++ b/core/src/replay_stage.rs @@ -3749,7 +3749,7 @@ pub mod tests { bank.slot(), 0, // index, parent_offset as u16, - Some(&gibberish), + &gibberish, true, // is_last_data false, // is_last_in_slot 0, // reference_tick diff --git a/core/src/retransmit_stage.rs b/core/src/retransmit_stage.rs index b3b51df34..4e8524480 100644 --- a/core/src/retransmit_stage.rs +++ b/core/src/retransmit_stage.rs @@ -535,37 +535,37 @@ mod tests { let slot = 1; let index = 5; let version = 0x40; - let shred = Shred::new_from_data(slot, index, 0, None, true, true, 0, version, 0); + let shred = Shred::new_from_data(slot, index, 0, &[], true, true, 0, version, 0); let shreds_received = Arc::new(Mutex::new((LruCache::new(100), PacketHasher::default()))); // unique shred for (1, 5) should pass assert!(!should_skip_retransmit(&shred, &shreds_received)); // duplicate shred for (1, 5) blocked assert!(should_skip_retransmit(&shred, &shreds_received)); - let shred = Shred::new_from_data(slot, index, 2, None, true, true, 0, version, 0); + let shred = Shred::new_from_data(slot, index, 2, &[], true, true, 0, version, 0); // first duplicate shred for (1, 5) passed assert!(!should_skip_retransmit(&shred, &shreds_received)); // then blocked assert!(should_skip_retransmit(&shred, &shreds_received)); - let shred = Shred::new_from_data(slot, index, 8, None, true, true, 0, version, 0); + let shred = Shred::new_from_data(slot, index, 8, &[], true, true, 0, version, 0); // 2nd duplicate shred for (1, 5) blocked assert!(should_skip_retransmit(&shred, &shreds_received)); assert!(should_skip_retransmit(&shred, &shreds_received)); - let shred = Shred::new_empty_coding(slot, index, 0, 1, 1, 0, version); + let shred = Shred::new_from_parity_shard(slot, index, &[], 0, 1, 1, 0, version); // Coding at (1, 5) passes assert!(!should_skip_retransmit(&shred, &shreds_received)); // then blocked assert!(should_skip_retransmit(&shred, &shreds_received)); - let shred = Shred::new_empty_coding(slot, index, 2, 1, 1, 0, version); + let shred = Shred::new_from_parity_shard(slot, index, &[], 2, 1, 1, 0, version); // 2nd unique coding at (1, 5) passes assert!(!should_skip_retransmit(&shred, &shreds_received)); // same again is blocked assert!(should_skip_retransmit(&shred, &shreds_received)); - let shred = Shred::new_empty_coding(slot, index, 3, 1, 1, 0, version); + let shred = Shred::new_from_parity_shard(slot, index, &[], 3, 1, 1, 0, version); // Another unique coding at (1, 5) always blocked assert!(should_skip_retransmit(&shred, &shreds_received)); assert!(should_skip_retransmit(&shred, &shreds_received)); diff --git a/core/src/serve_repair.rs b/core/src/serve_repair.rs index 11f12c271..99b317875 100644 --- a/core/src/serve_repair.rs +++ b/core/src/serve_repair.rs @@ -876,7 +876,7 @@ mod tests { nonce, ); assert!(rv.is_none()); - let shred = Shred::new_from_data(slot, 1, 1, None, false, false, 0, 2, 0); + let shred = Shred::new_from_data(slot, 1, 1, &[], false, false, 0, 2, 0); blockstore .insert_shreds(vec![shred], None, false) @@ -1306,7 +1306,7 @@ mod tests { #[test] fn test_verify_shred_response() { fn new_test_data_shred(slot: Slot, index: u32) -> Shred { - Shred::new_from_data(slot, index, 1, None, false, false, 0, 0, 0) + Shred::new_from_data(slot, index, 1, &[], false, false, 0, 0, 0) } let repair = ShredRepairType::Orphan(9); // Ensure new options are addded to this test diff --git a/core/src/shred_fetch_stage.rs b/core/src/shred_fetch_stage.rs index 3ec3a3d77..30c35c725 100644 --- a/core/src/shred_fetch_stage.rs +++ b/core/src/shred_fetch_stage.rs @@ -228,9 +228,10 @@ mod tests { let slot = 1; let shred = Shred::new_from_data( - slot, 3, // shred index + slot, + 3, // shred index 0, // parent offset - None, // data + &[], // data true, // is_last_in_fec_set true, // is_last_in_slot 0, // reference_tick @@ -299,7 +300,7 @@ mod tests { ); assert_eq!(stats.index_overrun, 1); assert!(packet.meta.discard()); - let shred = Shred::new_from_data(1, 3, 0, None, true, true, 0, 0, 0); + let shred = Shred::new_from_data(1, 3, 0, &[], true, true, 0, 0, 0); shred.copy_to_packet(&mut packet); // rejected slot is 1, root is 3 @@ -341,7 +342,7 @@ mod tests { ); assert!(packet.meta.discard()); - let shred = Shred::new_from_data(1_000_000, 3, 0, None, true, true, 0, 0, 0); + let shred = Shred::new_from_data(1_000_000, 3, 0, &[], true, true, 0, 0, 0); shred.copy_to_packet(&mut packet); // Slot 1 million is too high @@ -358,7 +359,7 @@ mod tests { assert!(packet.meta.discard()); let index = MAX_DATA_SHREDS_PER_SLOT as u32; - let shred = Shred::new_from_data(5, index, 0, None, true, true, 0, 0, 0); + let shred = Shred::new_from_data(5, index, 0, &[], true, true, 0, 0, 0); shred.copy_to_packet(&mut packet); ShredFetchStage::process_packet( &mut packet, diff --git a/core/src/sigverify_shreds.rs b/core/src/sigverify_shreds.rs index 2175fe511..006fcb960 100644 --- a/core/src/sigverify_shreds.rs +++ b/core/src/sigverify_shreds.rs @@ -82,7 +82,7 @@ pub mod tests { 0xdead_c0de, 0xc0de, 0xdead, - Some(&[1, 2, 3, 4]), + &[1, 2, 3, 4], true, true, 0, @@ -101,7 +101,7 @@ pub mod tests { 0xc0de_dead, 0xc0de, 0xdead, - Some(&[1, 2, 3, 4]), + &[1, 2, 3, 4], true, true, 0, @@ -131,32 +131,14 @@ pub mod tests { let mut batches = vec![PacketBatch::default()]; batches[0].packets.resize(2, Packet::default()); - let mut shred = Shred::new_from_data( - 0, - 0xc0de, - 0xdead, - Some(&[1, 2, 3, 4]), - true, - true, - 0, - 0, - 0xc0de, - ); + let mut shred = + Shred::new_from_data(0, 0xc0de, 0xdead, &[1, 2, 3, 4], true, true, 0, 0, 0xc0de); shred.sign(&leader_keypair); batches[0].packets[0].data[0..shred.payload().len()].copy_from_slice(shred.payload()); batches[0].packets[0].meta.size = shred.payload().len(); - let mut shred = Shred::new_from_data( - 0, - 0xbeef, - 0xc0de, - Some(&[1, 2, 3, 4]), - true, - true, - 0, - 0, - 0xc0de, - ); + let mut shred = + Shred::new_from_data(0, 0xbeef, 0xc0de, &[1, 2, 3, 4], true, true, 0, 0, 0xc0de); let wrong_keypair = Keypair::new(); shred.sign(&wrong_keypair); batches[0].packets[1].data[0..shred.payload().len()].copy_from_slice(shred.payload()); diff --git a/core/src/window_service.rs b/core/src/window_service.rs index 45ee1a913..8791d93e0 100644 --- a/core/src/window_service.rs +++ b/core/src/window_service.rs @@ -863,14 +863,15 @@ mod test { )); // coding shreds don't contain parent slot information, test that slot >= root - let mut coding_shred = Shred::new_empty_coding( - 5, // slot - 5, // index - 5, // fec_set_index - 6, // num_data_shreds - 6, // num_coding_shreds - 3, // position - 0, // version + let mut coding_shred = Shred::new_from_parity_shard( + 5, // slot + 5, // index + &[], // parity_shard + 5, // fec_set_index + 6, // num_data_shreds + 6, // num_coding_shreds + 3, // position + 0, // version ); coding_shred.sign(&leader_keypair); // shred.slot() > root, shred continues @@ -945,14 +946,15 @@ mod test { std::net::{IpAddr, Ipv4Addr}, }; solana_logger::setup(); - let shred = Shred::new_empty_coding( - 5, // slot - 5, // index - 5, // fec_set_index - 6, // num_data_shreds - 6, // num_coding_shreds - 4, // position - 0, // version + let shred = Shred::new_from_parity_shard( + 5, // slot + 5, // index + &[], // parity_shard + 5, // fec_set_index + 6, // num_data_shreds + 6, // num_coding_shreds + 4, // position + 0, // version ); let mut shreds = vec![shred.clone(), shred.clone(), shred]; let _from_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); diff --git a/ledger/benches/sigverify_shreds.rs b/ledger/benches/sigverify_shreds.rs index 16bfd7200..cda35c4d9 100644 --- a/ledger/benches/sigverify_shreds.rs +++ b/ledger/benches/sigverify_shreds.rs @@ -32,7 +32,7 @@ fn bench_sigverify_shreds_sign_gpu(bencher: &mut Bencher) { slot, 0xc0de, 0xdead, - Some(&[5; SIZE_OF_DATA_SHRED_PAYLOAD]), + &[5; SIZE_OF_DATA_SHRED_PAYLOAD], true, true, 1, @@ -64,7 +64,7 @@ fn bench_sigverify_shreds_sign_cpu(bencher: &mut Bencher) { slot, 0xc0de, 0xdead, - Some(&[5; SIZE_OF_DATA_SHRED_PAYLOAD]), + &[5; SIZE_OF_DATA_SHRED_PAYLOAD], true, true, 1, diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 15fcfd5ea..2bda1480d 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -5705,7 +5705,7 @@ pub mod tests { slot, (i * gap) as u32, 0, - None, + &[], false, false, i as u8, @@ -5837,7 +5837,7 @@ pub mod tests { let parent_offset = shred5.slot() - shred5.parent().unwrap(); parent_offset as u16 }, - None, // data + &[], // data true, // is_last_data true, // is_last_in_slot 0, // reference_tick @@ -5938,13 +5938,15 @@ pub mod tests { let blockstore = Blockstore::open(ledger_path.path()).unwrap(); let slot = 1; - let coding_shred = Shred::new_empty_coding( - slot, 11, // index - 11, // fec_set_index - 11, // num_data_shreds - 11, // num_coding_shreds - 8, // position - 0, // version + let coding_shred = Shred::new_from_parity_shard( + slot, + 11, // index + &[], // parity_shard + 11, // fec_set_index + 11, // num_data_shreds + 11, // num_coding_shreds + 8, // position + 0, // version ); let mut erasure_metas = HashMap::new(); @@ -5994,13 +5996,15 @@ pub mod tests { let last_root = RwLock::new(0); let slot = 1; - let mut coding_shred = Shred::new_empty_coding( - slot, 11, // index - 11, // fec_set_index - 11, // num_data_shreds - 11, // num_coding_shreds - 8, // position - 0, // version + let mut coding_shred = Shred::new_from_parity_shard( + slot, + 11, // index + &[], // parity_shard + 11, // fec_set_index + 11, // num_data_shreds + 11, // num_coding_shreds + 8, // position + 0, // version ); // Insert a good coding shred @@ -6237,7 +6241,7 @@ pub mod tests { slot, next_shred_index as u32, 1, - Some(&[1, 1, 1]), + &[1, 1, 1], true, true, 0, diff --git a/ledger/src/shred.rs b/ledger/src/shred.rs index bb3ffe8d9..d0834260a 100644 --- a/ledger/src/shred.rs +++ b/ledger/src/shred.rs @@ -269,15 +269,14 @@ impl Shred { slot: Slot, index: u32, parent_offset: u16, - data: Option<&[u8]>, + data: &[u8], is_last_data: bool, is_last_in_slot: bool, reference_tick: u8, version: u16, fec_set_index: u32, ) -> Self { - let payload_size = SHRED_PAYLOAD_SIZE; - let mut payload = vec![0; payload_size]; + let mut payload = vec![0; SHRED_PAYLOAD_SIZE]; let common_header = ShredCommonHeader { slot, index, @@ -286,9 +285,7 @@ impl Shred { ..ShredCommonHeader::default() }; - let size = (data.map(|d| d.len()).unwrap_or(0) - + SIZE_OF_DATA_SHRED_HEADER - + SIZE_OF_COMMON_SHRED_HEADER) as u16; + let size = (data.len() + SIZE_OF_DATA_SHRED_HEADER + SIZE_OF_COMMON_SHRED_HEADER) as u16; let mut data_header = DataShredHeader { parent_offset, flags: reference_tick.min(SHRED_TICK_REFERENCE_MASK), @@ -320,10 +317,7 @@ impl Shred { ) .expect("Failed to write data header into shred buffer"); // TODO: Need to check if data is too large! - if let Some(data) = data { - payload[start..start + data.len()].clone_from_slice(data); - } - + payload[start..start + data.len()].copy_from_slice(data); Self { common_header, data_header, @@ -362,9 +356,10 @@ impl Shred { shred.sanitize().map(|_| shred) } - pub fn new_empty_coding( + pub fn new_from_parity_shard( slot: Slot, index: u32, + parity_shard: &[u8], fec_set_index: u32, num_data_shreds: u16, num_coding_shreds: u16, @@ -400,6 +395,10 @@ impl Shred { &coding_header, ) .expect("Failed to write coding header into shred buffer"); + // Tests may have an empty parity_shard. + if !parity_shard.is_empty() { + payload[SIZE_OF_CODING_SHRED_HEADERS..].copy_from_slice(parity_shard); + } Shred { common_header, data_header: DataShredHeader::default(), @@ -835,7 +834,7 @@ impl Shredder { self.slot, shred_index, parent_offset as u16, - Some(data), + data, is_last_data, is_last_in_slot, self.reference_tick, @@ -965,17 +964,16 @@ impl Shredder { .enumerate() .map(|(i, parity)| { let index = next_code_index + u32::try_from(i).unwrap(); - let mut shred = Shred::new_empty_coding( + Shred::new_from_parity_shard( slot, index, + parity, fec_set_index, num_data, num_coding, u16::try_from(i).unwrap(), // position version, - ); - shred.payload[SIZE_OF_CODING_SHRED_HEADERS..].copy_from_slice(parity); - shred + ) }) .collect() } @@ -1947,7 +1945,7 @@ mod tests { #[test] fn test_invalid_parent_offset() { - let shred = Shred::new_from_data(10, 0, 1000, Some(&[1, 2, 3]), false, false, 0, 1, 0); + let shred = Shred::new_from_data(10, 0, 1000, &[1, 2, 3], false, false, 0, 1, 0); let mut packet = Packet::default(); shred.copy_to_packet(&mut packet); let shred_res = Shred::new_from_serialized_shred(packet.data.to_vec()); @@ -1971,7 +1969,7 @@ mod tests { fn test_shred_offsets() { solana_logger::setup(); let mut packet = Packet::default(); - let shred = Shred::new_from_data(1, 3, 0, None, true, true, 0, 0, 0); + let shred = Shred::new_from_data(1, 3, 0, &[], true, true, 0, 0, 0); shred.copy_to_packet(&mut packet); let mut stats = ShredFetchStats::default(); let ret = get_shred_slot_index_type(&packet, &mut stats); @@ -2001,9 +1999,10 @@ mod tests { ); assert_eq!(stats.index_overrun, 4); - let shred = Shred::new_empty_coding( + let shred = Shred::new_from_parity_shard( 8, // slot 2, // index + &[], // parity_shard 10, // fec_set_index 30, // num_data 4, // num_code @@ -2016,14 +2015,15 @@ mod tests { get_shred_slot_index_type(&packet, &mut stats) ); - let shred = Shred::new_from_data(1, std::u32::MAX - 10, 0, None, true, true, 0, 0, 0); + let shred = Shred::new_from_data(1, std::u32::MAX - 10, 0, &[], true, true, 0, 0, 0); shred.copy_to_packet(&mut packet); assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats)); assert_eq!(1, stats.index_out_of_bounds); - let shred = Shred::new_empty_coding( + let shred = Shred::new_from_parity_shard( 8, // slot 2, // index + &[], // parity_shard 10, // fec_set_index 30, // num_data_shreds 4, // num_coding_shreds @@ -2072,8 +2072,7 @@ mod tests { 420, // slot 19, // index 5, // parent_offset - Some(&data), - true, // is_last_data + &data, true, // is_last_data false, // is_last_in_slot 3, // reference_tick 1, // version @@ -2133,14 +2132,15 @@ mod tests { #[test] fn test_sanitize_coding_shred() { - let mut shred = Shred::new_empty_coding( - 1, // slot - 12, // index - 11, // fec_set_index - 11, // num_data_shreds - 11, // num_coding_shreds - 8, // position - 0, // version + let mut shred = Shred::new_from_parity_shard( + 1, // slot + 12, // index + &[], // parity_shard + 11, // fec_set_index + 11, // num_data_shreds + 11, // num_coding_shreds + 8, // position + 0, // version ); assert_matches!(shred.sanitize(), Ok(())); // index < position is invalid. diff --git a/ledger/src/sigverify_shreds.rs b/ledger/src/sigverify_shreds.rs index e5c9dabbe..7f7bc6c44 100644 --- a/ledger/src/sigverify_shreds.rs +++ b/ledger/src/sigverify_shreds.rs @@ -470,7 +470,7 @@ pub mod tests { slot, 0xc0de, 0xdead, - Some(&[1, 2, 3, 4]), + &[1, 2, 3, 4], true, true, 0, @@ -516,7 +516,7 @@ pub mod tests { slot, 0xc0de, 0xdead, - Some(&[1, 2, 3, 4]), + &[1, 2, 3, 4], true, true, 0, @@ -571,7 +571,7 @@ pub mod tests { slot, 0xc0de, 0xdead, - Some(&[1, 2, 3, 4]), + &[1, 2, 3, 4], true, true, 0, @@ -639,7 +639,7 @@ pub mod tests { slot, 0xc0de, i as u16, - Some(&[5; SIZE_OF_DATA_SHRED_PAYLOAD]), + &[5; SIZE_OF_DATA_SHRED_PAYLOAD], true, true, 1, @@ -685,7 +685,7 @@ pub mod tests { slot, 0xc0de, 0xdead, - Some(&[1, 2, 3, 4]), + &[1, 2, 3, 4], true, true, 0,