diff --git a/core/benches/cluster_info.rs b/core/benches/cluster_info.rs index 644e86bf6..ea12f7d08 100644 --- a/core/benches/cluster_info.rs +++ b/core/benches/cluster_info.rs @@ -16,7 +16,7 @@ use { }, solana_ledger::{ genesis_utils::{create_genesis_config, GenesisConfigInfo}, - shred::Shred, + shred::{Shred, ShredFlags}, }, solana_runtime::{bank::Bank, bank_forks::BankForks}, solana_sdk::{ @@ -51,7 +51,7 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) { let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); const NUM_SHREDS: usize = 32; - let shred = Shred::new_from_data(0, 0, 0, &[], false, false, 0, 0, 0); + let shred = Shred::new_from_data(0, 0, 0, &[], ShredFlags::empty(), 0, 0, 0); let shreds = vec![shred; NUM_SHREDS]; let mut stakes = HashMap::new(); const NUM_PEERS: usize = 200; diff --git a/core/benches/cluster_nodes.rs b/core/benches/cluster_nodes.rs index d566703d7..7f69930d0 100644 --- a/core/benches/cluster_nodes.rs +++ b/core/benches/cluster_nodes.rs @@ -11,7 +11,7 @@ use { solana_gossip::contact_info::ContactInfo, solana_ledger::{ genesis_utils::{create_genesis_config, GenesisConfigInfo}, - shred::Shred, + shred::{Shred, ShredFlags}, }, solana_runtime::bank::Bank, solana_sdk::{clock::Slot, pubkey::Pubkey}, @@ -39,7 +39,16 @@ fn get_retransmit_peers_deterministic( let parent_offset = if slot == 0 { 0 } else { 1 }; for i in 0..num_simulated_shreds { let index = i as u32; - let shred = Shred::new_from_data(slot, index, parent_offset, &[], false, false, 0, 0, 0); + let shred = Shred::new_from_data( + slot, + index, + parent_offset, + &[], + ShredFlags::empty(), + 0, + 0, + 0, + ); let (_neighbors, _children) = cluster_nodes.get_retransmit_peers( *slot_leader, &shred, diff --git a/core/benches/shredder.rs b/core/benches/shredder.rs index 04b891b8c..e18b67860 100644 --- a/core/benches/shredder.rs +++ b/core/benches/shredder.rs @@ -8,8 +8,8 @@ use { raptorq::{Decoder, Encoder}, solana_entry::entry::{create_ticks, Entry}, solana_ledger::shred::{ - max_entries_per_n_shred, max_ticks_per_n_shreds, ProcessShredsStats, Shred, Shredder, - MAX_DATA_SHREDS_PER_FEC_BLOCK, SIZE_OF_DATA_SHRED_PAYLOAD, + max_entries_per_n_shred, max_ticks_per_n_shreds, ProcessShredsStats, Shred, ShredFlags, + Shredder, MAX_DATA_SHREDS_PER_FEC_BLOCK, SIZE_OF_DATA_SHRED_PAYLOAD, }, solana_perf::test_tx, solana_sdk::{hash::Hash, packet::PACKET_DATA_SIZE, signature::Keypair}, @@ -123,7 +123,7 @@ fn bench_deshredder(bencher: &mut Bencher) { fn bench_deserialize_hdr(bencher: &mut Bencher) { let data = vec![0; SIZE_OF_DATA_SHRED_PAYLOAD]; - let shred = Shred::new_from_data(2, 1, 1, &data, true, true, 0, 0, 1); + let shred = Shred::new_from_data(2, 1, 1, &data, ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 1); bencher.iter(|| { let payload = shred.payload().clone(); diff --git a/core/src/broadcast_stage/standard_broadcast_run.rs b/core/src/broadcast_stage/standard_broadcast_run.rs index ee57f0577..993500dfa 100644 --- a/core/src/broadcast_stage/standard_broadcast_run.rs +++ b/core/src/broadcast_stage/standard_broadcast_run.rs @@ -76,9 +76,8 @@ impl StandardBroadcastRun { state.slot, state.next_shred_index, parent_offset as u16, - &[], // data - true, // is_last_in_fec_set - true, // is_last_in_slot + &[], // data + ShredFlags::LAST_SHRED_IN_SLOT, reference_tick, self.shred_version, fec_set_index.unwrap(), diff --git a/core/src/outstanding_requests.rs b/core/src/outstanding_requests.rs index d521a242a..73ccfd439 100644 --- a/core/src/outstanding_requests.rs +++ b/core/src/outstanding_requests.rs @@ -85,7 +85,9 @@ pub struct RequestStatus { #[cfg(test)] pub(crate) mod tests { use { - super::*, crate::serve_repair::ShredRepairType, solana_ledger::shred::Shred, + super::*, + crate::serve_repair::ShredRepairType, + solana_ledger::shred::{Shred, ShredFlags}, solana_sdk::timing::timestamp, }; @@ -107,7 +109,7 @@ pub(crate) mod tests { let repair_type = ShredRepairType::Orphan(9); let mut outstanding_requests = OutstandingRequests::default(); let nonce = outstanding_requests.add_request(repair_type, timestamp()); - let shred = Shred::new_from_data(0, 0, 0, &[], false, false, 0, 0, 0); + let shred = Shred::new_from_data(0, 0, 0, &[], ShredFlags::empty(), 0, 0, 0); let expire_timestamp = outstanding_requests .requests @@ -127,7 +129,7 @@ pub(crate) mod tests { let mut outstanding_requests = OutstandingRequests::default(); let nonce = outstanding_requests.add_request(repair_type, timestamp()); - let shred = Shred::new_from_data(0, 0, 0, &[], false, false, 0, 0, 0); + let shred = Shred::new_from_data(0, 0, 0, &[], ShredFlags::empty(), 0, 0, 0); let mut expire_timestamp = outstanding_requests .requests .get(&nonce) diff --git a/core/src/repair_response.rs b/core/src/repair_response.rs index 262dc4d03..71d0be27b 100644 --- a/core/src/repair_response.rs +++ b/core/src/repair_response.rs @@ -52,7 +52,10 @@ pub fn nonce(buf: &[u8]) -> Option { mod test { use { super::*, - solana_ledger::{shred::Shred, sigverify_shreds::verify_shred_cpu}, + solana_ledger::{ + shred::{Shred, ShredFlags}, + sigverify_shreds::verify_shred_cpu, + }, solana_sdk::{ packet::PacketFlags, signature::{Keypair, Signer}, @@ -70,8 +73,7 @@ mod test { 0xc0de, 0xdead, &[1, 2, 3, 4], - true, - true, + ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 0xc0de, diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs index 8529e00bf..024b75b77 100644 --- a/core/src/replay_stage.rs +++ b/core/src/replay_stage.rs @@ -3123,7 +3123,7 @@ impl ReplayStage { } #[cfg(test)] -pub mod tests { +pub(crate) mod tests { use { super::*, crate::{ @@ -3142,7 +3142,7 @@ pub mod tests { create_new_tmp_ledger, genesis_utils::{create_genesis_config, create_genesis_config_with_leader}, get_tmp_ledger_path, - shred::{Shred, SIZE_OF_DATA_SHRED_PAYLOAD}, + shred::{Shred, ShredFlags, SIZE_OF_DATA_SHRED_PAYLOAD}, }, solana_rpc::{ optimistically_confirmed_bank_tracker::OptimisticallyConfirmedBank, @@ -3748,11 +3748,10 @@ pub mod tests { 0, // index, parent_offset as u16, &gibberish, - true, // is_last_data - false, // is_last_in_slot - 0, // reference_tick - 0, // version - 0, // fec_set_index + ShredFlags::DATA_COMPLETE_SHRED, + 0, // reference_tick + 0, // version + 0, // fec_set_index ); vec![shred] }); diff --git a/core/src/retransmit_stage.rs b/core/src/retransmit_stage.rs index 4e8524480..c05a434d1 100644 --- a/core/src/retransmit_stage.rs +++ b/core/src/retransmit_stage.rs @@ -528,27 +528,54 @@ impl RetransmitStage { #[cfg(test)] mod tests { - use super::*; + use {super::*, solana_ledger::shred::ShredFlags}; #[test] fn test_already_received() { let slot = 1; let index = 5; let version = 0x40; - let shred = Shred::new_from_data(slot, index, 0, &[], true, true, 0, version, 0); + let shred = Shred::new_from_data( + slot, + index, + 0, + &[], + ShredFlags::LAST_SHRED_IN_SLOT, + 0, + version, + 0, + ); let shreds_received = Arc::new(Mutex::new((LruCache::new(100), PacketHasher::default()))); // unique shred for (1, 5) should pass assert!(!should_skip_retransmit(&shred, &shreds_received)); // duplicate shred for (1, 5) blocked assert!(should_skip_retransmit(&shred, &shreds_received)); - let shred = Shred::new_from_data(slot, index, 2, &[], true, true, 0, version, 0); + let shred = Shred::new_from_data( + slot, + index, + 2, + &[], + ShredFlags::LAST_SHRED_IN_SLOT, + 0, + version, + 0, + ); // first duplicate shred for (1, 5) passed assert!(!should_skip_retransmit(&shred, &shreds_received)); // then blocked assert!(should_skip_retransmit(&shred, &shreds_received)); - let shred = Shred::new_from_data(slot, index, 8, &[], true, true, 0, version, 0); + let shred = Shred::new_from_data( + slot, + index, + 8, + &[], + ShredFlags::LAST_SHRED_IN_SLOT, + 0, + version, + 0, + ); // 2nd duplicate shred for (1, 5) blocked assert!(should_skip_retransmit(&shred, &shreds_received)); assert!(should_skip_retransmit(&shred, &shreds_received)); diff --git a/core/src/serve_repair.rs b/core/src/serve_repair.rs index 99b317875..a05996cd6 100644 --- a/core/src/serve_repair.rs +++ b/core/src/serve_repair.rs @@ -761,7 +761,7 @@ mod tests { blockstore::make_many_slot_entries, blockstore_processor::fill_blockstore_slot_with_ticks, get_tmp_ledger_path, - shred::{max_ticks_per_n_shreds, Shred}, + shred::{max_ticks_per_n_shreds, Shred, ShredFlags}, }, solana_perf::packet::Packet, solana_sdk::{hash::Hash, pubkey::Pubkey, signature::Keypair, timing::timestamp}, @@ -876,7 +876,7 @@ mod tests { nonce, ); assert!(rv.is_none()); - let shred = Shred::new_from_data(slot, 1, 1, &[], false, false, 0, 2, 0); + let shred = Shred::new_from_data(slot, 1, 1, &[], ShredFlags::empty(), 0, 2, 0); blockstore .insert_shreds(vec![shred], None, false) @@ -1306,7 +1306,7 @@ mod tests { #[test] fn test_verify_shred_response() { fn new_test_data_shred(slot: Slot, index: u32) -> Shred { - Shred::new_from_data(slot, index, 1, &[], false, false, 0, 0, 0) + Shred::new_from_data(slot, index, 1, &[], ShredFlags::empty(), 0, 0, 0) } let repair = ShredRepairType::Orphan(9); // Ensure new options are addded to this test diff --git a/core/src/shred_fetch_stage.rs b/core/src/shred_fetch_stage.rs index 30c35c725..287723995 100644 --- a/core/src/shred_fetch_stage.rs +++ b/core/src/shred_fetch_stage.rs @@ -216,7 +216,10 @@ impl ShredFetchStage { mod tests { use { super::*, - solana_ledger::{blockstore::MAX_DATA_SHREDS_PER_SLOT, shred::Shred}, + solana_ledger::{ + blockstore::MAX_DATA_SHREDS_PER_SLOT, + shred::{Shred, ShredFlags}, + }, }; #[test] @@ -229,14 +232,13 @@ mod tests { let slot = 1; let shred = Shred::new_from_data( slot, - 3, // shred index - 0, // parent offset - &[], // data - true, // is_last_in_fec_set - true, // is_last_in_slot - 0, // reference_tick - 0, // version - 3, // fec_set_index + 3, // shred index + 0, // parent offset + &[], // data + ShredFlags::LAST_SHRED_IN_SLOT, + 0, // reference_tick + 0, // version + 3, // fec_set_index ); shred.copy_to_packet(&mut packet); @@ -300,7 +302,7 @@ mod tests { ); assert_eq!(stats.index_overrun, 1); assert!(packet.meta.discard()); - let shred = Shred::new_from_data(1, 3, 0, &[], true, true, 0, 0, 0); + let shred = Shred::new_from_data(1, 3, 0, &[], ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 0); shred.copy_to_packet(&mut packet); // rejected slot is 1, root is 3 @@ -342,7 +344,16 @@ mod tests { ); assert!(packet.meta.discard()); - let shred = Shred::new_from_data(1_000_000, 3, 0, &[], true, true, 0, 0, 0); + let shred = Shred::new_from_data( + 1_000_000, + 3, + 0, + &[], + ShredFlags::LAST_SHRED_IN_SLOT, + 0, + 0, + 0, + ); shred.copy_to_packet(&mut packet); // Slot 1 million is too high @@ -359,7 +370,7 @@ mod tests { assert!(packet.meta.discard()); let index = MAX_DATA_SHREDS_PER_SLOT as u32; - let shred = Shred::new_from_data(5, index, 0, &[], true, true, 0, 0, 0); + let shred = Shred::new_from_data(5, index, 0, &[], ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 0); shred.copy_to_packet(&mut packet); ShredFetchStage::process_packet( &mut packet, diff --git a/core/src/sigverify_shreds.rs b/core/src/sigverify_shreds.rs index 006fcb960..e5f6567c4 100644 --- a/core/src/sigverify_shreds.rs +++ b/core/src/sigverify_shreds.rs @@ -69,7 +69,10 @@ impl SigVerifier for ShredSigVerifier { pub mod tests { use { super::*, - solana_ledger::{genesis_utils::create_genesis_config_with_leader, shred::Shred}, + solana_ledger::{ + genesis_utils::create_genesis_config_with_leader, + shred::{Shred, ShredFlags}, + }, solana_perf::packet::Packet, solana_runtime::bank::Bank, solana_sdk::signature::{Keypair, Signer}, @@ -83,8 +86,7 @@ pub mod tests { 0xc0de, 0xdead, &[1, 2, 3, 4], - true, - true, + ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 0xc0de, @@ -102,8 +104,7 @@ pub mod tests { 0xc0de, 0xdead, &[1, 2, 3, 4], - true, - true, + ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 0xc0de, @@ -131,14 +132,30 @@ pub mod tests { let mut batches = vec![PacketBatch::default()]; batches[0].packets.resize(2, Packet::default()); - let mut shred = - Shred::new_from_data(0, 0xc0de, 0xdead, &[1, 2, 3, 4], true, true, 0, 0, 0xc0de); + let mut shred = Shred::new_from_data( + 0, + 0xc0de, + 0xdead, + &[1, 2, 3, 4], + ShredFlags::LAST_SHRED_IN_SLOT, + 0, + 0, + 0xc0de, + ); shred.sign(&leader_keypair); batches[0].packets[0].data[0..shred.payload().len()].copy_from_slice(shred.payload()); batches[0].packets[0].meta.size = shred.payload().len(); - let mut shred = - Shred::new_from_data(0, 0xbeef, 0xc0de, &[1, 2, 3, 4], true, true, 0, 0, 0xc0de); + let mut shred = Shred::new_from_data( + 0, + 0xbeef, + 0xc0de, + &[1, 2, 3, 4], + ShredFlags::LAST_SHRED_IN_SLOT, + 0, + 0, + 0xc0de, + ); let wrong_keypair = Keypair::new(); shred.sign(&wrong_keypair); batches[0].packets[1].data[0..shred.payload().len()].copy_from_slice(shred.payload()); diff --git a/ledger/benches/sigverify_shreds.rs b/ledger/benches/sigverify_shreds.rs index cda35c4d9..4b27d6ad3 100644 --- a/ledger/benches/sigverify_shreds.rs +++ b/ledger/benches/sigverify_shreds.rs @@ -3,7 +3,7 @@ extern crate test; use { solana_ledger::{ - shred::{Shred, SIZE_OF_DATA_SHRED_PAYLOAD}, + shred::{Shred, ShredFlags, SIZE_OF_DATA_SHRED_PAYLOAD}, sigverify_shreds::{sign_shreds_cpu, sign_shreds_gpu, sign_shreds_gpu_pinned_keypair}, }, solana_perf::{ @@ -33,8 +33,7 @@ fn bench_sigverify_shreds_sign_gpu(bencher: &mut Bencher) { 0xc0de, 0xdead, &[5; SIZE_OF_DATA_SHRED_PAYLOAD], - true, - true, + ShredFlags::LAST_SHRED_IN_SLOT, 1, 2, 0, @@ -65,8 +64,7 @@ fn bench_sigverify_shreds_sign_cpu(bencher: &mut Bencher) { 0xc0de, 0xdead, &[5; SIZE_OF_DATA_SHRED_PAYLOAD], - true, - true, + ShredFlags::LAST_SHRED_IN_SLOT, 1, 2, 0, diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index f7992f941..5d985d8ca 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -4286,7 +4286,7 @@ pub mod tests { blockstore_db::BlockstoreRocksFifoOptions, genesis_utils::{create_genesis_config, GenesisConfigInfo}, leader_schedule::{FixedSchedule, LeaderSchedule}, - shred::max_ticks_per_n_shreds, + shred::{max_ticks_per_n_shreds, ShredFlags}, }, assert_matches::assert_matches, bincode::serialize, @@ -5719,8 +5719,7 @@ pub mod tests { (i * gap) as u32, 0, &[], - false, - false, + ShredFlags::empty(), i as u8, 0, (i * gap) as u32, @@ -5850,10 +5849,9 @@ pub mod tests { let parent_offset = shred5.slot() - shred5.parent().unwrap(); parent_offset as u16 }, - &[], // data - true, // is_last_data - true, // is_last_in_slot - 0, // reference_tick + &[], // data + ShredFlags::LAST_SHRED_IN_SLOT, + 0, // reference_tick shred5.version(), shred5.fec_set_index(), ); @@ -6255,8 +6253,7 @@ pub mod tests { next_shred_index as u32, 1, &[1, 1, 1], - true, - true, + ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, next_shred_index as u32, @@ -8776,26 +8773,6 @@ pub mod tests { } } - #[test] - fn test_remove_shred_data_complete_flag() { - let ledger_path = get_tmp_ledger_path_auto_delete!(); - let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - - let (mut shreds, entries) = make_slot_entries(0, 0, 1); - - // Remove the data complete flag from the last shred - shreds[0].unset_data_complete(); - - blockstore.insert_shreds(shreds, None, false).unwrap(); - - // Check that the `data_complete` flag was unset in the stored shred, but the - // `last_in_slot` flag is set. - let stored_shred = &blockstore.get_data_shreds_for_slot(0, 0).unwrap()[0]; - assert!(!stored_shred.data_complete()); - assert!(stored_shred.last_in_slot()); - assert_eq!(entries, blockstore.get_any_valid_slot_entries(0, 0)); - } - fn make_large_tx_entry(num_txs: usize) -> Entry { let txs: Vec<_> = (0..num_txs) .into_iter() diff --git a/ledger/src/shred.rs b/ledger/src/shred.rs index 65af685ac..1b923eec3 100644 --- a/ledger/src/shred.rs +++ b/ledger/src/shred.rs @@ -109,12 +109,14 @@ const ENCODED_PAYLOAD_SIZE: usize = SHRED_PAYLOAD_SIZE - SIZE_OF_CODING_SHRED_HE pub const MAX_DATA_SHREDS_PER_FEC_BLOCK: u32 = 32; +// LAST_SHRED_IN_SLOT also implies DATA_COMPLETE_SHRED. +// So it cannot be LAST_SHRED_IN_SLOT if not also DATA_COMPLETE_SHRED. bitflags! { #[derive(Default, Serialize, Deserialize)] pub struct ShredFlags:u8 { const SHRED_TICK_REFERENCE_MASK = 0b0011_1111; - const DATA_COMPLETE_SHRED = 0b0100_0000; - const LAST_SHRED_IN_SLOT = 0b1000_0000; + const DATA_COMPLETE_SHRED = 0b0100_0000; + const LAST_SHRED_IN_SLOT = 0b1100_0000; } } @@ -138,6 +140,8 @@ pub enum Error { InvalidParentSlot { slot: Slot, parent_slot: Slot }, #[error("Invalid payload size: {0}")] InvalidPayloadSize(/*payload size:*/ usize), + #[error("Invalid shred flags: {0}")] + InvalidShredFlags(u8), #[error("Invalid shred type")] InvalidShredType, } @@ -240,8 +244,7 @@ impl Shred { index: u32, parent_offset: u16, data: &[u8], - is_last_data: bool, - is_last_in_slot: bool, + flags: ShredFlags, reference_tick: u8, version: u16, fec_set_index: u32, @@ -257,26 +260,19 @@ impl Shred { }; let size = (data.len() + SIZE_OF_DATA_SHRED_HEADER + SIZE_OF_COMMON_SHRED_HEADER) as u16; - let mut data_header = DataShredHeader { - parent_offset, - flags: unsafe { + let flags = flags + | unsafe { ShredFlags::from_bits_unchecked( ShredFlags::SHRED_TICK_REFERENCE_MASK .bits() .min(reference_tick), ) - }, + }; + let data_header = DataShredHeader { + parent_offset, + flags, size, }; - - if is_last_data { - data_header.flags |= ShredFlags::DATA_COMPLETE_SHRED - } - - if is_last_in_slot { - data_header.flags |= ShredFlags::LAST_SHRED_IN_SLOT - } - let mut cursor = Cursor::new(&mut payload[..]); bincode::serialize_into(&mut cursor, &common_header).unwrap(); bincode::serialize_into(&mut cursor, &data_header).unwrap(); @@ -496,6 +492,12 @@ impl Shred { payload: self.payload.len(), }); } + let flags = self.data_header.flags; + if flags.intersects(ShredFlags::LAST_SHRED_IN_SLOT) + && !flags.contains(ShredFlags::DATA_COMPLETE_SHRED) + { + return Err(Error::InvalidShredFlags(self.data_header.flags.bits())); + } } ShredType::Code => { let num_coding_shreds = u32::from(self.coding_header.num_coding_shreds); @@ -638,18 +640,6 @@ impl Shred { bincode::serialize_into(buffer, &self.data_header).unwrap(); } - #[cfg(test)] - pub(crate) fn unset_data_complete(&mut self) { - if self.is_data() { - self.data_header - .flags - .remove(ShredFlags::DATA_COMPLETE_SHRED); - } - // Data header starts after the shred common header - let buffer = &mut self.payload[SIZE_OF_COMMON_SHRED_HEADER..]; - bincode::serialize_into(buffer, &self.data_header).unwrap(); - } - pub fn data_complete(&self) -> bool { if self.is_data() { self.data_header @@ -909,7 +899,7 @@ mod tests { #[test] fn test_invalid_parent_offset() { - let shred = Shred::new_from_data(10, 0, 1000, &[1, 2, 3], false, false, 0, 1, 0); + let shred = Shred::new_from_data(10, 0, 1000, &[1, 2, 3], ShredFlags::empty(), 0, 1, 0); let mut packet = Packet::default(); shred.copy_to_packet(&mut packet); let shred_res = Shred::new_from_serialized_shred(packet.data.to_vec()); @@ -933,7 +923,7 @@ mod tests { fn test_shred_offsets() { solana_logger::setup(); let mut packet = Packet::default(); - let shred = Shred::new_from_data(1, 3, 0, &[], true, true, 0, 0, 0); + let shred = Shred::new_from_data(1, 3, 0, &[], ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 0); shred.copy_to_packet(&mut packet); let mut stats = ShredFetchStats::default(); let ret = get_shred_slot_index_type(&packet, &mut stats); @@ -979,7 +969,16 @@ mod tests { get_shred_slot_index_type(&packet, &mut stats) ); - let shred = Shred::new_from_data(1, std::u32::MAX - 10, 0, &[], true, true, 0, 0, 0); + let shred = Shred::new_from_data( + 1, + std::u32::MAX - 10, + 0, + &[], + ShredFlags::LAST_SHRED_IN_SLOT, + 0, + 0, + 0, + ); shred.copy_to_packet(&mut packet); assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats)); assert_eq!(1, stats.index_out_of_bounds); @@ -1036,11 +1035,11 @@ mod tests { 420, // slot 19, // index 5, // parent_offset - &data, true, // is_last_data - false, // is_last_in_slot - 3, // reference_tick - 1, // version - 16, // fec_set_index + &data, + ShredFlags::DATA_COMPLETE_SHRED, + 3, // reference_tick + 1, // version + 16, // fec_set_index ); assert_matches!(shred.sanitize(), Ok(())); // Corrupt shred by making it too large @@ -1076,6 +1075,15 @@ mod tests { shred.common_header.index = MAX_DATA_SHREDS_PER_SLOT as u32; assert_matches!(shred.sanitize(), Err(Error::InvalidDataShredIndex(32768))); } + { + let mut shred = shred.clone(); + shred.data_header.flags |= ShredFlags::LAST_SHRED_IN_SLOT; + assert_matches!(shred.sanitize(), Ok(())); + shred.data_header.flags &= !ShredFlags::DATA_COMPLETE_SHRED; + assert_matches!(shred.sanitize(), Err(Error::InvalidShredFlags(131u8))); + shred.data_header.flags |= ShredFlags::SHRED_TICK_REFERENCE_MASK; + assert_matches!(shred.sanitize(), Err(Error::InvalidShredFlags(191u8))); + } { shred.data_header.size = shred.payload().len() as u16 + 1; assert_matches!( @@ -1163,10 +1171,10 @@ mod tests { #[test] fn test_serde_compat_shred_data() { const SEED: &str = "6qG9NGWEtoTugS4Zgs46u8zTccEJuRHtrNMiUayLHCxt"; - const PAYLOAD: &str = "Cuk5B7qosCx42HcZjwcUKPpeqE43sDhx1RFut5rEAk54dV\ - JFPrGEBYuPJSwaaNNbGyas9AuLS66NcFxNAcBzmBcb3gSmNrfQzmTUev5jSEeQRyqE5WG\ - rGwC67mS5QXmokCtVEdXu9B1SLQFBMB62CQGVEjqV1r6jz4xd5Zg1AyyVCjGpRe8ooqMB\ - 1doeATLCEBhjnPPYLLyZGqPfrqfz5huq8BCoVC8DCMWFJhxtPLA5XHPPpxieAhDBmS"; + const PAYLOAD: &str = "hNX8YgJCQwSFGJkZ6qZLiepwPjpctC9UCsMD1SNNQurBXv\ + rm7KKfLmPRMM9CpWHt6MsJuEWpDXLGwH9qdziJzGKhBMfYH63avcchjdaUiMqzVip7cUD\ + kqZ9zZJMrHCCUDnxxKMupsJWKroUSjKeo7hrug2KfHah85VckXpRna4R9QpH7tf2WVBTD\ + M4m3EerctsEQs8eZaTRxzTVkhtJYdNf74KZbH58dc3Yn2qUxF1mexWoPS6L5oZBatx"; let mut rng = { let seed = <[u8; 32]>::try_from(bs58_decode(SEED)).unwrap(); ChaChaRng::from_seed(seed) @@ -1179,11 +1187,10 @@ mod tests { 28685, // index 36390, // parent_offset &data, // data - false, // is_last_data - true, // is_last_in_slot - 37, // reference_tick - 45189, // version - 28657, // fec_set_index + ShredFlags::LAST_SHRED_IN_SLOT, + 37, // reference_tick + 45189, // version + 28657, // fec_set_index ); shred.sign(&keypair); assert!(shred.verify(&keypair.pubkey())); @@ -1225,11 +1232,10 @@ mod tests { 21443, // index 51279, // parent_offset &[], // data - true, // is_last_data - false, // is_last_in_slot - 49, // reference_tick - 59445, // version - 21414, // fec_set_index + ShredFlags::DATA_COMPLETE_SHRED, + 49, // reference_tick + 59445, // version + 21414, // fec_set_index ); shred.sign(&keypair); assert!(shred.verify(&keypair.pubkey())); @@ -1298,13 +1304,20 @@ mod tests { #[test] fn test_shred_flags() { fn make_shred(is_last_data: bool, is_last_in_slot: bool, reference_tick: u8) -> Shred { + let flags = if is_last_in_slot { + assert!(is_last_data); + ShredFlags::LAST_SHRED_IN_SLOT + } else if is_last_data { + ShredFlags::DATA_COMPLETE_SHRED + } else { + ShredFlags::empty() + }; Shred::new_from_data( 0, // slot 0, // index 0, // parent_offset &[], // data - is_last_data, - is_last_in_slot, + flags, reference_tick, 0, // version 0, // fec_set_index @@ -1326,15 +1339,37 @@ mod tests { } for is_last_data in [false, true] { for is_last_in_slot in [false, true] { + // LAST_SHRED_IN_SLOT also implies DATA_COMPLETE_SHRED. So it + // cannot be LAST_SHRED_IN_SLOT if not DATA_COMPLETE_SHRED. + let is_last_in_slot = is_last_in_slot && is_last_data; for reference_tick in [0, 37, 63, 64, 80, 128, 255] { let mut shred = make_shred(is_last_data, is_last_in_slot, reference_tick); check_shred_flags(&shred, is_last_data, is_last_in_slot, reference_tick); shred.set_last_in_slot(); - check_shred_flags(&shred, is_last_data, true, reference_tick); - shred.unset_data_complete(); - check_shred_flags(&shred, false, true, reference_tick); + check_shred_flags(&shred, true, true, reference_tick); } } } } + + #[test] + fn test_shred_flags_serde() { + let flags: ShredFlags = bincode::deserialize(&[0b0111_0001]).unwrap(); + assert!(flags.contains(ShredFlags::DATA_COMPLETE_SHRED)); + assert!(!flags.contains(ShredFlags::LAST_SHRED_IN_SLOT)); + assert_eq!((flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits(), 49u8); + assert_eq!(bincode::serialize(&flags).unwrap(), [0b0111_0001]); + + let flags: ShredFlags = bincode::deserialize(&[0b1110_0101]).unwrap(); + assert!(flags.contains(ShredFlags::DATA_COMPLETE_SHRED)); + assert!(flags.contains(ShredFlags::LAST_SHRED_IN_SLOT)); + assert_eq!((flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits(), 37u8); + assert_eq!(bincode::serialize(&flags).unwrap(), [0b1110_0101]); + + let flags: ShredFlags = bincode::deserialize(&[0b1011_1101]).unwrap(); + assert!(!flags.contains(ShredFlags::DATA_COMPLETE_SHRED)); + assert!(!flags.contains(ShredFlags::LAST_SHRED_IN_SLOT)); + assert_eq!((flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits(), 61u8); + assert_eq!(bincode::serialize(&flags).unwrap(), [0b1011_1101]); + } } diff --git a/ledger/src/shredder.rs b/ledger/src/shredder.rs index e4496a2b5..24e950f30 100644 --- a/ledger/src/shredder.rs +++ b/ledger/src/shredder.rs @@ -1,6 +1,8 @@ use { crate::{ - shred::{Error, Shred, MAX_DATA_SHREDS_PER_FEC_BLOCK, SIZE_OF_DATA_SHRED_PAYLOAD}, + shred::{ + Error, Shred, ShredFlags, MAX_DATA_SHREDS_PER_FEC_BLOCK, SIZE_OF_DATA_SHRED_PAYLOAD, + }, shred_stats::ProcessShredsStats, }, rayon::{prelude::*, ThreadPool}, @@ -113,8 +115,14 @@ impl Shredder { let last_shred_index = next_shred_index + num_shreds as u32 - 1; // 1) Generate data shreds let make_data_shred = |shred_index: u32, data| { - let is_last_data = shred_index == last_shred_index; - let is_last_in_slot = is_last_data && is_last_in_slot; + let flags = if shred_index != last_shred_index { + ShredFlags::empty() + } else if is_last_in_slot { + // LAST_SHRED_IN_SLOT also implies DATA_COMPLETE_SHRED. + ShredFlags::LAST_SHRED_IN_SLOT + } else { + ShredFlags::DATA_COMPLETE_SHRED + }; let parent_offset = self.slot - self.parent_slot; let fec_set_index = Self::fec_set_index(shred_index, fec_set_offset); let mut shred = Shred::new_from_data( @@ -122,8 +130,7 @@ impl Shredder { shred_index, parent_offset as u16, data, - is_last_data, - is_last_in_slot, + flags, self.reference_tick, self.version, fec_set_index.unwrap(), @@ -351,8 +358,7 @@ mod tests { use { super::*, crate::shred::{ - max_entries_per_n_shred, max_ticks_per_n_shreds, verify_test_data_shred, ShredFlags, - ShredType, + max_entries_per_n_shred, max_ticks_per_n_shreds, verify_test_data_shred, ShredType, }, bincode::serialized_size, matches::assert_matches, diff --git a/ledger/src/sigverify_shreds.rs b/ledger/src/sigverify_shreds.rs index 7f7bc6c44..719fa210e 100644 --- a/ledger/src/sigverify_shreds.rs +++ b/ledger/src/sigverify_shreds.rs @@ -459,7 +459,7 @@ pub fn sign_shreds_gpu( pub mod tests { use { super::*, - crate::shred::{Shred, SIZE_OF_DATA_SHRED_PAYLOAD}, + crate::shred::{Shred, ShredFlags, SIZE_OF_DATA_SHRED_PAYLOAD}, solana_sdk::signature::{Keypair, Signer}, }; @@ -471,8 +471,7 @@ pub mod tests { 0xc0de, 0xdead, &[1, 2, 3, 4], - true, - true, + ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 0xc0de, @@ -517,8 +516,7 @@ pub mod tests { 0xc0de, 0xdead, &[1, 2, 3, 4], - true, - true, + ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 0xc0de, @@ -572,8 +570,7 @@ pub mod tests { 0xc0de, 0xdead, &[1, 2, 3, 4], - true, - true, + ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 0xc0de, @@ -640,8 +637,7 @@ pub mod tests { 0xc0de, i as u16, &[5; SIZE_OF_DATA_SHRED_PAYLOAD], - true, - true, + ShredFlags::LAST_SHRED_IN_SLOT, 1, 2, 0xc0de, @@ -686,8 +682,7 @@ pub mod tests { 0xc0de, 0xdead, &[1, 2, 3, 4], - true, - true, + ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 0xc0de,