enforces that LAST_SHRED_IN_SLOT is also DATA_COMPLETE_SHRED (#24892)
A data shred cannot be LAST_SHRED_IN_SLOT if not also DATA_COMPLETE_SHRED. So LAST_SHRED_IN_SLOT should also imply DATA_COMPLETE_SHRED: https://github.com/solana-labs/solana/blob/74b586ae7/ledger/src/shredder.rs#L116-L117 https://github.com/solana-labs/solana/blob/74b586ae7/core/src/broadcast_stage/standard_broadcast_run.rs#L80-L81 However current shred constructs allow specifying a shred which is LAST_SHRED_IN_SLOT but not DATA_COMPLETE_SHRED: https://github.com/solana-labs/solana/blob/74b586ae7/ledger/src/shred.rs#L117-L118 https://github.com/solana-labs/solana/blob/74b586ae7/ledger/src/shred.rs#L272-L273 The commit updates ShredFlags so that if a shred is not DATA_COMPLETE_SHRED it cannot be LAST_SHRED_IN_SLOT either.
This commit is contained in:
parent
e83efe678c
commit
eff59193db
|
@ -16,7 +16,7 @@ use {
|
|||
},
|
||||
solana_ledger::{
|
||||
genesis_utils::{create_genesis_config, GenesisConfigInfo},
|
||||
shred::Shred,
|
||||
shred::{Shred, ShredFlags},
|
||||
},
|
||||
solana_runtime::{bank::Bank, bank_forks::BankForks},
|
||||
solana_sdk::{
|
||||
|
@ -51,7 +51,7 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
|
|||
let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
|
||||
|
||||
const NUM_SHREDS: usize = 32;
|
||||
let shred = Shred::new_from_data(0, 0, 0, &[], false, false, 0, 0, 0);
|
||||
let shred = Shred::new_from_data(0, 0, 0, &[], ShredFlags::empty(), 0, 0, 0);
|
||||
let shreds = vec![shred; NUM_SHREDS];
|
||||
let mut stakes = HashMap::new();
|
||||
const NUM_PEERS: usize = 200;
|
||||
|
|
|
@ -11,7 +11,7 @@ use {
|
|||
solana_gossip::contact_info::ContactInfo,
|
||||
solana_ledger::{
|
||||
genesis_utils::{create_genesis_config, GenesisConfigInfo},
|
||||
shred::Shred,
|
||||
shred::{Shred, ShredFlags},
|
||||
},
|
||||
solana_runtime::bank::Bank,
|
||||
solana_sdk::{clock::Slot, pubkey::Pubkey},
|
||||
|
@ -39,7 +39,16 @@ fn get_retransmit_peers_deterministic(
|
|||
let parent_offset = if slot == 0 { 0 } else { 1 };
|
||||
for i in 0..num_simulated_shreds {
|
||||
let index = i as u32;
|
||||
let shred = Shred::new_from_data(slot, index, parent_offset, &[], false, false, 0, 0, 0);
|
||||
let shred = Shred::new_from_data(
|
||||
slot,
|
||||
index,
|
||||
parent_offset,
|
||||
&[],
|
||||
ShredFlags::empty(),
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
);
|
||||
let (_neighbors, _children) = cluster_nodes.get_retransmit_peers(
|
||||
*slot_leader,
|
||||
&shred,
|
||||
|
|
|
@ -8,8 +8,8 @@ use {
|
|||
raptorq::{Decoder, Encoder},
|
||||
solana_entry::entry::{create_ticks, Entry},
|
||||
solana_ledger::shred::{
|
||||
max_entries_per_n_shred, max_ticks_per_n_shreds, ProcessShredsStats, Shred, Shredder,
|
||||
MAX_DATA_SHREDS_PER_FEC_BLOCK, SIZE_OF_DATA_SHRED_PAYLOAD,
|
||||
max_entries_per_n_shred, max_ticks_per_n_shreds, ProcessShredsStats, Shred, ShredFlags,
|
||||
Shredder, MAX_DATA_SHREDS_PER_FEC_BLOCK, SIZE_OF_DATA_SHRED_PAYLOAD,
|
||||
},
|
||||
solana_perf::test_tx,
|
||||
solana_sdk::{hash::Hash, packet::PACKET_DATA_SIZE, signature::Keypair},
|
||||
|
@ -123,7 +123,7 @@ fn bench_deshredder(bencher: &mut Bencher) {
|
|||
fn bench_deserialize_hdr(bencher: &mut Bencher) {
|
||||
let data = vec![0; SIZE_OF_DATA_SHRED_PAYLOAD];
|
||||
|
||||
let shred = Shred::new_from_data(2, 1, 1, &data, true, true, 0, 0, 1);
|
||||
let shred = Shred::new_from_data(2, 1, 1, &data, ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 1);
|
||||
|
||||
bencher.iter(|| {
|
||||
let payload = shred.payload().clone();
|
||||
|
|
|
@ -76,9 +76,8 @@ impl StandardBroadcastRun {
|
|||
state.slot,
|
||||
state.next_shred_index,
|
||||
parent_offset as u16,
|
||||
&[], // data
|
||||
true, // is_last_in_fec_set
|
||||
true, // is_last_in_slot
|
||||
&[], // data
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
reference_tick,
|
||||
self.shred_version,
|
||||
fec_set_index.unwrap(),
|
||||
|
|
|
@ -85,7 +85,9 @@ pub struct RequestStatus<T> {
|
|||
#[cfg(test)]
|
||||
pub(crate) mod tests {
|
||||
use {
|
||||
super::*, crate::serve_repair::ShredRepairType, solana_ledger::shred::Shred,
|
||||
super::*,
|
||||
crate::serve_repair::ShredRepairType,
|
||||
solana_ledger::shred::{Shred, ShredFlags},
|
||||
solana_sdk::timing::timestamp,
|
||||
};
|
||||
|
||||
|
@ -107,7 +109,7 @@ pub(crate) mod tests {
|
|||
let repair_type = ShredRepairType::Orphan(9);
|
||||
let mut outstanding_requests = OutstandingRequests::default();
|
||||
let nonce = outstanding_requests.add_request(repair_type, timestamp());
|
||||
let shred = Shred::new_from_data(0, 0, 0, &[], false, false, 0, 0, 0);
|
||||
let shred = Shred::new_from_data(0, 0, 0, &[], ShredFlags::empty(), 0, 0, 0);
|
||||
|
||||
let expire_timestamp = outstanding_requests
|
||||
.requests
|
||||
|
@ -127,7 +129,7 @@ pub(crate) mod tests {
|
|||
let mut outstanding_requests = OutstandingRequests::default();
|
||||
let nonce = outstanding_requests.add_request(repair_type, timestamp());
|
||||
|
||||
let shred = Shred::new_from_data(0, 0, 0, &[], false, false, 0, 0, 0);
|
||||
let shred = Shred::new_from_data(0, 0, 0, &[], ShredFlags::empty(), 0, 0, 0);
|
||||
let mut expire_timestamp = outstanding_requests
|
||||
.requests
|
||||
.get(&nonce)
|
||||
|
|
|
@ -52,7 +52,10 @@ pub fn nonce(buf: &[u8]) -> Option<Nonce> {
|
|||
mod test {
|
||||
use {
|
||||
super::*,
|
||||
solana_ledger::{shred::Shred, sigverify_shreds::verify_shred_cpu},
|
||||
solana_ledger::{
|
||||
shred::{Shred, ShredFlags},
|
||||
sigverify_shreds::verify_shred_cpu,
|
||||
},
|
||||
solana_sdk::{
|
||||
packet::PacketFlags,
|
||||
signature::{Keypair, Signer},
|
||||
|
@ -70,8 +73,7 @@ mod test {
|
|||
0xc0de,
|
||||
0xdead,
|
||||
&[1, 2, 3, 4],
|
||||
true,
|
||||
true,
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
0,
|
||||
0,
|
||||
0xc0de,
|
||||
|
|
|
@ -3123,7 +3123,7 @@ impl ReplayStage {
|
|||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub mod tests {
|
||||
pub(crate) mod tests {
|
||||
use {
|
||||
super::*,
|
||||
crate::{
|
||||
|
@ -3142,7 +3142,7 @@ pub mod tests {
|
|||
create_new_tmp_ledger,
|
||||
genesis_utils::{create_genesis_config, create_genesis_config_with_leader},
|
||||
get_tmp_ledger_path,
|
||||
shred::{Shred, SIZE_OF_DATA_SHRED_PAYLOAD},
|
||||
shred::{Shred, ShredFlags, SIZE_OF_DATA_SHRED_PAYLOAD},
|
||||
},
|
||||
solana_rpc::{
|
||||
optimistically_confirmed_bank_tracker::OptimisticallyConfirmedBank,
|
||||
|
@ -3748,11 +3748,10 @@ pub mod tests {
|
|||
0, // index,
|
||||
parent_offset as u16,
|
||||
&gibberish,
|
||||
true, // is_last_data
|
||||
false, // is_last_in_slot
|
||||
0, // reference_tick
|
||||
0, // version
|
||||
0, // fec_set_index
|
||||
ShredFlags::DATA_COMPLETE_SHRED,
|
||||
0, // reference_tick
|
||||
0, // version
|
||||
0, // fec_set_index
|
||||
);
|
||||
vec![shred]
|
||||
});
|
||||
|
|
|
@ -528,27 +528,54 @@ impl RetransmitStage {
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use {super::*, solana_ledger::shred::ShredFlags};
|
||||
|
||||
#[test]
|
||||
fn test_already_received() {
|
||||
let slot = 1;
|
||||
let index = 5;
|
||||
let version = 0x40;
|
||||
let shred = Shred::new_from_data(slot, index, 0, &[], true, true, 0, version, 0);
|
||||
let shred = Shred::new_from_data(
|
||||
slot,
|
||||
index,
|
||||
0,
|
||||
&[],
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
0,
|
||||
version,
|
||||
0,
|
||||
);
|
||||
let shreds_received = Arc::new(Mutex::new((LruCache::new(100), PacketHasher::default())));
|
||||
// unique shred for (1, 5) should pass
|
||||
assert!(!should_skip_retransmit(&shred, &shreds_received));
|
||||
// duplicate shred for (1, 5) blocked
|
||||
assert!(should_skip_retransmit(&shred, &shreds_received));
|
||||
|
||||
let shred = Shred::new_from_data(slot, index, 2, &[], true, true, 0, version, 0);
|
||||
let shred = Shred::new_from_data(
|
||||
slot,
|
||||
index,
|
||||
2,
|
||||
&[],
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
0,
|
||||
version,
|
||||
0,
|
||||
);
|
||||
// first duplicate shred for (1, 5) passed
|
||||
assert!(!should_skip_retransmit(&shred, &shreds_received));
|
||||
// then blocked
|
||||
assert!(should_skip_retransmit(&shred, &shreds_received));
|
||||
|
||||
let shred = Shred::new_from_data(slot, index, 8, &[], true, true, 0, version, 0);
|
||||
let shred = Shred::new_from_data(
|
||||
slot,
|
||||
index,
|
||||
8,
|
||||
&[],
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
0,
|
||||
version,
|
||||
0,
|
||||
);
|
||||
// 2nd duplicate shred for (1, 5) blocked
|
||||
assert!(should_skip_retransmit(&shred, &shreds_received));
|
||||
assert!(should_skip_retransmit(&shred, &shreds_received));
|
||||
|
|
|
@ -761,7 +761,7 @@ mod tests {
|
|||
blockstore::make_many_slot_entries,
|
||||
blockstore_processor::fill_blockstore_slot_with_ticks,
|
||||
get_tmp_ledger_path,
|
||||
shred::{max_ticks_per_n_shreds, Shred},
|
||||
shred::{max_ticks_per_n_shreds, Shred, ShredFlags},
|
||||
},
|
||||
solana_perf::packet::Packet,
|
||||
solana_sdk::{hash::Hash, pubkey::Pubkey, signature::Keypair, timing::timestamp},
|
||||
|
@ -876,7 +876,7 @@ mod tests {
|
|||
nonce,
|
||||
);
|
||||
assert!(rv.is_none());
|
||||
let shred = Shred::new_from_data(slot, 1, 1, &[], false, false, 0, 2, 0);
|
||||
let shred = Shred::new_from_data(slot, 1, 1, &[], ShredFlags::empty(), 0, 2, 0);
|
||||
|
||||
blockstore
|
||||
.insert_shreds(vec![shred], None, false)
|
||||
|
@ -1306,7 +1306,7 @@ mod tests {
|
|||
#[test]
|
||||
fn test_verify_shred_response() {
|
||||
fn new_test_data_shred(slot: Slot, index: u32) -> Shred {
|
||||
Shred::new_from_data(slot, index, 1, &[], false, false, 0, 0, 0)
|
||||
Shred::new_from_data(slot, index, 1, &[], ShredFlags::empty(), 0, 0, 0)
|
||||
}
|
||||
let repair = ShredRepairType::Orphan(9);
|
||||
// Ensure new options are addded to this test
|
||||
|
|
|
@ -216,7 +216,10 @@ impl ShredFetchStage {
|
|||
mod tests {
|
||||
use {
|
||||
super::*,
|
||||
solana_ledger::{blockstore::MAX_DATA_SHREDS_PER_SLOT, shred::Shred},
|
||||
solana_ledger::{
|
||||
blockstore::MAX_DATA_SHREDS_PER_SLOT,
|
||||
shred::{Shred, ShredFlags},
|
||||
},
|
||||
};
|
||||
|
||||
#[test]
|
||||
|
@ -229,14 +232,13 @@ mod tests {
|
|||
let slot = 1;
|
||||
let shred = Shred::new_from_data(
|
||||
slot,
|
||||
3, // shred index
|
||||
0, // parent offset
|
||||
&[], // data
|
||||
true, // is_last_in_fec_set
|
||||
true, // is_last_in_slot
|
||||
0, // reference_tick
|
||||
0, // version
|
||||
3, // fec_set_index
|
||||
3, // shred index
|
||||
0, // parent offset
|
||||
&[], // data
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
0, // reference_tick
|
||||
0, // version
|
||||
3, // fec_set_index
|
||||
);
|
||||
shred.copy_to_packet(&mut packet);
|
||||
|
||||
|
@ -300,7 +302,7 @@ mod tests {
|
|||
);
|
||||
assert_eq!(stats.index_overrun, 1);
|
||||
assert!(packet.meta.discard());
|
||||
let shred = Shred::new_from_data(1, 3, 0, &[], true, true, 0, 0, 0);
|
||||
let shred = Shred::new_from_data(1, 3, 0, &[], ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 0);
|
||||
shred.copy_to_packet(&mut packet);
|
||||
|
||||
// rejected slot is 1, root is 3
|
||||
|
@ -342,7 +344,16 @@ mod tests {
|
|||
);
|
||||
assert!(packet.meta.discard());
|
||||
|
||||
let shred = Shred::new_from_data(1_000_000, 3, 0, &[], true, true, 0, 0, 0);
|
||||
let shred = Shred::new_from_data(
|
||||
1_000_000,
|
||||
3,
|
||||
0,
|
||||
&[],
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
);
|
||||
shred.copy_to_packet(&mut packet);
|
||||
|
||||
// Slot 1 million is too high
|
||||
|
@ -359,7 +370,7 @@ mod tests {
|
|||
assert!(packet.meta.discard());
|
||||
|
||||
let index = MAX_DATA_SHREDS_PER_SLOT as u32;
|
||||
let shred = Shred::new_from_data(5, index, 0, &[], true, true, 0, 0, 0);
|
||||
let shred = Shred::new_from_data(5, index, 0, &[], ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 0);
|
||||
shred.copy_to_packet(&mut packet);
|
||||
ShredFetchStage::process_packet(
|
||||
&mut packet,
|
||||
|
|
|
@ -69,7 +69,10 @@ impl SigVerifier for ShredSigVerifier {
|
|||
pub mod tests {
|
||||
use {
|
||||
super::*,
|
||||
solana_ledger::{genesis_utils::create_genesis_config_with_leader, shred::Shred},
|
||||
solana_ledger::{
|
||||
genesis_utils::create_genesis_config_with_leader,
|
||||
shred::{Shred, ShredFlags},
|
||||
},
|
||||
solana_perf::packet::Packet,
|
||||
solana_runtime::bank::Bank,
|
||||
solana_sdk::signature::{Keypair, Signer},
|
||||
|
@ -83,8 +86,7 @@ pub mod tests {
|
|||
0xc0de,
|
||||
0xdead,
|
||||
&[1, 2, 3, 4],
|
||||
true,
|
||||
true,
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
0,
|
||||
0,
|
||||
0xc0de,
|
||||
|
@ -102,8 +104,7 @@ pub mod tests {
|
|||
0xc0de,
|
||||
0xdead,
|
||||
&[1, 2, 3, 4],
|
||||
true,
|
||||
true,
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
0,
|
||||
0,
|
||||
0xc0de,
|
||||
|
@ -131,14 +132,30 @@ pub mod tests {
|
|||
let mut batches = vec![PacketBatch::default()];
|
||||
batches[0].packets.resize(2, Packet::default());
|
||||
|
||||
let mut shred =
|
||||
Shred::new_from_data(0, 0xc0de, 0xdead, &[1, 2, 3, 4], true, true, 0, 0, 0xc0de);
|
||||
let mut shred = Shred::new_from_data(
|
||||
0,
|
||||
0xc0de,
|
||||
0xdead,
|
||||
&[1, 2, 3, 4],
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
0,
|
||||
0,
|
||||
0xc0de,
|
||||
);
|
||||
shred.sign(&leader_keypair);
|
||||
batches[0].packets[0].data[0..shred.payload().len()].copy_from_slice(shred.payload());
|
||||
batches[0].packets[0].meta.size = shred.payload().len();
|
||||
|
||||
let mut shred =
|
||||
Shred::new_from_data(0, 0xbeef, 0xc0de, &[1, 2, 3, 4], true, true, 0, 0, 0xc0de);
|
||||
let mut shred = Shred::new_from_data(
|
||||
0,
|
||||
0xbeef,
|
||||
0xc0de,
|
||||
&[1, 2, 3, 4],
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
0,
|
||||
0,
|
||||
0xc0de,
|
||||
);
|
||||
let wrong_keypair = Keypair::new();
|
||||
shred.sign(&wrong_keypair);
|
||||
batches[0].packets[1].data[0..shred.payload().len()].copy_from_slice(shred.payload());
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
extern crate test;
|
||||
use {
|
||||
solana_ledger::{
|
||||
shred::{Shred, SIZE_OF_DATA_SHRED_PAYLOAD},
|
||||
shred::{Shred, ShredFlags, SIZE_OF_DATA_SHRED_PAYLOAD},
|
||||
sigverify_shreds::{sign_shreds_cpu, sign_shreds_gpu, sign_shreds_gpu_pinned_keypair},
|
||||
},
|
||||
solana_perf::{
|
||||
|
@ -33,8 +33,7 @@ fn bench_sigverify_shreds_sign_gpu(bencher: &mut Bencher) {
|
|||
0xc0de,
|
||||
0xdead,
|
||||
&[5; SIZE_OF_DATA_SHRED_PAYLOAD],
|
||||
true,
|
||||
true,
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
1,
|
||||
2,
|
||||
0,
|
||||
|
@ -65,8 +64,7 @@ fn bench_sigverify_shreds_sign_cpu(bencher: &mut Bencher) {
|
|||
0xc0de,
|
||||
0xdead,
|
||||
&[5; SIZE_OF_DATA_SHRED_PAYLOAD],
|
||||
true,
|
||||
true,
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
1,
|
||||
2,
|
||||
0,
|
||||
|
|
|
@ -4286,7 +4286,7 @@ pub mod tests {
|
|||
blockstore_db::BlockstoreRocksFifoOptions,
|
||||
genesis_utils::{create_genesis_config, GenesisConfigInfo},
|
||||
leader_schedule::{FixedSchedule, LeaderSchedule},
|
||||
shred::max_ticks_per_n_shreds,
|
||||
shred::{max_ticks_per_n_shreds, ShredFlags},
|
||||
},
|
||||
assert_matches::assert_matches,
|
||||
bincode::serialize,
|
||||
|
@ -5719,8 +5719,7 @@ pub mod tests {
|
|||
(i * gap) as u32,
|
||||
0,
|
||||
&[],
|
||||
false,
|
||||
false,
|
||||
ShredFlags::empty(),
|
||||
i as u8,
|
||||
0,
|
||||
(i * gap) as u32,
|
||||
|
@ -5850,10 +5849,9 @@ pub mod tests {
|
|||
let parent_offset = shred5.slot() - shred5.parent().unwrap();
|
||||
parent_offset as u16
|
||||
},
|
||||
&[], // data
|
||||
true, // is_last_data
|
||||
true, // is_last_in_slot
|
||||
0, // reference_tick
|
||||
&[], // data
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
0, // reference_tick
|
||||
shred5.version(),
|
||||
shred5.fec_set_index(),
|
||||
);
|
||||
|
@ -6255,8 +6253,7 @@ pub mod tests {
|
|||
next_shred_index as u32,
|
||||
1,
|
||||
&[1, 1, 1],
|
||||
true,
|
||||
true,
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
0,
|
||||
0,
|
||||
next_shred_index as u32,
|
||||
|
@ -8776,26 +8773,6 @@ pub mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_remove_shred_data_complete_flag() {
|
||||
let ledger_path = get_tmp_ledger_path_auto_delete!();
|
||||
let blockstore = Blockstore::open(ledger_path.path()).unwrap();
|
||||
|
||||
let (mut shreds, entries) = make_slot_entries(0, 0, 1);
|
||||
|
||||
// Remove the data complete flag from the last shred
|
||||
shreds[0].unset_data_complete();
|
||||
|
||||
blockstore.insert_shreds(shreds, None, false).unwrap();
|
||||
|
||||
// Check that the `data_complete` flag was unset in the stored shred, but the
|
||||
// `last_in_slot` flag is set.
|
||||
let stored_shred = &blockstore.get_data_shreds_for_slot(0, 0).unwrap()[0];
|
||||
assert!(!stored_shred.data_complete());
|
||||
assert!(stored_shred.last_in_slot());
|
||||
assert_eq!(entries, blockstore.get_any_valid_slot_entries(0, 0));
|
||||
}
|
||||
|
||||
fn make_large_tx_entry(num_txs: usize) -> Entry {
|
||||
let txs: Vec<_> = (0..num_txs)
|
||||
.into_iter()
|
||||
|
|
|
@ -109,12 +109,14 @@ const ENCODED_PAYLOAD_SIZE: usize = SHRED_PAYLOAD_SIZE - SIZE_OF_CODING_SHRED_HE
|
|||
|
||||
pub const MAX_DATA_SHREDS_PER_FEC_BLOCK: u32 = 32;
|
||||
|
||||
// LAST_SHRED_IN_SLOT also implies DATA_COMPLETE_SHRED.
|
||||
// So it cannot be LAST_SHRED_IN_SLOT if not also DATA_COMPLETE_SHRED.
|
||||
bitflags! {
|
||||
#[derive(Default, Serialize, Deserialize)]
|
||||
pub struct ShredFlags:u8 {
|
||||
const SHRED_TICK_REFERENCE_MASK = 0b0011_1111;
|
||||
const DATA_COMPLETE_SHRED = 0b0100_0000;
|
||||
const LAST_SHRED_IN_SLOT = 0b1000_0000;
|
||||
const DATA_COMPLETE_SHRED = 0b0100_0000;
|
||||
const LAST_SHRED_IN_SLOT = 0b1100_0000;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -138,6 +140,8 @@ pub enum Error {
|
|||
InvalidParentSlot { slot: Slot, parent_slot: Slot },
|
||||
#[error("Invalid payload size: {0}")]
|
||||
InvalidPayloadSize(/*payload size:*/ usize),
|
||||
#[error("Invalid shred flags: {0}")]
|
||||
InvalidShredFlags(u8),
|
||||
#[error("Invalid shred type")]
|
||||
InvalidShredType,
|
||||
}
|
||||
|
@ -240,8 +244,7 @@ impl Shred {
|
|||
index: u32,
|
||||
parent_offset: u16,
|
||||
data: &[u8],
|
||||
is_last_data: bool,
|
||||
is_last_in_slot: bool,
|
||||
flags: ShredFlags,
|
||||
reference_tick: u8,
|
||||
version: u16,
|
||||
fec_set_index: u32,
|
||||
|
@ -257,26 +260,19 @@ impl Shred {
|
|||
};
|
||||
|
||||
let size = (data.len() + SIZE_OF_DATA_SHRED_HEADER + SIZE_OF_COMMON_SHRED_HEADER) as u16;
|
||||
let mut data_header = DataShredHeader {
|
||||
parent_offset,
|
||||
flags: unsafe {
|
||||
let flags = flags
|
||||
| unsafe {
|
||||
ShredFlags::from_bits_unchecked(
|
||||
ShredFlags::SHRED_TICK_REFERENCE_MASK
|
||||
.bits()
|
||||
.min(reference_tick),
|
||||
)
|
||||
},
|
||||
};
|
||||
let data_header = DataShredHeader {
|
||||
parent_offset,
|
||||
flags,
|
||||
size,
|
||||
};
|
||||
|
||||
if is_last_data {
|
||||
data_header.flags |= ShredFlags::DATA_COMPLETE_SHRED
|
||||
}
|
||||
|
||||
if is_last_in_slot {
|
||||
data_header.flags |= ShredFlags::LAST_SHRED_IN_SLOT
|
||||
}
|
||||
|
||||
let mut cursor = Cursor::new(&mut payload[..]);
|
||||
bincode::serialize_into(&mut cursor, &common_header).unwrap();
|
||||
bincode::serialize_into(&mut cursor, &data_header).unwrap();
|
||||
|
@ -496,6 +492,12 @@ impl Shred {
|
|||
payload: self.payload.len(),
|
||||
});
|
||||
}
|
||||
let flags = self.data_header.flags;
|
||||
if flags.intersects(ShredFlags::LAST_SHRED_IN_SLOT)
|
||||
&& !flags.contains(ShredFlags::DATA_COMPLETE_SHRED)
|
||||
{
|
||||
return Err(Error::InvalidShredFlags(self.data_header.flags.bits()));
|
||||
}
|
||||
}
|
||||
ShredType::Code => {
|
||||
let num_coding_shreds = u32::from(self.coding_header.num_coding_shreds);
|
||||
|
@ -638,18 +640,6 @@ impl Shred {
|
|||
bincode::serialize_into(buffer, &self.data_header).unwrap();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn unset_data_complete(&mut self) {
|
||||
if self.is_data() {
|
||||
self.data_header
|
||||
.flags
|
||||
.remove(ShredFlags::DATA_COMPLETE_SHRED);
|
||||
}
|
||||
// Data header starts after the shred common header
|
||||
let buffer = &mut self.payload[SIZE_OF_COMMON_SHRED_HEADER..];
|
||||
bincode::serialize_into(buffer, &self.data_header).unwrap();
|
||||
}
|
||||
|
||||
pub fn data_complete(&self) -> bool {
|
||||
if self.is_data() {
|
||||
self.data_header
|
||||
|
@ -909,7 +899,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn test_invalid_parent_offset() {
|
||||
let shred = Shred::new_from_data(10, 0, 1000, &[1, 2, 3], false, false, 0, 1, 0);
|
||||
let shred = Shred::new_from_data(10, 0, 1000, &[1, 2, 3], ShredFlags::empty(), 0, 1, 0);
|
||||
let mut packet = Packet::default();
|
||||
shred.copy_to_packet(&mut packet);
|
||||
let shred_res = Shred::new_from_serialized_shred(packet.data.to_vec());
|
||||
|
@ -933,7 +923,7 @@ mod tests {
|
|||
fn test_shred_offsets() {
|
||||
solana_logger::setup();
|
||||
let mut packet = Packet::default();
|
||||
let shred = Shred::new_from_data(1, 3, 0, &[], true, true, 0, 0, 0);
|
||||
let shred = Shred::new_from_data(1, 3, 0, &[], ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 0);
|
||||
shred.copy_to_packet(&mut packet);
|
||||
let mut stats = ShredFetchStats::default();
|
||||
let ret = get_shred_slot_index_type(&packet, &mut stats);
|
||||
|
@ -979,7 +969,16 @@ mod tests {
|
|||
get_shred_slot_index_type(&packet, &mut stats)
|
||||
);
|
||||
|
||||
let shred = Shred::new_from_data(1, std::u32::MAX - 10, 0, &[], true, true, 0, 0, 0);
|
||||
let shred = Shred::new_from_data(
|
||||
1,
|
||||
std::u32::MAX - 10,
|
||||
0,
|
||||
&[],
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
);
|
||||
shred.copy_to_packet(&mut packet);
|
||||
assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
|
||||
assert_eq!(1, stats.index_out_of_bounds);
|
||||
|
@ -1036,11 +1035,11 @@ mod tests {
|
|||
420, // slot
|
||||
19, // index
|
||||
5, // parent_offset
|
||||
&data, true, // is_last_data
|
||||
false, // is_last_in_slot
|
||||
3, // reference_tick
|
||||
1, // version
|
||||
16, // fec_set_index
|
||||
&data,
|
||||
ShredFlags::DATA_COMPLETE_SHRED,
|
||||
3, // reference_tick
|
||||
1, // version
|
||||
16, // fec_set_index
|
||||
);
|
||||
assert_matches!(shred.sanitize(), Ok(()));
|
||||
// Corrupt shred by making it too large
|
||||
|
@ -1076,6 +1075,15 @@ mod tests {
|
|||
shred.common_header.index = MAX_DATA_SHREDS_PER_SLOT as u32;
|
||||
assert_matches!(shred.sanitize(), Err(Error::InvalidDataShredIndex(32768)));
|
||||
}
|
||||
{
|
||||
let mut shred = shred.clone();
|
||||
shred.data_header.flags |= ShredFlags::LAST_SHRED_IN_SLOT;
|
||||
assert_matches!(shred.sanitize(), Ok(()));
|
||||
shred.data_header.flags &= !ShredFlags::DATA_COMPLETE_SHRED;
|
||||
assert_matches!(shred.sanitize(), Err(Error::InvalidShredFlags(131u8)));
|
||||
shred.data_header.flags |= ShredFlags::SHRED_TICK_REFERENCE_MASK;
|
||||
assert_matches!(shred.sanitize(), Err(Error::InvalidShredFlags(191u8)));
|
||||
}
|
||||
{
|
||||
shred.data_header.size = shred.payload().len() as u16 + 1;
|
||||
assert_matches!(
|
||||
|
@ -1163,10 +1171,10 @@ mod tests {
|
|||
#[test]
|
||||
fn test_serde_compat_shred_data() {
|
||||
const SEED: &str = "6qG9NGWEtoTugS4Zgs46u8zTccEJuRHtrNMiUayLHCxt";
|
||||
const PAYLOAD: &str = "Cuk5B7qosCx42HcZjwcUKPpeqE43sDhx1RFut5rEAk54dV\
|
||||
JFPrGEBYuPJSwaaNNbGyas9AuLS66NcFxNAcBzmBcb3gSmNrfQzmTUev5jSEeQRyqE5WG\
|
||||
rGwC67mS5QXmokCtVEdXu9B1SLQFBMB62CQGVEjqV1r6jz4xd5Zg1AyyVCjGpRe8ooqMB\
|
||||
1doeATLCEBhjnPPYLLyZGqPfrqfz5huq8BCoVC8DCMWFJhxtPLA5XHPPpxieAhDBmS";
|
||||
const PAYLOAD: &str = "hNX8YgJCQwSFGJkZ6qZLiepwPjpctC9UCsMD1SNNQurBXv\
|
||||
rm7KKfLmPRMM9CpWHt6MsJuEWpDXLGwH9qdziJzGKhBMfYH63avcchjdaUiMqzVip7cUD\
|
||||
kqZ9zZJMrHCCUDnxxKMupsJWKroUSjKeo7hrug2KfHah85VckXpRna4R9QpH7tf2WVBTD\
|
||||
M4m3EerctsEQs8eZaTRxzTVkhtJYdNf74KZbH58dc3Yn2qUxF1mexWoPS6L5oZBatx";
|
||||
let mut rng = {
|
||||
let seed = <[u8; 32]>::try_from(bs58_decode(SEED)).unwrap();
|
||||
ChaChaRng::from_seed(seed)
|
||||
|
@ -1179,11 +1187,10 @@ mod tests {
|
|||
28685, // index
|
||||
36390, // parent_offset
|
||||
&data, // data
|
||||
false, // is_last_data
|
||||
true, // is_last_in_slot
|
||||
37, // reference_tick
|
||||
45189, // version
|
||||
28657, // fec_set_index
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
37, // reference_tick
|
||||
45189, // version
|
||||
28657, // fec_set_index
|
||||
);
|
||||
shred.sign(&keypair);
|
||||
assert!(shred.verify(&keypair.pubkey()));
|
||||
|
@ -1225,11 +1232,10 @@ mod tests {
|
|||
21443, // index
|
||||
51279, // parent_offset
|
||||
&[], // data
|
||||
true, // is_last_data
|
||||
false, // is_last_in_slot
|
||||
49, // reference_tick
|
||||
59445, // version
|
||||
21414, // fec_set_index
|
||||
ShredFlags::DATA_COMPLETE_SHRED,
|
||||
49, // reference_tick
|
||||
59445, // version
|
||||
21414, // fec_set_index
|
||||
);
|
||||
shred.sign(&keypair);
|
||||
assert!(shred.verify(&keypair.pubkey()));
|
||||
|
@ -1298,13 +1304,20 @@ mod tests {
|
|||
#[test]
|
||||
fn test_shred_flags() {
|
||||
fn make_shred(is_last_data: bool, is_last_in_slot: bool, reference_tick: u8) -> Shred {
|
||||
let flags = if is_last_in_slot {
|
||||
assert!(is_last_data);
|
||||
ShredFlags::LAST_SHRED_IN_SLOT
|
||||
} else if is_last_data {
|
||||
ShredFlags::DATA_COMPLETE_SHRED
|
||||
} else {
|
||||
ShredFlags::empty()
|
||||
};
|
||||
Shred::new_from_data(
|
||||
0, // slot
|
||||
0, // index
|
||||
0, // parent_offset
|
||||
&[], // data
|
||||
is_last_data,
|
||||
is_last_in_slot,
|
||||
flags,
|
||||
reference_tick,
|
||||
0, // version
|
||||
0, // fec_set_index
|
||||
|
@ -1326,15 +1339,37 @@ mod tests {
|
|||
}
|
||||
for is_last_data in [false, true] {
|
||||
for is_last_in_slot in [false, true] {
|
||||
// LAST_SHRED_IN_SLOT also implies DATA_COMPLETE_SHRED. So it
|
||||
// cannot be LAST_SHRED_IN_SLOT if not DATA_COMPLETE_SHRED.
|
||||
let is_last_in_slot = is_last_in_slot && is_last_data;
|
||||
for reference_tick in [0, 37, 63, 64, 80, 128, 255] {
|
||||
let mut shred = make_shred(is_last_data, is_last_in_slot, reference_tick);
|
||||
check_shred_flags(&shred, is_last_data, is_last_in_slot, reference_tick);
|
||||
shred.set_last_in_slot();
|
||||
check_shred_flags(&shred, is_last_data, true, reference_tick);
|
||||
shred.unset_data_complete();
|
||||
check_shred_flags(&shred, false, true, reference_tick);
|
||||
check_shred_flags(&shred, true, true, reference_tick);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_shred_flags_serde() {
|
||||
let flags: ShredFlags = bincode::deserialize(&[0b0111_0001]).unwrap();
|
||||
assert!(flags.contains(ShredFlags::DATA_COMPLETE_SHRED));
|
||||
assert!(!flags.contains(ShredFlags::LAST_SHRED_IN_SLOT));
|
||||
assert_eq!((flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits(), 49u8);
|
||||
assert_eq!(bincode::serialize(&flags).unwrap(), [0b0111_0001]);
|
||||
|
||||
let flags: ShredFlags = bincode::deserialize(&[0b1110_0101]).unwrap();
|
||||
assert!(flags.contains(ShredFlags::DATA_COMPLETE_SHRED));
|
||||
assert!(flags.contains(ShredFlags::LAST_SHRED_IN_SLOT));
|
||||
assert_eq!((flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits(), 37u8);
|
||||
assert_eq!(bincode::serialize(&flags).unwrap(), [0b1110_0101]);
|
||||
|
||||
let flags: ShredFlags = bincode::deserialize(&[0b1011_1101]).unwrap();
|
||||
assert!(!flags.contains(ShredFlags::DATA_COMPLETE_SHRED));
|
||||
assert!(!flags.contains(ShredFlags::LAST_SHRED_IN_SLOT));
|
||||
assert_eq!((flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits(), 61u8);
|
||||
assert_eq!(bincode::serialize(&flags).unwrap(), [0b1011_1101]);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,6 +1,8 @@
|
|||
use {
|
||||
crate::{
|
||||
shred::{Error, Shred, MAX_DATA_SHREDS_PER_FEC_BLOCK, SIZE_OF_DATA_SHRED_PAYLOAD},
|
||||
shred::{
|
||||
Error, Shred, ShredFlags, MAX_DATA_SHREDS_PER_FEC_BLOCK, SIZE_OF_DATA_SHRED_PAYLOAD,
|
||||
},
|
||||
shred_stats::ProcessShredsStats,
|
||||
},
|
||||
rayon::{prelude::*, ThreadPool},
|
||||
|
@ -113,8 +115,14 @@ impl Shredder {
|
|||
let last_shred_index = next_shred_index + num_shreds as u32 - 1;
|
||||
// 1) Generate data shreds
|
||||
let make_data_shred = |shred_index: u32, data| {
|
||||
let is_last_data = shred_index == last_shred_index;
|
||||
let is_last_in_slot = is_last_data && is_last_in_slot;
|
||||
let flags = if shred_index != last_shred_index {
|
||||
ShredFlags::empty()
|
||||
} else if is_last_in_slot {
|
||||
// LAST_SHRED_IN_SLOT also implies DATA_COMPLETE_SHRED.
|
||||
ShredFlags::LAST_SHRED_IN_SLOT
|
||||
} else {
|
||||
ShredFlags::DATA_COMPLETE_SHRED
|
||||
};
|
||||
let parent_offset = self.slot - self.parent_slot;
|
||||
let fec_set_index = Self::fec_set_index(shred_index, fec_set_offset);
|
||||
let mut shred = Shred::new_from_data(
|
||||
|
@ -122,8 +130,7 @@ impl Shredder {
|
|||
shred_index,
|
||||
parent_offset as u16,
|
||||
data,
|
||||
is_last_data,
|
||||
is_last_in_slot,
|
||||
flags,
|
||||
self.reference_tick,
|
||||
self.version,
|
||||
fec_set_index.unwrap(),
|
||||
|
@ -351,8 +358,7 @@ mod tests {
|
|||
use {
|
||||
super::*,
|
||||
crate::shred::{
|
||||
max_entries_per_n_shred, max_ticks_per_n_shreds, verify_test_data_shred, ShredFlags,
|
||||
ShredType,
|
||||
max_entries_per_n_shred, max_ticks_per_n_shreds, verify_test_data_shred, ShredType,
|
||||
},
|
||||
bincode::serialized_size,
|
||||
matches::assert_matches,
|
||||
|
|
|
@ -459,7 +459,7 @@ pub fn sign_shreds_gpu(
|
|||
pub mod tests {
|
||||
use {
|
||||
super::*,
|
||||
crate::shred::{Shred, SIZE_OF_DATA_SHRED_PAYLOAD},
|
||||
crate::shred::{Shred, ShredFlags, SIZE_OF_DATA_SHRED_PAYLOAD},
|
||||
solana_sdk::signature::{Keypair, Signer},
|
||||
};
|
||||
|
||||
|
@ -471,8 +471,7 @@ pub mod tests {
|
|||
0xc0de,
|
||||
0xdead,
|
||||
&[1, 2, 3, 4],
|
||||
true,
|
||||
true,
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
0,
|
||||
0,
|
||||
0xc0de,
|
||||
|
@ -517,8 +516,7 @@ pub mod tests {
|
|||
0xc0de,
|
||||
0xdead,
|
||||
&[1, 2, 3, 4],
|
||||
true,
|
||||
true,
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
0,
|
||||
0,
|
||||
0xc0de,
|
||||
|
@ -572,8 +570,7 @@ pub mod tests {
|
|||
0xc0de,
|
||||
0xdead,
|
||||
&[1, 2, 3, 4],
|
||||
true,
|
||||
true,
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
0,
|
||||
0,
|
||||
0xc0de,
|
||||
|
@ -640,8 +637,7 @@ pub mod tests {
|
|||
0xc0de,
|
||||
i as u16,
|
||||
&[5; SIZE_OF_DATA_SHRED_PAYLOAD],
|
||||
true,
|
||||
true,
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
1,
|
||||
2,
|
||||
0xc0de,
|
||||
|
@ -686,8 +682,7 @@ pub mod tests {
|
|||
0xc0de,
|
||||
0xdead,
|
||||
&[1, 2, 3, 4],
|
||||
true,
|
||||
true,
|
||||
ShredFlags::LAST_SHRED_IN_SLOT,
|
||||
0,
|
||||
0,
|
||||
0xc0de,
|
||||
|
|
Loading…
Reference in New Issue