blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

enforces that LAST_SHRED_IN_SLOT is also DATA_COMPLETE_SHRED (#24892) 

A data shred cannot be LAST_SHRED_IN_SLOT if not also DATA_COMPLETE_SHRED.
So LAST_SHRED_IN_SLOT should also imply DATA_COMPLETE_SHRED:
https://github.com/solana-labs/solana/blob/74b586ae7/ledger/src/shredder.rs#L116-L117
https://github.com/solana-labs/solana/blob/74b586ae7/core/src/broadcast_stage/standard_broadcast_run.rs#L80-L81

However current shred constructs allow specifying a shred which is
LAST_SHRED_IN_SLOT but not DATA_COMPLETE_SHRED:
https://github.com/solana-labs/solana/blob/74b586ae7/ledger/src/shred.rs#L117-L118
https://github.com/solana-labs/solana/blob/74b586ae7/ledger/src/shred.rs#L272-L273

The commit updates ShredFlags so that if a shred is not
DATA_COMPLETE_SHRED it cannot be LAST_SHRED_IN_SLOT either.
This commit is contained in:
behzad nouri 2022-05-02 23:33:53 +00:00 committed by GitHub
parent e83efe678c
commit eff59193db
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 236 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
core/benches/cluster_info.rs
View File

@ -16,7 +16,7 @@ use {
},
solana_ledger::{
genesis_utils::{create_genesis_config, GenesisConfigInfo},
shred::Shred,
shred::{Shred, ShredFlags},
},
solana_runtime::{bank::Bank, bank_forks::BankForks},
solana_sdk::{
@ -51,7 +51,7 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
const NUM_SHREDS: usize = 32;
let shred = Shred::new_from_data(0, 0, 0, &[], false, false, 0, 0, 0);
let shred = Shred::new_from_data(0, 0, 0, &[], ShredFlags::empty(), 0, 0, 0);
let shreds = vec![shred; NUM_SHREDS];
let mut stakes = HashMap::new();
const NUM_PEERS: usize = 200;

13
core/benches/cluster_nodes.rs
View File

@ -11,7 +11,7 @@ use {
solana_gossip::contact_info::ContactInfo,
solana_ledger::{
genesis_utils::{create_genesis_config, GenesisConfigInfo},
shred::Shred,
shred::{Shred, ShredFlags},
},
solana_runtime::bank::Bank,
solana_sdk::{clock::Slot, pubkey::Pubkey},
@ -39,7 +39,16 @@ fn get_retransmit_peers_deterministic(
let parent_offset = if slot == 0 { 0 } else { 1 };
for i in 0..num_simulated_shreds {
let index = i as u32;
let shred = Shred::new_from_data(slot, index, parent_offset, &[], false, false, 0, 0, 0);
let shred = Shred::new_from_data(
slot,
index,
parent_offset,
&[],
ShredFlags::empty(),
0,
0,
0,
);
let (_neighbors, _children) = cluster_nodes.get_retransmit_peers(
*slot_leader,
&shred,

6
core/benches/shredder.rs
View File

@ -8,8 +8,8 @@ use {
raptorq::{Decoder, Encoder},
solana_entry::entry::{create_ticks, Entry},
solana_ledger::shred::{
max_entries_per_n_shred, max_ticks_per_n_shreds, ProcessShredsStats, Shred, Shredder,
MAX_DATA_SHREDS_PER_FEC_BLOCK, SIZE_OF_DATA_SHRED_PAYLOAD,
max_entries_per_n_shred, max_ticks_per_n_shreds, ProcessShredsStats, Shred, ShredFlags,
Shredder, MAX_DATA_SHREDS_PER_FEC_BLOCK, SIZE_OF_DATA_SHRED_PAYLOAD,
},
solana_perf::test_tx,
solana_sdk::{hash::Hash, packet::PACKET_DATA_SIZE, signature::Keypair},
@ -123,7 +123,7 @@ fn bench_deshredder(bencher: &mut Bencher) {
fn bench_deserialize_hdr(bencher: &mut Bencher) {
let data = vec![0; SIZE_OF_DATA_SHRED_PAYLOAD];
let shred = Shred::new_from_data(2, 1, 1, &data, true, true, 0, 0, 1);
let shred = Shred::new_from_data(2, 1, 1, &data, ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 1);
bencher.iter(|| {
let payload = shred.payload().clone();

5
core/src/broadcast_stage/standard_broadcast_run.rs
View File

@ -76,9 +76,8 @@ impl StandardBroadcastRun {
state.slot,
state.next_shred_index,
parent_offset as u16,
&[], // data
true, // is_last_in_fec_set
true, // is_last_in_slot
&[], // data
ShredFlags::LAST_SHRED_IN_SLOT,
reference_tick,
self.shred_version,
fec_set_index.unwrap(),

8
core/src/outstanding_requests.rs
View File

@ -85,7 +85,9 @@ pub struct RequestStatus<T> {
#[cfg(test)]
pub(crate) mod tests {
use {
super::*, crate::serve_repair::ShredRepairType, solana_ledger::shred::Shred,
super::*,
crate::serve_repair::ShredRepairType,
solana_ledger::shred::{Shred, ShredFlags},
solana_sdk::timing::timestamp,
};
@ -107,7 +109,7 @@ pub(crate) mod tests {
let repair_type = ShredRepairType::Orphan(9);
let mut outstanding_requests = OutstandingRequests::default();
let nonce = outstanding_requests.add_request(repair_type, timestamp());
let shred = Shred::new_from_data(0, 0, 0, &[], false, false, 0, 0, 0);
let shred = Shred::new_from_data(0, 0, 0, &[], ShredFlags::empty(), 0, 0, 0);
let expire_timestamp = outstanding_requests
.requests
@ -127,7 +129,7 @@ pub(crate) mod tests {
let mut outstanding_requests = OutstandingRequests::default();
let nonce = outstanding_requests.add_request(repair_type, timestamp());
let shred = Shred::new_from_data(0, 0, 0, &[], false, false, 0, 0, 0);
let shred = Shred::new_from_data(0, 0, 0, &[], ShredFlags::empty(), 0, 0, 0);
let mut expire_timestamp = outstanding_requests
.requests
.get(&nonce)

8
core/src/repair_response.rs
View File

@ -52,7 +52,10 @@ pub fn nonce(buf: &[u8]) -> Option<Nonce> {
mod test {
use {
super::*,
solana_ledger::{shred::Shred, sigverify_shreds::verify_shred_cpu},
solana_ledger::{
shred::{Shred, ShredFlags},
sigverify_shreds::verify_shred_cpu,
},
solana_sdk::{
packet::PacketFlags,
signature::{Keypair, Signer},
@ -70,8 +73,7 @@ mod test {
0xc0de,
0xdead,
&[1, 2, 3, 4],
true,
true,
ShredFlags::LAST_SHRED_IN_SLOT,
0,
0,
0xc0de,

13
core/src/replay_stage.rs
View File

@ -3123,7 +3123,7 @@ impl ReplayStage {
}
#[cfg(test)]
pub mod tests {
pub(crate) mod tests {
use {
super::*,
crate::{
@ -3142,7 +3142,7 @@ pub mod tests {
create_new_tmp_ledger,
genesis_utils::{create_genesis_config, create_genesis_config_with_leader},
get_tmp_ledger_path,
shred::{Shred, SIZE_OF_DATA_SHRED_PAYLOAD},
shred::{Shred, ShredFlags, SIZE_OF_DATA_SHRED_PAYLOAD},
},
solana_rpc::{
optimistically_confirmed_bank_tracker::OptimisticallyConfirmedBank,
@ -3748,11 +3748,10 @@ pub mod tests {
0, // index,
parent_offset as u16,
&gibberish,
true, // is_last_data
false, // is_last_in_slot
0, // reference_tick
0, // version
0, // fec_set_index
ShredFlags::DATA_COMPLETE_SHRED,
0, // reference_tick
0, // version
0, // fec_set_index
);
vec![shred]
});

35
core/src/retransmit_stage.rs
View File

@ -528,27 +528,54 @@ impl RetransmitStage {
#[cfg(test)]
mod tests {
use super::*;
use {super::*, solana_ledger::shred::ShredFlags};
#[test]
fn test_already_received() {
let slot = 1;
let index = 5;
let version = 0x40;
let shred = Shred::new_from_data(slot, index, 0, &[], true, true, 0, version, 0);
let shred = Shred::new_from_data(
slot,
index,
0,
&[],
ShredFlags::LAST_SHRED_IN_SLOT,
0,
version,
0,
);
let shreds_received = Arc::new(Mutex::new((LruCache::new(100), PacketHasher::default())));
// unique shred for (1, 5) should pass
assert!(!should_skip_retransmit(&shred, &shreds_received));
// duplicate shred for (1, 5) blocked
assert!(should_skip_retransmit(&shred, &shreds_received));
let shred = Shred::new_from_data(slot, index, 2, &[], true, true, 0, version, 0);
let shred = Shred::new_from_data(
slot,
index,
2,
&[],
ShredFlags::LAST_SHRED_IN_SLOT,
0,
version,
0,
);
// first duplicate shred for (1, 5) passed
assert!(!should_skip_retransmit(&shred, &shreds_received));
// then blocked
assert!(should_skip_retransmit(&shred, &shreds_received));
let shred = Shred::new_from_data(slot, index, 8, &[], true, true, 0, version, 0);
let shred = Shred::new_from_data(
slot,
index,
8,
&[],
ShredFlags::LAST_SHRED_IN_SLOT,
0,
version,
0,
);
// 2nd duplicate shred for (1, 5) blocked
assert!(should_skip_retransmit(&shred, &shreds_received));
assert!(should_skip_retransmit(&shred, &shreds_received));

6
core/src/serve_repair.rs
View File

@ -761,7 +761,7 @@ mod tests {
blockstore::make_many_slot_entries,
blockstore_processor::fill_blockstore_slot_with_ticks,
get_tmp_ledger_path,
shred::{max_ticks_per_n_shreds, Shred},
shred::{max_ticks_per_n_shreds, Shred, ShredFlags},
},
solana_perf::packet::Packet,
solana_sdk::{hash::Hash, pubkey::Pubkey, signature::Keypair, timing::timestamp},
@ -876,7 +876,7 @@ mod tests {
nonce,
);
assert!(rv.is_none());
let shred = Shred::new_from_data(slot, 1, 1, &[], false, false, 0, 2, 0);
let shred = Shred::new_from_data(slot, 1, 1, &[], ShredFlags::empty(), 0, 2, 0);
blockstore
.insert_shreds(vec![shred], None, false)
@ -1306,7 +1306,7 @@ mod tests {
#[test]
fn test_verify_shred_response() {
fn new_test_data_shred(slot: Slot, index: u32) -> Shred {
Shred::new_from_data(slot, index, 1, &[], false, false, 0, 0, 0)
Shred::new_from_data(slot, index, 1, &[], ShredFlags::empty(), 0, 0, 0)
}
let repair = ShredRepairType::Orphan(9);
// Ensure new options are addded to this test

35
core/src/shred_fetch_stage.rs
View File

@ -216,7 +216,10 @@ impl ShredFetchStage {
mod tests {
use {
super::*,
solana_ledger::{blockstore::MAX_DATA_SHREDS_PER_SLOT, shred::Shred},
solana_ledger::{
blockstore::MAX_DATA_SHREDS_PER_SLOT,
shred::{Shred, ShredFlags},
},
};
#[test]
@ -229,14 +232,13 @@ mod tests {
let slot = 1;
let shred = Shred::new_from_data(
slot,
3, // shred index
0, // parent offset
&[], // data
true, // is_last_in_fec_set
true, // is_last_in_slot
0, // reference_tick
0, // version
3, // fec_set_index
3, // shred index
0, // parent offset
&[], // data
ShredFlags::LAST_SHRED_IN_SLOT,
0, // reference_tick
0, // version
3, // fec_set_index
);
shred.copy_to_packet(&mut packet);
@ -300,7 +302,7 @@ mod tests {
);
assert_eq!(stats.index_overrun, 1);
assert!(packet.meta.discard());
let shred = Shred::new_from_data(1, 3, 0, &[], true, true, 0, 0, 0);
let shred = Shred::new_from_data(1, 3, 0, &[], ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 0);
shred.copy_to_packet(&mut packet);
// rejected slot is 1, root is 3
@ -342,7 +344,16 @@ mod tests {
);
assert!(packet.meta.discard());
let shred = Shred::new_from_data(1_000_000, 3, 0, &[], true, true, 0, 0, 0);
let shred = Shred::new_from_data(
1_000_000,
3,
0,
&[],
ShredFlags::LAST_SHRED_IN_SLOT,
0,
0,
0,
);
shred.copy_to_packet(&mut packet);
// Slot 1 million is too high
@ -359,7 +370,7 @@ mod tests {
assert!(packet.meta.discard());
let index = MAX_DATA_SHREDS_PER_SLOT as u32;
let shred = Shred::new_from_data(5, index, 0, &[], true, true, 0, 0, 0);
let shred = Shred::new_from_data(5, index, 0, &[], ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 0);
shred.copy_to_packet(&mut packet);
ShredFetchStage::process_packet(
&mut packet,

35
core/src/sigverify_shreds.rs
View File

@ -69,7 +69,10 @@ impl SigVerifier for ShredSigVerifier {
pub mod tests {
use {
super::*,
solana_ledger::{genesis_utils::create_genesis_config_with_leader, shred::Shred},
solana_ledger::{
genesis_utils::create_genesis_config_with_leader,
shred::{Shred, ShredFlags},
},
solana_perf::packet::Packet,
solana_runtime::bank::Bank,
solana_sdk::signature::{Keypair, Signer},
@ -83,8 +86,7 @@ pub mod tests {
0xc0de,
0xdead,
&[1, 2, 3, 4],
true,
true,
ShredFlags::LAST_SHRED_IN_SLOT,
0,
0,
0xc0de,
@ -102,8 +104,7 @@ pub mod tests {
0xc0de,
0xdead,
&[1, 2, 3, 4],
true,
true,
ShredFlags::LAST_SHRED_IN_SLOT,
0,
0,
0xc0de,
@ -131,14 +132,30 @@ pub mod tests {
let mut batches = vec![PacketBatch::default()];
batches[0].packets.resize(2, Packet::default());
let mut shred =
Shred::new_from_data(0, 0xc0de, 0xdead, &[1, 2, 3, 4], true, true, 0, 0, 0xc0de);
let mut shred = Shred::new_from_data(
0,
0xc0de,
0xdead,
&[1, 2, 3, 4],
ShredFlags::LAST_SHRED_IN_SLOT,
0,
0,
0xc0de,
);
shred.sign(&leader_keypair);
batches[0].packets[0].data[0..shred.payload().len()].copy_from_slice(shred.payload());
batches[0].packets[0].meta.size = shred.payload().len();
let mut shred =
Shred::new_from_data(0, 0xbeef, 0xc0de, &[1, 2, 3, 4], true, true, 0, 0, 0xc0de);
let mut shred = Shred::new_from_data(
0,
0xbeef,
0xc0de,
&[1, 2, 3, 4],
ShredFlags::LAST_SHRED_IN_SLOT,
0,
0,
0xc0de,
);
let wrong_keypair = Keypair::new();
shred.sign(&wrong_keypair);
batches[0].packets[1].data[0..shred.payload().len()].copy_from_slice(shred.payload());

8
ledger/benches/sigverify_shreds.rs
View File

@ -3,7 +3,7 @@
extern crate test;
use {
solana_ledger::{
shred::{Shred, SIZE_OF_DATA_SHRED_PAYLOAD},
shred::{Shred, ShredFlags, SIZE_OF_DATA_SHRED_PAYLOAD},
sigverify_shreds::{sign_shreds_cpu, sign_shreds_gpu, sign_shreds_gpu_pinned_keypair},
},
solana_perf::{
@ -33,8 +33,7 @@ fn bench_sigverify_shreds_sign_gpu(bencher: &mut Bencher) {
0xc0de,
0xdead,
&[5; SIZE_OF_DATA_SHRED_PAYLOAD],
true,
true,
ShredFlags::LAST_SHRED_IN_SLOT,
1,
2,
0,
@ -65,8 +64,7 @@ fn bench_sigverify_shreds_sign_cpu(bencher: &mut Bencher) {
0xc0de,
0xdead,
&[5; SIZE_OF_DATA_SHRED_PAYLOAD],
true,
true,
ShredFlags::LAST_SHRED_IN_SLOT,
1,
2,
0,

35
ledger/src/blockstore.rs
View File

@ -4286,7 +4286,7 @@ pub mod tests {
blockstore_db::BlockstoreRocksFifoOptions,
genesis_utils::{create_genesis_config, GenesisConfigInfo},
leader_schedule::{FixedSchedule, LeaderSchedule},
shred::max_ticks_per_n_shreds,
shred::{max_ticks_per_n_shreds, ShredFlags},
},
assert_matches::assert_matches,
bincode::serialize,
@ -5719,8 +5719,7 @@ pub mod tests {
(i * gap) as u32,
0,
&[],
false,
false,
ShredFlags::empty(),
i as u8,
0,
(i * gap) as u32,
@ -5850,10 +5849,9 @@ pub mod tests {
let parent_offset = shred5.slot() - shred5.parent().unwrap();
parent_offset as u16
},
&[], // data
true, // is_last_data
true, // is_last_in_slot
0, // reference_tick
&[], // data
ShredFlags::LAST_SHRED_IN_SLOT,
0, // reference_tick
shred5.version(),
shred5.fec_set_index(),
);
@ -6255,8 +6253,7 @@ pub mod tests {
next_shred_index as u32,
1,
&[1, 1, 1],
true,
true,
ShredFlags::LAST_SHRED_IN_SLOT,
0,
0,
next_shred_index as u32,
@ -8776,26 +8773,6 @@ pub mod tests {
}
}
#[test]
fn test_remove_shred_data_complete_flag() {
let ledger_path = get_tmp_ledger_path_auto_delete!();
let blockstore = Blockstore::open(ledger_path.path()).unwrap();
let (mut shreds, entries) = make_slot_entries(0, 0, 1);
// Remove the data complete flag from the last shred
shreds[0].unset_data_complete();
blockstore.insert_shreds(shreds, None, false).unwrap();
// Check that the `data_complete` flag was unset in the stored shred, but the
// `last_in_slot` flag is set.
let stored_shred = &blockstore.get_data_shreds_for_slot(0, 0).unwrap()[0];
assert!(!stored_shred.data_complete());
assert!(stored_shred.last_in_slot());
assert_eq!(entries, blockstore.get_any_valid_slot_entries(0, 0));
}
fn make_large_tx_entry(num_txs: usize) -> Entry {
let txs: Vec<_> = (0..num_txs)
.into_iter()

147
ledger/src/shred.rs
View File

@ -109,12 +109,14 @@ const ENCODED_PAYLOAD_SIZE: usize = SHRED_PAYLOAD_SIZE - SIZE_OF_CODING_SHRED_HE
pub const MAX_DATA_SHREDS_PER_FEC_BLOCK: u32 = 32;
// LAST_SHRED_IN_SLOT also implies DATA_COMPLETE_SHRED.
// So it cannot be LAST_SHRED_IN_SLOT if not also DATA_COMPLETE_SHRED.
bitflags! {
#[derive(Default, Serialize, Deserialize)]
pub struct ShredFlags:u8 {
const SHRED_TICK_REFERENCE_MASK = 0b0011_1111;
const DATA_COMPLETE_SHRED = 0b0100_0000;
const LAST_SHRED_IN_SLOT = 0b1000_0000;
const DATA_COMPLETE_SHRED = 0b0100_0000;
const LAST_SHRED_IN_SLOT = 0b1100_0000;
}
}
@ -138,6 +140,8 @@ pub enum Error {
InvalidParentSlot { slot: Slot, parent_slot: Slot },
#[error("Invalid payload size: {0}")]
InvalidPayloadSize(/*payload size:*/ usize),
#[error("Invalid shred flags: {0}")]
InvalidShredFlags(u8),
#[error("Invalid shred type")]
InvalidShredType,
}
@ -240,8 +244,7 @@ impl Shred {
index: u32,
parent_offset: u16,
data: &[u8],
is_last_data: bool,
is_last_in_slot: bool,
flags: ShredFlags,
reference_tick: u8,
version: u16,
fec_set_index: u32,
@ -257,26 +260,19 @@ impl Shred {
};
let size = (data.len() + SIZE_OF_DATA_SHRED_HEADER + SIZE_OF_COMMON_SHRED_HEADER) as u16;
let mut data_header = DataShredHeader {
parent_offset,
flags: unsafe {
let flags = flags
| unsafe {
ShredFlags::from_bits_unchecked(
ShredFlags::SHRED_TICK_REFERENCE_MASK
.bits()
.min(reference_tick),
)
},
};
let data_header = DataShredHeader {
parent_offset,
flags,
size,
};
if is_last_data {
data_header.flags |= ShredFlags::DATA_COMPLETE_SHRED
}
if is_last_in_slot {
data_header.flags |= ShredFlags::LAST_SHRED_IN_SLOT
}
let mut cursor = Cursor::new(&mut payload[..]);
bincode::serialize_into(&mut cursor, &common_header).unwrap();
bincode::serialize_into(&mut cursor, &data_header).unwrap();
@ -496,6 +492,12 @@ impl Shred {
payload: self.payload.len(),
});
}
let flags = self.data_header.flags;
if flags.intersects(ShredFlags::LAST_SHRED_IN_SLOT)
&& !flags.contains(ShredFlags::DATA_COMPLETE_SHRED)
{
return Err(Error::InvalidShredFlags(self.data_header.flags.bits()));
}
}
ShredType::Code => {
let num_coding_shreds = u32::from(self.coding_header.num_coding_shreds);
@ -638,18 +640,6 @@ impl Shred {
bincode::serialize_into(buffer, &self.data_header).unwrap();
}
#[cfg(test)]
pub(crate) fn unset_data_complete(&mut self) {
if self.is_data() {
self.data_header
.flags
.remove(ShredFlags::DATA_COMPLETE_SHRED);
}
// Data header starts after the shred common header
let buffer = &mut self.payload[SIZE_OF_COMMON_SHRED_HEADER..];
bincode::serialize_into(buffer, &self.data_header).unwrap();
}
pub fn data_complete(&self) -> bool {
if self.is_data() {
self.data_header
@ -909,7 +899,7 @@ mod tests {
#[test]
fn test_invalid_parent_offset() {
let shred = Shred::new_from_data(10, 0, 1000, &[1, 2, 3], false, false, 0, 1, 0);
let shred = Shred::new_from_data(10, 0, 1000, &[1, 2, 3], ShredFlags::empty(), 0, 1, 0);
let mut packet = Packet::default();
shred.copy_to_packet(&mut packet);
let shred_res = Shred::new_from_serialized_shred(packet.data.to_vec());
@ -933,7 +923,7 @@ mod tests {
fn test_shred_offsets() {
solana_logger::setup();
let mut packet = Packet::default();
let shred = Shred::new_from_data(1, 3, 0, &[], true, true, 0, 0, 0);
let shred = Shred::new_from_data(1, 3, 0, &[], ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 0);
shred.copy_to_packet(&mut packet);
let mut stats = ShredFetchStats::default();
let ret = get_shred_slot_index_type(&packet, &mut stats);
@ -979,7 +969,16 @@ mod tests {
get_shred_slot_index_type(&packet, &mut stats)
);
let shred = Shred::new_from_data(1, std::u32::MAX - 10, 0, &[], true, true, 0, 0, 0);
let shred = Shred::new_from_data(
1,
std::u32::MAX - 10,
0,
&[],
ShredFlags::LAST_SHRED_IN_SLOT,
0,
0,
0,
);
shred.copy_to_packet(&mut packet);
assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
assert_eq!(1, stats.index_out_of_bounds);
@ -1036,11 +1035,11 @@ mod tests {
420, // slot
19, // index
5, // parent_offset
&data, true, // is_last_data
false, // is_last_in_slot
3, // reference_tick
1, // version
16, // fec_set_index
&data,
ShredFlags::DATA_COMPLETE_SHRED,
3, // reference_tick
1, // version
16, // fec_set_index
);
assert_matches!(shred.sanitize(), Ok(()));
// Corrupt shred by making it too large
@ -1076,6 +1075,15 @@ mod tests {
shred.common_header.index = MAX_DATA_SHREDS_PER_SLOT as u32;
assert_matches!(shred.sanitize(), Err(Error::InvalidDataShredIndex(32768)));
}
{
let mut shred = shred.clone();
shred.data_header.flags |= ShredFlags::LAST_SHRED_IN_SLOT;
assert_matches!(shred.sanitize(), Ok(()));
shred.data_header.flags &= !ShredFlags::DATA_COMPLETE_SHRED;
assert_matches!(shred.sanitize(), Err(Error::InvalidShredFlags(131u8)));
shred.data_header.flags |= ShredFlags::SHRED_TICK_REFERENCE_MASK;
assert_matches!(shred.sanitize(), Err(Error::InvalidShredFlags(191u8)));
}
{
shred.data_header.size = shred.payload().len() as u16 + 1;
assert_matches!(
@ -1163,10 +1171,10 @@ mod tests {
#[test]
fn test_serde_compat_shred_data() {
const SEED: &str = "6qG9NGWEtoTugS4Zgs46u8zTccEJuRHtrNMiUayLHCxt";
const PAYLOAD: &str = "Cuk5B7qosCx42HcZjwcUKPpeqE43sDhx1RFut5rEAk54dV\
JFPrGEBYuPJSwaaNNbGyas9AuLS66NcFxNAcBzmBcb3gSmNrfQzmTUev5jSEeQRyqE5WG\
rGwC67mS5QXmokCtVEdXu9B1SLQFBMB62CQGVEjqV1r6jz4xd5Zg1AyyVCjGpRe8ooqMB\
1doeATLCEBhjnPPYLLyZGqPfrqfz5huq8BCoVC8DCMWFJhxtPLA5XHPPpxieAhDBmS";
const PAYLOAD: &str = "hNX8YgJCQwSFGJkZ6qZLiepwPjpctC9UCsMD1SNNQurBXv\
rm7KKfLmPRMM9CpWHt6MsJuEWpDXLGwH9qdziJzGKhBMfYH63avcchjdaUiMqzVip7cUD\
kqZ9zZJMrHCCUDnxxKMupsJWKroUSjKeo7hrug2KfHah85VckXpRna4R9QpH7tf2WVBTD\
M4m3EerctsEQs8eZaTRxzTVkhtJYdNf74KZbH58dc3Yn2qUxF1mexWoPS6L5oZBatx";
let mut rng = {
let seed = <[u8; 32]>::try_from(bs58_decode(SEED)).unwrap();
ChaChaRng::from_seed(seed)
@ -1179,11 +1187,10 @@ mod tests {
28685, // index
36390, // parent_offset
&data, // data
false, // is_last_data
true, // is_last_in_slot
37, // reference_tick
45189, // version
28657, // fec_set_index
ShredFlags::LAST_SHRED_IN_SLOT,
37, // reference_tick
45189, // version
28657, // fec_set_index
);
shred.sign(&keypair);
assert!(shred.verify(&keypair.pubkey()));
@ -1225,11 +1232,10 @@ mod tests {
21443, // index
51279, // parent_offset
&[], // data
true, // is_last_data
false, // is_last_in_slot
49, // reference_tick
59445, // version
21414, // fec_set_index
ShredFlags::DATA_COMPLETE_SHRED,
49, // reference_tick
59445, // version
21414, // fec_set_index
);
shred.sign(&keypair);
assert!(shred.verify(&keypair.pubkey()));
@ -1298,13 +1304,20 @@ mod tests {
#[test]
fn test_shred_flags() {
fn make_shred(is_last_data: bool, is_last_in_slot: bool, reference_tick: u8) -> Shred {
let flags = if is_last_in_slot {
assert!(is_last_data);
ShredFlags::LAST_SHRED_IN_SLOT
} else if is_last_data {
ShredFlags::DATA_COMPLETE_SHRED
} else {
ShredFlags::empty()
};
Shred::new_from_data(
0, // slot
0, // index
0, // parent_offset
&[], // data
is_last_data,
is_last_in_slot,
flags,
reference_tick,
0, // version
0, // fec_set_index
@ -1326,15 +1339,37 @@ mod tests {
}
for is_last_data in [false, true] {
for is_last_in_slot in [false, true] {
// LAST_SHRED_IN_SLOT also implies DATA_COMPLETE_SHRED. So it
// cannot be LAST_SHRED_IN_SLOT if not DATA_COMPLETE_SHRED.
let is_last_in_slot = is_last_in_slot && is_last_data;
for reference_tick in [0, 37, 63, 64, 80, 128, 255] {
let mut shred = make_shred(is_last_data, is_last_in_slot, reference_tick);
check_shred_flags(&shred, is_last_data, is_last_in_slot, reference_tick);
shred.set_last_in_slot();
check_shred_flags(&shred, is_last_data, true, reference_tick);
shred.unset_data_complete();
check_shred_flags(&shred, false, true, reference_tick);
check_shred_flags(&shred, true, true, reference_tick);
}
}
}
}
#[test]
fn test_shred_flags_serde() {
let flags: ShredFlags = bincode::deserialize(&[0b0111_0001]).unwrap();
assert!(flags.contains(ShredFlags::DATA_COMPLETE_SHRED));
assert!(!flags.contains(ShredFlags::LAST_SHRED_IN_SLOT));
assert_eq!((flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits(), 49u8);
assert_eq!(bincode::serialize(&flags).unwrap(), [0b0111_0001]);
let flags: ShredFlags = bincode::deserialize(&[0b1110_0101]).unwrap();
assert!(flags.contains(ShredFlags::DATA_COMPLETE_SHRED));
assert!(flags.contains(ShredFlags::LAST_SHRED_IN_SLOT));
assert_eq!((flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits(), 37u8);
assert_eq!(bincode::serialize(&flags).unwrap(), [0b1110_0101]);
let flags: ShredFlags = bincode::deserialize(&[0b1011_1101]).unwrap();
assert!(!flags.contains(ShredFlags::DATA_COMPLETE_SHRED));
assert!(!flags.contains(ShredFlags::LAST_SHRED_IN_SLOT));
assert_eq!((flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits(), 61u8);
assert_eq!(bincode::serialize(&flags).unwrap(), [0b1011_1101]);
}
}

20
ledger/src/shredder.rs
View File

@ -1,6 +1,8 @@
use {
crate::{
shred::{Error, Shred, MAX_DATA_SHREDS_PER_FEC_BLOCK, SIZE_OF_DATA_SHRED_PAYLOAD},
shred::{
Error, Shred, ShredFlags, MAX_DATA_SHREDS_PER_FEC_BLOCK, SIZE_OF_DATA_SHRED_PAYLOAD,
},
shred_stats::ProcessShredsStats,
},
rayon::{prelude::*, ThreadPool},
@ -113,8 +115,14 @@ impl Shredder {
let last_shred_index = next_shred_index + num_shreds as u32 - 1;
// 1) Generate data shreds
let make_data_shred = |shred_index: u32, data| {
let is_last_data = shred_index == last_shred_index;
let is_last_in_slot = is_last_data && is_last_in_slot;
let flags = if shred_index != last_shred_index {
ShredFlags::empty()
} else if is_last_in_slot {
// LAST_SHRED_IN_SLOT also implies DATA_COMPLETE_SHRED.
ShredFlags::LAST_SHRED_IN_SLOT
} else {
ShredFlags::DATA_COMPLETE_SHRED
};
let parent_offset = self.slot - self.parent_slot;
let fec_set_index = Self::fec_set_index(shred_index, fec_set_offset);
let mut shred = Shred::new_from_data(
@ -122,8 +130,7 @@ impl Shredder {
shred_index,
parent_offset as u16,
data,
is_last_data,
is_last_in_slot,
flags,
self.reference_tick,
self.version,
fec_set_index.unwrap(),
@ -351,8 +358,7 @@ mod tests {
use {
super::*,
crate::shred::{
max_entries_per_n_shred, max_ticks_per_n_shreds, verify_test_data_shred, ShredFlags,
ShredType,
max_entries_per_n_shred, max_ticks_per_n_shreds, verify_test_data_shred, ShredType,
},
bincode::serialized_size,
matches::assert_matches,

17
ledger/src/sigverify_shreds.rs
View File

@ -459,7 +459,7 @@ pub fn sign_shreds_gpu(
pub mod tests {
use {
super::*,
crate::shred::{Shred, SIZE_OF_DATA_SHRED_PAYLOAD},
crate::shred::{Shred, ShredFlags, SIZE_OF_DATA_SHRED_PAYLOAD},
solana_sdk::signature::{Keypair, Signer},
};
@ -471,8 +471,7 @@ pub mod tests {
0xc0de,
0xdead,
&[1, 2, 3, 4],
true,
true,
ShredFlags::LAST_SHRED_IN_SLOT,
0,
0,
0xc0de,
@ -517,8 +516,7 @@ pub mod tests {
0xc0de,
0xdead,
&[1, 2, 3, 4],
true,
true,
ShredFlags::LAST_SHRED_IN_SLOT,
0,
0,
0xc0de,
@ -572,8 +570,7 @@ pub mod tests {
0xc0de,
0xdead,
&[1, 2, 3, 4],
true,
true,
ShredFlags::LAST_SHRED_IN_SLOT,
0,
0,
0xc0de,
@ -640,8 +637,7 @@ pub mod tests {
0xc0de,
i as u16,
&[5; SIZE_OF_DATA_SHRED_PAYLOAD],
true,
true,
ShredFlags::LAST_SHRED_IN_SLOT,
1,
2,
0xc0de,
@ -686,8 +682,7 @@ pub mod tests {
0xc0de,
0xdead,
&[1, 2, 3, 4],
true,
true,
ShredFlags::LAST_SHRED_IN_SLOT,
0,
0,
0xc0de,