blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

replaces Shred::new_empty_coding with Shred::new_from_parity_shard (#24749) 

Removing implementation details of shreds and payload offsets from
shredder, so that shredder does not need to mutate payload:
https://github.com/solana-labs/solana/blob/71ad12128/ledger/src/shred.rs#L968-L977

Also, Shred::new_from_data can simply obtain a slice as opposed to
Option<&[u8]>:
https://github.com/solana-labs/solana/blob/71ad12128/ledger/src/shred.rs#L268-L278
This commit is contained in:
behzad nouri 2022-04-27 18:04:10 +00:00 committed by GitHub
parent cdce19be29
commit 0f60665100
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 105 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/benches/cluster_info.rs
View File

@ -51,7 +51,7 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
const NUM_SHREDS: usize = 32;
let shred = Shred::new_from_data(0, 0, 0, None, false, false, 0, 0, 0);
let shred = Shred::new_from_data(0, 0, 0, &[], false, false, 0, 0, 0);
let shreds = vec![shred; NUM_SHREDS];
let mut stakes = HashMap::new();
const NUM_PEERS: usize = 200;

2
core/benches/cluster_nodes.rs
View File

@ -39,7 +39,7 @@ fn get_retransmit_peers_deterministic(
let parent_offset = if slot == 0 { 0 } else { 1 };
for i in 0..num_simulated_shreds {
let index = i as u32;
let shred = Shred::new_from_data(slot, index, parent_offset, None, false, false, 0, 0, 0);
let shred = Shred::new_from_data(slot, index, parent_offset, &[], false, false, 0, 0, 0);
let (_neighbors, _children) = cluster_nodes.get_retransmit_peers(
*slot_leader,
&shred,

2
core/benches/shredder.rs
View File

@ -123,7 +123,7 @@ fn bench_deshredder(bencher: &mut Bencher) {
fn bench_deserialize_hdr(bencher: &mut Bencher) {
let data = vec![0; SIZE_OF_DATA_SHRED_PAYLOAD];
let shred = Shred::new_from_data(2, 1, 1, Some(&data), true, true, 0, 0, 1);
let shred = Shred::new_from_data(2, 1, 1, &data, true, true, 0, 0, 1);
bencher.iter(|| {
let payload = shred.payload().clone();

2
core/src/broadcast_stage/standard_broadcast_run.rs
View File

@ -76,7 +76,7 @@ impl StandardBroadcastRun {
state.slot,
state.next_shred_index,
parent_offset as u16,
None, // data
&[], // data
true, // is_last_in_fec_set
true, // is_last_in_slot
reference_tick,

4
core/src/outstanding_requests.rs
View File

@ -107,7 +107,7 @@ pub(crate) mod tests {
let repair_type = ShredRepairType::Orphan(9);
let mut outstanding_requests = OutstandingRequests::default();
let nonce = outstanding_requests.add_request(repair_type, timestamp());
let shred = Shred::new_from_data(0, 0, 0, None, false, false, 0, 0, 0);
let shred = Shred::new_from_data(0, 0, 0, &[], false, false, 0, 0, 0);
let expire_timestamp = outstanding_requests
.requests
@ -127,7 +127,7 @@ pub(crate) mod tests {
let mut outstanding_requests = OutstandingRequests::default();
let nonce = outstanding_requests.add_request(repair_type, timestamp());
let shred = Shred::new_from_data(0, 0, 0, None, false, false, 0, 0, 0);
let shred = Shred::new_from_data(0, 0, 0, &[], false, false, 0, 0, 0);
let mut expire_timestamp = outstanding_requests
.requests
.get(&nonce)

2
core/src/repair_response.rs
View File

@ -69,7 +69,7 @@ mod test {
slot,
0xc0de,
0xdead,
Some(&[1, 2, 3, 4]),
&[1, 2, 3, 4],
true,
true,
0,

2
core/src/replay_stage.rs
View File

@ -3749,7 +3749,7 @@ pub mod tests {
bank.slot(),
0, // index,
parent_offset as u16,
Some(&gibberish),
&gibberish,
true, // is_last_data
false, // is_last_in_slot
0, // reference_tick

12
core/src/retransmit_stage.rs
View File

@ -535,37 +535,37 @@ mod tests {
let slot = 1;
let index = 5;
let version = 0x40;
let shred = Shred::new_from_data(slot, index, 0, None, true, true, 0, version, 0);
let shred = Shred::new_from_data(slot, index, 0, &[], true, true, 0, version, 0);
let shreds_received = Arc::new(Mutex::new((LruCache::new(100), PacketHasher::default())));
// unique shred for (1, 5) should pass
assert!(!should_skip_retransmit(&shred, &shreds_received));
// duplicate shred for (1, 5) blocked
assert!(should_skip_retransmit(&shred, &shreds_received));
let shred = Shred::new_from_data(slot, index, 2, None, true, true, 0, version, 0);
let shred = Shred::new_from_data(slot, index, 2, &[], true, true, 0, version, 0);
// first duplicate shred for (1, 5) passed
assert!(!should_skip_retransmit(&shred, &shreds_received));
// then blocked
assert!(should_skip_retransmit(&shred, &shreds_received));
let shred = Shred::new_from_data(slot, index, 8, None, true, true, 0, version, 0);
let shred = Shred::new_from_data(slot, index, 8, &[], true, true, 0, version, 0);
// 2nd duplicate shred for (1, 5) blocked
assert!(should_skip_retransmit(&shred, &shreds_received));
assert!(should_skip_retransmit(&shred, &shreds_received));
let shred = Shred::new_empty_coding(slot, index, 0, 1, 1, 0, version);
let shred = Shred::new_from_parity_shard(slot, index, &[], 0, 1, 1, 0, version);
// Coding at (1, 5) passes
assert!(!should_skip_retransmit(&shred, &shreds_received));
// then blocked
assert!(should_skip_retransmit(&shred, &shreds_received));
let shred = Shred::new_empty_coding(slot, index, 2, 1, 1, 0, version);
let shred = Shred::new_from_parity_shard(slot, index, &[], 2, 1, 1, 0, version);
// 2nd unique coding at (1, 5) passes
assert!(!should_skip_retransmit(&shred, &shreds_received));
// same again is blocked
assert!(should_skip_retransmit(&shred, &shreds_received));
let shred = Shred::new_empty_coding(slot, index, 3, 1, 1, 0, version);
let shred = Shred::new_from_parity_shard(slot, index, &[], 3, 1, 1, 0, version);
// Another unique coding at (1, 5) always blocked
assert!(should_skip_retransmit(&shred, &shreds_received));
assert!(should_skip_retransmit(&shred, &shreds_received));

4
core/src/serve_repair.rs
View File

@ -876,7 +876,7 @@ mod tests {
nonce,
);
assert!(rv.is_none());
let shred = Shred::new_from_data(slot, 1, 1, None, false, false, 0, 2, 0);
let shred = Shred::new_from_data(slot, 1, 1, &[], false, false, 0, 2, 0);
blockstore
.insert_shreds(vec![shred], None, false)
@ -1306,7 +1306,7 @@ mod tests {
#[test]
fn test_verify_shred_response() {
fn new_test_data_shred(slot: Slot, index: u32) -> Shred {
Shred::new_from_data(slot, index, 1, None, false, false, 0, 0, 0)
Shred::new_from_data(slot, index, 1, &[], false, false, 0, 0, 0)
}
let repair = ShredRepairType::Orphan(9);
// Ensure new options are addded to this test

11
core/src/shred_fetch_stage.rs
View File

@ -228,9 +228,10 @@ mod tests {
let slot = 1;
let shred = Shred::new_from_data(
slot, 3, // shred index
slot,
3, // shred index
0, // parent offset
None, // data
&[], // data
true, // is_last_in_fec_set
true, // is_last_in_slot
0, // reference_tick
@ -299,7 +300,7 @@ mod tests {
);
assert_eq!(stats.index_overrun, 1);
assert!(packet.meta.discard());
let shred = Shred::new_from_data(1, 3, 0, None, true, true, 0, 0, 0);
let shred = Shred::new_from_data(1, 3, 0, &[], true, true, 0, 0, 0);
shred.copy_to_packet(&mut packet);
// rejected slot is 1, root is 3
@ -341,7 +342,7 @@ mod tests {
);
assert!(packet.meta.discard());
let shred = Shred::new_from_data(1_000_000, 3, 0, None, true, true, 0, 0, 0);
let shred = Shred::new_from_data(1_000_000, 3, 0, &[], true, true, 0, 0, 0);
shred.copy_to_packet(&mut packet);
// Slot 1 million is too high
@ -358,7 +359,7 @@ mod tests {
assert!(packet.meta.discard());
let index = MAX_DATA_SHREDS_PER_SLOT as u32;
let shred = Shred::new_from_data(5, index, 0, None, true, true, 0, 0, 0);
let shred = Shred::new_from_data(5, index, 0, &[], true, true, 0, 0, 0);
shred.copy_to_packet(&mut packet);
ShredFetchStage::process_packet(
&mut packet,

30
core/src/sigverify_shreds.rs
View File

@ -82,7 +82,7 @@ pub mod tests {
0xdead_c0de,
0xc0de,
0xdead,
Some(&[1, 2, 3, 4]),
&[1, 2, 3, 4],
true,
true,
0,
@ -101,7 +101,7 @@ pub mod tests {
0xc0de_dead,
0xc0de,
0xdead,
Some(&[1, 2, 3, 4]),
&[1, 2, 3, 4],
true,
true,
0,
@ -131,32 +131,14 @@ pub mod tests {
let mut batches = vec![PacketBatch::default()];
batches[0].packets.resize(2, Packet::default());
let mut shred = Shred::new_from_data(
0,
0xc0de,
0xdead,
Some(&[1, 2, 3, 4]),
true,
true,
0,
0,
0xc0de,
);
let mut shred =
Shred::new_from_data(0, 0xc0de, 0xdead, &[1, 2, 3, 4], true, true, 0, 0, 0xc0de);
shred.sign(&leader_keypair);
batches[0].packets[0].data[0..shred.payload().len()].copy_from_slice(shred.payload());
batches[0].packets[0].meta.size = shred.payload().len();
let mut shred = Shred::new_from_data(
0,
0xbeef,
0xc0de,
Some(&[1, 2, 3, 4]),
true,
true,
0,
0,
0xc0de,
);
let mut shred =
Shred::new_from_data(0, 0xbeef, 0xc0de, &[1, 2, 3, 4], true, true, 0, 0, 0xc0de);
let wrong_keypair = Keypair::new();
shred.sign(&wrong_keypair);
batches[0].packets[1].data[0..shred.payload().len()].copy_from_slice(shred.payload());

34
core/src/window_service.rs
View File

@ -863,14 +863,15 @@ mod test {
));
// coding shreds don't contain parent slot information, test that slot >= root
let mut coding_shred = Shred::new_empty_coding(
5, // slot
5, // index
5, // fec_set_index
6, // num_data_shreds
6, // num_coding_shreds
3, // position
0, // version
let mut coding_shred = Shred::new_from_parity_shard(
5, // slot
5, // index
&[], // parity_shard
5, // fec_set_index
6, // num_data_shreds
6, // num_coding_shreds
3, // position
0, // version
);
coding_shred.sign(&leader_keypair);
// shred.slot() > root, shred continues
@ -945,14 +946,15 @@ mod test {
std::net::{IpAddr, Ipv4Addr},
};
solana_logger::setup();
let shred = Shred::new_empty_coding(
5, // slot
5, // index
5, // fec_set_index
6, // num_data_shreds
6, // num_coding_shreds
4, // position
0, // version
let shred = Shred::new_from_parity_shard(
5, // slot
5, // index
&[], // parity_shard
5, // fec_set_index
6, // num_data_shreds
6, // num_coding_shreds
4, // position
0, // version
);
let mut shreds = vec![shred.clone(), shred.clone(), shred];
let _from_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080);

4
ledger/benches/sigverify_shreds.rs
View File

@ -32,7 +32,7 @@ fn bench_sigverify_shreds_sign_gpu(bencher: &mut Bencher) {
slot,
0xc0de,
0xdead,
Some(&[5; SIZE_OF_DATA_SHRED_PAYLOAD]),
&[5; SIZE_OF_DATA_SHRED_PAYLOAD],
true,
true,
1,
@ -64,7 +64,7 @@ fn bench_sigverify_shreds_sign_cpu(bencher: &mut Bencher) {
slot,
0xc0de,
0xdead,
Some(&[5; SIZE_OF_DATA_SHRED_PAYLOAD]),
&[5; SIZE_OF_DATA_SHRED_PAYLOAD],
true,
true,
1,

38
ledger/src/blockstore.rs
View File

@ -5705,7 +5705,7 @@ pub mod tests {
slot,
(i * gap) as u32,
0,
None,
&[],
false,
false,
i as u8,
@ -5837,7 +5837,7 @@ pub mod tests {
let parent_offset = shred5.slot() - shred5.parent().unwrap();
parent_offset as u16
},
None, // data
&[], // data
true, // is_last_data
true, // is_last_in_slot
0, // reference_tick
@ -5938,13 +5938,15 @@ pub mod tests {
let blockstore = Blockstore::open(ledger_path.path()).unwrap();
let slot = 1;
let coding_shred = Shred::new_empty_coding(
slot, 11, // index
11, // fec_set_index
11, // num_data_shreds
11, // num_coding_shreds
8, // position
0, // version
let coding_shred = Shred::new_from_parity_shard(
slot,
11, // index
&[], // parity_shard
11, // fec_set_index
11, // num_data_shreds
11, // num_coding_shreds
8, // position
0, // version
);
let mut erasure_metas = HashMap::new();
@ -5994,13 +5996,15 @@ pub mod tests {
let last_root = RwLock::new(0);
let slot = 1;
let mut coding_shred = Shred::new_empty_coding(
slot, 11, // index
11, // fec_set_index
11, // num_data_shreds
11, // num_coding_shreds
8, // position
0, // version
let mut coding_shred = Shred::new_from_parity_shard(
slot,
11, // index
&[], // parity_shard
11, // fec_set_index
11, // num_data_shreds
11, // num_coding_shreds
8, // position
0, // version
);
// Insert a good coding shred
@ -6237,7 +6241,7 @@ pub mod tests {
slot,
next_shred_index as u32,
1,
Some(&[1, 1, 1]),
&[1, 1, 1],
true,
true,
0,

62
ledger/src/shred.rs
View File

@ -269,15 +269,14 @@ impl Shred {
slot: Slot,
index: u32,
parent_offset: u16,
data: Option<&[u8]>,
data: &[u8],
is_last_data: bool,
is_last_in_slot: bool,
reference_tick: u8,
version: u16,
fec_set_index: u32,
) -> Self {
let payload_size = SHRED_PAYLOAD_SIZE;
let mut payload = vec![0; payload_size];
let mut payload = vec![0; SHRED_PAYLOAD_SIZE];
let common_header = ShredCommonHeader {
slot,
index,
@ -286,9 +285,7 @@ impl Shred {
..ShredCommonHeader::default()
};
let size = (data.map(|d| d.len()).unwrap_or(0)
+ SIZE_OF_DATA_SHRED_HEADER
+ SIZE_OF_COMMON_SHRED_HEADER) as u16;
let size = (data.len() + SIZE_OF_DATA_SHRED_HEADER + SIZE_OF_COMMON_SHRED_HEADER) as u16;
let mut data_header = DataShredHeader {
parent_offset,
flags: reference_tick.min(SHRED_TICK_REFERENCE_MASK),
@ -320,10 +317,7 @@ impl Shred {
)
.expect("Failed to write data header into shred buffer");
// TODO: Need to check if data is too large!
if let Some(data) = data {
payload[start..start + data.len()].clone_from_slice(data);
}
payload[start..start + data.len()].copy_from_slice(data);
Self {
common_header,
data_header,
@ -362,9 +356,10 @@ impl Shred {
shred.sanitize().map(|_| shred)
}
pub fn new_empty_coding(
pub fn new_from_parity_shard(
slot: Slot,
index: u32,
parity_shard: &[u8],
fec_set_index: u32,
num_data_shreds: u16,
num_coding_shreds: u16,
@ -400,6 +395,10 @@ impl Shred {
&coding_header,
)
.expect("Failed to write coding header into shred buffer");
// Tests may have an empty parity_shard.
if !parity_shard.is_empty() {
payload[SIZE_OF_CODING_SHRED_HEADERS..].copy_from_slice(parity_shard);
}
Shred {
common_header,
data_header: DataShredHeader::default(),
@ -835,7 +834,7 @@ impl Shredder {
self.slot,
shred_index,
parent_offset as u16,
Some(data),
data,
is_last_data,
is_last_in_slot,
self.reference_tick,
@ -965,17 +964,16 @@ impl Shredder {
.enumerate()
.map(|(i, parity)| {
let index = next_code_index + u32::try_from(i).unwrap();
let mut shred = Shred::new_empty_coding(
Shred::new_from_parity_shard(
slot,
index,
parity,
fec_set_index,
num_data,
num_coding,
u16::try_from(i).unwrap(), // position
version,
);
shred.payload[SIZE_OF_CODING_SHRED_HEADERS..].copy_from_slice(parity);
shred
)
})
.collect()
}
@ -1947,7 +1945,7 @@ mod tests {
#[test]
fn test_invalid_parent_offset() {
let shred = Shred::new_from_data(10, 0, 1000, Some(&[1, 2, 3]), false, false, 0, 1, 0);
let shred = Shred::new_from_data(10, 0, 1000, &[1, 2, 3], false, false, 0, 1, 0);
let mut packet = Packet::default();
shred.copy_to_packet(&mut packet);
let shred_res = Shred::new_from_serialized_shred(packet.data.to_vec());
@ -1971,7 +1969,7 @@ mod tests {
fn test_shred_offsets() {
solana_logger::setup();
let mut packet = Packet::default();
let shred = Shred::new_from_data(1, 3, 0, None, true, true, 0, 0, 0);
let shred = Shred::new_from_data(1, 3, 0, &[], true, true, 0, 0, 0);
shred.copy_to_packet(&mut packet);
let mut stats = ShredFetchStats::default();
let ret = get_shred_slot_index_type(&packet, &mut stats);
@ -2001,9 +1999,10 @@ mod tests {
);
assert_eq!(stats.index_overrun, 4);
let shred = Shred::new_empty_coding(
let shred = Shred::new_from_parity_shard(
8, // slot
2, // index
&[], // parity_shard
10, // fec_set_index
30, // num_data
4, // num_code
@ -2016,14 +2015,15 @@ mod tests {
get_shred_slot_index_type(&packet, &mut stats)
);
let shred = Shred::new_from_data(1, std::u32::MAX - 10, 0, None, true, true, 0, 0, 0);
let shred = Shred::new_from_data(1, std::u32::MAX - 10, 0, &[], true, true, 0, 0, 0);
shred.copy_to_packet(&mut packet);
assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
assert_eq!(1, stats.index_out_of_bounds);
let shred = Shred::new_empty_coding(
let shred = Shred::new_from_parity_shard(
8, // slot
2, // index
&[], // parity_shard
10, // fec_set_index
30, // num_data_shreds
4, // num_coding_shreds
@ -2072,8 +2072,7 @@ mod tests {
420, // slot
19, // index
5, // parent_offset
Some(&data),
true, // is_last_data
&data, true, // is_last_data
false, // is_last_in_slot
3, // reference_tick
1, // version
@ -2133,14 +2132,15 @@ mod tests {
#[test]
fn test_sanitize_coding_shred() {
let mut shred = Shred::new_empty_coding(
1, // slot
12, // index
11, // fec_set_index
11, // num_data_shreds
11, // num_coding_shreds
8, // position
0, // version
let mut shred = Shred::new_from_parity_shard(
1, // slot
12, // index
&[], // parity_shard
11, // fec_set_index
11, // num_data_shreds
11, // num_coding_shreds
8, // position
0, // version
);
assert_matches!(shred.sanitize(), Ok(()));
// index < position is invalid.

10
ledger/src/sigverify_shreds.rs
View File

@ -470,7 +470,7 @@ pub mod tests {
slot,
0xc0de,
0xdead,
Some(&[1, 2, 3, 4]),
&[1, 2, 3, 4],
true,
true,
0,
@ -516,7 +516,7 @@ pub mod tests {
slot,
0xc0de,
0xdead,
Some(&[1, 2, 3, 4]),
&[1, 2, 3, 4],
true,
true,
0,
@ -571,7 +571,7 @@ pub mod tests {
slot,
0xc0de,
0xdead,
Some(&[1, 2, 3, 4]),
&[1, 2, 3, 4],
true,
true,
0,
@ -639,7 +639,7 @@ pub mod tests {
slot,
0xc0de,
i as u16,
Some(&[5; SIZE_OF_DATA_SHRED_PAYLOAD]),
&[5; SIZE_OF_DATA_SHRED_PAYLOAD],
true,
true,
1,
@ -685,7 +685,7 @@ pub mod tests {
slot,
0xc0de,
0xdead,
Some(&[1, 2, 3, 4]),
&[1, 2, 3, 4],
true,
true,
0,