blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add reference tick to data shreds (#6772) 

* Add reference tick to data shreds

* fix tests
This commit is contained in:
Pankaj Garg 2019-11-06 13:27:58 -08:00 committed by GitHub
parent b3a75a60a4
commit 0ace79939b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 144 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
core/benches/shredder.rs
View File

@ -35,7 +35,7 @@ fn bench_shredder_ticks(bencher: &mut Bencher) {
let num_ticks = max_ticks_per_n_shreds(1) * num_shreds as u64;
let entries = create_ticks(num_ticks, 0, Hash::default());
bencher.iter(|| {
let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone()).unwrap();
let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone(), 0).unwrap();
shredder.entries_to_shreds(&entries, true, 0);
})
}
@ -50,7 +50,7 @@ fn bench_shredder_large_entries(bencher: &mut Bencher) {
let entries = make_large_unchained_entries(txs_per_entry, num_entries);
// 1Mb
bencher.iter(|| {
let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone()).unwrap();
let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone(), 0).unwrap();
shredder.entries_to_shreds(&entries, true, 0);
})
}
@ -63,7 +63,7 @@ fn bench_deshredder(bencher: &mut Bencher) {
let num_shreds = ((10000 * 1000) + (shred_size - 1)) / shred_size;
let num_ticks = max_ticks_per_n_shreds(1) * num_shreds as u64;
let entries = create_ticks(num_ticks, 0, Hash::default());
let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp).unwrap();
let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp, 0).unwrap();
let data_shreds = shredder.entries_to_shreds(&entries, true, 0).0;
bencher.iter(|| {
let raw = &mut Shredder::deshred(&data_shreds).unwrap();
@ -75,7 +75,7 @@ fn bench_deshredder(bencher: &mut Bencher) {
fn bench_deserialize_hdr(bencher: &mut Bencher) {
let data = vec![0; SIZE_OF_DATA_SHRED_PAYLOAD];
let shred = Shred::new_from_data(2, 1, 1, Some(&data), true, true);
let shred = Shred::new_from_data(2, 1, 1, Some(&data), true, true, 0);
bencher.iter(|| {
let payload = shred.payload.clone();

1
core/src/broadcast_stage/broadcast_fake_blobs_run.rs
View File

@ -44,6 +44,7 @@ impl BroadcastRun for BroadcastFakeBlobsRun {
bank.parent().unwrap().slot(),
RECOMMENDED_FEC_RATE,
keypair.clone(),
(bank.tick_height() % bank.ticks_per_slot()) as u8,
)
.expect("Expected to create a new shredder");

1
core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs
View File

@ -42,6 +42,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
bank.parent().unwrap().slot(),
RECOMMENDED_FEC_RATE,
keypair.clone(),
(bank.tick_height() % bank.ticks_per_slot()) as u8,
)
.expect("Expected to create a new shredder");

13
core/src/broadcast_stage/standard_broadcast_run.rs
View File

@ -2,7 +2,7 @@ use super::broadcast_utils::{self, ReceiveResults};
use super::*;
use crate::broadcast_stage::broadcast_utils::UnfinishedSlotInfo;
use solana_ledger::entry::Entry;
use solana_ledger::shred::{Shred, Shredder, RECOMMENDED_FEC_RATE};
use solana_ledger::shred::{Shred, Shredder, RECOMMENDED_FEC_RATE, SHRED_TICK_REFERENCE_MASK};
use solana_sdk::pubkey::Pubkey;
use solana_sdk::signature::Keypair;
use solana_sdk::timing::duration_as_us;
@ -48,7 +48,7 @@ impl StandardBroadcastRun {
}
}
fn check_for_interrupted_slot(&mut self) -> Option<Shred> {
fn check_for_interrupted_slot(&mut self, max_ticks_in_slot: u8) -> Option<Shred> {
let (slot, _) = self.current_slot_and_parent.unwrap();
let mut last_unfinished_slot_shred = self
.unfinished_slot
@ -62,6 +62,7 @@ impl StandardBroadcastRun {
None,
true,
true,
max_ticks_in_slot & SHRED_TICK_REFERENCE_MASK,
))
} else {
None
@ -83,6 +84,7 @@ impl StandardBroadcastRun {
blocktree: &Blocktree,
entries: &[Entry],
is_slot_end: bool,
reference_tick: u8,
) -> (Vec<Shred>, Vec<Shred>) {
let (slot, parent_slot) = self.current_slot_and_parent.unwrap();
let shredder = Shredder::new(
@ -90,6 +92,7 @@ impl StandardBroadcastRun {
parent_slot,
RECOMMENDED_FEC_RATE,
self.keypair.clone(),
reference_tick,
)
.expect("Expected to create a new shredder");
@ -149,13 +152,15 @@ impl StandardBroadcastRun {
let to_shreds_start = Instant::now();
// 1) Check if slot was interrupted
let last_unfinished_slot_shred = self.check_for_interrupted_slot();
let last_unfinished_slot_shred =
self.check_for_interrupted_slot(bank.ticks_per_slot() as u8);
// 2) Convert entries to shreds and coding shreds
let (mut data_shreds, coding_shreds) = self.entries_to_shreds(
blocktree,
&receive_results.entries,
last_tick_height == bank.max_tick_height(),
(bank.tick_height() % bank.ticks_per_slot()) as u8,
);
if let Some(last_shred) = last_unfinished_slot_shred {
data_shreds.push(last_shred);
@ -353,7 +358,7 @@ mod test {
// Slot 2 interrupted slot 1
let shred = run
.check_for_interrupted_slot()
.check_for_interrupted_slot(0)
.expect("Expected a shred that signals an interrupt");
// Validate the shred

2
core/src/chacha.rs
View File

@ -166,7 +166,7 @@ mod tests {
hasher.hash(&buf[..size]);
// golden needs to be updated if blob stuff changes....
let golden: Hash = "BdmY3efqu7zbnFuGRAeFANwa35HkDdQ7hwhYez3xGXiM"
let golden: Hash = "HLzH7Nrh4q2K5WTh3e9vPNFZ1QVYhVDRMN9u5v51GqpJ"
.parse()
.unwrap();

2
core/src/replay_stage.rs
View File

@ -1137,7 +1137,7 @@ mod test {
let payload_len = SIZE_OF_DATA_SHRED_PAYLOAD;
let gibberish = [0xa5u8; PACKET_DATA_SIZE];
let mut data_header = DataShredHeader::default();
data_header.flags = DATA_COMPLETE_SHRED;
data_header.flags |= DATA_COMPLETE_SHRED;
let mut shred = Shred::new_empty_from_header(
ShredCommonHeader::default(),
data_header,

26
core/src/sigverify_shreds.rs
View File

@ -93,8 +93,15 @@ pub mod tests {
#[test]
fn test_sigverify_shreds_read_slots() {
solana_logger::setup();
let mut shred =
Shred::new_from_data(0xdeadc0de, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true);
let mut shred = Shred::new_from_data(
0xdeadc0de,
0xc0de,
0xdead,
Some(&[1, 2, 3, 4]),
true,
true,
0,
);
let mut batch = [Packets::default(), Packets::default()];
let keypair = Keypair::new();
@ -103,8 +110,15 @@ pub mod tests {
batch[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload);
batch[0].packets[0].meta.size = shred.payload.len();
let mut shred =
Shred::new_from_data(0xc0dedead, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true);
let mut shred = Shred::new_from_data(
0xc0dedead,
0xc0de,
0xdead,
Some(&[1, 2, 3, 4]),
true,
true,
0,
);
Shredder::sign_shred(&keypair, &mut shred);
batch[1].packets.resize(1, Packet::default());
batch[1].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload);
@ -127,12 +141,12 @@ pub mod tests {
let mut batch = vec![Packets::default()];
batch[0].packets.resize(2, Packet::default());
let mut shred = Shred::new_from_data(0, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true);
let mut shred = Shred::new_from_data(0, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true, 0);
Shredder::sign_shred(&leader_keypair, &mut shred);
batch[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload);
batch[0].packets[0].meta.size = shred.payload.len();
let mut shred = Shred::new_from_data(0, 0xbeef, 0xc0de, Some(&[1, 2, 3, 4]), true, true);
let mut shred = Shred::new_from_data(0, 0xbeef, 0xc0de, Some(&[1, 2, 3, 4]), true, true, 0);
let wrong_keypair = Keypair::new();
Shredder::sign_shred(&wrong_keypair, &mut shred);
batch[0].packets[1].data[0..shred.payload.len()].copy_from_slice(&shred.payload);

2
core/src/window_service.rs
View File

@ -315,7 +315,7 @@ mod test {
parent: Slot,
keypair: &Arc<Keypair>,
) -> Vec<Shred> {
let shredder = Shredder::new(slot, parent, 0.0, keypair.clone())
let shredder = Shredder::new(slot, parent, 0.0, keypair.clone(), 0)
.expect("Failed to create entry shredder");
shredder.entries_to_shreds(&entries, true, 0).0
}

17
ledger/src/blocktree.rs
View File

@ -897,7 +897,7 @@ impl Blocktree {
let mut remaining_ticks_in_slot = num_slots * ticks_per_slot - num_ticks_in_start_slot;
let mut current_slot = start_slot;
let mut shredder = Shredder::new(current_slot, parent_slot, 0.0, keypair.clone())
let mut shredder = Shredder::new(current_slot, parent_slot, 0.0, keypair.clone(), 0)
.expect("Failed to create entry shredder");
let mut all_shreds = vec![];
let mut slot_entries = vec![];
@ -920,8 +920,14 @@ impl Blocktree {
shredder.entries_to_shreds(&current_entries, true, start_index);
all_shreds.append(&mut data_shreds);
all_shreds.append(&mut coding_shreds);
shredder = Shredder::new(current_slot, parent_slot, 0.0, keypair.clone())
.expect("Failed to create entry shredder");
shredder = Shredder::new(
current_slot,
parent_slot,
0.0,
keypair.clone(),
(ticks_per_slot - remaining_ticks_in_slot) as u8,
)
.expect("Failed to create entry shredder");
}
if entry.is_tick() {
@ -1695,7 +1701,7 @@ pub fn create_new_ledger(ledger_path: &Path, genesis_block: &GenesisBlock) -> Re
let entries = create_ticks(ticks_per_slot, hashes_per_tick, genesis_block.hash());
let last_hash = entries.last().unwrap().hash;
let shredder = Shredder::new(0, 0, 0.0, Arc::new(Keypair::new()))
let shredder = Shredder::new(0, 0, 0.0, Arc::new(Keypair::new()), 0)
.expect("Failed to create entry shredder");
let shreds = shredder.entries_to_shreds(&entries, true, 0).0;
assert!(shreds.last().unwrap().last_in_slot());
@ -1780,7 +1786,7 @@ pub fn entries_to_test_shreds(
parent_slot: Slot,
is_full_slot: bool,
) -> Vec<Shred> {
let shredder = Shredder::new(slot, parent_slot, 0.0, Arc::new(Keypair::new()))
let shredder = Shredder::new(slot, parent_slot, 0.0, Arc::new(Keypair::new()), 0)
.expect("Failed to create entry shredder");
shredder.entries_to_shreds(&entries, is_full_slot, 0).0
@ -3807,6 +3813,7 @@ pub mod tests {
Some(&[1, 1, 1]),
true,
true,
0,
)];
// With the corruption, nothing should be returned, even though an

96
ledger/src/shred.rs
View File

@ -51,8 +51,9 @@ pub const MAX_DATA_SHREDS_PER_FEC_BLOCK: u32 = 16;
/// Based on rse benchmarks, the optimal erasure config uses 16 data shreds and 4 coding shreds
pub const RECOMMENDED_FEC_RATE: f32 = 0.25;
const LAST_SHRED_IN_SLOT: u8 = 0b0000_0001;
pub const DATA_COMPLETE_SHRED: u8 = 0b0000_0010;
pub const SHRED_TICK_REFERENCE_MASK: u8 = 0b0011_1111;
const LAST_SHRED_IN_SLOT: u8 = 0b1000_0000;
pub const DATA_COMPLETE_SHRED: u8 = 0b0100_0000;
#[derive(Debug)]
pub enum ShredError {
@ -143,6 +144,7 @@ impl Shred {
data: Option<&[u8]>,
is_last_data: bool,
is_last_in_slot: bool,
reference_tick: u8,
) -> Self {
let mut payload = vec![0; PACKET_DATA_SIZE];
let mut common_header = ShredCommonHeader::default();
@ -151,6 +153,7 @@ impl Shred {
let mut data_header = DataShredHeader::default();
data_header.parent_offset = parent_offset;
data_header.flags = reference_tick.min(SHRED_TICK_REFERENCE_MASK);
if is_last_data {
data_header.flags |= DATA_COMPLETE_SHRED
@ -336,6 +339,14 @@ impl Shred {
}
}
pub fn reference_tick(&self) -> u8 {
if self.is_data() {
self.data_header.flags & SHRED_TICK_REFERENCE_MASK
} else {
SHRED_TICK_REFERENCE_MASK
}
}
pub fn verify(&self, pubkey: &Pubkey) -> bool {
self.signature()
.verify(pubkey.as_ref(), &self.payload[SIZE_OF_SIGNATURE..])
@ -349,6 +360,7 @@ pub struct Shredder {
fec_rate: f32,
keypair: Arc<Keypair>,
pub signing_coding_time: u128,
reference_tick: u8,
}
impl Shredder {
@ -357,6 +369,7 @@ impl Shredder {
parent_slot: Slot,
fec_rate: f32,
keypair: Arc<Keypair>,
reference_tick: u8,
) -> Result<Self> {
if fec_rate > 1.0 || fec_rate < 0.0 {
Err(ShredError::InvalidFecRate(fec_rate))
@ -369,6 +382,7 @@ impl Shredder {
fec_rate,
keypair,
signing_coding_time: 0,
reference_tick,
})
}
}
@ -412,6 +426,7 @@ impl Shredder {
Some(shred_data),
is_last_data,
is_last_in_slot,
self.reference_tick,
);
Shredder::sign_shred(&self.keypair, &mut shred);
@ -814,7 +829,7 @@ pub mod tests {
// Test that parent cannot be > current slot
assert_matches!(
Shredder::new(slot, slot + 1, 1.00, keypair.clone()),
Shredder::new(slot, slot + 1, 1.00, keypair.clone(), 0),
Err(ShredError::SlotTooLow {
slot: _,
parent_slot: _,
@ -822,7 +837,7 @@ pub mod tests {
);
// Test that slot - parent cannot be > u16 MAX
assert_matches!(
Shredder::new(slot, slot - 1 - 0xffff, 1.00, keypair.clone()),
Shredder::new(slot, slot - 1 - 0xffff, 1.00, keypair.clone(), 0),
Err(ShredError::SlotTooLow {
slot: _,
parent_slot: _,
@ -831,7 +846,7 @@ pub mod tests {
let fec_rate = 0.25;
let parent_slot = slot - 5;
let shredder = Shredder::new(slot, parent_slot, fec_rate, keypair.clone())
let shredder = Shredder::new(slot, parent_slot, fec_rate, keypair.clone(), 0)
.expect("Failed in creating shredder");
let entries: Vec<_> = (0..5)
@ -906,7 +921,7 @@ pub mod tests {
let slot = 1;
let parent_slot = 0;
let shredder = Shredder::new(slot, parent_slot, 0.0, keypair.clone())
let shredder = Shredder::new(slot, parent_slot, 0.0, keypair.clone(), 0)
.expect("Failed in creating shredder");
let entries: Vec<_> = (0..5)
@ -926,6 +941,67 @@ pub mod tests {
assert_eq!(deserialized_shred, *data_shreds.last().unwrap());
}
#[test]
fn test_shred_reference_tick() {
let keypair = Arc::new(Keypair::new());
let slot = 1;
let parent_slot = 0;
let shredder = Shredder::new(slot, parent_slot, 0.0, keypair.clone(), 5)
.expect("Failed in creating shredder");
let entries: Vec<_> = (0..5)
.map(|_| {
let keypair0 = Keypair::new();
let keypair1 = Keypair::new();
let tx0 =
system_transaction::transfer(&keypair0, &keypair1.pubkey(), 1, Hash::default());
Entry::new(&Hash::default(), 1, vec![tx0])
})
.collect();
let data_shreds = shredder.entries_to_shreds(&entries, true, 0).0;
data_shreds.iter().for_each(|s| {
assert_eq!(s.reference_tick(), 5);
});
let deserialized_shred =
Shred::new_from_serialized_shred(data_shreds.last().unwrap().payload.clone()).unwrap();
assert_eq!(deserialized_shred.reference_tick(), 5);
}
#[test]
fn test_shred_reference_tick_overflow() {
let keypair = Arc::new(Keypair::new());
let slot = 1;
let parent_slot = 0;
let shredder = Shredder::new(slot, parent_slot, 0.0, keypair.clone(), u8::max_value())
.expect("Failed in creating shredder");
let entries: Vec<_> = (0..5)
.map(|_| {
let keypair0 = Keypair::new();
let keypair1 = Keypair::new();
let tx0 =
system_transaction::transfer(&keypair0, &keypair1.pubkey(), 1, Hash::default());
Entry::new(&Hash::default(), 1, vec![tx0])
})
.collect();
let data_shreds = shredder.entries_to_shreds(&entries, true, 0).0;
data_shreds.iter().for_each(|s| {
assert_eq!(s.reference_tick(), SHRED_TICK_REFERENCE_MASK);
});
let deserialized_shred =
Shred::new_from_serialized_shred(data_shreds.last().unwrap().payload.clone()).unwrap();
assert_eq!(
deserialized_shred.reference_tick(),
SHRED_TICK_REFERENCE_MASK
);
}
#[test]
fn test_data_and_code_shredder() {
let keypair = Arc::new(Keypair::new());
@ -933,11 +1009,11 @@ pub mod tests {
let slot = 0x123456789abcdef0;
// Test that FEC rate cannot be > 1.0
assert_matches!(
Shredder::new(slot, slot - 5, 1.001, keypair.clone()),
Shredder::new(slot, slot - 5, 1.001, keypair.clone(), 0),
Err(ShredError::InvalidFecRate(_))
);
let shredder = Shredder::new(0x123456789abcdef0, slot - 5, 1.0, keypair.clone())
let shredder = Shredder::new(0x123456789abcdef0, slot - 5, 1.0, keypair.clone(), 0)
.expect("Failed in creating shredder");
// Create enough entries to make > 1 shred
@ -979,7 +1055,7 @@ pub mod tests {
fn test_recovery_and_reassembly() {
let keypair = Arc::new(Keypair::new());
let slot = 0x123456789abcdef0;
let shredder = Shredder::new(slot, slot - 5, 1.0, keypair.clone())
let shredder = Shredder::new(slot, slot - 5, 1.0, keypair.clone(), 0)
.expect("Failed in creating shredder");
let keypair0 = Keypair::new();
@ -1225,7 +1301,7 @@ pub mod tests {
fn test_multi_fec_block_coding() {
let keypair = Arc::new(Keypair::new());
let slot = 0x123456789abcdef0;
let shredder = Shredder::new(slot, slot - 5, 1.0, keypair.clone())
let shredder = Shredder::new(slot, slot - 5, 1.0, keypair.clone(), 0)
.expect("Failed in creating shredder");
let num_fec_sets = 100;

13
ledger/src/sigverify_shreds.rs
View File

@ -494,7 +494,8 @@ pub mod tests {
solana_logger::setup();
let mut packet = Packet::default();
let slot = 0xdeadc0de;
let mut shred = Shred::new_from_data(slot, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true);
let mut shred =
Shred::new_from_data(slot, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true, 0);
assert_eq!(shred.slot(), slot);
let keypair = Keypair::new();
Shredder::sign_shred(&keypair, &mut shred);
@ -527,7 +528,8 @@ pub mod tests {
solana_logger::setup();
let mut batch = [Packets::default()];
let slot = 0xdeadc0de;
let mut shred = Shred::new_from_data(slot, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true);
let mut shred =
Shred::new_from_data(slot, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true, 0);
let keypair = Keypair::new();
Shredder::sign_shred(&keypair, &mut shred);
batch[0].packets.resize(1, Packet::default());
@ -570,7 +572,8 @@ pub mod tests {
let mut batch = [Packets::default()];
let slot = 0xdeadc0de;
let mut shred = Shred::new_from_data(slot, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true);
let mut shred =
Shred::new_from_data(slot, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true, 0);
let keypair = Keypair::new();
Shredder::sign_shred(&keypair, &mut shred);
batch[0].packets.resize(1, Packet::default());
@ -623,7 +626,7 @@ pub mod tests {
let mut batch = [Packets::default()];
let slot = 0xdeadc0de;
let keypair = Keypair::new();
let shred = Shred::new_from_data(slot, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true);
let shred = Shred::new_from_data(slot, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true, 0);
batch[0].packets.resize(1, Packet::default());
batch[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload);
batch[0].packets[0].meta.size = shred.payload.len();
@ -666,7 +669,7 @@ pub mod tests {
let mut batch = [Packets::default()];
let slot = 0xdeadc0de;
let keypair = Keypair::new();
let shred = Shred::new_from_data(slot, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true);
let shred = Shred::new_from_data(slot, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true, 0);
batch[0].packets.resize(1, Packet::default());
batch[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload);
batch[0].packets[0].meta.size = shred.payload.len();