blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

adds chained_merkle_root to shredder arguments (#34952) 

Working towards chaining Merkle root of erasure batches, the commit adds
chained_merkle_root to shredder arguments.
This commit is contained in:
behzad nouri 2024-01-27 15:04:31 +00:00 committed by GitHub
parent d4fdcd940a
commit 79bbe4381a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 108 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
core/benches/shredder.rs
View File

@ -4,7 +4,7 @@
extern crate test;
use {
rand::seq::SliceRandom,
rand::{seq::SliceRandom, Rng},
raptorq::{Decoder, Encoder},
solana_entry::entry::{create_ticks, Entry},
solana_ledger::shred::{
@ -50,6 +50,7 @@ fn make_shreds(num_shreds: usize) -> Vec<Shred> {
&Keypair::new(),
&entries,
true, // is_last_in_slot
None, // chained_merkle_root
0, // next_shred_index
0, // next_code_index
false, // merkle_variant
@ -80,12 +81,14 @@ fn bench_shredder_ticks(bencher: &mut Bencher) {
let num_ticks = max_ticks_per_n_shreds(1, Some(LEGACY_SHRED_DATA_CAPACITY)) * num_shreds as u64;
let entries = create_ticks(num_ticks, 0, Hash::default());
let reed_solomon_cache = ReedSolomonCache::default();
let chained_merkle_root = Some(Hash::new_from_array(rand::thread_rng().gen()));
bencher.iter(|| {
let shredder = Shredder::new(1, 0, 0, 0).unwrap();
shredder.entries_to_shreds(
&kp,
&entries,
true,
chained_merkle_root,
0,
0,
true, // merkle_variant
@ -107,6 +110,7 @@ fn bench_shredder_large_entries(bencher: &mut Bencher) {
Some(shred_size),
);
let entries = make_large_unchained_entries(txs_per_entry, num_entries);
let chained_merkle_root = Some(Hash::new_from_array(rand::thread_rng().gen()));
let reed_solomon_cache = ReedSolomonCache::default();
// 1Mb
bencher.iter(|| {
@ -115,6 +119,7 @@ fn bench_shredder_large_entries(bencher: &mut Bencher) {
&kp,
&entries,
true,
chained_merkle_root,
0,
0,
true, // merkle_variant
@ -133,10 +138,12 @@ fn bench_deshredder(bencher: &mut Bencher) {
let num_ticks = max_ticks_per_n_shreds(1, Some(shred_size)) * num_shreds as u64;
let entries = create_ticks(num_ticks, 0, Hash::default());
let shredder = Shredder::new(1, 0, 0, 0).unwrap();
let chained_merkle_root = Some(Hash::new_from_array(rand::thread_rng().gen()));
let (data_shreds, _) = shredder.entries_to_shreds(
&kp,
&entries,
true,
chained_merkle_root,
0,
0,
true, // merkle_variant

1
core/src/window_service.rs
View File

@ -582,6 +582,7 @@ mod test {
keypair,
entries,
true, // is_last_in_slot
None, // chained_merkle_root
0, // next_shred_index
0, // next_code_index
true, // merkle_variant

2
gossip/src/duplicate_shred.rs
View File

@ -409,6 +409,8 @@ pub(crate) mod tests {
keypair,
&entries,
is_last_in_slot,
// chained_merkle_root
Some(Hash::new_from_array(rng.gen())),
next_shred_index,
next_code_index, // next_code_index
merkle_variant,

41
ledger/src/blockstore.rs
View File

@ -29,6 +29,7 @@ use {
crossbeam_channel::{bounded, Receiver, Sender, TrySendError},
dashmap::DashSet,
log::*,
rand::Rng,
rayon::{
iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator},
ThreadPool,
@ -1956,6 +1957,7 @@ impl Blockstore {
let mut all_shreds = vec![];
let mut slot_entries = vec![];
let reed_solomon_cache = ReedSolomonCache::default();
let mut chained_merkle_root = Some(Hash::new_from_array(rand::thread_rng().gen()));
// Find all the entries for start_slot
for entry in entries.into_iter() {
if remaining_ticks_in_slot == 0 {
@ -1973,7 +1975,8 @@ impl Blockstore {
let (mut data_shreds, mut coding_shreds) = shredder.entries_to_shreds(
keypair,
&current_entries,
true, // is_last_in_slot
true, // is_last_in_slot
chained_merkle_root,
start_index, // next_shred_index
start_index, // next_code_index
true, // merkle_variant
@ -1982,6 +1985,7 @@ impl Blockstore {
);
all_shreds.append(&mut data_shreds);
all_shreds.append(&mut coding_shreds);
chained_merkle_root = Some(coding_shreds.last().unwrap().merkle_root().unwrap());
shredder = Shredder::new(
current_slot,
parent_slot,
@ -2002,6 +2006,7 @@ impl Blockstore {
keypair,
&slot_entries,
is_full_slot,
chained_merkle_root,
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
@ -4285,6 +4290,8 @@ pub fn create_new_ledger(
&Keypair::new(),
&entries,
true, // is_last_in_slot
// chained_merkle_root
Some(Hash::new_from_array(rand::thread_rng().gen())),
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
@ -4546,6 +4553,8 @@ pub fn entries_to_test_shreds(
&Keypair::new(),
entries,
is_full_slot,
// chained_merkle_root
Some(Hash::new_from_array(rand::thread_rng().gen())),
0, // next_shred_index,
0, // next_code_index
merkle_variant,
@ -4806,6 +4815,7 @@ pub mod tests {
InnerInstruction, InnerInstructions, Reward, Rewards, TransactionTokenBalance,
},
std::{cmp::Ordering, thread::Builder, time::Duration},
test_case::test_case,
};
// used for tests only
@ -7434,7 +7444,7 @@ pub mod tests {
#[test]
fn test_insert_multiple_is_last() {
solana_logger::setup();
let (shreds, _) = make_slot_entries(0, 0, 20, /*merkle_variant:*/ true);
let (shreds, _) = make_slot_entries(0, 0, 19, /*merkle_variant:*/ true);
let num_shreds = shreds.len() as u64;
let ledger_path = get_tmp_ledger_path_auto_delete!();
let blockstore = Blockstore::open(ledger_path.path()).unwrap();
@ -7448,6 +7458,7 @@ pub mod tests {
assert!(slot_meta.is_full());
let (shreds, _) = make_slot_entries(0, 0, 22, /*merkle_variant:*/ true);
assert!(shreds.len() > num_shreds as usize);
blockstore.insert_shreds(shreds, None, false).unwrap();
let slot_meta = blockstore.meta(0).unwrap().unwrap();
@ -9863,7 +9874,9 @@ pub mod tests {
let (data_shreds, coding_shreds) = shredder.entries_to_shreds(
&leader_keypair,
&entries,
true, // is_last_in_slot
true, // is_last_in_slot
// chained_merkle_root
Some(Hash::new_from_array(rand::thread_rng().gen())),
fec_set_index, // next_shred_index
fec_set_index, // next_code_index
true, // merkle_variant
@ -9916,18 +9929,21 @@ pub mod tests {
assert_eq!(num_coding_in_index, num_coding);
}
#[test]
fn test_duplicate_slot() {
#[test_case(false)]
#[test_case(true)]
fn test_duplicate_slot(chained: bool) {
let slot = 0;
let entries1 = make_slot_entries_with_transactions(1);
let entries2 = make_slot_entries_with_transactions(1);
let leader_keypair = Arc::new(Keypair::new());
let reed_solomon_cache = ReedSolomonCache::default();
let shredder = Shredder::new(slot, 0, 0, 0).unwrap();
let chained_merkle_root = chained.then(|| Hash::new_from_array(rand::thread_rng().gen()));
let (shreds, _) = shredder.entries_to_shreds(
&leader_keypair,
&entries1,
true, // is_last_in_slot
chained_merkle_root,
0, // next_shred_index
0, // next_code_index,
true, // merkle_variant
@ -9938,6 +9954,7 @@ pub mod tests {
&leader_keypair,
&entries2,
true, // is_last_in_slot
chained_merkle_root,
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
@ -10323,7 +10340,11 @@ pub mod tests {
let num_unique_entries = max_ticks_per_n_shreds(1, None) + 1;
let (mut original_shreds, original_entries) =
make_slot_entries(0, 0, num_unique_entries, /*merkle_variant:*/ true);
let mut duplicate_shreds = original_shreds.clone();
// Mutate signature so that payloads are not the same as the originals.
for shred in &mut duplicate_shreds {
shred.sign(&Keypair::new());
}
// Discard first shred, so that the slot is not full
assert!(original_shreds.len() > 1);
let last_index = original_shreds.last().unwrap().index() as u64;
@ -10345,14 +10366,6 @@ pub mod tests {
assert!(!blockstore.is_full(0));
}
let duplicate_shreds = entries_to_test_shreds(
&original_entries,
0, // slot
0, // parent_slot
true, // is_full_slot
0, // version
true, // merkle_variant
);
let num_shreds = duplicate_shreds.len() as u64;
blockstore
.insert_shreds(duplicate_shreds, None, false)

69
ledger/src/shredder.rs
View File

@ -13,7 +13,7 @@ use {
solana_entry::entry::Entry,
solana_measure::measure::Measure,
solana_rayon_threadlimit::get_thread_count,
solana_sdk::{clock::Slot, signature::Keypair},
solana_sdk::{clock::Slot, hash::Hash, signature::Keypair},
std::{
borrow::Borrow,
fmt::Debug,
@ -69,11 +69,13 @@ impl Shredder {
}
}
#[allow(clippy::too_many_arguments)]
pub fn entries_to_shreds(
&self,
keypair: &Keypair,
entries: &[Entry],
is_last_in_slot: bool,
chained_merkle_root: Option<Hash>,
next_shred_index: u32,
next_code_index: u32,
merkle_variant: bool,
@ -93,7 +95,7 @@ impl Shredder {
self.version,
self.reference_tick,
is_last_in_slot,
None, // chained_merkle_root
chained_merkle_root,
next_shred_index,
next_code_index,
reed_solomon_cache,
@ -500,6 +502,7 @@ mod tests {
system_transaction,
},
std::{collections::HashSet, convert::TryInto, iter::repeat_with, sync::Arc},
test_case::test_case,
};
fn verify_test_code_shred(shred: &Shred, index: u32, slot: Slot, pk: &Pubkey, verify: bool) {
@ -510,7 +513,7 @@ mod tests {
assert_eq!(verify, shred.verify(pk));
}
fn run_test_data_shredder(slot: Slot) {
fn run_test_data_shredder(slot: Slot, chained: bool) {
let keypair = Arc::new(Keypair::new());
// Test that parent cannot be > current slot
@ -548,6 +551,8 @@ mod tests {
&keypair,
&entries,
is_last_in_slot,
// chained_merkle_root
chained.then(|| Hash::new_from_array(rand::thread_rng().gen())),
start_index, // next_shred_index
start_index, // next_code_index
true, // merkle_variant
@ -602,13 +607,15 @@ mod tests {
assert_eq!(entries, deshred_entries);
}
#[test]
fn test_data_shredder() {
run_test_data_shredder(0x1234_5678_9abc_def0);
#[test_case(false)]
#[test_case(true)]
fn test_data_shredder(chained: bool) {
run_test_data_shredder(0x1234_5678_9abc_def0, chained);
}
#[test]
fn test_deserialize_shred_payload() {
#[test_case(false)]
#[test_case(true)]
fn test_deserialize_shred_payload(chained: bool) {
let keypair = Arc::new(Keypair::new());
let slot = 1;
let parent_slot = 0;
@ -627,6 +634,8 @@ mod tests {
&keypair,
&entries,
true, // is_last_in_slot
// chained_merkle_root
chained.then(|| Hash::new_from_array(rand::thread_rng().gen())),
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
@ -639,8 +648,9 @@ mod tests {
assert_eq!(deserialized_shred, *data_shreds.last().unwrap());
}
#[test]
fn test_shred_reference_tick() {
#[test_case(false)]
#[test_case(true)]
fn test_shred_reference_tick(chained: bool) {
let keypair = Arc::new(Keypair::new());
let slot = 1;
let parent_slot = 0;
@ -659,6 +669,8 @@ mod tests {
&keypair,
&entries,
true, // is_last_in_slot
// chained_merkle_root,
chained.then(|| Hash::new_from_array(rand::thread_rng().gen())),
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
@ -676,8 +688,9 @@ mod tests {
assert_eq!(deserialized_shred.reference_tick(), 5);
}
#[test]
fn test_shred_reference_tick_overflow() {
#[test_case(false)]
#[test_case(true)]
fn test_shred_reference_tick_overflow(chained: bool) {
let keypair = Arc::new(Keypair::new());
let slot = 1;
let parent_slot = 0;
@ -696,6 +709,8 @@ mod tests {
&keypair,
&entries,
true, // is_last_in_slot
// chained_merkle_root
chained.then(|| Hash::new_from_array(rand::thread_rng().gen())),
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
@ -722,7 +737,7 @@ mod tests {
);
}
fn run_test_data_and_code_shredder(slot: Slot) {
fn run_test_data_and_code_shredder(slot: Slot, chained: bool) {
let keypair = Arc::new(Keypair::new());
let shredder = Shredder::new(slot, slot - 5, 0, 0).unwrap();
// Create enough entries to make > 1 shred
@ -742,6 +757,8 @@ mod tests {
&keypair,
&entries,
true, // is_last_in_slot
// chained_merkle_root
chained.then(|| Hash::new_from_array(rand::thread_rng().gen())),
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
@ -766,9 +783,10 @@ mod tests {
}
}
#[test]
fn test_data_and_code_shredder() {
run_test_data_and_code_shredder(0x1234_5678_9abc_def0);
#[test_case(false)]
#[test_case(true)]
fn test_data_and_code_shredder(chained: bool) {
run_test_data_and_code_shredder(0x1234_5678_9abc_def0, chained);
}
fn run_test_recovery_and_reassembly(slot: Slot, is_last_in_slot: bool) {
@ -799,6 +817,7 @@ mod tests {
&keypair,
&entries,
is_last_in_slot,
None, // chained_merkle_root
0, // next_shred_index
0, // next_code_index
false, // merkle_variant
@ -936,6 +955,7 @@ mod tests {
&keypair,
&entries,
true, // is_last_in_slot
None, // chained_merkle_root
25, // next_shred_index,
25, // next_code_index
false, // merkle_variant
@ -1032,6 +1052,7 @@ mod tests {
&keypair,
&[entry],
is_last_in_slot,
None, // chained_merkle_root
next_shred_index,
next_shred_index, // next_code_index
false, // merkle_variant
@ -1073,8 +1094,9 @@ mod tests {
}
}
#[test]
fn test_shred_version() {
#[test_case(false)]
#[test_case(true)]
fn test_shred_version(chained: bool) {
let keypair = Arc::new(Keypair::new());
let hash = hash(Hash::default().as_ref());
let version = shred_version::version_from_hash(&hash);
@ -1094,6 +1116,8 @@ mod tests {
&keypair,
&entries,
true, // is_last_in_slot
// chained_merkle_root
chained.then(|| Hash::new_from_array(rand::thread_rng().gen())),
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
@ -1106,8 +1130,9 @@ mod tests {
.any(|s| s.version() != version));
}
#[test]
fn test_shred_fec_set_index() {
#[test_case(false)]
#[test_case(true)]
fn test_shred_fec_set_index(chained: bool) {
let keypair = Arc::new(Keypair::new());
let hash = hash(Hash::default().as_ref());
let version = shred_version::version_from_hash(&hash);
@ -1127,7 +1152,9 @@ mod tests {
let (data_shreds, coding_shreds) = shredder.entries_to_shreds(
&keypair,
&entries,
true, // is_last_in_slot
true, // is_last_in_slot
// chained_merkle_root
chained.then(|| Hash::new_from_array(rand::thread_rng().gen())),
start_index, // next_shred_index
start_index, // next_code_index
true, // merkle_variant

4
ledger/src/sigverify_shreds.rs
View File

@ -739,7 +739,9 @@ mod tests {
.entries_to_shreds(
keypair,
&make_entries(rng, num_entries),
rng.gen(), // is_last_in_slot
rng.gen(), // is_last_in_slot
// chained_merkle_root
rng.gen::<bool>().then(|| Hash::new_from_array(rng.gen())),
rng.gen_range(0..2671), // next_shred_index
rng.gen_range(0..2781), // next_code_index
rng.gen(), // merkle_variant,

2
ledger/tests/shred.rs
View File

@ -53,6 +53,7 @@ fn test_multi_fec_block_coding() {
&keypair,
&entries,
true, // is_last_in_slot
None, // chained_merkle_root
0, // next_shred_index
0, // next_code_index
false, // merkle_variant
@ -226,6 +227,7 @@ fn setup_different_sized_fec_blocks(
&keypair,
&entries,
is_last,
None, // chained_merkle_root
next_shred_index,
next_code_index,
false, // merkle_variant

1
local-cluster/tests/local_cluster.rs
View File

@ -5610,6 +5610,7 @@ fn test_invalid_forks_persisted_on_restart() {
&majority_keypair,
&entries,
true, // is_full_slot
None, // chained_merkle_root
0, // next_shred_index,
0, // next_code_index
false, // merkle_variant

3
turbine/benches/retransmit_stage.rs
View File

@ -6,6 +6,7 @@ extern crate test;
use {
crossbeam_channel::unbounded,
log::*,
rand::Rng,
solana_entry::entry::Entry,
solana_gossip::{
cluster_info::{ClusterInfo, Node},
@ -105,6 +106,8 @@ fn bench_retransmitter(bencher: &mut Bencher) {
&keypair,
&entries,
true, // is_last_in_slot
// chained_merkle_root
Some(Hash::new_from_array(rand::thread_rng().gen())),
0, // next_shred_index
0, // next_code_index
true, // merkle_variant

3
turbine/src/broadcast_stage.rs
View File

@ -503,6 +503,7 @@ pub mod test {
use {
super::*,
crossbeam_channel::unbounded,
rand::Rng,
solana_entry::entry::create_ticks,
solana_gossip::cluster_info::{ClusterInfo, Node},
solana_ledger::{
@ -544,6 +545,8 @@ pub mod test {
&Keypair::new(),
&entries,
true, // is_last_in_slot
// chained_merkle_root
Some(Hash::new_from_array(rand::thread_rng().gen())),
0, // next_shred_index,
0, // next_code_index
true, // merkle_variant

3
turbine/src/broadcast_stage/broadcast_duplicates_run.rs
View File

@ -173,6 +173,7 @@ impl BroadcastRun for BroadcastDuplicatesRun {
keypair,
&receive_results.entries,
last_tick_height == bank.max_tick_height() && last_entries.is_none(),
None, // chained_merkle_root
self.next_shred_index,
self.next_code_index,
false, // merkle_variant
@ -190,6 +191,7 @@ impl BroadcastRun for BroadcastDuplicatesRun {
keypair,
&[original_last_entry],
true,
None, // chained_merkle_root
self.next_shred_index,
self.next_code_index,
false, // merkle_variant
@ -203,6 +205,7 @@ impl BroadcastRun for BroadcastDuplicatesRun {
keypair,
&duplicate_extra_last_entries,
true,
None, // chained_merkle_root
self.next_shred_index,
self.next_code_index,
false, // merkle_variant

2
turbine/src/broadcast_stage/broadcast_fake_shreds_run.rs
View File

@ -60,6 +60,7 @@ impl BroadcastRun for BroadcastFakeShredsRun {
keypair,
&receive_results.entries,
last_tick_height == bank.max_tick_height(),
None, // chained_merkle_root
next_shred_index,
self.next_code_index,
true, // merkle_variant
@ -81,6 +82,7 @@ impl BroadcastRun for BroadcastFakeShredsRun {
keypair,
&fake_entries,
last_tick_height == bank.max_tick_height(),
None, // chained_merkle_root
next_shred_index,
self.next_code_index,
true, // merkle_variant

3
turbine/src/broadcast_stage/fail_entry_verification_broadcast_run.rs
View File

@ -92,6 +92,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
keypair,
&receive_results.entries,
last_tick_height == bank.max_tick_height() && last_entries.is_none(),
None, // chained_merkle_root
self.next_shred_index,
self.next_code_index,
true, // merkle_variant
@ -108,6 +109,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
keypair,
&[good_last_entry],
true,
None, // chained_merkle_root
self.next_shred_index,
self.next_code_index,
true, // merkle_variant
@ -121,6 +123,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
keypair,
&[bad_last_entry],
false,
None, // chained_merkle_root
self.next_shred_index,
self.next_code_index,
true, // merkle_variant

2
turbine/src/broadcast_stage/standard_broadcast_run.rs
View File

@ -85,6 +85,7 @@ impl StandardBroadcastRun {
keypair,
&[], // entries
true, // is_last_in_slot,
None, // chained_merkle_root
state.next_shred_index,
state.next_code_index,
true, // merkle_variant
@ -143,6 +144,7 @@ impl StandardBroadcastRun {
keypair,
entries,
is_slot_end,
None, // chained_merkle_root
next_shred_index,
next_code_index,
true, // merkle_variant