blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

plumbs through flag to generate merkle variant of shreds

This commit is contained in:
behzad nouri 2022-09-15 17:51:41 -04:00
parent 7d3f3b2f7d
commit 97c9af4c6b
27 changed files with 459 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/benches/retransmit_stage.rs
View File

@ -106,6 +106,7 @@ fn bench_retransmitter(bencher: &mut Bencher) {
true, // is_last_in_slot
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
&mut ProcessShredsStats::default(),
);

10
core/benches/shredder.rs
View File

@ -49,9 +49,10 @@ fn make_shreds(num_shreds: usize) -> Vec<Shred> {
let (data_shreds, _) = shredder.entries_to_shreds(
&Keypair::new(),
&entries,
true, // is_last_in_slot
0, // next_shred_index
0, // next_code_index
true, // is_last_in_slot
0, // next_shred_index
0, // next_code_index
false, // merkle_variant
&mut ProcessShredsStats::default(),
);
assert!(data_shreds.len() >= num_shreds);
@ -85,6 +86,7 @@ fn bench_shredder_ticks(bencher: &mut Bencher) {
true,
0,
0,
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
})
@ -111,6 +113,7 @@ fn bench_shredder_large_entries(bencher: &mut Bencher) {
true,
0,
0,
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
})
@ -131,6 +134,7 @@ fn bench_deshredder(bencher: &mut Bencher) {
true,
0,
0,
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
bencher.iter(|| {

18
core/src/banking_stage.rs
View File

@ -3626,7 +3626,14 @@ mod tests {
poh_recorder.write().unwrap().set_bank(&bank, false);
let shreds = entries_to_test_shreds(&entries, bank.slot(), 0, true, 0);
let shreds = entries_to_test_shreds(
&entries,
bank.slot(),
0, // parent_slot
true, // is_full_slot
0, // version
true, // merkle_variant
);
blockstore.insert_shreds(shreds, None, false).unwrap();
blockstore.set_roots(std::iter::once(&bank.slot())).unwrap();
@ -3788,7 +3795,14 @@ mod tests {
poh_recorder.write().unwrap().set_bank(&bank, false);
let shreds = entries_to_test_shreds(&entries, bank.slot(), 0, true, 0);
let shreds = entries_to_test_shreds(
&entries,
bank.slot(),
0, // parent_slot
true, // is_full_slot
0, // version
true, // merkle_variant
);
blockstore.insert_shreds(shreds, None, false).unwrap();
blockstore.set_roots(std::iter::once(&bank.slot())).unwrap();

1
core/src/broadcast_stage.rs
View File

@ -481,6 +481,7 @@ pub mod test {
true, // is_last_in_slot
0, // next_shred_index,
0, // next_code_index
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
(

3
core/src/broadcast_stage/broadcast_duplicates_run.rs
View File

@ -163,6 +163,7 @@ impl BroadcastRun for BroadcastDuplicatesRun {
last_tick_height == bank.max_tick_height() && last_entries.is_none(),
self.next_shred_index,
self.next_code_index,
false, // merkle_variant
&mut ProcessShredsStats::default(),
);
@ -178,6 +179,7 @@ impl BroadcastRun for BroadcastDuplicatesRun {
true,
self.next_shred_index,
self.next_code_index,
false, // merkle_variant
&mut ProcessShredsStats::default(),
);
// Don't mark the last shred as last so that validators won't
@ -189,6 +191,7 @@ impl BroadcastRun for BroadcastDuplicatesRun {
true,
self.next_shred_index,
self.next_code_index,
false, // merkle_variant
&mut ProcessShredsStats::default(),
);
let sigs: Vec<_> = partition_last_data_shred

2
core/src/broadcast_stage/broadcast_fake_shreds_run.rs
View File

@ -60,6 +60,7 @@ impl BroadcastRun for BroadcastFakeShredsRun {
last_tick_height == bank.max_tick_height(),
next_shred_index,
self.next_code_index,
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
@ -79,6 +80,7 @@ impl BroadcastRun for BroadcastFakeShredsRun {
last_tick_height == bank.max_tick_height(),
next_shred_index,
self.next_code_index,
true, // merkle_variant
&mut ProcessShredsStats::default(),
);

3
core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs
View File

@ -91,6 +91,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
last_tick_height == bank.max_tick_height() && last_entries.is_none(),
self.next_shred_index,
self.next_code_index,
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
@ -105,6 +106,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
true,
self.next_shred_index,
self.next_code_index,
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
// Don't mark the last shred as last so that validators won't know
@ -116,6 +118,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
false,
self.next_shred_index,
self.next_code_index,
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
self.next_shred_index += 1;

2
core/src/broadcast_stage/standard_broadcast_run.rs
View File

@ -76,6 +76,7 @@ impl StandardBroadcastRun {
true, // is_last_in_slot,
state.next_shred_index,
state.next_code_index,
false, // merkle_variant
stats,
);
self.report_and_reset_stats(true);
@ -124,6 +125,7 @@ impl StandardBroadcastRun {
is_slot_end,
next_shred_index,
next_code_index,
false, // merkle_variant
process_stats,
);
let next_shred_index = match data_shreds.iter().map(Shred::index).max() {

16
core/src/consensus.rs
View File

@ -2848,11 +2848,11 @@ pub mod test {
{
let blockstore = Blockstore::open(&blockstore_path).unwrap();
let (shreds, _) = make_slot_entries(1, 0, 42);
let (shreds, _) = make_slot_entries(1, 0, 42, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
let (shreds, _) = make_slot_entries(3, 1, 42);
let (shreds, _) = make_slot_entries(3, 1, 42, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
let (shreds, _) = make_slot_entries(4, 1, 42);
let (shreds, _) = make_slot_entries(4, 1, 42, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
assert!(!blockstore.is_root(0));
assert!(!blockstore.is_root(1));
@ -2886,11 +2886,11 @@ pub mod test {
{
let blockstore = Blockstore::open(&blockstore_path).unwrap();
let (shreds, _) = make_slot_entries(1, 0, 42);
let (shreds, _) = make_slot_entries(1, 0, 42, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
let (shreds, _) = make_slot_entries(3, 1, 42);
let (shreds, _) = make_slot_entries(3, 1, 42, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
let (shreds, _) = make_slot_entries(4, 1, 42);
let (shreds, _) = make_slot_entries(4, 1, 42, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
blockstore.set_roots(std::iter::once(&3)).unwrap();
assert!(!blockstore.is_root(0));
@ -2917,9 +2917,9 @@ pub mod test {
{
let blockstore = Blockstore::open(&blockstore_path).unwrap();
let (shreds, _) = make_slot_entries(1, 0, 42);
let (shreds, _) = make_slot_entries(1, 0, 42, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
let (shreds, _) = make_slot_entries(3, 1, 42);
let (shreds, _) = make_slot_entries(3, 1, 42, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
assert!(!blockstore.is_root(0));
assert!(!blockstore.is_root(1));

1
core/src/repair_generic_traversal.rs
View File

@ -307,6 +307,7 @@ pub mod test {
parent.unwrap_or(slot),
is_slot_complete,
0,
true, // merkle_variant
);
// remove next to last shred

40
core/src/repair_service.rs
View File

@ -774,8 +774,8 @@ mod test {
let blockstore = Blockstore::open(&blockstore_path).unwrap();
// Create some orphan slots
let (mut shreds, _) = make_slot_entries(1, 0, 1);
let (shreds2, _) = make_slot_entries(5, 2, 1);
let (mut shreds, _) = make_slot_entries(1, 0, 1, /*merkle_variant:*/ true);
let (shreds2, _) = make_slot_entries(5, 2, 1, /*merkle_variant:*/ true);
shreds.extend(shreds2);
blockstore.insert_shreds(shreds, None, false).unwrap();
let mut repair_weight = RepairWeight::new(0);
@ -808,7 +808,7 @@ mod test {
{
let blockstore = Blockstore::open(&blockstore_path).unwrap();
let (shreds, _) = make_slot_entries(2, 0, 1);
let (shreds, _) = make_slot_entries(2, 0, 1, /*merkle_variant:*/ true);
// Write this shred to slot 2, should chain to slot 0, which we haven't received
// any shreds for
@ -854,7 +854,11 @@ mod test {
let mut missing_indexes_per_slot = vec![];
for i in (0..num_shreds).rev() {
let index = i % num_shreds_per_slot;
if index % nth == 0 {
// get_best_repair_shreds only returns missing shreds in
// between shreds received; So this should either insert the
// last shred in each slot, or exclude missing shreds after the
// last inserted shred from expected repairs.
if index % nth == 0 || index + 1 == num_shreds_per_slot {
shreds_to_write.insert(0, shreds.remove(i as usize));
} else if i < num_shreds_per_slot {
missing_indexes_per_slot.insert(0, index);
@ -918,7 +922,12 @@ mod test {
let num_entries_per_slot = 100;
// Create some shreds
let (mut shreds, _) = make_slot_entries(0, 0, num_entries_per_slot as u64);
let (mut shreds, _) = make_slot_entries(
0, // slot
0, // parent_slot
num_entries_per_slot as u64,
true, // merkle_variant
);
let num_shreds_per_slot = shreds.len() as u64;
// Remove last shred (which is also last in slot) so that slot is not complete
@ -1014,7 +1023,12 @@ mod test {
// Create some shreds in slots 0..num_slots
for i in start..start + num_slots {
let parent = if i > 0 { i - 1 } else { 0 };
let (shreds, _) = make_slot_entries(i, parent, num_entries_per_slot as u64);
let (shreds, _) = make_slot_entries(
i, // slot
parent,
num_entries_per_slot as u64,
true, // merkle_variant
);
blockstore.insert_shreds(shreds, None, false).unwrap();
}
@ -1054,7 +1068,12 @@ mod test {
// Insert some shreds to create a SlotMeta, should make repairs
let num_entries_per_slot = max_ticks_per_n_shreds(1, None) + 1;
let (mut shreds, _) = make_slot_entries(dead_slot, dead_slot - 1, num_entries_per_slot);
let (mut shreds, _) = make_slot_entries(
dead_slot, // slot
dead_slot - 1, // parent_slot
num_entries_per_slot,
true, // merkle_variant
);
blockstore
.insert_shreds(shreds[..shreds.len() - 1].to_vec(), None, false)
.unwrap();
@ -1094,7 +1113,12 @@ mod test {
// Insert some shreds to create a SlotMeta,
let num_entries_per_slot = max_ticks_per_n_shreds(1, None) + 1;
let (mut shreds, _) = make_slot_entries(dead_slot, dead_slot - 1, num_entries_per_slot);
let (mut shreds, _) = make_slot_entries(
dead_slot,
dead_slot - 1,
num_entries_per_slot,
true, // merkle_variant
);
blockstore
.insert_shreds(shreds[..shreds.len() - 1].to_vec(), None, false)
.unwrap();

26
core/src/repair_weighted_traversal.rs
View File

@ -140,7 +140,10 @@ pub fn get_best_repair_shreds<'a>(
pub mod test {
use {
super::*,
solana_ledger::{get_tmp_ledger_path, shred::Shred},
solana_ledger::{
get_tmp_ledger_path,
shred::{Shred, ShredFlags},
},
solana_runtime::bank_utils,
solana_sdk::hash::Hash,
trees::tr,
@ -272,15 +275,18 @@ pub mod test {
let completed_shreds: Vec<Shred> = [0, 2, 4, 6]
.iter()
.map(|slot| {
let mut shred = Shred::new_from_serialized_shred(
blockstore
.get_data_shred(*slot, last_shred - 1)
.unwrap()
.unwrap(),
)
.unwrap();
shred.set_index(last_shred as u32);
shred.set_last_in_slot();
let parent_offset = if *slot == 0 { 0 } else { 1 };
let shred = Shred::new_from_data(
*slot,
last_shred as u32, // index
parent_offset,
&[0u8; 8], // data
ShredFlags::LAST_SHRED_IN_SLOT,
8, // reference_tick
0, // version
last_shred as u32, // fec_set_index
);
assert!(shred.sanitize().is_ok());
shred
})
.collect();

45
core/src/replay_stage.rs
View File

@ -3705,7 +3705,12 @@ pub(crate) mod tests {
// Insert shreds for slot NUM_CONSECUTIVE_LEADER_SLOTS,
// chaining to slot 1
let (shreds, _) = make_slot_entries(NUM_CONSECUTIVE_LEADER_SLOTS, 1, 8);
let (shreds, _) = make_slot_entries(
NUM_CONSECUTIVE_LEADER_SLOTS, // slot
1, // parent_slot
8, // num_entries
true, // merkle_variant
);
blockstore.insert_shreds(shreds, None, false).unwrap();
assert!(bank_forks
.read()
@ -3729,7 +3734,12 @@ pub(crate) mod tests {
// Insert shreds for slot 2 * NUM_CONSECUTIVE_LEADER_SLOTS,
// chaining to slot 1
let (shreds, _) = make_slot_entries(2 * NUM_CONSECUTIVE_LEADER_SLOTS, 1, 8);
let (shreds, _) = make_slot_entries(
2 * NUM_CONSECUTIVE_LEADER_SLOTS,
1,
8,
true, // merkle_variant
);
blockstore.insert_shreds(shreds, None, false).unwrap();
assert!(bank_forks
.read()
@ -3945,7 +3955,14 @@ pub(crate) mod tests {
), // should cause AccountNotFound error
],
);
entries_to_test_shreds(&[entry], slot, slot.saturating_sub(1), false, 0)
entries_to_test_shreds(
&[entry],
slot,
slot.saturating_sub(1), // parent_slot
false, // is_full_slot
0, // version
true, // merkle_variant
)
});
assert_matches!(
@ -3975,7 +3992,14 @@ pub(crate) mod tests {
blockhash,
)],
);
entries_to_test_shreds(&[entry], slot, slot.saturating_sub(1), false, 0)
entries_to_test_shreds(
&[entry],
slot,
slot.saturating_sub(1), // parent_slot
false, // is_full_slot
0, // version
true, // merkle_variant
)
});
if let Err(BlockstoreProcessorError::InvalidBlock(block_error)) = res {
@ -4000,6 +4024,7 @@ pub(crate) mod tests {
slot.saturating_sub(1),
false,
0,
true, // merkle_variant
)
});
@ -4024,6 +4049,7 @@ pub(crate) mod tests {
slot.saturating_sub(1),
false,
0,
true, // merkle_variant
)
});
@ -4044,6 +4070,7 @@ pub(crate) mod tests {
slot.saturating_sub(1),
true,
0,
true, // merkle_variant
)
});
@ -4066,6 +4093,7 @@ pub(crate) mod tests {
slot.saturating_sub(1),
false,
0,
true, // merkle_variant
)
});
@ -4089,7 +4117,14 @@ pub(crate) mod tests {
let tx = system_transaction::transfer(funded_keypair, &keypair.pubkey(), 2, blockhash);
let trailing_entry = entry::next_entry(&last_entry_hash, 1, vec![tx]);
entries.push(trailing_entry);
entries_to_test_shreds(&entries, slot, slot.saturating_sub(1), true, 0)
entries_to_test_shreds(
&entries,
slot,
slot.saturating_sub(1), // parent_slot
true, // is_full_slot
0, // version
true, // merkle_variant
)
});
if let Err(BlockstoreProcessorError::InvalidBlock(block_error)) = res {

9
core/src/validator.rs
View File

@ -2223,7 +2223,14 @@ mod tests {
info!("creating shreds");
let mut last_print = Instant::now();
for i in 1..10 {
let shreds = blockstore::entries_to_test_shreds(&entries, i, i - 1, true, 1);
let shreds = blockstore::entries_to_test_shreds(
&entries,
i, // slot
i - 1, // parent_slot
true, // is_full_slot
1, // version
true, // merkle_variant
);
blockstore.insert_shreds(shreds, None, true).unwrap();
if last_print.elapsed().as_millis() > 5000 {
info!("inserted {}", i);

8
core/src/window_service.rs
View File

@ -506,6 +506,7 @@ mod test {
true, // is_last_in_slot
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
data_shreds
@ -539,8 +540,11 @@ mod test {
blockstore
.insert_shreds(shreds.clone(), None, false)
.unwrap();
let mut duplicate_shred = shreds[1].clone();
duplicate_shred.set_slot(shreds[0].slot());
let duplicate_shred = {
let (mut shreds, _) = make_many_slot_entries(5, 1, 10);
shreds.swap_remove(0)
};
assert_eq!(duplicate_shred.slot(), shreds[0].slot());
let duplicate_shred_slot = duplicate_shred.slot();
sender.send(duplicate_shred).unwrap();
assert!(!blockstore.has_duplicate_shreds_in_slot(duplicate_shred_slot));

12
core/tests/ledger_cleanup.rs
View File

@ -479,8 +479,6 @@ mod tests {
let mut num_shreds = 0;
let mut max_speed = 0f32;
let mut min_speed = f32::MAX;
let (mut shreds_with_parent, _) = make_many_slot_shreds(
1, batch_size_slots, shreds_per_slot);
let (first_shreds, _) = make_many_slot_shreds(
0, batch_size_slots, shreds_per_slot);
loop {
@ -502,14 +500,10 @@ mod tests {
break;
}
} else {
let mut slot_id = start_slot;
let slot_id = start_slot;
if slot_id > 0 {
for shred in shreds_with_parent.iter_mut() {
shred.set_slot(slot_id);
if shred.index() as u64 == shreds_per_slot - 1 {
slot_id += 1;
}
}
let (shreds_with_parent, _) = make_many_slot_shreds(
slot_id, batch_size_slots, shreds_per_slot);
total += shreds_with_parent.len();
cloned_blockstore.insert_shreds(
shreds_with_parent.clone(), None, false).unwrap()

1
gossip/src/duplicate_shred.rs
View File

@ -342,6 +342,7 @@ pub(crate) mod tests {
true, // is_last_in_slot
next_shred_index,
next_shred_index, // next_code_index
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
data_shreds.swap_remove(0)

4
ledger-tool/tests/basic.rs
View File

@ -79,7 +79,9 @@ fn insert_test_shreds(ledger_path: &Path, ending_slot: u64) {
let blockstore = Blockstore::open(ledger_path).unwrap();
for i in 1..ending_slot {
let entries = entry::create_ticks(1, 0, Hash::default());
let shreds = blockstore::entries_to_test_shreds(&entries, i, 0, false, 0);
let shreds = blockstore::entries_to_test_shreds(
&entries, i, 0, false, 0, /*merkle_variant:*/ true,
);
blockstore.insert_shreds(shreds, None, false).unwrap();
}
}

15
ledger/benches/blockstore.rs
View File

@ -20,7 +20,7 @@ fn bench_write_shreds(bench: &mut Bencher, entries: Vec<Entry>, ledger_path: &Pa
let blockstore =
Blockstore::open(ledger_path).expect("Expected to be able to open database ledger");
bench.iter(move || {
let shreds = entries_to_test_shreds(&entries, 0, 0, true, 0);
let shreds = entries_to_test_shreds(&entries, 0, 0, true, 0, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
});
@ -42,7 +42,14 @@ fn setup_read_bench(
);
// Convert the entries to shreds, write the shreds to the ledger
let shreds = entries_to_test_shreds(&entries, slot, slot.saturating_sub(1), true, 0);
let shreds = entries_to_test_shreds(
&entries,
slot,
slot.saturating_sub(1), // parent_slot
true, // is_full_slot
0, // version
true, // merkle_variant
);
blockstore
.insert_shreds(shreds, None, false)
.expect("Expectd successful insertion of shreds into ledger");
@ -135,7 +142,7 @@ fn bench_insert_data_shred_small(bench: &mut Bencher) {
let num_entries = 32 * 1024;
let entries = create_ticks(num_entries, 0, Hash::default());
bench.iter(move || {
let shreds = entries_to_test_shreds(&entries, 0, 0, true, 0);
let shreds = entries_to_test_shreds(&entries, 0, 0, true, 0, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
});
Blockstore::destroy(&ledger_path).expect("Expected successful database destruction");
@ -150,7 +157,7 @@ fn bench_insert_data_shred_big(bench: &mut Bencher) {
let num_entries = 32 * 1024;
let entries = create_ticks(num_entries, 0, Hash::default());
bench.iter(move || {
let shreds = entries_to_test_shreds(&entries, 0, 0, true, 0);
let shreds = entries_to_test_shreds(&entries, 0, 0, true, 0, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
});
Blockstore::destroy(&ledger_path).expect("Expected successful database destruction");

6
ledger/src/ancestor_iterator.rs
View File

@ -124,11 +124,11 @@ mod tests {
let ledger_path = get_tmp_ledger_path_auto_delete!();
let blockstore = Blockstore::open(ledger_path.path()).unwrap();
let (shreds, _) = make_slot_entries(0, 0, 42);
let (shreds, _) = make_slot_entries(0, 0, 42, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
let (shreds, _) = make_slot_entries(1, 0, 42);
let (shreds, _) = make_slot_entries(1, 0, 42, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
let (shreds, _) = make_slot_entries(2, 1, 42);
let (shreds, _) = make_slot_entries(2, 1, 42, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
assert_eq!(

284
ledger/src/blockstore.rs
View File

@ -401,6 +401,7 @@ impl Blockstore {
parent.unwrap_or(slot),
is_slot_complete,
0,
true, // merkle_variant
);
self.insert_shreds(shreds, None, false).unwrap();
}
@ -1736,6 +1737,7 @@ impl Blockstore {
true, // is_last_in_slot
start_index, // next_shred_index
start_index, // next_code_index
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
all_shreds.append(&mut data_shreds);
@ -1760,8 +1762,9 @@ impl Blockstore {
keypair,
&slot_entries,
is_full_slot,
0, // next_shred_index
0, // next_code_index
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
all_shreds.append(&mut data_shreds);
@ -3876,6 +3879,7 @@ pub fn create_new_ledger(
true, // is_last_in_slot
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
assert!(shreds.last().unwrap().last_in_slot());
@ -4131,6 +4135,7 @@ pub fn entries_to_test_shreds(
parent_slot: Slot,
is_full_slot: bool,
version: u16,
merkle_variant: bool,
) -> Vec<Shred> {
Shredder::new(slot, parent_slot, 0, version)
.unwrap()
@ -4140,6 +4145,7 @@ pub fn entries_to_test_shreds(
is_full_slot,
0, // next_shred_index,
0, // next_code_index
merkle_variant,
&mut ProcessShredsStats::default(),
)
.0
@ -4150,9 +4156,10 @@ pub fn make_slot_entries(
slot: Slot,
parent_slot: Slot,
num_entries: u64,
merkle_variant: bool,
) -> (Vec<Shred>, Vec<Entry>) {
let entries = create_ticks(num_entries, 0, Hash::default());
let shreds = entries_to_test_shreds(&entries, slot, parent_slot, true, 0);
let shreds = entries_to_test_shreds(&entries, slot, parent_slot, true, 0, merkle_variant);
(shreds, entries)
}
@ -4167,7 +4174,12 @@ pub fn make_many_slot_entries(
for slot in start_slot..start_slot + num_slots {
let parent_slot = if slot == 0 { 0 } else { slot - 1 };
let (slot_shreds, slot_entries) = make_slot_entries(slot, parent_slot, entries_per_slot);
let (slot_shreds, slot_entries) = make_slot_entries(
slot,
parent_slot,
entries_per_slot,
true, // merkle_variant
);
shreds.extend(slot_shreds);
entries.extend(slot_entries);
}
@ -4296,7 +4308,12 @@ pub fn make_chaining_slot_entries(
}
};
let result = make_slot_entries(*slot, parent_slot, entries_per_slot);
let result = make_slot_entries(
*slot,
parent_slot,
entries_per_slot,
true, // merkle_variant
);
slots_shreds_and_entries.push(result);
}
@ -4407,7 +4424,12 @@ pub mod tests {
}
fn make_and_insert_slot(blockstore: &Blockstore, slot: Slot, parent_slot: Slot) {
let (shreds, _) = make_slot_entries(slot, parent_slot, 100);
let (shreds, _) = make_slot_entries(
slot,
parent_slot,
100, // num_entries
true, // merkle_variant
);
blockstore.insert_shreds(shreds, None, true).unwrap();
let meta = blockstore.meta(slot).unwrap().unwrap();
@ -4471,7 +4493,12 @@ pub mod tests {
let num_entries = max_ticks_per_n_shreds(1, None) + 1;
assert!(num_entries > 1);
let (mut shreds, _) = make_slot_entries(0, 0, num_entries);
let (mut shreds, _) = make_slot_entries(
0, // slot
0, // parent_slot
num_entries,
true, // merkle_variant
);
let ledger_path = get_tmp_ledger_path_auto_delete!();
let blockstore = Blockstore::open(ledger_path.path()).unwrap();
@ -4671,7 +4698,7 @@ pub mod tests {
#[test]
fn test_read_shred_bytes() {
let slot = 0;
let (shreds, _) = make_slot_entries(slot, 0, 100);
let (shreds, _) = make_slot_entries(slot, 0, 100, /*merkle_variant:*/ true);
let num_shreds = shreds.len() as u64;
let shred_bufs: Vec<_> = shreds.iter().map(Shred::payload).cloned().collect();
@ -4727,7 +4754,7 @@ pub mod tests {
#[test]
fn test_shred_cleanup_check() {
let slot = 1;
let (shreds, _) = make_slot_entries(slot, 0, 100);
let (shreds, _) = make_slot_entries(slot, 0, 100, /*merkle_variant:*/ true);
let ledger_path = get_tmp_ledger_path_auto_delete!();
let blockstore = Blockstore::open(ledger_path.path()).unwrap();
@ -4752,7 +4779,12 @@ pub mod tests {
let num_entries = max_ticks_per_n_shreds(1, None) + 1;
assert!(num_entries > 1);
let (mut shreds, entries) = make_slot_entries(0, 0, num_entries);
let (mut shreds, entries) = make_slot_entries(
0, // slot
0, // parent_slot
num_entries,
true, // merkle_variant
);
let num_shreds = shreds.len() as u64;
let ledger_path = get_tmp_ledger_path_auto_delete!();
@ -4795,7 +4827,12 @@ pub mod tests {
fn test_insert_data_shreds_reverse() {
let num_shreds = 10;
let num_entries = max_ticks_per_n_shreds(num_shreds, None);
let (mut shreds, entries) = make_slot_entries(0, 0, num_entries);
let (mut shreds, entries) = make_slot_entries(
0, // slot
0, // parent_slot
num_entries,
true, // merkle_variant
);
let num_shreds = shreds.len() as u64;
let ledger_path = get_tmp_ledger_path_auto_delete!();
@ -4873,12 +4910,26 @@ pub mod tests {
let ledger_path = get_tmp_ledger_path_auto_delete!();
let blockstore = Blockstore::open(ledger_path.path()).unwrap();
let entries = create_ticks(8, 0, Hash::default());
let shreds = entries_to_test_shreds(&entries[0..4], 1, 0, false, 0);
let shreds = entries_to_test_shreds(
&entries[0..4],
1,
0,
false,
0,
true, // merkle_variant
);
blockstore
.insert_shreds(shreds, None, false)
.expect("Expected successful write of shreds");
let mut shreds1 = entries_to_test_shreds(&entries[4..], 1, 0, false, 0);
let mut shreds1 = entries_to_test_shreds(
&entries[4..],
1,
0,
false,
0,
false, // merkle_variant
);
for (i, b) in shreds1.iter_mut().enumerate() {
b.set_index(8 + i as u32);
}
@ -4906,8 +4957,14 @@ pub mod tests {
for slot in 0..num_slots {
let entries = create_ticks(slot + 1, 0, Hash::default());
let last_entry = entries.last().unwrap().clone();
let mut shreds =
entries_to_test_shreds(&entries, slot, slot.saturating_sub(1), false, 0);
let mut shreds = entries_to_test_shreds(
&entries,
slot,
slot.saturating_sub(1),
false,
0,
true, // merkle_variant
);
for b in shreds.iter_mut() {
b.set_index(index);
b.set_slot(slot as u64);
@ -4940,7 +4997,14 @@ pub mod tests {
// Write entries
for slot in 0..num_slots {
let entries = create_ticks(entries_per_slot, 0, Hash::default());
let shreds = entries_to_test_shreds(&entries, slot, slot.saturating_sub(1), false, 0);
let shreds = entries_to_test_shreds(
&entries,
slot,
slot.saturating_sub(1),
false,
0,
true, // merkle_variant
);
assert!(shreds.len() as u64 >= shreds_per_slot);
blockstore
.insert_shreds(shreds, None, false)
@ -4960,7 +5024,12 @@ pub mod tests {
let parent_slot = if i == 0 { 0 } else { i - 1 };
// Write entries
let num_entries = min_entries * (i + 1);
let (shreds, original_entries) = make_slot_entries(slot, parent_slot, num_entries);
let (shreds, original_entries) = make_slot_entries(
slot,
parent_slot,
num_entries,
true, // merkle_variant
);
let num_shreds = shreds.len() as u64;
assert!(num_shreds > 1);
@ -5018,7 +5087,8 @@ pub mod tests {
let slot = 0;
let num_entries = max_ticks_per_n_shreds(1, None) + 1;
let entries = create_ticks(num_entries, slot, Hash::default());
let shreds = entries_to_test_shreds(&entries, slot, 0, true, 0);
let shreds =
entries_to_test_shreds(&entries, slot, 0, true, 0, /*merkle_variant:*/ true);
let num_shreds = shreds.len();
assert!(num_shreds > 1);
assert!(blockstore
@ -5057,7 +5127,12 @@ pub mod tests {
let entries_per_slot = 50;
// Create entries for slot 0
let (mut shreds, _) = make_slot_entries(0, 0, entries_per_slot);
let (mut shreds, _) = make_slot_entries(
0, // slot
0, // parent_slot
entries_per_slot,
false, // merkle_variant
);
let shreds_per_slot = shreds.len() as u64;
// Insert second shred, but we're missing the first shred, so no consecutive
@ -5087,7 +5162,12 @@ pub mod tests {
let mut shreds = vec![];
let mut missing_shreds = vec![];
for slot in 1..num_slots + 1 {
let (mut slot_shreds, _) = make_slot_entries(slot, slot - 1, entries_per_slot);
let (mut slot_shreds, _) = make_slot_entries(
slot,
slot - 1, // parent_slot
entries_per_slot,
false, // merkle_variant
);
let missing_shred = slot_shreds.remove(slot as usize - 1);
shreds.extend(slot_shreds);
missing_shreds.push(missing_shred);
@ -5101,7 +5181,12 @@ pub mod tests {
// should get no updates
let shreds: Vec<_> = (1..num_slots + 1)
.flat_map(|slot| {
let (mut shred, _) = make_slot_entries(slot, slot - 1, 1);
let (mut shred, _) = make_slot_entries(
slot,
slot - 1, // parent_slot
1, // num_entries
false, // merkle_variant
);
shred[0].set_index(2 * num_slots as u32);
shred
})
@ -5141,7 +5226,8 @@ pub mod tests {
let entries_per_slot = 10;
// Create shreds for slot 0
let (mut shreds, _) = make_slot_entries(0, 0, entries_per_slot);
let (mut shreds, _) =
make_slot_entries(0, 0, entries_per_slot, /*merkle_variant:*/ true);
let shred0 = shreds.remove(0);
// Insert all but the first shred in the slot, should not be considered complete
@ -5216,7 +5302,12 @@ pub mod tests {
let (shreds0, _) = all_shreds.remove(0);
let (shreds1, _) = all_shreds.remove(0);
let (shreds2, _) = all_shreds.remove(0);
let (shreds3, _) = make_slot_entries(disconnected_slot, 1, entries_per_slot);
let (shreds3, _) = make_slot_entries(
disconnected_slot,
1, // parent_slot
entries_per_slot,
true, // merkle_variant
);
let mut all_shreds: Vec<_> = vec![shreds0, shreds1, shreds2, shreds3]
.into_iter()
@ -5315,7 +5406,12 @@ pub mod tests {
slot - 1
}
};
let (slot_shreds, _) = make_slot_entries(slot, parent_slot, entries_per_slot);
let (slot_shreds, _) = make_slot_entries(
slot,
parent_slot,
entries_per_slot,
true, // merkle_variant
);
shreds_per_slot = slot_shreds.len();
if slot % 2 == 1 {
@ -5688,8 +5784,8 @@ pub mod tests {
// Write some slot that also chains to existing slots and orphan,
// nothing should change
let (shred4, _) = make_slot_entries(4, 0, 1);
let (shred5, _) = make_slot_entries(5, 1, 1);
let (shred4, _) = make_slot_entries(4, 0, 1, /*merkle_variant:*/ true);
let (shred5, _) = make_slot_entries(5, 1, 1, /*merkle_variant:*/ true);
blockstore.insert_shreds(shred4, None, false).unwrap();
blockstore.insert_shreds(shred5, None, false).unwrap();
assert_eq!(
@ -5728,7 +5824,8 @@ pub mod tests {
}
};
let (mut shred, entry) = make_slot_entries(slot, parent_slot, 1);
let (mut shred, entry) =
make_slot_entries(slot, parent_slot, 1, /*merkle_variant:*/ false);
num_shreds_per_slot = shred.len() as u64;
shred
.iter_mut()
@ -5780,7 +5877,8 @@ pub mod tests {
// Create enough entries to ensure there are at least two shreds created
let num_entries = max_ticks_per_n_shreds(1, None) + 1;
let entries = create_ticks(num_entries, 0, Hash::default());
let mut shreds = entries_to_test_shreds(&entries, slot, 0, true, 0);
let mut shreds =
entries_to_test_shreds(&entries, slot, 0, true, 0, /*merkle_variant:*/ false);
let num_shreds = shreds.len();
assert!(num_shreds > 1);
for (i, s) in shreds.iter_mut().enumerate() {
@ -5918,7 +6016,8 @@ pub mod tests {
);
let entries = create_ticks(100, 0, Hash::default());
let mut shreds = entries_to_test_shreds(&entries, slot, 0, true, 0);
let mut shreds =
entries_to_test_shreds(&entries, slot, 0, true, 0, /*merkle_variant:*/ false);
assert!(shreds.len() > 2);
shreds.drain(2..);
@ -5957,7 +6056,8 @@ pub mod tests {
// Write entries
let num_entries = 10;
let entries = create_ticks(num_entries, 0, Hash::default());
let shreds = entries_to_test_shreds(&entries, slot, 0, true, 0);
let shreds =
entries_to_test_shreds(&entries, slot, 0, true, 0, /*merkle_variant:*/ true);
let num_shreds = shreds.len();
blockstore.insert_shreds(shreds, None, false).unwrap();
@ -5991,7 +6091,7 @@ pub mod tests {
#[test]
fn test_should_insert_data_shred() {
solana_logger::setup();
let (mut shreds, _) = make_slot_entries(0, 0, 200);
let (mut shreds, _) = make_slot_entries(0, 0, 200, /*merkle_variant:*/ false);
let ledger_path = get_tmp_ledger_path_auto_delete!();
let blockstore = Blockstore::open(ledger_path.path()).unwrap();
@ -6077,7 +6177,7 @@ pub mod tests {
#[test]
fn test_is_data_shred_present() {
let (shreds, _) = make_slot_entries(0, 0, 200);
let (shreds, _) = make_slot_entries(0, 0, 200, /*merkle_variant:*/ true);
let ledger_path = get_tmp_ledger_path_auto_delete!();
let blockstore = Blockstore::open(ledger_path.path()).unwrap();
let index_cf = &blockstore.index_cf;
@ -6237,7 +6337,7 @@ pub mod tests {
#[test]
fn test_insert_multiple_is_last() {
solana_logger::setup();
let (shreds, _) = make_slot_entries(0, 0, 20);
let (shreds, _) = make_slot_entries(0, 0, 20, /*merkle_variant:*/ true);
let num_shreds = shreds.len() as u64;
let ledger_path = get_tmp_ledger_path_auto_delete!();
let blockstore = Blockstore::open(ledger_path.path()).unwrap();
@ -6250,7 +6350,7 @@ pub mod tests {
assert_eq!(slot_meta.last_index, Some(num_shreds - 1));
assert!(slot_meta.is_full());
let (shreds, _) = make_slot_entries(0, 0, 22);
let (shreds, _) = make_slot_entries(0, 0, 22, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
let slot_meta = blockstore.meta(0).unwrap().unwrap();
@ -6403,7 +6503,8 @@ pub mod tests {
let num_ticks = 8;
let entries = create_ticks(num_ticks, 0, Hash::default());
let slot = 1;
let shreds = entries_to_test_shreds(&entries, slot, 0, false, 0);
let shreds =
entries_to_test_shreds(&entries, slot, 0, false, 0, /*merkle_variant:*/ true);
let next_shred_index = shreds.len();
blockstore
.insert_shreds(shreds, None, false)
@ -6437,7 +6538,7 @@ pub mod tests {
fn test_no_insert_but_modify_slot_meta() {
// This tests correctness of the SlotMeta in various cases in which a shred
// that gets filtered out by checks
let (shreds0, _) = make_slot_entries(0, 0, 200);
let (shreds0, _) = make_slot_entries(0, 0, 200, /*merkle_variant:*/ true);
let ledger_path = get_tmp_ledger_path_auto_delete!();
let blockstore = Blockstore::open(ledger_path.path()).unwrap();
@ -6449,8 +6550,8 @@ pub mod tests {
// Insert a repetitive shred for slot 's', should get ignored, but also
// insert shreds that chains to 's', should see the update in the SlotMeta
// for 's'.
let (mut shreds2, _) = make_slot_entries(2, 0, 200);
let (mut shreds3, _) = make_slot_entries(3, 0, 200);
let (mut shreds2, _) = make_slot_entries(2, 0, 200, /*merkle_variant:*/ true);
let (mut shreds3, _) = make_slot_entries(3, 0, 200, /*merkle_variant:*/ true);
shreds2.push(shreds0[1].clone());
shreds3.insert(0, shreds0[1].clone());
blockstore.insert_shreds(shreds2, None, false).unwrap();
@ -6467,7 +6568,7 @@ pub mod tests {
let blockstore = Blockstore::open(ledger_path.path()).unwrap();
// Make shred for slot 1
let (shreds1, _) = make_slot_entries(1, 0, 1);
let (shreds1, _) = make_slot_entries(1, 0, 1, /*merkle_variant:*/ true);
let last_root = 100;
blockstore.set_roots(std::iter::once(&last_root)).unwrap();
@ -6496,7 +6597,14 @@ pub mod tests {
assert_eq!(blockstore.lowest_slot(), 0);
for slot in 1..4 {
let entries = make_slot_entries_with_transactions(100);
let shreds = entries_to_test_shreds(&entries, slot, slot - 1, true, 0);
let shreds = entries_to_test_shreds(
&entries,
slot,
slot - 1, // parent_slot
true, // is_full_slot
0, // version
true, // merkle_variant
);
blockstore.insert_shreds(shreds, None, false).unwrap();
blockstore.set_roots(vec![slot].iter()).unwrap();
}
@ -6515,9 +6623,30 @@ pub mod tests {
let slot = 10;
let entries = make_slot_entries_with_transactions(100);
let blockhash = get_last_hash(entries.iter()).unwrap();
let shreds = entries_to_test_shreds(&entries, slot, slot - 1, true, 0);
let more_shreds = entries_to_test_shreds(&entries, slot + 1, slot, true, 0);
let unrooted_shreds = entries_to_test_shreds(&entries, slot + 2, slot + 1, true, 0);
let shreds = entries_to_test_shreds(
&entries,
slot,
slot - 1, // parent_slot
true, // is_full_slot
0, // version
true, // merkle_variant
);
let more_shreds = entries_to_test_shreds(
&entries,
slot + 1,
slot, // parent_slot
true, // is_full_slot
0, // version
true, // merkle_variant
);
let unrooted_shreds = entries_to_test_shreds(
&entries,
slot + 2,
slot + 1, // parent_slot
true, // is_full_slot
0, // version
true, // merkle_variant
);
let ledger_path = get_tmp_ledger_path_auto_delete!();
let blockstore = Blockstore::open(ledger_path.path()).unwrap();
blockstore.insert_shreds(shreds, None, false).unwrap();
@ -7427,7 +7556,14 @@ pub mod tests {
fn test_get_rooted_transaction() {
let slot = 2;
let entries = make_slot_entries_with_transactions(5);
let shreds = entries_to_test_shreds(&entries, slot, slot - 1, true, 0);
let shreds = entries_to_test_shreds(
&entries,
slot,
slot - 1, // parent_slot
true, // is_full_slot
0, // version
true, // merkle_variant
);
let ledger_path = get_tmp_ledger_path_auto_delete!();
let blockstore = Blockstore::open(ledger_path.path()).unwrap();
blockstore.insert_shreds(shreds, None, false).unwrap();
@ -7540,7 +7676,14 @@ pub mod tests {
let slot = 2;
let entries = make_slot_entries_with_transactions(5);
let shreds = entries_to_test_shreds(&entries, slot, slot - 1, true, 0);
let shreds = entries_to_test_shreds(
&entries,
slot,
slot - 1, // parent_slot
true, // is_full_slot
0, // version
true, // merkle_variant
);
blockstore.insert_shreds(shreds, None, false).unwrap();
let expected_transactions: Vec<VersionedTransactionWithStatusMeta> = entries
@ -7876,7 +8019,7 @@ pub mod tests {
let ledger_path = get_tmp_ledger_path_auto_delete!();
let blockstore = Blockstore::open(ledger_path.path()).unwrap();
let (shreds, _) = make_slot_entries(1, 0, 4);
let (shreds, _) = make_slot_entries(1, 0, 4, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
fn make_slot_entries_with_transaction_addresses(addresses: &[Pubkey]) -> Vec<Entry> {
@ -7903,7 +8046,14 @@ pub mod tests {
let entries = make_slot_entries_with_transaction_addresses(&[
address0, address1, address0, address1,
]);
let shreds = entries_to_test_shreds(&entries, slot, slot - 1, true, 0);
let shreds = entries_to_test_shreds(
&entries,
slot,
slot - 1, // parent_slot
true, // is_full_slot
0, // version
true, // merkle_variant
);
blockstore.insert_shreds(shreds, None, false).unwrap();
for entry in entries.into_iter() {
@ -7927,7 +8077,8 @@ pub mod tests {
let entries = make_slot_entries_with_transaction_addresses(&[
address0, address1, address0, address1,
]);
let shreds = entries_to_test_shreds(&entries, slot, 8, true, 0);
let shreds =
entries_to_test_shreds(&entries, slot, 8, true, 0, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
for entry in entries.into_iter() {
@ -8435,7 +8586,7 @@ pub mod tests {
assert_eq!(blockstore.lowest_slot(), 0);
for slot in 0..10 {
let (shreds, _) = make_slot_entries(slot, 0, 1);
let (shreds, _) = make_slot_entries(slot, 0, 1, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
}
assert_eq!(blockstore.lowest_slot(), 1);
@ -8451,7 +8602,7 @@ pub mod tests {
assert_eq!(blockstore.highest_slot().unwrap(), None);
for slot in 0..10 {
let (shreds, _) = make_slot_entries(slot, 0, 1);
let (shreds, _) = make_slot_entries(slot, 0, 1, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
assert_eq!(blockstore.highest_slot().unwrap(), Some(slot));
}
@ -8643,6 +8794,7 @@ pub mod tests {
true, // is_last_in_slot
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
@ -8704,6 +8856,7 @@ pub mod tests {
true, // is_last_in_slot
0, // next_shred_index
0, // next_code_index,
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
let (duplicate_shreds, _) = shredder.entries_to_shreds(
@ -8712,6 +8865,7 @@ pub mod tests {
true, // is_last_in_slot
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
let shred = shreds[0].clone();
@ -9046,7 +9200,14 @@ pub mod tests {
let parent = 0;
let num_txs = 20;
let entry = make_large_tx_entry(num_txs);
let shreds = entries_to_test_shreds(&[entry], slot, parent, true, 0);
let shreds = entries_to_test_shreds(
&[entry],
slot,
parent,
true, // is_full_slot
0, // version
false, // merkle_variant
);
assert!(shreds.len() > 1);
let ledger_path = get_tmp_ledger_path_auto_delete!();
@ -9079,7 +9240,8 @@ pub mod tests {
// Create enough entries to ensure there are at least two shreds created
let num_unique_entries = max_ticks_per_n_shreds(1, None) + 1;
let (mut original_shreds, original_entries) = make_slot_entries(0, 0, num_unique_entries);
let (mut original_shreds, original_entries) =
make_slot_entries(0, 0, num_unique_entries, /*merkle_variant:*/ true);
// Discard first shred, so that the slot is not full
assert!(original_shreds.len() > 1);
@ -9102,7 +9264,14 @@ pub mod tests {
assert!(!blockstore.is_full(0));
}
let duplicate_shreds = entries_to_test_shreds(&original_entries, 0, 0, true, 0);
let duplicate_shreds = entries_to_test_shreds(
&original_entries,
0, // slot
0, // parent_slot
true, // is_full_slot
0, // version
true, // merkle_variant
);
let num_shreds = duplicate_shreds.len() as u64;
blockstore
.insert_shreds(duplicate_shreds, None, false)
@ -9124,7 +9293,8 @@ pub mod tests {
let num_shreds = 2;
let num_entries = max_ticks_per_n_shreds(num_shreds, None);
let slot = 1;
let (mut shreds, _) = make_slot_entries(slot, 0, num_entries);
let (mut shreds, _) =
make_slot_entries(slot, 0, num_entries, /*merkle_variant:*/ false);
// Mark both as last shred
shreds[0].set_last_in_slot();
@ -9145,7 +9315,8 @@ pub mod tests {
let setup_test_shreds = |slot: Slot| -> Vec<Shred> {
let num_entries = max_ticks_per_n_shreds(num_shreds, None);
let (mut shreds, _) = make_slot_entries(slot, 0, num_entries);
let (mut shreds, _) =
make_slot_entries(slot, 0, num_entries, /*merkle_variant:*/ false);
shreds[smaller_last_shred_index].set_last_in_slot();
shreds[larger_last_shred_index].set_last_in_slot();
shreds
@ -9319,7 +9490,8 @@ pub mod tests {
let num_shreds = 10;
let middle_shred_index = 5;
let num_entries = max_ticks_per_n_shreds(num_shreds, None);
let (shreds, _) = make_slot_entries(slot, 0, num_entries);
let (shreds, _) =
make_slot_entries(slot, 0, num_entries, /*merkle_variant:*/ false);
// Reverse shreds so that last shred gets inserted first and sets meta.received
let mut shreds: Vec<Shred> = shreds.into_iter().rev().collect();

27
ledger/src/blockstore/blockstore_purge.rs
View File

@ -803,7 +803,14 @@ pub mod tests {
for x in 0..index0_max_slot + 1 {
let entries = make_slot_entries_with_transactions(1);
let shreds = entries_to_test_shreds(&entries, x, x.saturating_sub(1), true, 0);
let shreds = entries_to_test_shreds(
&entries,
x, // slot
x.saturating_sub(1), // parent_slot
true, // is_full_slot
0, // version
true, // merkle_variant
);
blockstore.insert_shreds(shreds, None, false).unwrap();
let signature = entries
.iter()
@ -839,7 +846,14 @@ pub mod tests {
for x in index0_max_slot + 1..index1_max_slot + 1 {
let entries = make_slot_entries_with_transactions(1);
let shreds = entries_to_test_shreds(&entries, x, x.saturating_sub(1), true, 0);
let shreds = entries_to_test_shreds(
&entries,
x, // slot
x.saturating_sub(1), // parent_slot
true, // is_full_slot
0, // version
true, // merkle_variant
);
blockstore.insert_shreds(shreds, None, false).unwrap();
let signature: Signature = entries
.iter()
@ -1239,7 +1253,14 @@ pub mod tests {
let mut tick = create_ticks(1, 0, hash(&serialize(&x).unwrap()));
entries.append(&mut tick);
}
let shreds = entries_to_test_shreds(&entries, slot, slot - 1, true, 0);
let shreds = entries_to_test_shreds(
&entries,
slot,
slot - 1, // parent_slot
true, // is_full_slot
0, // version
true, // merkle_variant
);
blockstore.insert_shreds(shreds, None, false).unwrap();
let mut write_batch = blockstore.db.batch().unwrap();

4
ledger/src/leader_schedule_cache.rs
View File

@ -451,7 +451,7 @@ mod tests {
// Write a shred into slot 2 that chains to slot 1,
// but slot 1 is empty so should not be skipped
let (shreds, _) = make_slot_entries(2, 1, 1);
let (shreds, _) = make_slot_entries(2, 1, 1, /*merkle_variant:*/ true);
blockstore.insert_shreds(shreds, None, false).unwrap();
assert_eq!(
cache
@ -462,7 +462,7 @@ mod tests {
);
// Write a shred into slot 1
let (shreds, _) = make_slot_entries(1, 0, 1);
let (shreds, _) = make_slot_entries(1, 0, 1, /*merkle_variant:*/ true);
// Check that slot 1 and 2 are skipped
blockstore.insert_shreds(shreds, None, false).unwrap();

23
ledger/src/shredder.rs
View File

@ -69,12 +69,13 @@ impl Shredder {
is_last_in_slot: bool,
next_shred_index: u32,
next_code_index: u32,
merkle_variant: bool,
stats: &mut ProcessShredsStats,
) -> (
Vec<Shred>, // data shreds
Vec<Shred>, // coding shreds
) {
if false {
if merkle_variant {
return shred::make_merkle_shreds_from_entries(
&PAR_THREAD_POOL,
keypair,
@ -462,6 +463,7 @@ mod tests {
true, // is_last_in_slot
start_index, // next_shred_index
start_index, // next_code_index
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
let next_index = data_shreds.last().unwrap().index() + 1;
@ -539,6 +541,7 @@ mod tests {
true, // is_last_in_slot
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
let deserialized_shred =
@ -569,6 +572,7 @@ mod tests {
true, // is_last_in_slot
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
data_shreds.iter().for_each(|s| {
@ -604,6 +608,7 @@ mod tests {
true, // is_last_in_slot
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
data_shreds.iter().for_each(|s| {
@ -648,6 +653,7 @@ mod tests {
true, // is_last_in_slot
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
for (i, s) in data_shreds.iter().enumerate() {
@ -700,8 +706,9 @@ mod tests {
&keypair,
&entries,
is_last_in_slot,
0, // next_shred_index
0, // next_code_index
0, // next_shred_index
0, // next_code_index
false, // merkle_variant
&mut ProcessShredsStats::default(),
);
let num_coding_shreds = coding_shreds.len();
@ -827,9 +834,10 @@ mod tests {
let (data_shreds, coding_shreds) = shredder.entries_to_shreds(
&keypair,
&entries,
true, // is_last_in_slot
25, // next_shred_index,
25, // next_code_index
true, // is_last_in_slot
25, // next_shred_index,
25, // next_code_index
false, // merkle_variant
&mut ProcessShredsStats::default(),
);
// We should have 10 shreds now
@ -921,6 +929,7 @@ mod tests {
is_last_in_slot,
next_shred_index,
next_shred_index, // next_code_index
false, // merkle_variant
&mut ProcessShredsStats::default(),
);
let num_data_shreds = data_shreds.len();
@ -981,6 +990,7 @@ mod tests {
true, // is_last_in_slot
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
assert!(!data_shreds
@ -1013,6 +1023,7 @@ mod tests {
true, // is_last_in_slot
start_index, // next_shred_index
start_index, // next_code_index
true, // merkle_variant
&mut ProcessShredsStats::default(),
);
const MIN_CHUNK_SIZE: usize = DATA_SHREDS_PER_FEC_BLOCK;

9
ledger/tests/blockstore.rs
View File

@ -21,7 +21,14 @@ fn test_multiple_threads_insert_shred() {
let threads: Vec<_> = (0..num_threads)
.map(|i| {
let entries = entry::create_ticks(1, 0, Hash::default());
let shreds = blockstore::entries_to_test_shreds(&entries, i + 1, 0, false, 0);
let shreds = blockstore::entries_to_test_shreds(
&entries,
i + 1,
0,
false,
0,
true, // merkle_variant
);
let blockstore_ = blockstore.clone();
Builder::new()
.name("blockstore-writer".to_string())

8
ledger/tests/shred.rs
View File

@ -51,9 +51,10 @@ fn test_multi_fec_block_coding() {
let (data_shreds, coding_shreds) = shredder.entries_to_shreds(
&keypair,
&entries,
true, // is_last_in_slot
0, // next_shred_index
0, // next_code_index
true, // is_last_in_slot
0, // next_shred_index
0, // next_code_index
false, // merkle_variant
&mut ProcessShredsStats::default(),
);
let next_index = data_shreds.last().unwrap().index() + 1;
@ -226,6 +227,7 @@ fn setup_different_sized_fec_blocks(
is_last,
next_shred_index,
next_code_index,
false, // merkle_variant
&mut ProcessShredsStats::default(),
);
for shred in &data_shreds {

10
rpc/src/rpc.rs
View File

@ -4567,8 +4567,14 @@ pub fn populate_blockstore_for_tests(
) {
let slot = bank.slot();
let parent_slot = bank.parent_slot();
let shreds =
solana_ledger::blockstore::entries_to_test_shreds(&entries, slot, parent_slot, true, 0);
let shreds = solana_ledger::blockstore::entries_to_test_shreds(
&entries,
slot,
parent_slot,
true,
0,
true, // merkle_variant
);
blockstore.insert_shreds(shreds, None, false).unwrap();
blockstore.set_roots(std::iter::once(&slot)).unwrap();