blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

restore timestamp() in find_missing_indexes (#30345)

This commit is contained in:
Jeff Biseda 2023-02-15 16:12:36 -08:00 committed by GitHub
parent 5680c76c86
commit 20614fa746
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 56 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
core/src/repair_generic_traversal.rs
View File

@ -114,7 +114,6 @@ pub fn get_closest_completion(
slot_meta_cache: &mut HashMap<Slot, Option<SlotMeta>>,
processed_slots: &mut HashSet<Slot>,
limit: usize,
now_timestamp: u64,
) -> Vec<ShredRepairType> {
let mut v: Vec<(Slot, u64)> = Vec::default();
let iter = GenericTraversal::new(tree);
@ -188,7 +187,6 @@ pub fn get_closest_completion(
slot,
slot_meta,
limit - repairs.len(),
now_timestamp,
);
repairs.extend(new_repairs);
}
@ -201,9 +199,9 @@ pub fn get_closest_completion(
pub mod test {
use {
super::*,
crate::repair_service::post_shred_deferment_timestamp,
crate::repair_service::sleep_shred_deferment_period,
solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path},
solana_sdk::{hash::Hash, timing::timestamp},
solana_sdk::hash::Hash,
trees::{tr, Tree, TreeWalk},
};
@ -240,7 +238,6 @@ pub mod test {
&mut slot_meta_cache,
&mut processed_slots,
10,
timestamp(),
);
assert_eq!(repairs, []);
@ -258,13 +255,13 @@ pub mod test {
let heaviest_subtree_fork_choice = HeaviestSubtreeForkChoice::new_from_tree(forks);
let mut slot_meta_cache = HashMap::default();
let mut processed_slots = HashSet::default();
sleep_shred_deferment_period();
let repairs = get_closest_completion(
&heaviest_subtree_fork_choice,
&blockstore,
&mut slot_meta_cache,
&mut processed_slots,
2,
post_shred_deferment_timestamp(),
);
assert_eq!(
repairs,

29
core/src/repair_service.rs
View File

@ -353,7 +353,6 @@ impl RepairService {
MAX_UNKNOWN_LAST_INDEX_REPAIRS,
MAX_CLOSEST_COMPLETION_REPAIRS,
&duplicate_slot_repair_statuses,
timestamp(),
&mut repair_timing,
&mut best_repairs_stats,
);
@ -530,7 +529,6 @@ impl RepairService {
slot: Slot,
slot_meta: &SlotMeta,
max_repairs: usize,
now_timestamp: u64,
) -> Vec<ShredRepairType> {
if max_repairs == 0 || slot_meta.is_full() {
vec![]
@ -543,8 +541,9 @@ impl RepairService {
.and_then(|shred| shred::layout::get_reference_tick(&shred).ok())
.map(u64::from)
{
// System time is not monotonic
let ticks_since_first_insert = DEFAULT_TICKS_PER_SECOND
* now_timestamp.saturating_sub(slot_meta.first_shred_timestamp)
* timestamp().saturating_sub(slot_meta.first_shred_timestamp)
/ 1_000;
if ticks_since_first_insert
< reference_tick.saturating_add(DEFER_REPAIR_THRESHOLD_TICKS)
@ -557,7 +556,6 @@ impl RepairService {
blockstore
.find_missing_data_indexes(
slot,
now_timestamp,
slot_meta.first_shred_timestamp,
DEFER_REPAIR_THRESHOLD_TICKS,
slot_meta.consumed,
@ -577,7 +575,6 @@ impl RepairService {
max_repairs: usize,
slot: Slot,
duplicate_slot_repair_statuses: &impl Contains<'a, Slot>,
now_timestamp: u64,
) {
let mut pending_slots = vec![slot];
while repairs.len() < max_repairs && !pending_slots.is_empty() {
@ -592,7 +589,6 @@ impl RepairService {
slot,
&slot_meta,
max_repairs - repairs.len(),
now_timestamp,
);
repairs.extend(new_repairs);
let next_slots = slot_meta.next_slots;
@ -609,7 +605,6 @@ impl RepairService {
blockstore: &Blockstore,
max_repairs: usize,
repair_range: &RepairSlotRange,
now_timestamp: u64,
) -> crate::result::Result<Vec<ShredRepairType>> {
// Slot height and shred indexes for shreds we want to repair
let mut repairs: Vec<ShredRepairType> = vec![];
@ -631,7 +626,6 @@ impl RepairService {
slot,
&meta,
max_repairs - repairs.len(),
now_timestamp,
);
repairs.extend(new_repairs);
}
@ -654,7 +648,6 @@ impl RepairService {
slot,
&slot_meta,
MAX_REPAIR_PER_DUPLICATE,
timestamp(),
))
}
} else {
@ -795,9 +788,11 @@ impl RepairService {
}
#[cfg(test)]
pub(crate) fn post_shred_deferment_timestamp() -> u64 {
// adjust timestamp to bypass shred deferment window
timestamp() + DEFAULT_MS_PER_SLOT + DEFER_REPAIR_THRESHOLD.as_millis() as u64
pub(crate) fn sleep_shred_deferment_period() {
// sleep to bypass shred deferment window
sleep(Duration::from_millis(
DEFAULT_MS_PER_SLOT + DEFER_REPAIR_THRESHOLD.as_millis() as u64,
));
}
#[cfg(test)]
@ -850,7 +845,6 @@ mod test {
MAX_UNKNOWN_LAST_INDEX_REPAIRS,
MAX_CLOSEST_COMPLETION_REPAIRS,
&HashSet::default(),
timestamp(),
&mut RepairTiming::default(),
&mut BestRepairsStats::default(),
),
@ -888,7 +882,6 @@ mod test {
MAX_UNKNOWN_LAST_INDEX_REPAIRS,
MAX_CLOSEST_COMPLETION_REPAIRS,
&HashSet::default(),
timestamp(),
&mut RepairTiming::default(),
&mut BestRepairsStats::default(),
),
@ -939,6 +932,7 @@ mod test {
.collect();
let mut repair_weight = RepairWeight::new(0);
sleep_shred_deferment_period();
assert_eq!(
repair_weight.get_best_weighted_repairs(
&blockstore,
@ -949,7 +943,6 @@ mod test {
MAX_UNKNOWN_LAST_INDEX_REPAIRS,
MAX_CLOSEST_COMPLETION_REPAIRS,
&HashSet::default(),
post_shred_deferment_timestamp(),
&mut RepairTiming::default(),
&mut BestRepairsStats::default(),
),
@ -966,7 +959,6 @@ mod test {
MAX_UNKNOWN_LAST_INDEX_REPAIRS,
MAX_CLOSEST_COMPLETION_REPAIRS,
&HashSet::default(),
post_shred_deferment_timestamp(),
&mut RepairTiming::default(),
&mut BestRepairsStats::default(),
)[..],
@ -1002,6 +994,7 @@ mod test {
let expected: Vec<ShredRepairType> =
vec![ShredRepairType::HighestShred(0, num_shreds_per_slot - 1)];
sleep_shred_deferment_period();
let mut repair_weight = RepairWeight::new(0);
assert_eq!(
repair_weight.get_best_weighted_repairs(
@ -1013,7 +1006,6 @@ mod test {
MAX_UNKNOWN_LAST_INDEX_REPAIRS,
MAX_CLOSEST_COMPLETION_REPAIRS,
&HashSet::default(),
post_shred_deferment_timestamp(),
&mut RepairTiming::default(),
&mut BestRepairsStats::default(),
),
@ -1057,12 +1049,12 @@ mod test {
})
.collect();
sleep_shred_deferment_period();
assert_eq!(
RepairService::generate_repairs_in_range(
&blockstore,
std::usize::MAX,
&repair_slot_range,
post_shred_deferment_timestamp(),
)
.unwrap(),
expected
@ -1111,7 +1103,6 @@ mod test {
&blockstore,
std::usize::MAX,
&repair_slot_range,
timestamp(),
)
.unwrap(),
expected

7
core/src/repair_weight.rs
View File

@ -157,7 +157,6 @@ impl RepairWeight {
max_unknown_last_index_repairs: usize,
max_closest_completion_repairs: usize,
ignore_slots: &impl Contains<'a, Slot>,
now_timestamp: u64,
repair_timing: &mut RepairTiming,
stats: &mut BestRepairsStats,
) -> Vec<ShredRepairType> {
@ -188,7 +187,6 @@ impl RepairWeight {
&mut best_shreds_repairs,
max_new_shreds,
ignore_slots,
now_timestamp,
);
let num_best_shreds_repairs = best_shreds_repairs.len();
let repair_slots_set: HashSet<Slot> =
@ -223,7 +221,6 @@ impl RepairWeight {
&mut slot_meta_cache,
&mut processed_slots,
max_closest_completion_repairs,
now_timestamp,
);
let num_closest_completion_repairs = closest_completion_repairs.len();
let num_closest_completion_slots = processed_slots.len() - pre_num_slots;
@ -353,7 +350,6 @@ impl RepairWeight {
repairs: &mut Vec<ShredRepairType>,
max_new_shreds: usize,
ignore_slots: &impl Contains<'a, Slot>,
now_timestamp: u64,
) {
let root_tree = self.trees.get(&self.root).expect("Root tree must exist");
repair_weighted_traversal::get_best_repair_shreds(
@ -363,7 +359,6 @@ impl RepairWeight {
repairs,
max_new_shreds,
ignore_slots,
now_timestamp,
);
}
@ -472,7 +467,6 @@ impl RepairWeight {
slot_meta_cache: &mut HashMap<Slot, Option<SlotMeta>>,
processed_slots: &mut HashSet<Slot>,
max_new_repairs: usize,
now_timestamp: u64,
) -> Vec<ShredRepairType> {
let mut repairs = Vec::default();
for (_slot, tree) in self.trees.iter() {
@ -485,7 +479,6 @@ impl RepairWeight {
slot_meta_cache,
processed_slots,
max_new_repairs - repairs.len(),
now_timestamp,
);
repairs.extend(new_repairs);
}

33
core/src/repair_weighted_traversal.rs
View File

@ -80,7 +80,6 @@ pub fn get_best_repair_shreds<'a>(
repairs: &mut Vec<ShredRepairType>,
max_new_shreds: usize,
ignore_slots: &impl Contains<'a, Slot>,
now_timestamp: u64,
) {
let initial_len = repairs.len();
let max_repairs = initial_len + max_new_shreds;
@ -107,7 +106,6 @@ pub fn get_best_repair_shreds<'a>(
slot,
slot_meta,
max_repairs - repairs.len(),
now_timestamp,
);
repairs.extend(new_repairs);
}
@ -129,7 +127,6 @@ pub fn get_best_repair_shreds<'a>(
max_repairs,
*new_child_slot,
ignore_slots,
now_timestamp,
);
}
visited_set.insert(*new_child_slot);
@ -144,13 +141,13 @@ pub fn get_best_repair_shreds<'a>(
pub mod test {
use {
super::*,
crate::repair_service::post_shred_deferment_timestamp,
crate::repair_service::sleep_shred_deferment_period,
solana_ledger::{
get_tmp_ledger_path,
shred::{Shred, ShredFlags},
},
solana_runtime::bank_utils,
solana_sdk::{hash::Hash, timing::timestamp},
solana_sdk::hash::Hash,
trees::tr,
};
@ -230,6 +227,7 @@ pub mod test {
let mut slot_meta_cache = HashMap::default();
let last_shred = blockstore.meta(0).unwrap().unwrap().received;
sleep_shred_deferment_period();
get_best_repair_shreds(
&heaviest_subtree_fork_choice,
&blockstore,
@ -237,18 +235,6 @@ pub mod test {
&mut repairs,
6,
&HashSet::default(),
timestamp().saturating_sub(1_000 * 60), // ensure deferment
);
assert_eq!(repairs, vec![]);
get_best_repair_shreds(
&heaviest_subtree_fork_choice,
&blockstore,
&mut slot_meta_cache,
&mut repairs,
6,
&HashSet::default(),
post_shred_deferment_timestamp(),
);
assert_eq!(
repairs,
@ -271,6 +257,7 @@ pub mod test {
2,
Hash::default(),
);
sleep_shred_deferment_period();
get_best_repair_shreds(
&heaviest_subtree_fork_choice,
&blockstore,
@ -278,7 +265,6 @@ pub mod test {
&mut repairs,
6,
&HashSet::default(),
post_shred_deferment_timestamp(),
);
assert_eq!(
repairs,
@ -312,6 +298,7 @@ pub mod test {
blockstore
.insert_shreds(completed_shreds, None, false)
.unwrap();
sleep_shred_deferment_period();
get_best_repair_shreds(
&heaviest_subtree_fork_choice,
&blockstore,
@ -319,7 +306,6 @@ pub mod test {
&mut repairs,
4,
&HashSet::default(),
post_shred_deferment_timestamp(),
);
assert_eq!(
repairs,
@ -334,6 +320,7 @@ pub mod test {
repairs = vec![];
slot_meta_cache = HashMap::default();
blockstore.add_tree(tr(2) / (tr(8)), true, false, 2, Hash::default());
sleep_shred_deferment_period();
get_best_repair_shreds(
&heaviest_subtree_fork_choice,
&blockstore,
@ -341,7 +328,6 @@ pub mod test {
&mut repairs,
4,
&HashSet::default(),
post_shred_deferment_timestamp(),
);
assert_eq!(
repairs,
@ -359,6 +345,7 @@ pub mod test {
// 4 again when the Unvisited(2) event happens
blockstore.add_tree(tr(2) / (tr(6) / tr(7)), true, false, 2, Hash::default());
sleep_shred_deferment_period();
let mut repairs = vec![];
let mut slot_meta_cache = HashMap::default();
get_best_repair_shreds(
@ -368,7 +355,6 @@ pub mod test {
&mut repairs,
std::usize::MAX,
&HashSet::default(),
post_shred_deferment_timestamp(),
);
let last_shred = blockstore.meta(0).unwrap().unwrap().received;
assert_eq!(
@ -383,6 +369,7 @@ pub mod test {
#[test]
fn test_get_best_repair_shreds_ignore() {
let (blockstore, heaviest_subtree_fork_choice) = setup_forks();
sleep_shred_deferment_period();
// Adding slots to ignore should remove them from the repair set, but
// should not remove their children
@ -397,7 +384,6 @@ pub mod test {
&mut repairs,
std::usize::MAX,
&ignore_set,
post_shred_deferment_timestamp(),
);
assert_eq!(
repairs,
@ -413,6 +399,7 @@ pub mod test {
slot_meta_cache = HashMap::default();
blockstore.add_tree(tr(2) / (tr(6) / tr(7)), true, false, 2, Hash::default());
ignore_set.insert(2);
sleep_shred_deferment_period();
get_best_repair_shreds(
&heaviest_subtree_fork_choice,
&blockstore,
@ -420,7 +407,6 @@ pub mod test {
&mut repairs,
std::usize::MAX,
&ignore_set,
post_shred_deferment_timestamp(),
);
assert_eq!(
repairs,
@ -442,7 +428,6 @@ pub mod test {
&mut repairs,
std::usize::MAX,
&ignore_set,
post_shred_deferment_timestamp(),
);
assert_eq!(
repairs,

87
ledger/src/blockstore.rs
View File

@ -1801,7 +1801,6 @@ impl Blockstore {
fn find_missing_indexes<C>(
db_iterator: &mut DBRawIterator,
slot: Slot,
now_timestamp: u64,
first_timestamp: u64,
defer_threshold_ticks: u64,
start_index: u64,
@ -1816,8 +1815,9 @@ impl Blockstore {
}
let mut missing_indexes = vec![];
// System time is not monotonic
let ticks_since_first_insert =
DEFAULT_TICKS_PER_SECOND * (now_timestamp - first_timestamp) / 1000;
DEFAULT_TICKS_PER_SECOND * timestamp().saturating_sub(first_timestamp) / 1000;
// Seek to the first shred with index >= start_index
db_iterator.seek(&C::key((slot, start_index)));
@ -1877,7 +1877,6 @@ impl Blockstore {
pub fn find_missing_data_indexes(
&self,
slot: Slot,
now_timestamp: u64,
first_timestamp: u64,
defer_threshold_ticks: u64,
start_index: u64,
@ -1891,7 +1890,6 @@ impl Blockstore {
Self::find_missing_indexes::<cf::ShredData>(
&mut db_iterator,
slot,
now_timestamp,
first_timestamp,
defer_threshold_ticks,
start_index,
@ -5916,7 +5914,6 @@ pub mod tests {
assert_eq!(
blockstore.find_missing_data_indexes(
slot,
timestamp(),
0, // first_timestamp
0, // defer_threshold_ticks
0, // start_index
@ -5928,7 +5925,6 @@ pub mod tests {
assert_eq!(
blockstore.find_missing_data_indexes(
slot,
timestamp(),
0, // first_timestamp
0, // defer_threshold_ticks
1, // start_index
@ -5940,7 +5936,6 @@ pub mod tests {
assert_eq!(
blockstore.find_missing_data_indexes(
slot,
timestamp(),
0, // first_timestmap
0, // defer_threshold_ticks
0, // start_index
@ -5952,7 +5947,6 @@ pub mod tests {
assert_eq!(
blockstore.find_missing_data_indexes(
slot,
timestamp(),
0, // first_timestmap
0, // defer_threshold_ticks
gap - 2, // start_index
@ -5963,8 +5957,7 @@ pub mod tests {
);
assert_eq!(
blockstore.find_missing_data_indexes(
slot,
timestamp(),
slot, // slot
0, // first_timestamp
0, // defer_threshold_ticks
gap - 2, // start_index
@ -5975,13 +5968,12 @@ pub mod tests {
);
assert_eq!(
blockstore.find_missing_data_indexes(
slot,
timestamp(),
0, // first_timestamp
0, // defer_threshold_ticks
0, // start_index
gap, // end_index
1, // max_missing
slot, // slot
0, // first_timestamp
0, // defer_threshold_ticks
0, // start_index
gap, // end_index
1, // max_missing
),
vec![1],
);
@ -5993,7 +5985,6 @@ pub mod tests {
assert_eq!(
blockstore.find_missing_data_indexes(
slot,
timestamp(),
0, // first_timestamp
0, // defer_threshold_ticks
0, // start_index
@ -6005,7 +5996,6 @@ pub mod tests {
assert_eq!(
blockstore.find_missing_data_indexes(
slot,
timestamp(),
0, // first_timestamp
0, // defer_threshold_ticks
0, // start_index
@ -6027,7 +6017,6 @@ pub mod tests {
assert_eq!(
blockstore.find_missing_data_indexes(
slot,
timestamp(),
0, // first_timestamp
0, // defer_threshold_ticks
j * gap, // start_index
@ -6068,7 +6057,6 @@ pub mod tests {
assert_eq!(
blockstore.find_missing_data_indexes(
slot,
timestamp(),
timestamp(), // first_timestamp
0, // defer_threshold_ticks
0, // start_index
@ -6081,7 +6069,6 @@ pub mod tests {
assert_eq!(
blockstore.find_missing_data_indexes(
slot,
timestamp(),
timestamp() - 400, // first_timestamp
0, // defer_threshold_ticks
0, // start_index
@ -6103,49 +6090,45 @@ pub mod tests {
let empty: Vec<u64> = vec![];
assert_eq!(
blockstore.find_missing_data_indexes(
slot,
timestamp(),
0, // first_timestamp
0, // defer_threshold_ticks
0, // start_index
0, // end_index
1, // max_missing
slot, // slot
0, // first_timestamp
0, // defer_threshold_ticks
0, // start_index
0, // end_index
1, // max_missing
),
empty
);
assert_eq!(
blockstore.find_missing_data_indexes(
slot,
timestamp(),
0, // first_timestamp
0, // defer_threshold_ticks
5, // start_index
5, // end_index
1, // max_missing
slot, // slot
0, // first_timestamp
0, // defer_threshold_ticks
5, // start_index
5, // end_index
1, // max_missing
),
empty
);
assert_eq!(
blockstore.find_missing_data_indexes(
slot,
timestamp(),
0, // first_timestamp
0, // defer_threshold_ticks
4, // start_index
3, // end_index
1, // max_missing
slot, // slot
0, // first_timestamp
0, // defer_threshold_ticks
4, // start_index
3, // end_index
1, // max_missing
),
empty
);
assert_eq!(
blockstore.find_missing_data_indexes(
slot,
timestamp(),
0, // first_timestamp
0, // defer_threshold_ticks
1, // start_index
2, // end_index
0, // max_missing
slot, // slot
0, // first_timestamp
0, // defer_threshold_ticks
1, // start_index
2, // end_index
0, // max_missing
),
empty
);
@ -6173,8 +6156,7 @@ pub mod tests {
// [i, first_index - 1]
for start in 0..STARTS {
let result = blockstore.find_missing_data_indexes(
slot,
timestamp(),
slot, // slot
0, // first_timestamp
0, // defer_threshold_ticks
start, // start_index
@ -6207,7 +6189,6 @@ pub mod tests {
assert_eq!(
blockstore.find_missing_data_indexes(
slot,
timestamp(),
0, // first_timestamp
0, // defer_threshold_ticks
j, // start_index