//! The `repair_service` module implements the tools necessary to generate a thread which //! regularly finds missing shreds in the ledger and sends repair requests for those shreds use { crate::{ ancestor_hashes_service::{AncestorHashesReplayUpdateReceiver, AncestorHashesService}, cluster_info_vote_listener::VerifiedVoteReceiver, cluster_slots::ClusterSlots, duplicate_repair_status::DuplicateSlotRepairStatus, outstanding_requests::OutstandingRequests, repair_weight::RepairWeight, result::Result, serve_repair::{ServeRepair, ShredRepairType, REPAIR_PEERS_CACHE_CAPACITY}, }, crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender}, lru::LruCache, solana_gossip::cluster_info::ClusterInfo, solana_ledger::blockstore::{Blockstore, SlotMeta}, solana_measure::measure::Measure, solana_runtime::{bank_forks::BankForks, contains::Contains}, solana_sdk::{ clock::Slot, epoch_schedule::EpochSchedule, hash::Hash, pubkey::Pubkey, signer::keypair::Keypair, }, solana_streamer::sendmmsg::{batch_send, SendPktsError}, std::{ collections::{HashMap, HashSet}, iter::Iterator, net::{SocketAddr, UdpSocket}, sync::{ atomic::{AtomicBool, Ordering}, Arc, RwLock, }, thread::{self, sleep, Builder, JoinHandle}, time::{Duration, Instant}, }, }; #[cfg(test)] use {solana_ledger::shred::Nonce, solana_sdk::timing::timestamp}; pub type DuplicateSlotsResetSender = CrossbeamSender>; pub type DuplicateSlotsResetReceiver = CrossbeamReceiver>; pub type ConfirmedSlotsSender = CrossbeamSender>; pub type ConfirmedSlotsReceiver = CrossbeamReceiver>; pub type OutstandingShredRepairs = OutstandingRequests; #[derive(Default, Debug)] pub struct SlotRepairs { highest_shred_index: u64, // map from pubkey to total number of requests pubkey_repairs: HashMap, } impl SlotRepairs { pub fn pubkey_repairs(&self) -> &HashMap { &self.pubkey_repairs } } #[derive(Default, Debug)] pub struct RepairStatsGroup { pub count: u64, pub min: u64, pub max: u64, pub slot_pubkeys: HashMap, } impl RepairStatsGroup { pub fn update(&mut self, repair_peer_id: &Pubkey, slot: Slot, shred_index: u64) { self.count += 1; let slot_repairs = self.slot_pubkeys.entry(slot).or_default(); // Increment total number of repairs of this type for this pubkey by 1 *slot_repairs .pubkey_repairs .entry(*repair_peer_id) .or_default() += 1; // Update the max requested shred index for this slot slot_repairs.highest_shred_index = std::cmp::max(slot_repairs.highest_shred_index, shred_index); self.min = std::cmp::min(self.min, slot); self.max = std::cmp::max(self.max, slot); } } #[derive(Default, Debug)] pub struct RepairStats { pub shred: RepairStatsGroup, pub highest_shred: RepairStatsGroup, pub orphan: RepairStatsGroup, pub get_best_orphans_us: u64, pub get_best_shreds_us: u64, } #[derive(Default, Debug)] pub struct RepairTiming { pub set_root_elapsed: u64, pub get_votes_elapsed: u64, pub add_votes_elapsed: u64, pub get_best_orphans_elapsed: u64, pub get_best_shreds_elapsed: u64, pub get_unknown_last_index_elapsed: u64, pub get_closest_completion_elapsed: u64, pub send_repairs_elapsed: u64, pub build_repairs_batch_elapsed: u64, pub batch_send_repairs_elapsed: u64, } impl RepairTiming { fn update( &mut self, set_root_elapsed: u64, get_votes_elapsed: u64, add_votes_elapsed: u64, build_repairs_batch_elapsed: u64, batch_send_repairs_elapsed: u64, ) { self.set_root_elapsed += set_root_elapsed; self.get_votes_elapsed += get_votes_elapsed; self.add_votes_elapsed += add_votes_elapsed; self.build_repairs_batch_elapsed += build_repairs_batch_elapsed; self.batch_send_repairs_elapsed += batch_send_repairs_elapsed; self.send_repairs_elapsed += build_repairs_batch_elapsed + batch_send_repairs_elapsed; } } #[derive(Default, Debug)] pub struct BestRepairsStats { pub call_count: u64, pub num_orphan_slots: u64, pub num_orphan_repairs: u64, pub num_best_shreds_slots: u64, pub num_best_shreds_repairs: u64, pub num_unknown_last_index_slots: u64, pub num_unknown_last_index_repairs: u64, pub num_closest_completion_slots: u64, pub num_closest_completion_repairs: u64, } impl BestRepairsStats { pub fn update( &mut self, num_orphan_slots: u64, num_orphan_repairs: u64, num_best_shreds_slots: u64, num_best_shreds_repairs: u64, num_unknown_last_index_slots: u64, num_unknown_last_index_repairs: u64, num_closest_completion_slots: u64, num_closest_completion_repairs: u64, ) { self.call_count += 1; self.num_orphan_slots += num_orphan_slots; self.num_orphan_repairs += num_orphan_repairs; self.num_best_shreds_slots += num_best_shreds_slots; self.num_best_shreds_repairs += num_best_shreds_repairs; self.num_unknown_last_index_slots += num_unknown_last_index_slots; self.num_unknown_last_index_repairs += num_unknown_last_index_repairs; self.num_closest_completion_slots += num_closest_completion_slots; self.num_closest_completion_repairs += num_closest_completion_repairs; } } pub const MAX_REPAIR_LENGTH: usize = 512; pub const MAX_REPAIR_PER_DUPLICATE: usize = 20; pub const MAX_DUPLICATE_WAIT_MS: usize = 10_000; pub const REPAIR_MS: u64 = 100; pub const MAX_ORPHANS: usize = 5; pub const MAX_UNKNOWN_LAST_INDEX_REPAIRS: usize = 10; pub const MAX_CLOSEST_COMPLETION_REPAIRS: usize = 100; #[derive(Clone)] pub struct RepairInfo { pub bank_forks: Arc>, pub cluster_info: Arc, pub cluster_slots: Arc, pub epoch_schedule: EpochSchedule, pub duplicate_slots_reset_sender: DuplicateSlotsResetSender, pub repair_validators: Option>, } pub struct RepairSlotRange { pub start: Slot, pub end: Slot, } impl Default for RepairSlotRange { fn default() -> Self { RepairSlotRange { start: 0, end: std::u64::MAX, } } } pub struct RepairService { t_repair: JoinHandle<()>, ancestor_hashes_service: AncestorHashesService, } impl RepairService { pub fn new( blockstore: Arc, exit: Arc, repair_socket: Arc, ancestor_hashes_socket: Arc, repair_info: RepairInfo, verified_vote_receiver: VerifiedVoteReceiver, outstanding_requests: Arc>, ancestor_hashes_replay_update_receiver: AncestorHashesReplayUpdateReceiver, ) -> Self { let t_repair = { let blockstore = blockstore.clone(); let exit = exit.clone(); let repair_info = repair_info.clone(); Builder::new() .name("solRepairSvc".to_string()) .spawn(move || { Self::run( &blockstore, &exit, &repair_socket, repair_info, verified_vote_receiver, &outstanding_requests, ) }) .unwrap() }; let ancestor_hashes_service = AncestorHashesService::new( exit, blockstore, ancestor_hashes_socket, repair_info, ancestor_hashes_replay_update_receiver, ); RepairService { t_repair, ancestor_hashes_service, } } fn run( blockstore: &Blockstore, exit: &AtomicBool, repair_socket: &UdpSocket, repair_info: RepairInfo, verified_vote_receiver: VerifiedVoteReceiver, outstanding_requests: &RwLock, ) { let mut repair_weight = RepairWeight::new(repair_info.bank_forks.read().unwrap().root()); let serve_repair = ServeRepair::new( repair_info.cluster_info.clone(), repair_info.bank_forks.clone(), ); let id = repair_info.cluster_info.id(); let mut repair_stats = RepairStats::default(); let mut repair_timing = RepairTiming::default(); let mut best_repairs_stats = BestRepairsStats::default(); let mut last_stats = Instant::now(); let duplicate_slot_repair_statuses: HashMap = HashMap::new(); let mut peers_cache = LruCache::new(REPAIR_PEERS_CACHE_CAPACITY); loop { if exit.load(Ordering::Relaxed) { break; } let mut set_root_elapsed; let mut get_votes_elapsed; let mut add_votes_elapsed; let root_bank = repair_info.bank_forks.read().unwrap().root_bank(); let repairs = { let new_root = root_bank.slot(); // Purge outdated slots from the weighting heuristic set_root_elapsed = Measure::start("set_root_elapsed"); repair_weight.set_root(new_root); set_root_elapsed.stop(); // Add new votes to the weighting heuristic get_votes_elapsed = Measure::start("get_votes_elapsed"); let mut slot_to_vote_pubkeys: HashMap> = HashMap::new(); verified_vote_receiver .try_iter() .for_each(|(vote_pubkey, vote_slots)| { for slot in vote_slots { slot_to_vote_pubkeys .entry(slot) .or_default() .push(vote_pubkey); } }); get_votes_elapsed.stop(); add_votes_elapsed = Measure::start("add_votes"); repair_weight.add_votes( blockstore, slot_to_vote_pubkeys.into_iter(), root_bank.epoch_stakes_map(), root_bank.epoch_schedule(), ); add_votes_elapsed.stop(); let repairs = repair_weight.get_best_weighted_repairs( blockstore, root_bank.epoch_stakes_map(), root_bank.epoch_schedule(), MAX_ORPHANS, MAX_REPAIR_LENGTH, MAX_UNKNOWN_LAST_INDEX_REPAIRS, MAX_CLOSEST_COMPLETION_REPAIRS, &duplicate_slot_repair_statuses, Some(&mut repair_timing), Some(&mut best_repairs_stats), ); repairs }; let identity_keypair: &Keypair = &repair_info.cluster_info.keypair().clone(); let mut build_repairs_batch_elapsed = Measure::start("build_repairs_batch_elapsed"); let batch: Vec<(Vec, SocketAddr)> = { let mut outstanding_requests = outstanding_requests.write().unwrap(); repairs .iter() .filter_map(|repair_request| { let (to, req) = serve_repair .repair_request( &repair_info.cluster_slots, *repair_request, &mut peers_cache, &mut repair_stats, &repair_info.repair_validators, &mut outstanding_requests, identity_keypair, ) .ok()?; Some((req, to)) }) .collect() }; build_repairs_batch_elapsed.stop(); let mut batch_send_repairs_elapsed = Measure::start("batch_send_repairs_elapsed"); if !batch.is_empty() { if let Err(SendPktsError::IoError(err, num_failed)) = batch_send(repair_socket, &batch) { error!( "{} batch_send failed to send {}/{} packets first error {:?}", id, num_failed, batch.len(), err ); } } batch_send_repairs_elapsed.stop(); repair_timing.update( set_root_elapsed.as_us(), get_votes_elapsed.as_us(), add_votes_elapsed.as_us(), build_repairs_batch_elapsed.as_us(), batch_send_repairs_elapsed.as_us(), ); if last_stats.elapsed().as_secs() > 2 { let repair_total = repair_stats.shred.count + repair_stats.highest_shred.count + repair_stats.orphan.count; let slot_to_count: Vec<_> = repair_stats .shred .slot_pubkeys .iter() .chain(repair_stats.highest_shred.slot_pubkeys.iter()) .chain(repair_stats.orphan.slot_pubkeys.iter()) .map(|(slot, slot_repairs)| { (slot, slot_repairs.pubkey_repairs.values().sum::()) }) .collect(); info!("repair_stats: {:?}", slot_to_count); if repair_total > 0 { datapoint_info!( "repair_service-my_requests", ("repair-total", repair_total, i64), ("shred-count", repair_stats.shred.count, i64), ("highest-shred-count", repair_stats.highest_shred.count, i64), ("orphan-count", repair_stats.orphan.count, i64), ("repair-highest-slot", repair_stats.highest_shred.max, i64), ("repair-orphan", repair_stats.orphan.max, i64), ); } datapoint_info!( "repair_service-repair_timing", ("set-root-elapsed", repair_timing.set_root_elapsed, i64), ("get-votes-elapsed", repair_timing.get_votes_elapsed, i64), ("add-votes-elapsed", repair_timing.add_votes_elapsed, i64), ( "get-best-orphans-elapsed", repair_timing.get_best_orphans_elapsed, i64 ), ( "get-best-shreds-elapsed", repair_timing.get_best_shreds_elapsed, i64 ), ( "get-unknown-last-index-elapsed", repair_timing.get_unknown_last_index_elapsed, i64 ), ( "get-closest-completion-elapsed", repair_timing.get_closest_completion_elapsed, i64 ), ( "send-repairs-elapsed", repair_timing.send_repairs_elapsed, i64 ), ( "build-repairs-batch-elapsed", repair_timing.build_repairs_batch_elapsed, i64 ), ( "batch-send-repairs-elapsed", repair_timing.batch_send_repairs_elapsed, i64 ), ); datapoint_info!( "serve_repair-best-repairs", ("call-count", best_repairs_stats.call_count, i64), ("orphan-slots", best_repairs_stats.num_orphan_slots, i64), ("orphan-repairs", best_repairs_stats.num_orphan_repairs, i64), ( "best-shreds-slots", best_repairs_stats.num_best_shreds_slots, i64 ), ( "best-shreds-repairs", best_repairs_stats.num_best_shreds_repairs, i64 ), ( "unknown-last-index-slots", best_repairs_stats.num_unknown_last_index_slots, i64 ), ( "unknown-last-index-repairs", best_repairs_stats.num_unknown_last_index_repairs, i64 ), ( "closest-completion-slots", best_repairs_stats.num_closest_completion_slots, i64 ), ( "closest-completion-repairs", best_repairs_stats.num_closest_completion_repairs, i64 ), ); repair_stats = RepairStats::default(); repair_timing = RepairTiming::default(); best_repairs_stats = BestRepairsStats::default(); last_stats = Instant::now(); } sleep(Duration::from_millis(REPAIR_MS)); } } // Generate repairs for all slots `x` in the repair_range.start <= x <= repair_range.end pub fn generate_repairs_in_range( blockstore: &Blockstore, max_repairs: usize, repair_range: &RepairSlotRange, ) -> Result> { // Slot height and shred indexes for shreds we want to repair let mut repairs: Vec = vec![]; for slot in repair_range.start..=repair_range.end { if repairs.len() >= max_repairs { break; } let meta = blockstore .meta(slot) .expect("Unable to lookup slot meta") .unwrap_or(SlotMeta { slot, ..SlotMeta::default() }); let new_repairs = Self::generate_repairs_for_slot( blockstore, slot, &meta, max_repairs - repairs.len(), ); repairs.extend(new_repairs); } Ok(repairs) } pub fn generate_repairs_for_slot( blockstore: &Blockstore, slot: Slot, slot_meta: &SlotMeta, max_repairs: usize, ) -> Vec { if max_repairs == 0 || slot_meta.is_full() { vec![] } else if slot_meta.consumed == slot_meta.received { vec![ShredRepairType::HighestShred(slot, slot_meta.received)] } else { let reqs = blockstore.find_missing_data_indexes( slot, slot_meta.first_shred_timestamp, slot_meta.consumed, slot_meta.received, max_repairs, ); reqs.into_iter() .map(|i| ShredRepairType::Shred(slot, i)) .collect() } } /// Repairs any fork starting at the input slot pub fn generate_repairs_for_fork<'a>( blockstore: &Blockstore, repairs: &mut Vec, max_repairs: usize, slot: Slot, duplicate_slot_repair_statuses: &impl Contains<'a, Slot>, ) { let mut pending_slots = vec![slot]; while repairs.len() < max_repairs && !pending_slots.is_empty() { let slot = pending_slots.pop().unwrap(); if duplicate_slot_repair_statuses.contains(&slot) { // These are repaired through a different path continue; } if let Some(slot_meta) = blockstore.meta(slot).unwrap() { let new_repairs = Self::generate_repairs_for_slot( blockstore, slot, &slot_meta, max_repairs - repairs.len(), ); repairs.extend(new_repairs); let next_slots = slot_meta.next_slots; pending_slots.extend(next_slots); } else { break; } } } #[cfg(test)] fn generate_duplicate_repairs_for_slot( blockstore: &Blockstore, slot: Slot, ) -> Option> { if let Some(slot_meta) = blockstore.meta(slot).unwrap() { if slot_meta.is_full() { // If the slot is full, no further need to repair this slot None } else { Some(Self::generate_repairs_for_slot( blockstore, slot, &slot_meta, MAX_REPAIR_PER_DUPLICATE, )) } } else { error!("Slot meta for duplicate slot does not exist, cannot generate repairs"); // Filter out this slot from the set of duplicates to be repaired as // the SlotMeta has to exist for duplicates to be generated None } } #[cfg(test)] fn generate_and_send_duplicate_repairs( duplicate_slot_repair_statuses: &mut HashMap, cluster_slots: &ClusterSlots, blockstore: &Blockstore, serve_repair: &ServeRepair, repair_stats: &mut RepairStats, repair_socket: &UdpSocket, repair_validators: &Option>, outstanding_requests: &RwLock, identity_keypair: &Keypair, ) { duplicate_slot_repair_statuses.retain(|slot, status| { Self::update_duplicate_slot_repair_addr( *slot, status, cluster_slots, serve_repair, repair_validators, ); if let Some((repair_pubkey, repair_addr)) = status.repair_pubkey_and_addr { let repairs = Self::generate_duplicate_repairs_for_slot(blockstore, *slot); if let Some(repairs) = repairs { let mut outstanding_requests = outstanding_requests.write().unwrap(); for repair_type in repairs { let nonce = outstanding_requests.add_request(repair_type, timestamp()); if let Err(e) = Self::serialize_and_send_request( &repair_type, repair_socket, &repair_pubkey, &repair_addr, serve_repair, repair_stats, nonce, identity_keypair, ) { info!( "repair req send_to {} ({}) error {:?}", repair_pubkey, repair_addr, e ); } } true } else { false } } else { true } }) } #[cfg(test)] fn serialize_and_send_request( repair_type: &ShredRepairType, repair_socket: &UdpSocket, repair_pubkey: &Pubkey, to: &SocketAddr, serve_repair: &ServeRepair, repair_stats: &mut RepairStats, nonce: Nonce, identity_keypair: &Keypair, ) -> Result<()> { let req = serve_repair.map_repair_request( repair_type, repair_pubkey, repair_stats, nonce, identity_keypair, )?; repair_socket.send_to(&req, to)?; Ok(()) } #[cfg(test)] fn update_duplicate_slot_repair_addr( slot: Slot, status: &mut DuplicateSlotRepairStatus, cluster_slots: &ClusterSlots, serve_repair: &ServeRepair, repair_validators: &Option>, ) { let now = timestamp(); if status.repair_pubkey_and_addr.is_none() || now.saturating_sub(status.start_ts) >= MAX_DUPLICATE_WAIT_MS as u64 { let repair_pubkey_and_addr = serve_repair.repair_request_duplicate_compute_best_peer( slot, cluster_slots, repair_validators, ); status.repair_pubkey_and_addr = repair_pubkey_and_addr.ok(); status.start_ts = timestamp(); } } #[cfg(test)] #[allow(dead_code)] fn initiate_repair_for_duplicate_slot( slot: Slot, duplicate_slot_repair_statuses: &mut HashMap, cluster_slots: &ClusterSlots, serve_repair: &ServeRepair, repair_validators: &Option>, ) { // If we're already in the middle of repairing this, ignore the signal. if duplicate_slot_repair_statuses.contains_key(&slot) { return; } // Mark this slot as special repair, try to download from single // validator to avoid corruption let repair_pubkey_and_addr = serve_repair .repair_request_duplicate_compute_best_peer(slot, cluster_slots, repair_validators) .ok(); let new_duplicate_slot_repair_status = DuplicateSlotRepairStatus { correct_ancestors_to_repair: vec![(slot, Hash::default())], repair_pubkey_and_addr, start_ts: timestamp(), }; duplicate_slot_repair_statuses.insert(slot, new_duplicate_slot_repair_status); } pub fn join(self) -> thread::Result<()> { self.t_repair.join()?; self.ancestor_hashes_service.join() } } #[cfg(test)] mod test { use { super::*, solana_gossip::{cluster_info::Node, contact_info::ContactInfo}, solana_ledger::{ blockstore::{ make_chaining_slot_entries, make_many_slot_entries, make_slot_entries, Blockstore, }, genesis_utils::{create_genesis_config, GenesisConfigInfo}, get_tmp_ledger_path, shred::max_ticks_per_n_shreds, }, solana_runtime::bank::Bank, solana_sdk::signature::Keypair, solana_streamer::socket::SocketAddrSpace, std::collections::HashSet, }; fn new_test_cluster_info(contact_info: ContactInfo) -> ClusterInfo { ClusterInfo::new( contact_info, Arc::new(Keypair::new()), SocketAddrSpace::Unspecified, ) } #[test] pub fn test_repair_orphan() { let blockstore_path = get_tmp_ledger_path!(); { let blockstore = Blockstore::open(&blockstore_path).unwrap(); // Create some orphan slots let (mut shreds, _) = make_slot_entries(1, 0, 1, /*merkle_variant:*/ true); let (shreds2, _) = make_slot_entries(5, 2, 1, /*merkle_variant:*/ true); shreds.extend(shreds2); blockstore.insert_shreds(shreds, None, false).unwrap(); let mut repair_weight = RepairWeight::new(0); assert_eq!( repair_weight.get_best_weighted_repairs( &blockstore, &HashMap::new(), &EpochSchedule::default(), MAX_ORPHANS, MAX_REPAIR_LENGTH, MAX_UNKNOWN_LAST_INDEX_REPAIRS, MAX_CLOSEST_COMPLETION_REPAIRS, &HashSet::default(), None, None, ), vec![ ShredRepairType::Orphan(2), ShredRepairType::HighestShred(0, 0) ] ); } Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] pub fn test_repair_empty_slot() { let blockstore_path = get_tmp_ledger_path!(); { let blockstore = Blockstore::open(&blockstore_path).unwrap(); let (shreds, _) = make_slot_entries(2, 0, 1, /*merkle_variant:*/ true); // Write this shred to slot 2, should chain to slot 0, which we haven't received // any shreds for blockstore.insert_shreds(shreds, None, false).unwrap(); let mut repair_weight = RepairWeight::new(0); // Check that repair tries to patch the empty slot assert_eq!( repair_weight.get_best_weighted_repairs( &blockstore, &HashMap::new(), &EpochSchedule::default(), MAX_ORPHANS, MAX_REPAIR_LENGTH, MAX_UNKNOWN_LAST_INDEX_REPAIRS, MAX_CLOSEST_COMPLETION_REPAIRS, &HashSet::default(), None, None, ), vec![ShredRepairType::HighestShred(0, 0)] ); } Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] pub fn test_generate_repairs() { let blockstore_path = get_tmp_ledger_path!(); { let blockstore = Blockstore::open(&blockstore_path).unwrap(); let nth = 3; let num_slots = 2; // Create some shreds let (mut shreds, _) = make_many_slot_entries(0, num_slots, 150); let num_shreds = shreds.len() as u64; let num_shreds_per_slot = num_shreds / num_slots; // write every nth shred let mut shreds_to_write = vec![]; let mut missing_indexes_per_slot = vec![]; for i in (0..num_shreds).rev() { let index = i % num_shreds_per_slot; // get_best_repair_shreds only returns missing shreds in // between shreds received; So this should either insert the // last shred in each slot, or exclude missing shreds after the // last inserted shred from expected repairs. if index % nth == 0 || index + 1 == num_shreds_per_slot { shreds_to_write.insert(0, shreds.remove(i as usize)); } else if i < num_shreds_per_slot { missing_indexes_per_slot.insert(0, index); } } blockstore .insert_shreds(shreds_to_write, None, false) .unwrap(); // sleep so that the holes are ready for repair sleep(Duration::from_secs(1)); let expected: Vec = (0..num_slots) .flat_map(|slot| { missing_indexes_per_slot .iter() .map(move |shred_index| ShredRepairType::Shred(slot, *shred_index)) }) .collect(); let mut repair_weight = RepairWeight::new(0); assert_eq!( repair_weight.get_best_weighted_repairs( &blockstore, &HashMap::new(), &EpochSchedule::default(), MAX_ORPHANS, MAX_REPAIR_LENGTH, MAX_UNKNOWN_LAST_INDEX_REPAIRS, MAX_CLOSEST_COMPLETION_REPAIRS, &HashSet::default(), None, None, ), expected ); assert_eq!( repair_weight.get_best_weighted_repairs( &blockstore, &HashMap::new(), &EpochSchedule::default(), MAX_ORPHANS, expected.len() - 2, MAX_UNKNOWN_LAST_INDEX_REPAIRS, MAX_CLOSEST_COMPLETION_REPAIRS, &HashSet::default(), None, None, )[..], expected[0..expected.len() - 2] ); } Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] pub fn test_generate_highest_repair() { let blockstore_path = get_tmp_ledger_path!(); { let blockstore = Blockstore::open(&blockstore_path).unwrap(); let num_entries_per_slot = 100; // Create some shreds let (mut shreds, _) = make_slot_entries( 0, // slot 0, // parent_slot num_entries_per_slot as u64, true, // merkle_variant ); let num_shreds_per_slot = shreds.len() as u64; // Remove last shred (which is also last in slot) so that slot is not complete shreds.pop(); blockstore.insert_shreds(shreds, None, false).unwrap(); // We didn't get the last shred for this slot, so ask for the highest shred for that slot let expected: Vec = vec![ShredRepairType::HighestShred(0, num_shreds_per_slot - 1)]; let mut repair_weight = RepairWeight::new(0); assert_eq!( repair_weight.get_best_weighted_repairs( &blockstore, &HashMap::new(), &EpochSchedule::default(), MAX_ORPHANS, MAX_REPAIR_LENGTH, MAX_UNKNOWN_LAST_INDEX_REPAIRS, MAX_CLOSEST_COMPLETION_REPAIRS, &HashSet::default(), None, None, ), expected ); } Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] pub fn test_repair_range() { let blockstore_path = get_tmp_ledger_path!(); { let blockstore = Blockstore::open(&blockstore_path).unwrap(); let slots: Vec = vec![1, 3, 5, 7, 8]; let num_entries_per_slot = max_ticks_per_n_shreds(1, None) + 1; let shreds = make_chaining_slot_entries(&slots, num_entries_per_slot); for (mut slot_shreds, _) in shreds.into_iter() { slot_shreds.remove(0); blockstore.insert_shreds(slot_shreds, None, false).unwrap(); } // sleep to make slot eligible for repair sleep(Duration::from_secs(1)); // Iterate through all possible combinations of start..end (inclusive on both // sides of the range) for start in 0..slots.len() { for end in start..slots.len() { let repair_slot_range = RepairSlotRange { start: slots[start], end: slots[end], }; let expected: Vec = (repair_slot_range.start ..=repair_slot_range.end) .map(|slot_index| { if slots.contains(&slot_index) { ShredRepairType::Shred(slot_index, 0) } else { ShredRepairType::HighestShred(slot_index, 0) } }) .collect(); assert_eq!( RepairService::generate_repairs_in_range( &blockstore, std::usize::MAX, &repair_slot_range, ) .unwrap(), expected ); } } } Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] pub fn test_repair_range_highest() { let blockstore_path = get_tmp_ledger_path!(); { let blockstore = Blockstore::open(&blockstore_path).unwrap(); let num_entries_per_slot = 10; let num_slots = 1; let start = 5; // Create some shreds in slots 0..num_slots for i in start..start + num_slots { let parent = if i > 0 { i - 1 } else { 0 }; let (shreds, _) = make_slot_entries( i, // slot parent, num_entries_per_slot as u64, true, // merkle_variant ); blockstore.insert_shreds(shreds, None, false).unwrap(); } let end = 4; let expected: Vec = vec![ ShredRepairType::HighestShred(end - 2, 0), ShredRepairType::HighestShred(end - 1, 0), ShredRepairType::HighestShred(end, 0), ]; let repair_slot_range = RepairSlotRange { start: 2, end }; assert_eq!( RepairService::generate_repairs_in_range( &blockstore, std::usize::MAX, &repair_slot_range, ) .unwrap(), expected ); } Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] pub fn test_generate_duplicate_repairs_for_slot() { let blockstore_path = get_tmp_ledger_path!(); let blockstore = Blockstore::open(&blockstore_path).unwrap(); let dead_slot = 9; // SlotMeta doesn't exist, should make no repairs assert!( RepairService::generate_duplicate_repairs_for_slot(&blockstore, dead_slot,).is_none() ); // Insert some shreds to create a SlotMeta, should make repairs let num_entries_per_slot = max_ticks_per_n_shreds(1, None) + 1; let (mut shreds, _) = make_slot_entries( dead_slot, // slot dead_slot - 1, // parent_slot num_entries_per_slot, true, // merkle_variant ); blockstore .insert_shreds(shreds[..shreds.len() - 1].to_vec(), None, false) .unwrap(); assert!( RepairService::generate_duplicate_repairs_for_slot(&blockstore, dead_slot,).is_some() ); // SlotMeta is full, should make no repairs blockstore .insert_shreds(vec![shreds.pop().unwrap()], None, false) .unwrap(); assert!( RepairService::generate_duplicate_repairs_for_slot(&blockstore, dead_slot,).is_none() ); } #[test] pub fn test_generate_and_send_duplicate_repairs() { let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); let bank = Bank::new_for_tests(&genesis_config); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); let blockstore_path = get_tmp_ledger_path!(); let blockstore = Blockstore::open(&blockstore_path).unwrap(); let cluster_slots = ClusterSlots::default(); let cluster_info = Arc::new(new_test_cluster_info(Node::new_localhost().info)); let identity_keypair = cluster_info.keypair().clone(); let serve_repair = ServeRepair::new(cluster_info, bank_forks); let mut duplicate_slot_repair_statuses = HashMap::new(); let dead_slot = 9; let receive_socket = &UdpSocket::bind("0.0.0.0:0").unwrap(); let duplicate_status = DuplicateSlotRepairStatus { correct_ancestors_to_repair: vec![(dead_slot, Hash::default())], start_ts: std::u64::MAX, repair_pubkey_and_addr: None, }; // Insert some shreds to create a SlotMeta, let num_entries_per_slot = max_ticks_per_n_shreds(1, None) + 1; let (mut shreds, _) = make_slot_entries( dead_slot, dead_slot - 1, num_entries_per_slot, true, // merkle_variant ); blockstore .insert_shreds(shreds[..shreds.len() - 1].to_vec(), None, false) .unwrap(); duplicate_slot_repair_statuses.insert(dead_slot, duplicate_status); // There is no repair_addr, so should not get filtered because the timeout // `std::u64::MAX` has not expired RepairService::generate_and_send_duplicate_repairs( &mut duplicate_slot_repair_statuses, &cluster_slots, &blockstore, &serve_repair, &mut RepairStats::default(), &UdpSocket::bind("0.0.0.0:0").unwrap(), &None, &RwLock::new(OutstandingRequests::default()), &identity_keypair, ); assert!(duplicate_slot_repair_statuses .get(&dead_slot) .unwrap() .repair_pubkey_and_addr .is_none()); assert!(duplicate_slot_repair_statuses.get(&dead_slot).is_some()); // Give the slot a repair address duplicate_slot_repair_statuses .get_mut(&dead_slot) .unwrap() .repair_pubkey_and_addr = Some((Pubkey::default(), receive_socket.local_addr().unwrap())); // Slot is not yet full, should not get filtered from `duplicate_slot_repair_statuses` RepairService::generate_and_send_duplicate_repairs( &mut duplicate_slot_repair_statuses, &cluster_slots, &blockstore, &serve_repair, &mut RepairStats::default(), &UdpSocket::bind("0.0.0.0:0").unwrap(), &None, &RwLock::new(OutstandingRequests::default()), &identity_keypair, ); assert_eq!(duplicate_slot_repair_statuses.len(), 1); assert!(duplicate_slot_repair_statuses.get(&dead_slot).is_some()); // Insert rest of shreds. Slot is full, should get filtered from // `duplicate_slot_repair_statuses` blockstore .insert_shreds(vec![shreds.pop().unwrap()], None, false) .unwrap(); RepairService::generate_and_send_duplicate_repairs( &mut duplicate_slot_repair_statuses, &cluster_slots, &blockstore, &serve_repair, &mut RepairStats::default(), &UdpSocket::bind("0.0.0.0:0").unwrap(), &None, &RwLock::new(OutstandingRequests::default()), &identity_keypair, ); assert!(duplicate_slot_repair_statuses.is_empty()); } #[test] pub fn test_update_duplicate_slot_repair_addr() { let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); let bank = Bank::new_for_tests(&genesis_config); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); let dummy_addr = Some(( Pubkey::default(), UdpSocket::bind("0.0.0.0:0").unwrap().local_addr().unwrap(), )); let cluster_info = Arc::new(new_test_cluster_info(Node::new_localhost().info)); let serve_repair = ServeRepair::new(cluster_info.clone(), bank_forks); let valid_repair_peer = Node::new_localhost().info; // Signal that this peer has confirmed the dead slot, and is thus // a valid target for repair let dead_slot = 9; let cluster_slots = ClusterSlots::default(); cluster_slots.insert_node_id(dead_slot, valid_repair_peer.id); cluster_info.insert_info(valid_repair_peer); // Not enough time has passed, should not update the // address let mut duplicate_status = DuplicateSlotRepairStatus { correct_ancestors_to_repair: vec![(dead_slot, Hash::default())], start_ts: std::u64::MAX, repair_pubkey_and_addr: dummy_addr, }; RepairService::update_duplicate_slot_repair_addr( dead_slot, &mut duplicate_status, &cluster_slots, &serve_repair, &None, ); assert_eq!(duplicate_status.repair_pubkey_and_addr, dummy_addr); // If the repair address is None, should try to update let mut duplicate_status = DuplicateSlotRepairStatus { correct_ancestors_to_repair: vec![(dead_slot, Hash::default())], start_ts: std::u64::MAX, repair_pubkey_and_addr: None, }; RepairService::update_duplicate_slot_repair_addr( dead_slot, &mut duplicate_status, &cluster_slots, &serve_repair, &None, ); assert!(duplicate_status.repair_pubkey_and_addr.is_some()); // If sufficient time has passed, should try to update let mut duplicate_status = DuplicateSlotRepairStatus { correct_ancestors_to_repair: vec![(dead_slot, Hash::default())], start_ts: timestamp() - MAX_DUPLICATE_WAIT_MS as u64, repair_pubkey_and_addr: dummy_addr, }; RepairService::update_duplicate_slot_repair_addr( dead_slot, &mut duplicate_status, &cluster_slots, &serve_repair, &None, ); assert_ne!(duplicate_status.repair_pubkey_and_addr, dummy_addr); } }