2019-02-07 15:10:54 -08:00
|
|
|
//! The `repair_service` module implements the tools necessary to generate a thread which
|
2019-11-14 11:49:31 -08:00
|
|
|
//! regularly finds missing shreds in the ledger and sends repair requests for those shreds
|
2020-01-31 14:23:51 -08:00
|
|
|
use crate::{
|
|
|
|
cluster_info::ClusterInfo,
|
2020-03-11 21:31:50 -07:00
|
|
|
cluster_slots::ClusterSlots,
|
2020-01-31 14:23:51 -08:00
|
|
|
result::Result,
|
|
|
|
serve_repair::{RepairType, ServeRepair},
|
|
|
|
};
|
2019-11-02 00:38:30 -07:00
|
|
|
use solana_ledger::{
|
|
|
|
bank_forks::BankForks,
|
2020-01-13 13:13:52 -08:00
|
|
|
blockstore::{Blockstore, CompletedSlotsReceiver, SlotMeta},
|
2019-11-02 00:38:30 -07:00
|
|
|
};
|
|
|
|
use solana_sdk::{clock::Slot, epoch_schedule::EpochSchedule, pubkey::Pubkey};
|
2019-10-08 22:34:26 -07:00
|
|
|
use std::{
|
2020-03-12 17:34:46 -07:00
|
|
|
collections::HashMap,
|
2020-03-04 18:10:30 -08:00
|
|
|
iter::Iterator,
|
2020-03-11 21:31:50 -07:00
|
|
|
net::SocketAddr,
|
2019-10-08 22:34:26 -07:00
|
|
|
net::UdpSocket,
|
|
|
|
sync::atomic::{AtomicBool, Ordering},
|
|
|
|
sync::{Arc, RwLock},
|
|
|
|
thread::sleep,
|
|
|
|
thread::{self, Builder, JoinHandle},
|
2020-03-29 14:43:58 -07:00
|
|
|
time::{Duration, Instant},
|
2019-10-08 22:34:26 -07:00
|
|
|
};
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2020-03-29 14:43:58 -07:00
|
|
|
#[derive(Default)]
|
|
|
|
pub struct RepairStatsGroup {
|
|
|
|
pub count: u64,
|
|
|
|
pub min: u64,
|
|
|
|
pub max: u64,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl RepairStatsGroup {
|
|
|
|
pub fn update(&mut self, slot: u64) {
|
|
|
|
self.count += 1;
|
|
|
|
self.min = std::cmp::min(self.min, slot);
|
|
|
|
self.max = std::cmp::max(self.max, slot);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Default)]
|
|
|
|
pub struct RepairStats {
|
|
|
|
pub shred: RepairStatsGroup,
|
|
|
|
pub highest_shred: RepairStatsGroup,
|
|
|
|
pub orphan: RepairStatsGroup,
|
|
|
|
}
|
|
|
|
|
2019-11-11 13:12:22 -08:00
|
|
|
pub const MAX_REPAIR_LENGTH: usize = 512;
|
|
|
|
pub const REPAIR_MS: u64 = 100;
|
2019-04-06 19:41:22 -07:00
|
|
|
pub const MAX_ORPHANS: usize = 5;
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-05-09 14:10:04 -07:00
|
|
|
pub enum RepairStrategy {
|
|
|
|
RepairRange(RepairSlotRange),
|
|
|
|
RepairAll {
|
|
|
|
bank_forks: Arc<RwLock<BankForks>>,
|
|
|
|
completed_slots_receiver: CompletedSlotsReceiver,
|
2019-05-13 15:37:50 -07:00
|
|
|
epoch_schedule: EpochSchedule,
|
2019-05-09 14:10:04 -07:00
|
|
|
},
|
|
|
|
}
|
|
|
|
|
2019-02-12 17:43:45 -08:00
|
|
|
pub struct RepairSlotRange {
|
2019-12-05 11:25:13 -08:00
|
|
|
pub start: Slot,
|
|
|
|
pub end: Slot,
|
2019-02-12 17:43:45 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
impl Default for RepairSlotRange {
|
|
|
|
fn default() -> Self {
|
|
|
|
RepairSlotRange {
|
|
|
|
start: 0,
|
|
|
|
end: std::u64::MAX,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-02-07 15:10:54 -08:00
|
|
|
pub struct RepairService {
|
|
|
|
t_repair: JoinHandle<()>,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl RepairService {
|
2019-05-08 13:50:32 -07:00
|
|
|
pub fn new(
|
2020-01-13 13:13:52 -08:00
|
|
|
blockstore: Arc<Blockstore>,
|
2019-05-24 19:20:09 -07:00
|
|
|
exit: Arc<AtomicBool>,
|
2019-05-08 13:50:32 -07:00
|
|
|
repair_socket: Arc<UdpSocket>,
|
2020-04-21 12:54:45 -07:00
|
|
|
cluster_info: Arc<ClusterInfo>,
|
2019-05-09 14:10:04 -07:00
|
|
|
repair_strategy: RepairStrategy,
|
2020-03-25 18:09:19 -07:00
|
|
|
cluster_slots: Arc<ClusterSlots>,
|
2019-05-08 13:50:32 -07:00
|
|
|
) -> Self {
|
|
|
|
let t_repair = Builder::new()
|
|
|
|
.name("solana-repair-service".to_string())
|
|
|
|
.spawn(move || {
|
|
|
|
Self::run(
|
2020-01-13 13:13:52 -08:00
|
|
|
&blockstore,
|
2019-05-24 19:20:09 -07:00
|
|
|
&exit,
|
2019-05-08 13:50:32 -07:00
|
|
|
&repair_socket,
|
|
|
|
&cluster_info,
|
2019-05-09 14:10:04 -07:00
|
|
|
repair_strategy,
|
2020-03-25 18:09:19 -07:00
|
|
|
&cluster_slots,
|
2019-05-08 13:50:32 -07:00
|
|
|
)
|
|
|
|
})
|
|
|
|
.unwrap();
|
|
|
|
|
2020-02-10 17:00:00 -08:00
|
|
|
RepairService { t_repair }
|
2019-05-08 13:50:32 -07:00
|
|
|
}
|
|
|
|
|
2019-02-07 15:10:54 -08:00
|
|
|
fn run(
|
2020-03-25 18:09:19 -07:00
|
|
|
blockstore: &Blockstore,
|
|
|
|
exit: &AtomicBool,
|
|
|
|
repair_socket: &UdpSocket,
|
2020-04-21 12:54:45 -07:00
|
|
|
cluster_info: &Arc<ClusterInfo>,
|
2019-05-09 14:10:04 -07:00
|
|
|
repair_strategy: RepairStrategy,
|
2020-03-25 18:09:19 -07:00
|
|
|
cluster_slots: &Arc<ClusterSlots>,
|
2019-02-07 15:10:54 -08:00
|
|
|
) {
|
2020-01-31 14:23:51 -08:00
|
|
|
let serve_repair = ServeRepair::new(cluster_info.clone());
|
2020-04-21 12:54:45 -07:00
|
|
|
let id = cluster_info.id();
|
2020-03-11 21:31:50 -07:00
|
|
|
if let RepairStrategy::RepairAll { .. } = repair_strategy {
|
|
|
|
Self::initialize_lowest_slot(id, blockstore, cluster_info);
|
2019-05-13 15:37:50 -07:00
|
|
|
}
|
2020-03-29 14:43:58 -07:00
|
|
|
let mut repair_stats = RepairStats::default();
|
|
|
|
let mut last_stats = Instant::now();
|
2020-04-16 19:32:19 -07:00
|
|
|
if let RepairStrategy::RepairAll {
|
|
|
|
ref completed_slots_receiver,
|
|
|
|
..
|
|
|
|
} = repair_strategy
|
|
|
|
{
|
|
|
|
Self::initialize_epoch_slots(blockstore, cluster_info, completed_slots_receiver);
|
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
loop {
|
|
|
|
if exit.load(Ordering::Relaxed) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2019-04-08 12:46:23 -07:00
|
|
|
let repairs = {
|
2019-05-09 14:10:04 -07:00
|
|
|
match repair_strategy {
|
|
|
|
RepairStrategy::RepairRange(ref repair_slot_range) => {
|
2019-10-21 10:29:37 -07:00
|
|
|
// Strategy used by archivers
|
2019-05-09 14:10:04 -07:00
|
|
|
Self::generate_repairs_in_range(
|
2020-01-13 13:13:52 -08:00
|
|
|
blockstore,
|
2019-05-09 14:10:04 -07:00
|
|
|
MAX_REPAIR_LENGTH,
|
|
|
|
repair_slot_range,
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
|
|
|
RepairStrategy::RepairAll {
|
|
|
|
ref completed_slots_receiver,
|
2020-03-11 21:31:50 -07:00
|
|
|
ref bank_forks,
|
2019-05-13 15:37:50 -07:00
|
|
|
..
|
2019-05-09 14:10:04 -07:00
|
|
|
} => {
|
2020-01-13 13:13:52 -08:00
|
|
|
let new_root = blockstore.last_root();
|
|
|
|
let lowest_slot = blockstore.lowest_slot();
|
2020-03-11 21:31:50 -07:00
|
|
|
Self::update_lowest_slot(&id, lowest_slot, &cluster_info);
|
2020-04-16 19:32:19 -07:00
|
|
|
Self::update_completed_slots(completed_slots_receiver, &cluster_info);
|
2020-03-11 21:31:50 -07:00
|
|
|
cluster_slots.update(new_root, cluster_info, bank_forks);
|
2020-01-13 13:13:52 -08:00
|
|
|
Self::generate_repairs(blockstore, new_root, MAX_REPAIR_LENGTH)
|
2019-05-09 14:10:04 -07:00
|
|
|
}
|
2019-04-08 12:46:23 -07:00
|
|
|
}
|
|
|
|
};
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
if let Ok(repairs) = repairs {
|
2020-03-12 17:34:46 -07:00
|
|
|
let mut cache = HashMap::new();
|
2020-03-11 21:31:50 -07:00
|
|
|
let reqs: Vec<((SocketAddr, Vec<u8>), RepairType)> = repairs
|
2019-02-07 15:10:54 -08:00
|
|
|
.into_iter()
|
2019-02-14 00:14:23 -08:00
|
|
|
.filter_map(|repair_request| {
|
2020-01-31 14:23:51 -08:00
|
|
|
serve_repair
|
2020-03-29 14:43:58 -07:00
|
|
|
.repair_request(
|
|
|
|
&cluster_slots,
|
|
|
|
&repair_request,
|
|
|
|
&mut cache,
|
|
|
|
&mut repair_stats,
|
|
|
|
)
|
2019-04-06 19:41:22 -07:00
|
|
|
.map(|result| (result, repair_request))
|
2019-02-07 15:10:54 -08:00
|
|
|
.ok()
|
|
|
|
})
|
|
|
|
.collect();
|
|
|
|
|
2019-11-11 14:25:25 -08:00
|
|
|
for ((to, req), _) in reqs {
|
2019-02-07 15:10:54 -08:00
|
|
|
repair_socket.send_to(&req, to).unwrap_or_else(|e| {
|
|
|
|
info!("{} repair req send_to({}) error {:?}", id, to, e);
|
|
|
|
0
|
|
|
|
});
|
|
|
|
}
|
|
|
|
}
|
2020-03-29 14:43:58 -07:00
|
|
|
if last_stats.elapsed().as_secs() > 1 {
|
|
|
|
let repair_total = repair_stats.shred.count
|
|
|
|
+ repair_stats.highest_shred.count
|
|
|
|
+ repair_stats.orphan.count;
|
|
|
|
if repair_total > 0 {
|
|
|
|
datapoint_info!(
|
|
|
|
"serve_repair-repair",
|
|
|
|
("repair-total", repair_total, i64),
|
|
|
|
("shred-count", repair_stats.shred.count, i64),
|
|
|
|
("highest-shred-count", repair_stats.highest_shred.count, i64),
|
|
|
|
("orphan-count", repair_stats.orphan.count, i64),
|
|
|
|
("repair-highest-slot", repair_stats.highest_shred.max, i64),
|
|
|
|
("repair-orphan", repair_stats.orphan.max, i64),
|
|
|
|
);
|
|
|
|
}
|
|
|
|
repair_stats = RepairStats::default();
|
|
|
|
last_stats = Instant::now();
|
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
sleep(Duration::from_millis(REPAIR_MS));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-05-09 19:57:51 -07:00
|
|
|
// Generate repairs for all slots `x` in the repair_range.start <= x <= repair_range.end
|
2019-06-17 18:12:13 -07:00
|
|
|
pub fn generate_repairs_in_range(
|
2020-01-13 13:13:52 -08:00
|
|
|
blockstore: &Blockstore,
|
2019-04-08 12:46:23 -07:00
|
|
|
max_repairs: usize,
|
|
|
|
repair_range: &RepairSlotRange,
|
2019-11-19 20:15:37 -08:00
|
|
|
) -> Result<Vec<RepairType>> {
|
2019-11-14 11:49:31 -08:00
|
|
|
// Slot height and shred indexes for shreds we want to repair
|
2019-04-08 12:46:23 -07:00
|
|
|
let mut repairs: Vec<RepairType> = vec![];
|
2019-05-15 11:37:20 -07:00
|
|
|
for slot in repair_range.start..=repair_range.end {
|
|
|
|
if repairs.len() >= max_repairs {
|
2019-04-08 12:46:23 -07:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2020-01-13 13:13:52 -08:00
|
|
|
let meta = blockstore
|
2019-05-15 11:37:20 -07:00
|
|
|
.meta(slot)
|
|
|
|
.expect("Unable to lookup slot meta")
|
|
|
|
.unwrap_or(SlotMeta {
|
|
|
|
slot,
|
|
|
|
..SlotMeta::default()
|
|
|
|
});
|
2019-04-08 12:46:23 -07:00
|
|
|
|
2019-05-15 11:37:20 -07:00
|
|
|
let new_repairs = Self::generate_repairs_for_slot(
|
2020-01-13 13:13:52 -08:00
|
|
|
blockstore,
|
2019-05-15 11:37:20 -07:00
|
|
|
slot,
|
|
|
|
&meta,
|
|
|
|
max_repairs - repairs.len(),
|
|
|
|
);
|
|
|
|
repairs.extend(new_repairs);
|
2019-04-08 12:46:23 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
Ok(repairs)
|
|
|
|
}
|
|
|
|
|
2019-05-20 19:04:18 -07:00
|
|
|
fn generate_repairs(
|
2020-01-13 13:13:52 -08:00
|
|
|
blockstore: &Blockstore,
|
2019-12-05 11:25:13 -08:00
|
|
|
root: Slot,
|
2019-05-20 19:04:18 -07:00
|
|
|
max_repairs: usize,
|
2019-11-19 20:15:37 -08:00
|
|
|
) -> Result<Vec<RepairType>> {
|
2019-11-14 11:49:31 -08:00
|
|
|
// Slot height and shred indexes for shreds we want to repair
|
2019-04-08 12:46:23 -07:00
|
|
|
let mut repairs: Vec<RepairType> = vec![];
|
2020-01-13 13:13:52 -08:00
|
|
|
Self::generate_repairs_for_fork(blockstore, &mut repairs, max_repairs, root);
|
2019-04-08 12:46:23 -07:00
|
|
|
|
|
|
|
// TODO: Incorporate gossip to determine priorities for repair?
|
|
|
|
|
2020-01-13 13:13:52 -08:00
|
|
|
// Try to resolve orphans in blockstore
|
2020-03-04 18:10:30 -08:00
|
|
|
let orphans = blockstore.orphans_iterator(root + 1).unwrap();
|
|
|
|
Self::generate_repairs_for_orphans(orphans, &mut repairs);
|
2019-04-08 12:46:23 -07:00
|
|
|
Ok(repairs)
|
|
|
|
}
|
|
|
|
|
2019-04-06 19:41:22 -07:00
|
|
|
fn generate_repairs_for_slot(
|
2020-01-13 13:13:52 -08:00
|
|
|
blockstore: &Blockstore,
|
2019-11-02 00:38:30 -07:00
|
|
|
slot: Slot,
|
2019-03-05 14:18:29 -08:00
|
|
|
slot_meta: &SlotMeta,
|
2019-02-07 15:10:54 -08:00
|
|
|
max_repairs: usize,
|
2019-02-14 00:14:23 -08:00
|
|
|
) -> Vec<RepairType> {
|
2019-03-05 14:18:29 -08:00
|
|
|
if slot_meta.is_full() {
|
2019-02-14 00:14:23 -08:00
|
|
|
vec![]
|
2019-03-05 14:18:29 -08:00
|
|
|
} else if slot_meta.consumed == slot_meta.received {
|
2019-11-14 11:49:31 -08:00
|
|
|
vec![RepairType::HighestShred(slot, slot_meta.received)]
|
2019-02-07 15:10:54 -08:00
|
|
|
} else {
|
2020-04-16 18:20:55 -07:00
|
|
|
let reqs = blockstore.find_missing_data_indexes_ts(
|
2019-03-05 14:18:29 -08:00
|
|
|
slot,
|
2019-11-07 11:08:09 -08:00
|
|
|
slot_meta.first_shred_timestamp,
|
2019-03-05 14:18:29 -08:00
|
|
|
slot_meta.consumed,
|
|
|
|
slot_meta.received,
|
2019-02-13 13:49:54 -08:00
|
|
|
max_repairs,
|
|
|
|
);
|
2019-02-14 00:14:23 -08:00
|
|
|
reqs.into_iter()
|
2019-11-07 11:08:09 -08:00
|
|
|
.map(|i| RepairType::Shred(slot, i))
|
2019-02-14 00:14:23 -08:00
|
|
|
.collect()
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-03-04 18:10:30 -08:00
|
|
|
fn generate_repairs_for_orphans(
|
|
|
|
orphans: impl Iterator<Item = u64>,
|
|
|
|
repairs: &mut Vec<RepairType>,
|
|
|
|
) {
|
|
|
|
repairs.extend(orphans.take(MAX_ORPHANS).map(RepairType::Orphan));
|
2019-04-06 19:41:22 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/// Repairs any fork starting at the input slot
|
|
|
|
fn generate_repairs_for_fork(
|
2020-01-13 13:13:52 -08:00
|
|
|
blockstore: &Blockstore,
|
2019-04-06 19:41:22 -07:00
|
|
|
repairs: &mut Vec<RepairType>,
|
|
|
|
max_repairs: usize,
|
2019-11-02 00:38:30 -07:00
|
|
|
slot: Slot,
|
2019-04-06 19:41:22 -07:00
|
|
|
) {
|
|
|
|
let mut pending_slots = vec![slot];
|
|
|
|
while repairs.len() < max_repairs && !pending_slots.is_empty() {
|
|
|
|
let slot = pending_slots.pop().unwrap();
|
2020-01-13 13:13:52 -08:00
|
|
|
if let Some(slot_meta) = blockstore.meta(slot).unwrap() {
|
2019-04-06 19:41:22 -07:00
|
|
|
let new_repairs = Self::generate_repairs_for_slot(
|
2020-01-13 13:13:52 -08:00
|
|
|
blockstore,
|
2019-04-06 19:41:22 -07:00
|
|
|
slot,
|
|
|
|
&slot_meta,
|
2019-02-13 13:49:54 -08:00
|
|
|
max_repairs - repairs.len(),
|
2019-02-14 00:14:23 -08:00
|
|
|
);
|
2019-02-13 13:49:54 -08:00
|
|
|
repairs.extend(new_repairs);
|
2019-04-06 19:41:22 -07:00
|
|
|
let next_slots = slot_meta.next_slots;
|
|
|
|
pending_slots.extend(next_slots);
|
|
|
|
} else {
|
|
|
|
break;
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2019-05-08 13:50:32 -07:00
|
|
|
|
2020-04-21 12:54:45 -07:00
|
|
|
fn initialize_lowest_slot(id: Pubkey, blockstore: &Blockstore, cluster_info: &ClusterInfo) {
|
2019-05-13 15:37:50 -07:00
|
|
|
// Safe to set into gossip because by this time, the leader schedule cache should
|
2020-01-13 13:13:52 -08:00
|
|
|
// also be updated with the latest root (done in blockstore_processor) and thus
|
2019-11-14 11:49:31 -08:00
|
|
|
// will provide a schedule to window_service for any incoming shreds up to the
|
2019-05-13 15:37:50 -07:00
|
|
|
// last_confirmed_epoch.
|
2020-04-21 12:54:45 -07:00
|
|
|
cluster_info.push_lowest_slot(id, blockstore.lowest_slot());
|
2019-05-13 15:37:50 -07:00
|
|
|
}
|
|
|
|
|
2020-03-11 21:31:50 -07:00
|
|
|
fn update_completed_slots(
|
2019-05-13 15:37:50 -07:00
|
|
|
completed_slots_receiver: &CompletedSlotsReceiver,
|
2020-04-21 12:54:45 -07:00
|
|
|
cluster_info: &ClusterInfo,
|
2019-05-13 15:37:50 -07:00
|
|
|
) {
|
2020-03-11 21:31:50 -07:00
|
|
|
let mut slots: Vec<Slot> = vec![];
|
|
|
|
while let Ok(mut more) = completed_slots_receiver.try_recv() {
|
|
|
|
slots.append(&mut more);
|
2020-02-17 12:39:30 -08:00
|
|
|
}
|
2020-03-11 21:31:50 -07:00
|
|
|
slots.sort();
|
|
|
|
if !slots.is_empty() {
|
2020-04-21 12:54:45 -07:00
|
|
|
cluster_info.push_epoch_slots(&slots);
|
2020-03-11 21:31:50 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-04-21 12:54:45 -07:00
|
|
|
fn update_lowest_slot(id: &Pubkey, lowest_slot: Slot, cluster_info: &ClusterInfo) {
|
|
|
|
cluster_info.push_lowest_slot(*id, lowest_slot);
|
2020-03-05 10:58:00 -08:00
|
|
|
}
|
|
|
|
|
2020-04-16 19:32:19 -07:00
|
|
|
fn initialize_epoch_slots(
|
|
|
|
blockstore: &Blockstore,
|
2020-04-21 12:54:45 -07:00
|
|
|
cluster_info: &ClusterInfo,
|
2020-04-16 19:32:19 -07:00
|
|
|
completed_slots_receiver: &CompletedSlotsReceiver,
|
|
|
|
) {
|
|
|
|
let root = blockstore.last_root();
|
|
|
|
let mut slots: Vec<_> = blockstore
|
|
|
|
.live_slots_iterator(root)
|
|
|
|
.filter_map(|(slot, slot_meta)| {
|
|
|
|
if slot_meta.is_full() {
|
|
|
|
Some(slot)
|
|
|
|
} else {
|
|
|
|
None
|
|
|
|
}
|
|
|
|
})
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
while let Ok(mut more) = completed_slots_receiver.try_recv() {
|
|
|
|
slots.append(&mut more);
|
|
|
|
}
|
|
|
|
slots.sort();
|
|
|
|
slots.dedup();
|
|
|
|
if !slots.is_empty() {
|
2020-04-21 12:54:45 -07:00
|
|
|
cluster_info.push_epoch_slots(&slots);
|
2020-04-16 19:32:19 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-11-13 10:12:09 -08:00
|
|
|
pub fn join(self) -> thread::Result<()> {
|
2020-02-10 17:00:00 -08:00
|
|
|
self.t_repair.join()
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
mod test {
|
|
|
|
use super::*;
|
2019-05-13 15:37:50 -07:00
|
|
|
use crate::cluster_info::Node;
|
2020-01-13 13:13:52 -08:00
|
|
|
use solana_ledger::blockstore::{
|
2019-10-18 09:28:51 -07:00
|
|
|
make_chaining_slot_entries, make_many_slot_entries, make_slot_entries,
|
|
|
|
};
|
|
|
|
use solana_ledger::shred::max_ticks_per_n_shreds;
|
2020-01-13 13:13:52 -08:00
|
|
|
use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
#[test]
|
2019-04-06 19:41:22 -07:00
|
|
|
pub fn test_repair_orphan() {
|
2020-01-13 13:13:52 -08:00
|
|
|
let blockstore_path = get_tmp_ledger_path!();
|
2019-02-07 15:10:54 -08:00
|
|
|
{
|
2020-01-13 13:13:52 -08:00
|
|
|
let blockstore = Blockstore::open(&blockstore_path).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-04-06 19:41:22 -07:00
|
|
|
// Create some orphan slots
|
2019-09-03 21:32:51 -07:00
|
|
|
let (mut shreds, _) = make_slot_entries(1, 0, 1);
|
|
|
|
let (shreds2, _) = make_slot_entries(5, 2, 1);
|
|
|
|
shreds.extend(shreds2);
|
2020-01-13 13:13:52 -08:00
|
|
|
blockstore.insert_shreds(shreds, None, false).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
assert_eq!(
|
2020-01-13 13:13:52 -08:00
|
|
|
RepairService::generate_repairs(&blockstore, 0, 2).unwrap(),
|
2019-11-14 11:49:31 -08:00
|
|
|
vec![RepairType::HighestShred(0, 0), RepairType::Orphan(2)]
|
2019-02-07 15:10:54 -08:00
|
|
|
);
|
|
|
|
}
|
|
|
|
|
2020-01-13 13:13:52 -08:00
|
|
|
Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
pub fn test_repair_empty_slot() {
|
2020-01-13 13:13:52 -08:00
|
|
|
let blockstore_path = get_tmp_ledger_path!();
|
2019-02-07 15:10:54 -08:00
|
|
|
{
|
2020-01-13 13:13:52 -08:00
|
|
|
let blockstore = Blockstore::open(&blockstore_path).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let (shreds, _) = make_slot_entries(2, 0, 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-11-14 11:49:31 -08:00
|
|
|
// Write this shred to slot 2, should chain to slot 0, which we haven't received
|
|
|
|
// any shreds for
|
2020-01-13 13:13:52 -08:00
|
|
|
blockstore.insert_shreds(shreds, None, false).unwrap();
|
2019-04-06 19:41:22 -07:00
|
|
|
|
2019-02-07 15:10:54 -08:00
|
|
|
// Check that repair tries to patch the empty slot
|
|
|
|
assert_eq!(
|
2020-01-13 13:13:52 -08:00
|
|
|
RepairService::generate_repairs(&blockstore, 0, 2).unwrap(),
|
2019-11-14 11:49:31 -08:00
|
|
|
vec![RepairType::HighestShred(0, 0)]
|
2019-02-07 15:10:54 -08:00
|
|
|
);
|
|
|
|
}
|
2020-01-13 13:13:52 -08:00
|
|
|
Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
pub fn test_generate_repairs() {
|
2020-01-13 13:13:52 -08:00
|
|
|
let blockstore_path = get_tmp_ledger_path!();
|
2019-02-07 15:10:54 -08:00
|
|
|
{
|
2020-01-13 13:13:52 -08:00
|
|
|
let blockstore = Blockstore::open(&blockstore_path).unwrap();
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-02-18 18:41:31 -08:00
|
|
|
let nth = 3;
|
2019-02-07 15:10:54 -08:00
|
|
|
let num_slots = 2;
|
|
|
|
|
2019-11-14 11:49:31 -08:00
|
|
|
// Create some shreds
|
2019-09-17 15:11:29 -07:00
|
|
|
let (mut shreds, _) = make_many_slot_entries(0, num_slots as u64, 150 as u64);
|
2019-08-21 15:27:42 -07:00
|
|
|
let num_shreds = shreds.len() as u64;
|
|
|
|
let num_shreds_per_slot = num_shreds / num_slots;
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-11-14 11:49:31 -08:00
|
|
|
// write every nth shred
|
2019-08-21 15:27:42 -07:00
|
|
|
let mut shreds_to_write = vec![];
|
|
|
|
let mut missing_indexes_per_slot = vec![];
|
|
|
|
for i in (0..num_shreds).rev() {
|
|
|
|
let index = i % num_shreds_per_slot;
|
|
|
|
if index % nth == 0 {
|
|
|
|
shreds_to_write.insert(0, shreds.remove(i as usize));
|
|
|
|
} else if i < num_shreds_per_slot {
|
|
|
|
missing_indexes_per_slot.insert(0, index);
|
|
|
|
}
|
|
|
|
}
|
2020-01-13 13:13:52 -08:00
|
|
|
blockstore
|
2019-11-14 00:32:07 -08:00
|
|
|
.insert_shreds(shreds_to_write, None, false)
|
|
|
|
.unwrap();
|
2019-11-07 11:08:09 -08:00
|
|
|
// sleep so that the holes are ready for repair
|
|
|
|
sleep(Duration::from_secs(1));
|
2019-02-14 00:14:23 -08:00
|
|
|
let expected: Vec<RepairType> = (0..num_slots)
|
2019-03-05 14:18:29 -08:00
|
|
|
.flat_map(|slot| {
|
2019-02-07 15:10:54 -08:00
|
|
|
missing_indexes_per_slot
|
|
|
|
.iter()
|
2019-11-14 11:49:31 -08:00
|
|
|
.map(move |shred_index| RepairType::Shred(slot as u64, *shred_index))
|
2019-02-07 15:10:54 -08:00
|
|
|
})
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
assert_eq!(
|
2020-01-13 13:13:52 -08:00
|
|
|
RepairService::generate_repairs(&blockstore, 0, std::usize::MAX).unwrap(),
|
2019-02-07 15:10:54 -08:00
|
|
|
expected
|
|
|
|
);
|
|
|
|
|
|
|
|
assert_eq!(
|
2020-01-13 13:13:52 -08:00
|
|
|
RepairService::generate_repairs(&blockstore, 0, expected.len() - 2).unwrap()[..],
|
2019-02-07 15:10:54 -08:00
|
|
|
expected[0..expected.len() - 2]
|
|
|
|
);
|
2019-02-18 18:41:31 -08:00
|
|
|
}
|
2020-01-13 13:13:52 -08:00
|
|
|
Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
|
2019-02-18 18:41:31 -08:00
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-02-18 18:41:31 -08:00
|
|
|
#[test]
|
|
|
|
pub fn test_generate_highest_repair() {
|
2020-01-13 13:13:52 -08:00
|
|
|
let blockstore_path = get_tmp_ledger_path!();
|
2019-02-18 18:41:31 -08:00
|
|
|
{
|
2020-01-13 13:13:52 -08:00
|
|
|
let blockstore = Blockstore::open(&blockstore_path).unwrap();
|
2019-02-14 00:14:23 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let num_entries_per_slot = 100;
|
2019-02-18 18:41:31 -08:00
|
|
|
|
2019-11-14 11:49:31 -08:00
|
|
|
// Create some shreds
|
2019-09-03 21:32:51 -07:00
|
|
|
let (mut shreds, _) = make_slot_entries(0, 0, num_entries_per_slot as u64);
|
|
|
|
let num_shreds_per_slot = shreds.len() as u64;
|
2019-02-18 18:41:31 -08:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
// Remove last shred (which is also last in slot) so that slot is not complete
|
|
|
|
shreds.pop();
|
2019-02-18 18:41:31 -08:00
|
|
|
|
2020-01-13 13:13:52 -08:00
|
|
|
blockstore.insert_shreds(shreds, None, false).unwrap();
|
2019-02-18 18:41:31 -08:00
|
|
|
|
2019-11-14 11:49:31 -08:00
|
|
|
// We didn't get the last shred for this slot, so ask for the highest shred for that slot
|
2019-09-03 21:32:51 -07:00
|
|
|
let expected: Vec<RepairType> =
|
2019-11-14 11:49:31 -08:00
|
|
|
vec![RepairType::HighestShred(0, num_shreds_per_slot - 1)];
|
2019-02-07 15:10:54 -08:00
|
|
|
|
|
|
|
assert_eq!(
|
2020-01-13 13:13:52 -08:00
|
|
|
RepairService::generate_repairs(&blockstore, 0, std::usize::MAX).unwrap(),
|
2019-02-07 15:10:54 -08:00
|
|
|
expected
|
|
|
|
);
|
|
|
|
}
|
2020-01-13 13:13:52 -08:00
|
|
|
Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|
2019-02-12 17:43:45 -08:00
|
|
|
|
2019-04-08 12:46:23 -07:00
|
|
|
#[test]
|
2019-02-12 17:43:45 -08:00
|
|
|
pub fn test_repair_range() {
|
2020-01-13 13:13:52 -08:00
|
|
|
let blockstore_path = get_tmp_ledger_path!();
|
2019-05-09 19:57:51 -07:00
|
|
|
{
|
2020-01-13 13:13:52 -08:00
|
|
|
let blockstore = Blockstore::open(&blockstore_path).unwrap();
|
2019-05-09 19:57:51 -07:00
|
|
|
|
|
|
|
let slots: Vec<u64> = vec![1, 3, 5, 7, 8];
|
2019-10-09 16:07:18 -07:00
|
|
|
let num_entries_per_slot = max_ticks_per_n_shreds(1) + 1;
|
2019-05-09 19:57:51 -07:00
|
|
|
|
2019-09-03 21:32:51 -07:00
|
|
|
let shreds = make_chaining_slot_entries(&slots, num_entries_per_slot);
|
2019-08-26 18:27:45 -07:00
|
|
|
for (mut slot_shreds, _) in shreds.into_iter() {
|
|
|
|
slot_shreds.remove(0);
|
2020-01-13 13:13:52 -08:00
|
|
|
blockstore.insert_shreds(slot_shreds, None, false).unwrap();
|
2019-05-09 19:57:51 -07:00
|
|
|
}
|
2019-11-07 11:08:09 -08:00
|
|
|
// sleep to make slot eligible for repair
|
|
|
|
sleep(Duration::from_secs(1));
|
2019-05-09 19:57:51 -07:00
|
|
|
// Iterate through all possible combinations of start..end (inclusive on both
|
|
|
|
// sides of the range)
|
|
|
|
for start in 0..slots.len() {
|
|
|
|
for end in start..slots.len() {
|
|
|
|
let mut repair_slot_range = RepairSlotRange::default();
|
|
|
|
repair_slot_range.start = slots[start];
|
|
|
|
repair_slot_range.end = slots[end];
|
2019-05-15 11:37:20 -07:00
|
|
|
let expected: Vec<RepairType> = (repair_slot_range.start
|
|
|
|
..=repair_slot_range.end)
|
|
|
|
.map(|slot_index| {
|
|
|
|
if slots.contains(&(slot_index as u64)) {
|
2019-11-07 11:08:09 -08:00
|
|
|
RepairType::Shred(slot_index as u64, 0)
|
2019-05-15 11:37:20 -07:00
|
|
|
} else {
|
2019-11-14 11:49:31 -08:00
|
|
|
RepairType::HighestShred(slot_index as u64, 0)
|
2019-05-15 11:37:20 -07:00
|
|
|
}
|
|
|
|
})
|
2019-05-09 19:57:51 -07:00
|
|
|
.collect();
|
|
|
|
|
|
|
|
assert_eq!(
|
|
|
|
RepairService::generate_repairs_in_range(
|
2020-01-13 13:13:52 -08:00
|
|
|
&blockstore,
|
2019-05-09 19:57:51 -07:00
|
|
|
std::usize::MAX,
|
|
|
|
&repair_slot_range
|
|
|
|
)
|
|
|
|
.unwrap(),
|
|
|
|
expected
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2020-01-13 13:13:52 -08:00
|
|
|
Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
|
2019-05-09 19:57:51 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
pub fn test_repair_range_highest() {
|
2020-01-13 13:13:52 -08:00
|
|
|
let blockstore_path = get_tmp_ledger_path!();
|
2019-02-12 17:43:45 -08:00
|
|
|
{
|
2020-01-13 13:13:52 -08:00
|
|
|
let blockstore = Blockstore::open(&blockstore_path).unwrap();
|
2019-02-12 17:43:45 -08:00
|
|
|
|
|
|
|
let num_entries_per_slot = 10;
|
|
|
|
|
|
|
|
let num_slots = 1;
|
|
|
|
let start = 5;
|
2019-05-09 19:57:51 -07:00
|
|
|
|
2019-11-14 11:49:31 -08:00
|
|
|
// Create some shreds in slots 0..num_slots
|
2019-02-12 17:43:45 -08:00
|
|
|
for i in start..start + num_slots {
|
|
|
|
let parent = if i > 0 { i - 1 } else { 0 };
|
2019-09-03 21:32:51 -07:00
|
|
|
let (shreds, _) = make_slot_entries(i, parent, num_entries_per_slot as u64);
|
2019-02-12 17:43:45 -08:00
|
|
|
|
2020-01-13 13:13:52 -08:00
|
|
|
blockstore.insert_shreds(shreds, None, false).unwrap();
|
2019-02-12 17:43:45 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
let end = 4;
|
2019-05-15 11:37:20 -07:00
|
|
|
let expected: Vec<RepairType> = vec![
|
2019-11-14 11:49:31 -08:00
|
|
|
RepairType::HighestShred(end - 2, 0),
|
|
|
|
RepairType::HighestShred(end - 1, 0),
|
|
|
|
RepairType::HighestShred(end, 0),
|
2019-05-15 11:37:20 -07:00
|
|
|
];
|
2019-02-12 17:43:45 -08:00
|
|
|
|
|
|
|
let mut repair_slot_range = RepairSlotRange::default();
|
|
|
|
repair_slot_range.start = 2;
|
|
|
|
repair_slot_range.end = end;
|
|
|
|
|
|
|
|
assert_eq!(
|
2019-04-08 12:46:23 -07:00
|
|
|
RepairService::generate_repairs_in_range(
|
2020-01-13 13:13:52 -08:00
|
|
|
&blockstore,
|
2019-04-08 12:46:23 -07:00
|
|
|
std::usize::MAX,
|
|
|
|
&repair_slot_range
|
|
|
|
)
|
|
|
|
.unwrap(),
|
2019-02-12 17:43:45 -08:00
|
|
|
expected
|
|
|
|
);
|
|
|
|
}
|
2020-01-13 13:13:52 -08:00
|
|
|
Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
|
2019-04-08 12:46:23 -07:00
|
|
|
}
|
2019-05-13 15:37:50 -07:00
|
|
|
|
|
|
|
#[test]
|
2020-03-11 21:31:50 -07:00
|
|
|
pub fn test_update_lowest_slot() {
|
|
|
|
let node_info = Node::new_localhost_with_pubkey(&Pubkey::default());
|
2020-04-21 12:54:45 -07:00
|
|
|
let cluster_info = ClusterInfo::new_with_invalid_keypair(node_info.info.clone());
|
2020-03-11 21:31:50 -07:00
|
|
|
RepairService::update_lowest_slot(&Pubkey::default(), 5, &cluster_info);
|
|
|
|
let lowest = cluster_info
|
2020-04-21 12:54:45 -07:00
|
|
|
.get_lowest_slot_for_node(&Pubkey::default(), None, |lowest_slot, _| {
|
|
|
|
lowest_slot.clone()
|
|
|
|
})
|
|
|
|
.unwrap();
|
2020-03-11 21:31:50 -07:00
|
|
|
assert_eq!(lowest.lowest, 5);
|
2020-03-05 10:58:00 -08:00
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
}
|