2021-01-14 21:45:11 -08:00
|
|
|
use {
|
2021-05-26 08:15:46 -07:00
|
|
|
solana_gossip::cluster_info::ClusterInfo,
|
2021-01-14 21:45:11 -08:00
|
|
|
solana_sdk::{clock::Slot, pubkey::Pubkey},
|
|
|
|
std::{
|
|
|
|
collections::HashSet,
|
2021-12-03 09:00:31 -08:00
|
|
|
sync::{
|
|
|
|
atomic::{AtomicBool, Ordering},
|
|
|
|
Arc,
|
|
|
|
},
|
2021-01-14 21:45:11 -08:00
|
|
|
},
|
2020-05-30 00:33:59 -07:00
|
|
|
};
|
|
|
|
|
2022-05-22 18:00:42 -07:00
|
|
|
#[derive(PartialEq, Eq, Clone, Copy, Debug)]
|
2020-05-30 00:33:59 -07:00
|
|
|
pub enum RpcHealthStatus {
|
|
|
|
Ok,
|
2021-11-12 10:57:55 -08:00
|
|
|
Behind { num_slots: Slot }, // Validator is behind its known validators
|
2021-03-04 21:18:08 -08:00
|
|
|
Unknown,
|
2020-05-30 00:33:59 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
pub struct RpcHealth {
|
|
|
|
cluster_info: Arc<ClusterInfo>,
|
2021-11-12 10:57:55 -08:00
|
|
|
known_validators: Option<HashSet<Pubkey>>,
|
2020-05-30 00:33:59 -07:00
|
|
|
health_check_slot_distance: u64,
|
|
|
|
override_health_check: Arc<AtomicBool>,
|
2022-06-29 14:48:33 -07:00
|
|
|
startup_verification_complete: Arc<AtomicBool>,
|
2020-06-01 13:53:37 -07:00
|
|
|
#[cfg(test)]
|
|
|
|
stub_health_status: std::sync::RwLock<Option<RpcHealthStatus>>,
|
2020-05-30 00:33:59 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
impl RpcHealth {
|
|
|
|
pub fn new(
|
|
|
|
cluster_info: Arc<ClusterInfo>,
|
2021-11-12 10:57:55 -08:00
|
|
|
known_validators: Option<HashSet<Pubkey>>,
|
2020-05-30 00:33:59 -07:00
|
|
|
health_check_slot_distance: u64,
|
|
|
|
override_health_check: Arc<AtomicBool>,
|
2022-06-29 14:48:33 -07:00
|
|
|
startup_verification_complete: Arc<AtomicBool>,
|
2020-05-30 00:33:59 -07:00
|
|
|
) -> Self {
|
|
|
|
Self {
|
|
|
|
cluster_info,
|
2021-11-12 10:57:55 -08:00
|
|
|
known_validators,
|
2020-05-30 00:33:59 -07:00
|
|
|
health_check_slot_distance,
|
|
|
|
override_health_check,
|
2022-06-29 14:48:33 -07:00
|
|
|
startup_verification_complete,
|
2020-06-01 13:53:37 -07:00
|
|
|
#[cfg(test)]
|
|
|
|
stub_health_status: std::sync::RwLock::new(None),
|
2020-05-30 00:33:59 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn check(&self) -> RpcHealthStatus {
|
2020-06-01 13:53:37 -07:00
|
|
|
#[cfg(test)]
|
|
|
|
{
|
|
|
|
if let Some(stub_health_status) = *self.stub_health_status.read().unwrap() {
|
|
|
|
return stub_health_status;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-07-15 12:29:56 -07:00
|
|
|
if !self.startup_verification_complete.load(Ordering::Acquire) {
|
2022-06-29 14:48:33 -07:00
|
|
|
return RpcHealthStatus::Unknown;
|
|
|
|
}
|
|
|
|
|
2020-05-30 00:33:59 -07:00
|
|
|
if self.override_health_check.load(Ordering::Relaxed) {
|
|
|
|
RpcHealthStatus::Ok
|
2021-11-12 10:57:55 -08:00
|
|
|
} else if let Some(known_validators) = &self.known_validators {
|
2021-03-04 21:18:08 -08:00
|
|
|
match (
|
|
|
|
self.cluster_info
|
|
|
|
.get_accounts_hash_for_node(&self.cluster_info.id(), |hashes| {
|
|
|
|
hashes
|
|
|
|
.iter()
|
|
|
|
.max_by(|a, b| a.0.cmp(&b.0))
|
|
|
|
.map(|slot_hash| slot_hash.0)
|
|
|
|
})
|
|
|
|
.flatten(),
|
2021-11-12 10:57:55 -08:00
|
|
|
known_validators
|
2021-03-04 21:18:08 -08:00
|
|
|
.iter()
|
2021-11-12 10:57:55 -08:00
|
|
|
.filter_map(|known_validator| {
|
2021-03-04 21:18:08 -08:00
|
|
|
self.cluster_info
|
2021-11-12 10:57:55 -08:00
|
|
|
.get_accounts_hash_for_node(known_validator, |hashes| {
|
2021-03-04 21:18:08 -08:00
|
|
|
hashes
|
|
|
|
.iter()
|
|
|
|
.max_by(|a, b| a.0.cmp(&b.0))
|
|
|
|
.map(|slot_hash| slot_hash.0)
|
|
|
|
})
|
|
|
|
.flatten()
|
|
|
|
})
|
|
|
|
.max(),
|
|
|
|
) {
|
2020-05-30 00:33:59 -07:00
|
|
|
(
|
2021-03-04 21:18:08 -08:00
|
|
|
Some(latest_account_hash_slot),
|
2021-11-12 10:57:55 -08:00
|
|
|
Some(latest_known_validator_account_hash_slot),
|
2021-03-04 21:18:08 -08:00
|
|
|
) => {
|
|
|
|
// The validator is considered healthy if its latest account hash slot is within
|
2021-11-12 10:57:55 -08:00
|
|
|
// `health_check_slot_distance` of the latest known validator's account hash slot
|
2021-03-04 21:18:08 -08:00
|
|
|
if latest_account_hash_slot
|
2021-11-12 10:57:55 -08:00
|
|
|
> latest_known_validator_account_hash_slot
|
2021-03-04 21:18:08 -08:00
|
|
|
.saturating_sub(self.health_check_slot_distance)
|
|
|
|
{
|
|
|
|
RpcHealthStatus::Ok
|
|
|
|
} else {
|
2021-11-12 10:57:55 -08:00
|
|
|
let num_slots = latest_known_validator_account_hash_slot
|
2021-03-04 21:18:08 -08:00
|
|
|
.saturating_sub(latest_account_hash_slot);
|
|
|
|
warn!(
|
2021-11-12 10:57:55 -08:00
|
|
|
"health check: behind by {} slots: me={}, latest known_validator={}",
|
2021-03-04 21:18:08 -08:00
|
|
|
num_slots,
|
|
|
|
latest_account_hash_slot,
|
2021-11-12 10:57:55 -08:00
|
|
|
latest_known_validator_account_hash_slot
|
2021-03-04 21:18:08 -08:00
|
|
|
);
|
|
|
|
RpcHealthStatus::Behind { num_slots }
|
|
|
|
}
|
|
|
|
}
|
2021-11-12 10:57:55 -08:00
|
|
|
(latest_account_hash_slot, latest_known_validator_account_hash_slot) => {
|
2021-06-21 10:37:34 -07:00
|
|
|
if latest_account_hash_slot.is_none() {
|
|
|
|
warn!("health check: latest_account_hash_slot not available");
|
|
|
|
}
|
2021-11-12 10:57:55 -08:00
|
|
|
if latest_known_validator_account_hash_slot.is_none() {
|
|
|
|
warn!(
|
|
|
|
"health check: latest_known_validator_account_hash_slot not available"
|
|
|
|
);
|
2021-06-21 10:37:34 -07:00
|
|
|
}
|
|
|
|
RpcHealthStatus::Unknown
|
|
|
|
}
|
2020-05-30 00:33:59 -07:00
|
|
|
}
|
|
|
|
} else {
|
2021-11-12 10:57:55 -08:00
|
|
|
// No known validator point of reference available, so this validator is healthy
|
2020-05-30 00:33:59 -07:00
|
|
|
// because it's running
|
|
|
|
RpcHealthStatus::Ok
|
|
|
|
}
|
|
|
|
}
|
2020-05-30 00:39:24 -07:00
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
pub(crate) fn stub() -> Arc<Self> {
|
2021-07-23 08:25:03 -07:00
|
|
|
use {
|
2023-01-08 08:00:55 -08:00
|
|
|
solana_gossip::legacy_contact_info::LegacyContactInfo as ContactInfo,
|
|
|
|
solana_sdk::signer::keypair::Keypair, solana_streamer::socket::SocketAddrSpace,
|
2021-07-23 08:25:03 -07:00
|
|
|
};
|
2020-05-30 00:39:24 -07:00
|
|
|
Arc::new(Self::new(
|
2021-07-23 08:25:03 -07:00
|
|
|
Arc::new(ClusterInfo::new(
|
|
|
|
ContactInfo::default(),
|
|
|
|
Arc::new(Keypair::new()),
|
|
|
|
SocketAddrSpace::Unspecified,
|
|
|
|
)),
|
2020-05-30 00:39:24 -07:00
|
|
|
None,
|
|
|
|
42,
|
|
|
|
Arc::new(AtomicBool::new(false)),
|
2022-06-29 14:48:33 -07:00
|
|
|
Arc::new(AtomicBool::new(true)),
|
2020-05-30 00:39:24 -07:00
|
|
|
))
|
|
|
|
}
|
2020-06-01 13:53:37 -07:00
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
pub(crate) fn stub_set_health_status(&self, stub_health_status: Option<RpcHealthStatus>) {
|
|
|
|
*self.stub_health_status.write().unwrap() = stub_health_status;
|
|
|
|
}
|
2020-05-30 00:33:59 -07:00
|
|
|
}
|