Refactor
This commit is contained in:
parent
a0ed3261c9
commit
23c5bb17c7
|
@ -10,11 +10,18 @@ use solana_clap_utils::{
|
|||
input_validators::{is_pubkey_or_keypair, is_url},
|
||||
};
|
||||
use solana_cli_config::{Config, CONFIG_FILE};
|
||||
use solana_client::rpc_client::RpcClient;
|
||||
use solana_client::{rpc_client::RpcClient, rpc_response::RpcVoteAccountStatus};
|
||||
use solana_metrics::{datapoint_error, datapoint_info};
|
||||
use solana_sdk::native_token::lamports_to_sol;
|
||||
use solana_sdk::{hash::Hash, native_token::lamports_to_sol};
|
||||
use std::{error, io, thread::sleep, time::Duration};
|
||||
|
||||
fn get_cluster_info(rpc_client: &RpcClient) -> io::Result<(u64, Hash, RpcVoteAccountStatus)> {
|
||||
let transaction_count = rpc_client.get_transaction_count()?;
|
||||
let recent_blockhash = rpc_client.get_recent_blockhash()?.0;
|
||||
let vote_accounts = rpc_client.get_vote_accounts()?;
|
||||
Ok((transaction_count, recent_blockhash, vote_accounts))
|
||||
}
|
||||
|
||||
fn main() -> Result<(), Box<dyn error::Error>> {
|
||||
let matches = App::new(crate_name!())
|
||||
.about(crate_description!())
|
||||
|
@ -95,151 +102,118 @@ fn main() -> Result<(), Box<dyn error::Error>> {
|
|||
|
||||
let notifier = Notifier::new();
|
||||
let mut last_transaction_count = 0;
|
||||
let mut last_check_notification_sent = false;
|
||||
let mut last_notification_msg = String::from("");
|
||||
let mut last_recent_blockhash = Hash::default();
|
||||
let mut last_notification_msg = "".into();
|
||||
|
||||
loop {
|
||||
let mut notify_msg = String::from("solana-watchtower: undefined error");
|
||||
let ok = rpc_client
|
||||
.get_transaction_count()
|
||||
.and_then(|transaction_count| {
|
||||
let failure = match get_cluster_info(&rpc_client) {
|
||||
Ok((transaction_count, recent_blockhash, vote_accounts)) => {
|
||||
info!("Current transaction count: {}", transaction_count);
|
||||
info!("Recent blockhash: {}", recent_blockhash);
|
||||
info!("Current validator count: {}", vote_accounts.current.len());
|
||||
info!(
|
||||
"Delinquent validator count: {}",
|
||||
vote_accounts.delinquent.len()
|
||||
);
|
||||
|
||||
let mut failures = vec![];
|
||||
|
||||
let total_current_stake = vote_accounts
|
||||
.current
|
||||
.iter()
|
||||
.fold(0, |acc, vote_account| acc + vote_account.activated_stake);
|
||||
let total_delinquent_stake = vote_accounts
|
||||
.delinquent
|
||||
.iter()
|
||||
.fold(0, |acc, vote_account| acc + vote_account.activated_stake);
|
||||
|
||||
let total_stake = total_current_stake + total_delinquent_stake;
|
||||
let current_stake_percent = total_current_stake * 100 / total_stake;
|
||||
info!(
|
||||
"Current stake: {}% | Total stake: {} SOL, current stake: {} SOL, delinquent: {} SOL",
|
||||
current_stake_percent,
|
||||
lamports_to_sol(total_stake),
|
||||
lamports_to_sol(total_current_stake),
|
||||
lamports_to_sol(total_delinquent_stake)
|
||||
);
|
||||
|
||||
if transaction_count > last_transaction_count {
|
||||
last_transaction_count = transaction_count;
|
||||
Ok(true)
|
||||
} else {
|
||||
Err(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
failures.push((
|
||||
"transaction-count",
|
||||
format!(
|
||||
"Transaction count is not advancing: {} <= {}",
|
||||
transaction_count, last_transaction_count
|
||||
),
|
||||
))
|
||||
));
|
||||
}
|
||||
})
|
||||
.unwrap_or_else(|err| {
|
||||
notify_msg = format!("solana-watchtower: {}", err.to_string());
|
||||
datapoint_error!(
|
||||
"watchtower-sanity-failure",
|
||||
("test", "transaction-count", String),
|
||||
("err", err.to_string(), String)
|
||||
);
|
||||
false
|
||||
})
|
||||
&& rpc_client
|
||||
.get_recent_blockhash()
|
||||
.and_then(|(blockhash, _fee_calculator)| {
|
||||
info!("Current blockhash: {}", blockhash);
|
||||
rpc_client.get_new_blockhash(&blockhash)
|
||||
})
|
||||
.and_then(|(blockhash, _fee_calculator)| {
|
||||
info!("New blockhash: {}", blockhash);
|
||||
Ok(true)
|
||||
})
|
||||
.unwrap_or_else(|err| {
|
||||
notify_msg = format!("solana-watchtower: {}", err.to_string());
|
||||
datapoint_error!(
|
||||
"watchtower-sanity-failure",
|
||||
("test", "blockhash", String),
|
||||
("err", err.to_string(), String)
|
||||
);
|
||||
false
|
||||
})
|
||||
&& rpc_client
|
||||
.get_vote_accounts()
|
||||
.and_then(|vote_accounts| {
|
||||
|
||||
let total_current_stake = vote_accounts
|
||||
.current
|
||||
.iter()
|
||||
.fold(0, |acc, vote_account| acc + vote_account.activated_stake);
|
||||
let total_delinquent_stake = vote_accounts
|
||||
.delinquent
|
||||
.iter()
|
||||
.fold(0, |acc, vote_account| acc + vote_account.activated_stake);
|
||||
if recent_blockhash != last_recent_blockhash {
|
||||
last_recent_blockhash = recent_blockhash;
|
||||
} else {
|
||||
failures.push((
|
||||
"recent-blockhash",
|
||||
format!("Unable to get new blockhash: {}", recent_blockhash),
|
||||
));
|
||||
}
|
||||
|
||||
let total_stake = total_current_stake + total_delinquent_stake;
|
||||
let current_stake_percent = total_current_stake * 100 / total_stake;
|
||||
info!(
|
||||
"Current stake: {}% | Total stake: {} SOL, current stake: {} SOL, delinquent: {} SOL",
|
||||
current_stake_percent,
|
||||
lamports_to_sol(total_stake),
|
||||
lamports_to_sol(total_current_stake),
|
||||
lamports_to_sol(total_delinquent_stake)
|
||||
);
|
||||
|
||||
info!("Current validator count: {}", vote_accounts.current.len());
|
||||
info!(
|
||||
"Delinquent validator count: {}",
|
||||
vote_accounts.delinquent.len()
|
||||
);
|
||||
|
||||
if validator_identity_pubkeys.is_empty() {
|
||||
if vote_accounts.delinquent.is_empty() {
|
||||
Ok(true)
|
||||
} else {
|
||||
Err(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("{} delinquent validators", vote_accounts.delinquent.len()),
|
||||
))
|
||||
}
|
||||
} else {
|
||||
let mut errors = vec![];
|
||||
for validator_identity in validator_identity_pubkeys.iter() {
|
||||
if vote_accounts
|
||||
.delinquent
|
||||
.iter()
|
||||
.any(|vai| vai.node_pubkey == *validator_identity)
|
||||
{
|
||||
errors.push(format!("{} delinquent", validator_identity));
|
||||
} else if !vote_accounts
|
||||
.current
|
||||
.iter()
|
||||
.any(|vai| vai.node_pubkey == *validator_identity)
|
||||
{
|
||||
errors.push(format!("{} missing", validator_identity));
|
||||
}
|
||||
}
|
||||
|
||||
if errors.is_empty() {
|
||||
Ok(true)
|
||||
} else {
|
||||
Err(io::Error::new(io::ErrorKind::Other, errors.join(",")))
|
||||
if validator_identity_pubkeys.is_empty() {
|
||||
if !vote_accounts.delinquent.is_empty() {
|
||||
failures.push((
|
||||
"delinquent",
|
||||
format!("{} delinquent validators", vote_accounts.delinquent.len()),
|
||||
));
|
||||
}
|
||||
} else {
|
||||
let mut errors = vec![];
|
||||
for validator_identity in validator_identity_pubkeys.iter() {
|
||||
if vote_accounts
|
||||
.delinquent
|
||||
.iter()
|
||||
.any(|vai| vai.node_pubkey == *validator_identity)
|
||||
{
|
||||
errors.push(format!("{} delinquent", validator_identity));
|
||||
} else if !vote_accounts
|
||||
.current
|
||||
.iter()
|
||||
.any(|vai| vai.node_pubkey == *validator_identity)
|
||||
{
|
||||
errors.push(format!("{} missing", validator_identity));
|
||||
}
|
||||
}
|
||||
})
|
||||
.unwrap_or_else(|err| {
|
||||
notify_msg = format!("solana-watchtower: {}", err.to_string());
|
||||
datapoint_error!(
|
||||
"watchtower-sanity-failure",
|
||||
("test", "delinquent-validators", String),
|
||||
("err", err.to_string(), String)
|
||||
);
|
||||
false
|
||||
});
|
||||
|
||||
datapoint_info!("watchtower-sanity", ("ok", ok, bool));
|
||||
if !ok {
|
||||
last_check_notification_sent = true;
|
||||
if no_duplicate_notifications {
|
||||
if last_notification_msg != notify_msg {
|
||||
notifier.send(¬ify_msg);
|
||||
last_notification_msg = notify_msg;
|
||||
} else {
|
||||
datapoint_info!(
|
||||
"watchtower-sanity",
|
||||
("Suppressing duplicate notification", ok, bool)
|
||||
);
|
||||
if !errors.is_empty() {
|
||||
failures.push(("delinquent", errors.join(",")));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
notifier.send(¬ify_msg);
|
||||
|
||||
failures.into_iter().next() // Only report the first failure if any
|
||||
}
|
||||
Err(err) => Some(("rpc", err.to_string())),
|
||||
};
|
||||
|
||||
datapoint_info!("watchtower-sanity", ("ok", failure.is_none(), bool));
|
||||
if let Some((failure_test_name, failure_error_message)) = &failure {
|
||||
let notification_msg = format!(
|
||||
"solana-watchtower: Error: {}: {}",
|
||||
failure_test_name, failure_error_message
|
||||
);
|
||||
if !no_duplicate_notifications || last_notification_msg != notification_msg {
|
||||
notifier.send(¬ification_msg);
|
||||
}
|
||||
datapoint_error!(
|
||||
"watchtower-sanity-failure",
|
||||
("test", failure_test_name, String),
|
||||
("err", failure_error_message, String)
|
||||
);
|
||||
last_notification_msg = notification_msg;
|
||||
} else {
|
||||
if last_check_notification_sent {
|
||||
notifier.send("solana-watchtower: All Clear");
|
||||
if !last_notification_msg.is_empty() {
|
||||
info!("All clear");
|
||||
notifier.send("solana-watchtower: All clear");
|
||||
}
|
||||
last_check_notification_sent = false;
|
||||
last_notification_msg = String::from("");
|
||||
last_notification_msg = "".into();
|
||||
}
|
||||
sleep(interval);
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue