project-serum
/
solana
mirror of https://github.com/project-serum/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Refactor

This commit is contained in:
Michael Vines 2020-03-09 22:02:40 -07:00
parent a0ed3261c9
commit 23c5bb17c7
1 changed files with 101 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

228
watchtower/src/main.rs
View File

@ -10,11 +10,18 @@ use solana_clap_utils::{
input_validators::{is_pubkey_or_keypair, is_url}, input_validators::{is_pubkey_or_keypair, is_url},
}; };
use solana_cli_config::{Config, CONFIG_FILE}; use solana_cli_config::{Config, CONFIG_FILE};
use solana_client::rpc_client::RpcClient; use solana_client::{rpc_client::RpcClient, rpc_response::RpcVoteAccountStatus};
use solana_metrics::{datapoint_error, datapoint_info}; use solana_metrics::{datapoint_error, datapoint_info};
use solana_sdk::native_token::lamports_to_sol; use solana_sdk::{hash::Hash, native_token::lamports_to_sol};
use std::{error, io, thread::sleep, time::Duration}; use std::{error, io, thread::sleep, time::Duration};
fn get_cluster_info(rpc_client: &RpcClient) -> io::Result<(u64, Hash, RpcVoteAccountStatus)> {
let transaction_count = rpc_client.get_transaction_count()?;
let recent_blockhash = rpc_client.get_recent_blockhash()?.0;
let vote_accounts = rpc_client.get_vote_accounts()?;
Ok((transaction_count, recent_blockhash, vote_accounts))
}
fn main() -> Result<(), Box<dyn error::Error>> { fn main() -> Result<(), Box<dyn error::Error>> {
let matches = App::new(crate_name!()) let matches = App::new(crate_name!())
.about(crate_description!()) .about(crate_description!())
@ -95,151 +102,118 @@ fn main() -> Result<(), Box<dyn error::Error>> {
let notifier = Notifier::new(); let notifier = Notifier::new();
let mut last_transaction_count = 0; let mut last_transaction_count = 0;
let mut last_check_notification_sent = false; let mut last_recent_blockhash = Hash::default();
let mut last_notification_msg = String::from(""); let mut last_notification_msg = "".into();
loop { loop {
let mut notify_msg = String::from("solana-watchtower: undefined error"); let failure = match get_cluster_info(&rpc_client) {
let ok = rpc_client Ok((transaction_count, recent_blockhash, vote_accounts)) => {
.get_transaction_count()
.and_then(|transaction_count| {
info!("Current transaction count: {}", transaction_count); info!("Current transaction count: {}", transaction_count);
info!("Recent blockhash: {}", recent_blockhash);
info!("Current validator count: {}", vote_accounts.current.len());
info!(
"Delinquent validator count: {}",
vote_accounts.delinquent.len()
);
let mut failures = vec![];
let total_current_stake = vote_accounts
.current
.iter()
.fold(0, |acc, vote_account| acc + vote_account.activated_stake);
let total_delinquent_stake = vote_accounts
.delinquent
.iter()
.fold(0, |acc, vote_account| acc + vote_account.activated_stake);
let total_stake = total_current_stake + total_delinquent_stake;
let current_stake_percent = total_current_stake * 100 / total_stake;
info!(
"Current stake: {}% | Total stake: {} SOL, current stake: {} SOL, delinquent: {} SOL",
current_stake_percent,
lamports_to_sol(total_stake),
lamports_to_sol(total_current_stake),
lamports_to_sol(total_delinquent_stake)
);
if transaction_count > last_transaction_count { if transaction_count > last_transaction_count {
last_transaction_count = transaction_count; last_transaction_count = transaction_count;
Ok(true)
} else { } else {
Err(io::Error::new( failures.push((
io::ErrorKind::Other, "transaction-count",
format!( format!(
"Transaction count is not advancing: {} <= {}", "Transaction count is not advancing: {} <= {}",
transaction_count, last_transaction_count transaction_count, last_transaction_count
), ),
)) ));
} }
})
.unwrap_or_else(|err| {
notify_msg = format!("solana-watchtower: {}", err.to_string());
datapoint_error!(
"watchtower-sanity-failure",
("test", "transaction-count", String),
("err", err.to_string(), String)
);
false
})
&& rpc_client
.get_recent_blockhash()
.and_then(|(blockhash, _fee_calculator)| {
info!("Current blockhash: {}", blockhash);
rpc_client.get_new_blockhash(&blockhash)
})
.and_then(|(blockhash, _fee_calculator)| {
info!("New blockhash: {}", blockhash);
Ok(true)
})
.unwrap_or_else(|err| {
notify_msg = format!("solana-watchtower: {}", err.to_string());
datapoint_error!(
"watchtower-sanity-failure",
("test", "blockhash", String),
("err", err.to_string(), String)
);
false
})
&& rpc_client
.get_vote_accounts()
.and_then(|vote_accounts| {
let total_current_stake = vote_accounts if recent_blockhash != last_recent_blockhash {
.current last_recent_blockhash = recent_blockhash;
.iter() } else {
.fold(0, |acc, vote_account| acc + vote_account.activated_stake); failures.push((
let total_delinquent_stake = vote_accounts "recent-blockhash",
.delinquent format!("Unable to get new blockhash: {}", recent_blockhash),
.iter() ));
.fold(0, |acc, vote_account| acc + vote_account.activated_stake); }
let total_stake = total_current_stake + total_delinquent_stake; if validator_identity_pubkeys.is_empty() {
let current_stake_percent = total_current_stake * 100 / total_stake; if !vote_accounts.delinquent.is_empty() {
info!( failures.push((
"Current stake: {}% | Total stake: {} SOL, current stake: {} SOL, delinquent: {} SOL", "delinquent",
current_stake_percent, format!("{} delinquent validators", vote_accounts.delinquent.len()),
lamports_to_sol(total_stake), ));
lamports_to_sol(total_current_stake), }
lamports_to_sol(total_delinquent_stake) } else {
); let mut errors = vec![];
for validator_identity in validator_identity_pubkeys.iter() {
info!("Current validator count: {}", vote_accounts.current.len()); if vote_accounts
info!( .delinquent
"Delinquent validator count: {}", .iter()
vote_accounts.delinquent.len() .any(|vai| vai.node_pubkey == *validator_identity)
); {
errors.push(format!("{} delinquent", validator_identity));
if validator_identity_pubkeys.is_empty() { } else if !vote_accounts
if vote_accounts.delinquent.is_empty() { .current
Ok(true) .iter()
} else { .any(|vai| vai.node_pubkey == *validator_identity)
Err(io::Error::new( {
io::ErrorKind::Other, errors.push(format!("{} missing", validator_identity));
format!("{} delinquent validators", vote_accounts.delinquent.len()),
))
}
} else {
let mut errors = vec![];
for validator_identity in validator_identity_pubkeys.iter() {
if vote_accounts
.delinquent
.iter()
.any(|vai| vai.node_pubkey == *validator_identity)
{
errors.push(format!("{} delinquent", validator_identity));
} else if !vote_accounts
.current
.iter()
.any(|vai| vai.node_pubkey == *validator_identity)
{
errors.push(format!("{} missing", validator_identity));
}
}
if errors.is_empty() {
Ok(true)
} else {
Err(io::Error::new(io::ErrorKind::Other, errors.join(",")))
} }
} }
})
.unwrap_or_else(|err| {
notify_msg = format!("solana-watchtower: {}", err.to_string());
datapoint_error!(
"watchtower-sanity-failure",
("test", "delinquent-validators", String),
("err", err.to_string(), String)
);
false
});
datapoint_info!("watchtower-sanity", ("ok", ok, bool)); if !errors.is_empty() {
if !ok { failures.push(("delinquent", errors.join(",")));
last_check_notification_sent = true; }
if no_duplicate_notifications {
if last_notification_msg != notify_msg {
notifier.send(&notify_msg);
last_notification_msg = notify_msg;
} else {
datapoint_info!(
"watchtower-sanity",
("Suppressing duplicate notification", ok, bool)
);
} }
} else {
notifier.send(&notify_msg); failures.into_iter().next() // Only report the first failure if any
} }
Err(err) => Some(("rpc", err.to_string())),
};
datapoint_info!("watchtower-sanity", ("ok", failure.is_none(), bool));
if let Some((failure_test_name, failure_error_message)) = &failure {
let notification_msg = format!(
"solana-watchtower: Error: {}: {}",
failure_test_name, failure_error_message
);
if !no_duplicate_notifications || last_notification_msg != notification_msg {
notifier.send(&notification_msg);
}
datapoint_error!(
"watchtower-sanity-failure",
("test", failure_test_name, String),
("err", failure_error_message, String)
);
last_notification_msg = notification_msg;
} else { } else {
if last_check_notification_sent { if !last_notification_msg.is_empty() {
notifier.send("solana-watchtower: All Clear"); info!("All clear");
notifier.send("solana-watchtower: All clear");
} }
last_check_notification_sent = false; last_notification_msg = "".into();
last_notification_msg = String::from("");
} }
sleep(interval); sleep(interval);
} }