From 23c5bb17c790f72f0e12eb3399bb63c8dc22d471 Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Mon, 9 Mar 2020 22:02:40 -0700 Subject: [PATCH] Refactor --- watchtower/src/main.rs | 228 ++++++++++++++++++----------------------- 1 file changed, 101 insertions(+), 127 deletions(-) diff --git a/watchtower/src/main.rs b/watchtower/src/main.rs index 4f2fc4b25..8f989e2b3 100644 --- a/watchtower/src/main.rs +++ b/watchtower/src/main.rs @@ -10,11 +10,18 @@ use solana_clap_utils::{ input_validators::{is_pubkey_or_keypair, is_url}, }; use solana_cli_config::{Config, CONFIG_FILE}; -use solana_client::rpc_client::RpcClient; +use solana_client::{rpc_client::RpcClient, rpc_response::RpcVoteAccountStatus}; use solana_metrics::{datapoint_error, datapoint_info}; -use solana_sdk::native_token::lamports_to_sol; +use solana_sdk::{hash::Hash, native_token::lamports_to_sol}; use std::{error, io, thread::sleep, time::Duration}; +fn get_cluster_info(rpc_client: &RpcClient) -> io::Result<(u64, Hash, RpcVoteAccountStatus)> { + let transaction_count = rpc_client.get_transaction_count()?; + let recent_blockhash = rpc_client.get_recent_blockhash()?.0; + let vote_accounts = rpc_client.get_vote_accounts()?; + Ok((transaction_count, recent_blockhash, vote_accounts)) +} + fn main() -> Result<(), Box> { let matches = App::new(crate_name!()) .about(crate_description!()) @@ -95,151 +102,118 @@ fn main() -> Result<(), Box> { let notifier = Notifier::new(); let mut last_transaction_count = 0; - let mut last_check_notification_sent = false; - let mut last_notification_msg = String::from(""); + let mut last_recent_blockhash = Hash::default(); + let mut last_notification_msg = "".into(); + loop { - let mut notify_msg = String::from("solana-watchtower: undefined error"); - let ok = rpc_client - .get_transaction_count() - .and_then(|transaction_count| { + let failure = match get_cluster_info(&rpc_client) { + Ok((transaction_count, recent_blockhash, vote_accounts)) => { info!("Current transaction count: {}", transaction_count); + info!("Recent blockhash: {}", recent_blockhash); + info!("Current validator count: {}", vote_accounts.current.len()); + info!( + "Delinquent validator count: {}", + vote_accounts.delinquent.len() + ); + + let mut failures = vec![]; + + let total_current_stake = vote_accounts + .current + .iter() + .fold(0, |acc, vote_account| acc + vote_account.activated_stake); + let total_delinquent_stake = vote_accounts + .delinquent + .iter() + .fold(0, |acc, vote_account| acc + vote_account.activated_stake); + + let total_stake = total_current_stake + total_delinquent_stake; + let current_stake_percent = total_current_stake * 100 / total_stake; + info!( + "Current stake: {}% | Total stake: {} SOL, current stake: {} SOL, delinquent: {} SOL", + current_stake_percent, + lamports_to_sol(total_stake), + lamports_to_sol(total_current_stake), + lamports_to_sol(total_delinquent_stake) + ); if transaction_count > last_transaction_count { last_transaction_count = transaction_count; - Ok(true) } else { - Err(io::Error::new( - io::ErrorKind::Other, + failures.push(( + "transaction-count", format!( "Transaction count is not advancing: {} <= {}", transaction_count, last_transaction_count ), - )) + )); } - }) - .unwrap_or_else(|err| { - notify_msg = format!("solana-watchtower: {}", err.to_string()); - datapoint_error!( - "watchtower-sanity-failure", - ("test", "transaction-count", String), - ("err", err.to_string(), String) - ); - false - }) - && rpc_client - .get_recent_blockhash() - .and_then(|(blockhash, _fee_calculator)| { - info!("Current blockhash: {}", blockhash); - rpc_client.get_new_blockhash(&blockhash) - }) - .and_then(|(blockhash, _fee_calculator)| { - info!("New blockhash: {}", blockhash); - Ok(true) - }) - .unwrap_or_else(|err| { - notify_msg = format!("solana-watchtower: {}", err.to_string()); - datapoint_error!( - "watchtower-sanity-failure", - ("test", "blockhash", String), - ("err", err.to_string(), String) - ); - false - }) - && rpc_client - .get_vote_accounts() - .and_then(|vote_accounts| { - let total_current_stake = vote_accounts - .current - .iter() - .fold(0, |acc, vote_account| acc + vote_account.activated_stake); - let total_delinquent_stake = vote_accounts - .delinquent - .iter() - .fold(0, |acc, vote_account| acc + vote_account.activated_stake); + if recent_blockhash != last_recent_blockhash { + last_recent_blockhash = recent_blockhash; + } else { + failures.push(( + "recent-blockhash", + format!("Unable to get new blockhash: {}", recent_blockhash), + )); + } - let total_stake = total_current_stake + total_delinquent_stake; - let current_stake_percent = total_current_stake * 100 / total_stake; - info!( - "Current stake: {}% | Total stake: {} SOL, current stake: {} SOL, delinquent: {} SOL", - current_stake_percent, - lamports_to_sol(total_stake), - lamports_to_sol(total_current_stake), - lamports_to_sol(total_delinquent_stake) - ); - - info!("Current validator count: {}", vote_accounts.current.len()); - info!( - "Delinquent validator count: {}", - vote_accounts.delinquent.len() - ); - - if validator_identity_pubkeys.is_empty() { - if vote_accounts.delinquent.is_empty() { - Ok(true) - } else { - Err(io::Error::new( - io::ErrorKind::Other, - format!("{} delinquent validators", vote_accounts.delinquent.len()), - )) - } - } else { - let mut errors = vec![]; - for validator_identity in validator_identity_pubkeys.iter() { - if vote_accounts - .delinquent - .iter() - .any(|vai| vai.node_pubkey == *validator_identity) - { - errors.push(format!("{} delinquent", validator_identity)); - } else if !vote_accounts - .current - .iter() - .any(|vai| vai.node_pubkey == *validator_identity) - { - errors.push(format!("{} missing", validator_identity)); - } - } - - if errors.is_empty() { - Ok(true) - } else { - Err(io::Error::new(io::ErrorKind::Other, errors.join(","))) + if validator_identity_pubkeys.is_empty() { + if !vote_accounts.delinquent.is_empty() { + failures.push(( + "delinquent", + format!("{} delinquent validators", vote_accounts.delinquent.len()), + )); + } + } else { + let mut errors = vec![]; + for validator_identity in validator_identity_pubkeys.iter() { + if vote_accounts + .delinquent + .iter() + .any(|vai| vai.node_pubkey == *validator_identity) + { + errors.push(format!("{} delinquent", validator_identity)); + } else if !vote_accounts + .current + .iter() + .any(|vai| vai.node_pubkey == *validator_identity) + { + errors.push(format!("{} missing", validator_identity)); } } - }) - .unwrap_or_else(|err| { - notify_msg = format!("solana-watchtower: {}", err.to_string()); - datapoint_error!( - "watchtower-sanity-failure", - ("test", "delinquent-validators", String), - ("err", err.to_string(), String) - ); - false - }); - datapoint_info!("watchtower-sanity", ("ok", ok, bool)); - if !ok { - last_check_notification_sent = true; - if no_duplicate_notifications { - if last_notification_msg != notify_msg { - notifier.send(¬ify_msg); - last_notification_msg = notify_msg; - } else { - datapoint_info!( - "watchtower-sanity", - ("Suppressing duplicate notification", ok, bool) - ); + if !errors.is_empty() { + failures.push(("delinquent", errors.join(","))); + } } - } else { - notifier.send(¬ify_msg); + + failures.into_iter().next() // Only report the first failure if any } + Err(err) => Some(("rpc", err.to_string())), + }; + + datapoint_info!("watchtower-sanity", ("ok", failure.is_none(), bool)); + if let Some((failure_test_name, failure_error_message)) = &failure { + let notification_msg = format!( + "solana-watchtower: Error: {}: {}", + failure_test_name, failure_error_message + ); + if !no_duplicate_notifications || last_notification_msg != notification_msg { + notifier.send(¬ification_msg); + } + datapoint_error!( + "watchtower-sanity-failure", + ("test", failure_test_name, String), + ("err", failure_error_message, String) + ); + last_notification_msg = notification_msg; } else { - if last_check_notification_sent { - notifier.send("solana-watchtower: All Clear"); + if !last_notification_msg.is_empty() { + info!("All clear"); + notifier.send("solana-watchtower: All clear"); } - last_check_notification_sent = false; - last_notification_msg = String::from(""); + last_notification_msg = "".into(); } sleep(interval); }