project-serum
/
solana
mirror of https://github.com/project-serum/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Refactor

This commit is contained in:
Michael Vines 2020-03-09 22:02:40 -07:00
parent a0ed3261c9
commit 23c5bb17c7
1 changed files with 101 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

228
watchtower/src/main.rs
View File

@ -10,11 +10,18 @@ use solana_clap_utils::{
input_validators::{is_pubkey_or_keypair, is_url},
};
use solana_cli_config::{Config, CONFIG_FILE};
use solana_client::rpc_client::RpcClient;
use solana_client::{rpc_client::RpcClient, rpc_response::RpcVoteAccountStatus};
use solana_metrics::{datapoint_error, datapoint_info};
use solana_sdk::native_token::lamports_to_sol;
use solana_sdk::{hash::Hash, native_token::lamports_to_sol};
use std::{error, io, thread::sleep, time::Duration};
fn get_cluster_info(rpc_client: &RpcClient) -> io::Result<(u64, Hash, RpcVoteAccountStatus)> {
let transaction_count = rpc_client.get_transaction_count()?;
let recent_blockhash = rpc_client.get_recent_blockhash()?.0;
let vote_accounts = rpc_client.get_vote_accounts()?;
Ok((transaction_count, recent_blockhash, vote_accounts))
}
fn main() -> Result<(), Box<dyn error::Error>> {
let matches = App::new(crate_name!())
.about(crate_description!())
@ -95,151 +102,118 @@ fn main() -> Result<(), Box<dyn error::Error>> {
let notifier = Notifier::new();
let mut last_transaction_count = 0;
let mut last_check_notification_sent = false;
let mut last_notification_msg = String::from("");
let mut last_recent_blockhash = Hash::default();
let mut last_notification_msg = "".into();
loop {
let mut notify_msg = String::from("solana-watchtower: undefined error");
let ok = rpc_client
.get_transaction_count()
.and_then(|transaction_count| {
let failure = match get_cluster_info(&rpc_client) {
Ok((transaction_count, recent_blockhash, vote_accounts)) => {
info!("Current transaction count: {}", transaction_count);
info!("Recent blockhash: {}", recent_blockhash);
info!("Current validator count: {}", vote_accounts.current.len());
info!(
"Delinquent validator count: {}",
vote_accounts.delinquent.len()
);
let mut failures = vec![];
let total_current_stake = vote_accounts
.current
.iter()
.fold(0, |acc, vote_account| acc + vote_account.activated_stake);
let total_delinquent_stake = vote_accounts
.delinquent
.iter()
.fold(0, |acc, vote_account| acc + vote_account.activated_stake);
let total_stake = total_current_stake + total_delinquent_stake;
let current_stake_percent = total_current_stake * 100 / total_stake;
info!(
"Current stake: {}% | Total stake: {} SOL, current stake: {} SOL, delinquent: {} SOL",
current_stake_percent,
lamports_to_sol(total_stake),
lamports_to_sol(total_current_stake),
lamports_to_sol(total_delinquent_stake)
);
if transaction_count > last_transaction_count {
last_transaction_count = transaction_count;
Ok(true)
} else {
Err(io::Error::new(
io::ErrorKind::Other,
failures.push((
"transaction-count",
format!(
"Transaction count is not advancing: {} <= {}",
transaction_count, last_transaction_count
),
))
));
}
})
.unwrap_or_else(|err| {
notify_msg = format!("solana-watchtower: {}", err.to_string());
datapoint_error!(
"watchtower-sanity-failure",
("test", "transaction-count", String),
("err", err.to_string(), String)
);
false
})
&& rpc_client
.get_recent_blockhash()
.and_then(|(blockhash, _fee_calculator)| {
info!("Current blockhash: {}", blockhash);
rpc_client.get_new_blockhash(&blockhash)
})
.and_then(|(blockhash, _fee_calculator)| {
info!("New blockhash: {}", blockhash);
Ok(true)
})
.unwrap_or_else(|err| {
notify_msg = format!("solana-watchtower: {}", err.to_string());
datapoint_error!(
"watchtower-sanity-failure",
("test", "blockhash", String),
("err", err.to_string(), String)
);
false
})
&& rpc_client
.get_vote_accounts()
.and_then(|vote_accounts| {
let total_current_stake = vote_accounts
.current
.iter()
.fold(0, |acc, vote_account| acc + vote_account.activated_stake);
let total_delinquent_stake = vote_accounts
.delinquent
.iter()
.fold(0, |acc, vote_account| acc + vote_account.activated_stake);
if recent_blockhash != last_recent_blockhash {
last_recent_blockhash = recent_blockhash;
} else {
failures.push((
"recent-blockhash",
format!("Unable to get new blockhash: {}", recent_blockhash),
));
}
let total_stake = total_current_stake + total_delinquent_stake;
let current_stake_percent = total_current_stake * 100 / total_stake;
info!(
"Current stake: {}% | Total stake: {} SOL, current stake: {} SOL, delinquent: {} SOL",
current_stake_percent,
lamports_to_sol(total_stake),
lamports_to_sol(total_current_stake),
lamports_to_sol(total_delinquent_stake)
);
info!("Current validator count: {}", vote_accounts.current.len());
info!(
"Delinquent validator count: {}",
vote_accounts.delinquent.len()
);
if validator_identity_pubkeys.is_empty() {
if vote_accounts.delinquent.is_empty() {
Ok(true)
} else {
Err(io::Error::new(
io::ErrorKind::Other,
format!("{} delinquent validators", vote_accounts.delinquent.len()),
))
}
} else {
let mut errors = vec![];
for validator_identity in validator_identity_pubkeys.iter() {
if vote_accounts
.delinquent
.iter()
.any(|vai| vai.node_pubkey == *validator_identity)
{
errors.push(format!("{} delinquent", validator_identity));
} else if !vote_accounts
.current
.iter()
.any(|vai| vai.node_pubkey == *validator_identity)
{
errors.push(format!("{} missing", validator_identity));
}
}
if errors.is_empty() {
Ok(true)
} else {
Err(io::Error::new(io::ErrorKind::Other, errors.join(",")))
if validator_identity_pubkeys.is_empty() {
if !vote_accounts.delinquent.is_empty() {
failures.push((
"delinquent",
format!("{} delinquent validators", vote_accounts.delinquent.len()),
));
}
} else {
let mut errors = vec![];
for validator_identity in validator_identity_pubkeys.iter() {
if vote_accounts
.delinquent
.iter()
.any(|vai| vai.node_pubkey == *validator_identity)
{
errors.push(format!("{} delinquent", validator_identity));
} else if !vote_accounts
.current
.iter()
.any(|vai| vai.node_pubkey == *validator_identity)
{
errors.push(format!("{} missing", validator_identity));
}
}
})
.unwrap_or_else(|err| {
notify_msg = format!("solana-watchtower: {}", err.to_string());
datapoint_error!(
"watchtower-sanity-failure",
("test", "delinquent-validators", String),
("err", err.to_string(), String)
);
false
});
datapoint_info!("watchtower-sanity", ("ok", ok, bool));
if !ok {
last_check_notification_sent = true;
if no_duplicate_notifications {
if last_notification_msg != notify_msg {
notifier.send(&notify_msg);
last_notification_msg = notify_msg;
} else {
datapoint_info!(
"watchtower-sanity",
("Suppressing duplicate notification", ok, bool)
);
if !errors.is_empty() {
failures.push(("delinquent", errors.join(",")));
}
}
} else {
notifier.send(&notify_msg);
failures.into_iter().next() // Only report the first failure if any
}
Err(err) => Some(("rpc", err.to_string())),
};
datapoint_info!("watchtower-sanity", ("ok", failure.is_none(), bool));
if let Some((failure_test_name, failure_error_message)) = &failure {
let notification_msg = format!(
"solana-watchtower: Error: {}: {}",
failure_test_name, failure_error_message
);
if !no_duplicate_notifications || last_notification_msg != notification_msg {
notifier.send(&notification_msg);
}
datapoint_error!(
"watchtower-sanity-failure",
("test", failure_test_name, String),
("err", failure_error_message, String)
);
last_notification_msg = notification_msg;
} else {
if last_check_notification_sent {
notifier.send("solana-watchtower: All Clear");
if !last_notification_msg.is_empty() {
info!("All clear");
notifier.send("solana-watchtower: All clear");
}
last_check_notification_sent = false;
last_notification_msg = String::from("");
last_notification_msg = "".into();
}
sleep(interval);
}