2021-02-05 22:39:23 -08:00
|
|
|
use {
|
2021-05-18 22:55:55 -07:00
|
|
|
crate::{admin_rpc_service, new_spinner_progress_bar, println_name_value, ProgressBar},
|
2021-02-05 22:39:23 -08:00
|
|
|
console::style,
|
|
|
|
solana_client::{
|
|
|
|
client_error, rpc_client::RpcClient, rpc_request, rpc_response::RpcContactInfo,
|
|
|
|
},
|
2021-03-04 13:01:11 -08:00
|
|
|
solana_core::validator::ValidatorStartProgress,
|
2021-02-05 22:39:23 -08:00
|
|
|
solana_sdk::{
|
2021-06-03 20:06:13 -07:00
|
|
|
clock::Slot, commitment_config::CommitmentConfig, exit::Exit, native_token::Sol,
|
|
|
|
pubkey::Pubkey,
|
2021-02-05 22:39:23 -08:00
|
|
|
},
|
|
|
|
std::{
|
|
|
|
io,
|
2021-03-04 13:01:11 -08:00
|
|
|
net::SocketAddr,
|
2021-02-05 22:39:23 -08:00
|
|
|
path::{Path, PathBuf},
|
2021-02-26 21:42:09 -08:00
|
|
|
sync::{
|
|
|
|
atomic::{AtomicBool, Ordering},
|
|
|
|
Arc,
|
|
|
|
},
|
2021-02-05 22:39:23 -08:00
|
|
|
thread,
|
2021-03-04 13:01:11 -08:00
|
|
|
time::{Duration, SystemTime},
|
2021-02-05 22:39:23 -08:00
|
|
|
},
|
|
|
|
};
|
|
|
|
|
|
|
|
pub struct Dashboard {
|
|
|
|
progress_bar: ProgressBar,
|
|
|
|
ledger_path: PathBuf,
|
2021-02-26 21:42:09 -08:00
|
|
|
exit: Arc<AtomicBool>,
|
2021-02-05 22:39:23 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
impl Dashboard {
|
2021-02-26 21:42:09 -08:00
|
|
|
pub fn new(
|
|
|
|
ledger_path: &Path,
|
|
|
|
log_path: Option<&Path>,
|
2021-06-03 20:06:13 -07:00
|
|
|
validator_exit: Option<&mut Exit>,
|
2021-02-26 21:42:09 -08:00
|
|
|
) -> Result<Self, io::Error> {
|
2021-02-05 22:39:23 -08:00
|
|
|
println_name_value("Ledger location:", &format!("{}", ledger_path.display()));
|
|
|
|
if let Some(log_path) = log_path {
|
|
|
|
println_name_value("Log:", &format!("{}", log_path.display()));
|
|
|
|
}
|
|
|
|
|
|
|
|
let progress_bar = new_spinner_progress_bar();
|
|
|
|
progress_bar.set_message("Initializing...");
|
|
|
|
|
2021-02-26 21:42:09 -08:00
|
|
|
let exit = Arc::new(AtomicBool::new(false));
|
|
|
|
if let Some(validator_exit) = validator_exit {
|
|
|
|
let exit = exit.clone();
|
|
|
|
validator_exit.register_exit(Box::new(move || exit.store(true, Ordering::Relaxed)));
|
|
|
|
}
|
|
|
|
|
2021-02-05 22:39:23 -08:00
|
|
|
Ok(Self {
|
2021-02-26 21:42:09 -08:00
|
|
|
exit,
|
2021-02-05 22:39:23 -08:00
|
|
|
ledger_path: ledger_path.to_path_buf(),
|
2021-02-26 21:42:09 -08:00
|
|
|
progress_bar,
|
2021-02-05 22:39:23 -08:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2021-03-06 11:53:21 -08:00
|
|
|
pub fn run(self, refresh_interval: Duration) {
|
2021-02-05 22:39:23 -08:00
|
|
|
let Self {
|
2021-02-26 21:42:09 -08:00
|
|
|
exit,
|
2021-02-05 22:39:23 -08:00
|
|
|
ledger_path,
|
2021-02-26 21:42:09 -08:00
|
|
|
progress_bar,
|
|
|
|
..
|
2021-02-05 22:39:23 -08:00
|
|
|
} = self;
|
|
|
|
drop(progress_bar);
|
|
|
|
|
2021-07-26 11:32:17 -07:00
|
|
|
let runtime = admin_rpc_service::runtime();
|
2021-02-26 21:42:09 -08:00
|
|
|
while !exit.load(Ordering::Relaxed) {
|
|
|
|
let progress_bar = new_spinner_progress_bar();
|
|
|
|
progress_bar.set_message("Connecting...");
|
2021-02-05 22:39:23 -08:00
|
|
|
|
2021-03-04 13:01:11 -08:00
|
|
|
let (rpc_addr, start_time) = match runtime.block_on(wait_for_validator_startup(
|
|
|
|
&ledger_path,
|
|
|
|
&exit,
|
|
|
|
progress_bar,
|
2021-03-06 11:53:21 -08:00
|
|
|
refresh_interval,
|
2021-03-04 13:01:11 -08:00
|
|
|
)) {
|
|
|
|
None => continue,
|
|
|
|
Some(results) => results,
|
|
|
|
};
|
2021-02-05 22:39:23 -08:00
|
|
|
|
2021-03-04 13:01:11 -08:00
|
|
|
let rpc_client = RpcClient::new_socket(rpc_addr);
|
|
|
|
let identity = match rpc_client.get_identity() {
|
|
|
|
Ok(identity) => identity,
|
|
|
|
Err(err) => {
|
|
|
|
println!("Failed to get validator identity over RPC: {}", err);
|
|
|
|
continue;
|
2021-02-05 22:39:23 -08:00
|
|
|
}
|
|
|
|
};
|
2021-02-26 21:42:09 -08:00
|
|
|
println_name_value("Identity:", &identity.to_string());
|
2021-05-13 15:44:46 -07:00
|
|
|
|
|
|
|
if let Ok(genesis_hash) = rpc_client.get_genesis_hash() {
|
|
|
|
println_name_value("Genesis Hash:", &genesis_hash.to_string());
|
|
|
|
}
|
|
|
|
|
2021-02-26 21:42:09 -08:00
|
|
|
if let Some(contact_info) = get_contact_info(&rpc_client, &identity) {
|
|
|
|
println_name_value(
|
|
|
|
"Version:",
|
|
|
|
&contact_info.version.unwrap_or_else(|| "?".to_string()),
|
|
|
|
);
|
2021-05-13 15:44:46 -07:00
|
|
|
if let Some(shred_version) = contact_info.shred_version {
|
|
|
|
println_name_value("Shred Version:", &shred_version.to_string());
|
|
|
|
}
|
2021-02-26 21:42:09 -08:00
|
|
|
if let Some(gossip) = contact_info.gossip {
|
|
|
|
println_name_value("Gossip Address:", &gossip.to_string());
|
|
|
|
}
|
|
|
|
if let Some(tpu) = contact_info.tpu {
|
|
|
|
println_name_value("TPU Address:", &tpu.to_string());
|
|
|
|
}
|
|
|
|
if let Some(rpc) = contact_info.rpc {
|
2021-10-22 21:25:54 -07:00
|
|
|
println_name_value("JSON RPC URL:", &format!("http://{}", rpc));
|
2021-02-26 21:42:09 -08:00
|
|
|
}
|
|
|
|
}
|
2021-02-05 22:39:23 -08:00
|
|
|
|
2021-02-26 21:42:09 -08:00
|
|
|
let progress_bar = new_spinner_progress_bar();
|
2021-09-02 13:25:42 -07:00
|
|
|
let mut snapshot_slot_info = None;
|
2021-02-26 21:42:09 -08:00
|
|
|
for i in 0.. {
|
|
|
|
if exit.load(Ordering::Relaxed) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if i % 10 == 0 {
|
2021-09-02 13:25:42 -07:00
|
|
|
snapshot_slot_info = rpc_client.get_highest_snapshot_slot().ok();
|
2021-02-26 21:42:09 -08:00
|
|
|
}
|
2021-02-05 22:39:23 -08:00
|
|
|
|
|
|
|
match get_validator_stats(&rpc_client, &identity) {
|
|
|
|
Ok((
|
2021-03-30 14:23:08 -07:00
|
|
|
max_retransmit_slot,
|
2021-02-05 22:39:23 -08:00
|
|
|
processed_slot,
|
|
|
|
confirmed_slot,
|
|
|
|
finalized_slot,
|
|
|
|
transaction_count,
|
|
|
|
identity_balance,
|
|
|
|
health,
|
|
|
|
)) => {
|
2021-02-26 21:42:09 -08:00
|
|
|
let uptime = {
|
|
|
|
let uptime =
|
|
|
|
chrono::Duration::from_std(start_time.elapsed().unwrap()).unwrap();
|
|
|
|
|
|
|
|
format!(
|
|
|
|
"{:02}:{:02}:{:02} ",
|
|
|
|
uptime.num_hours(),
|
|
|
|
uptime.num_minutes() % 60,
|
|
|
|
uptime.num_seconds() % 60
|
|
|
|
)
|
2021-02-05 22:39:23 -08:00
|
|
|
};
|
|
|
|
|
2021-06-01 10:38:02 -07:00
|
|
|
progress_bar.set_message(format!(
|
2021-03-30 14:23:08 -07:00
|
|
|
"{}{}{}| \
|
2021-02-26 21:42:09 -08:00
|
|
|
Processed Slot: {} | Confirmed Slot: {} | Finalized Slot: {} | \
|
2021-09-09 18:18:10 -07:00
|
|
|
Full Snapshot Slot: {} | Incremental Snapshot Slot: {} | \
|
2021-02-26 21:42:09 -08:00
|
|
|
Transactions: {} | {}",
|
2021-02-05 22:39:23 -08:00
|
|
|
uptime,
|
|
|
|
if health == "ok" {
|
|
|
|
"".to_string()
|
|
|
|
} else {
|
|
|
|
format!("| {} ", style(health).bold().red())
|
|
|
|
},
|
2021-03-30 14:23:08 -07:00
|
|
|
if max_retransmit_slot == 0 {
|
|
|
|
"".to_string()
|
|
|
|
} else {
|
|
|
|
format!("| Max Slot: {} ", max_retransmit_slot)
|
|
|
|
},
|
2021-02-05 22:39:23 -08:00
|
|
|
processed_slot,
|
|
|
|
confirmed_slot,
|
|
|
|
finalized_slot,
|
2021-09-02 13:25:42 -07:00
|
|
|
snapshot_slot_info
|
|
|
|
.as_ref()
|
|
|
|
.map(|snapshot_slot_info| snapshot_slot_info.full.to_string())
|
|
|
|
.unwrap_or_else(|| '-'.to_string()),
|
|
|
|
snapshot_slot_info
|
|
|
|
.as_ref()
|
|
|
|
.map(|snapshot_slot_info| snapshot_slot_info
|
|
|
|
.incremental
|
|
|
|
.map(|incremental| incremental.to_string()))
|
|
|
|
.flatten()
|
|
|
|
.unwrap_or_else(|| '-'.to_string()),
|
2021-02-05 22:39:23 -08:00
|
|
|
transaction_count,
|
|
|
|
identity_balance
|
|
|
|
));
|
2021-03-06 11:53:21 -08:00
|
|
|
thread::sleep(refresh_interval);
|
2021-02-05 22:39:23 -08:00
|
|
|
}
|
|
|
|
Err(err) => {
|
2021-02-26 21:42:09 -08:00
|
|
|
progress_bar
|
2021-06-01 10:38:02 -07:00
|
|
|
.abandon_with_message(format!("RPC connection failure: {}", err));
|
2021-02-26 21:42:09 -08:00
|
|
|
break;
|
2021-02-05 22:39:23 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2021-02-26 21:42:09 -08:00
|
|
|
|
2021-03-04 13:01:11 -08:00
|
|
|
async fn wait_for_validator_startup(
|
|
|
|
ledger_path: &Path,
|
|
|
|
exit: &Arc<AtomicBool>,
|
|
|
|
progress_bar: ProgressBar,
|
2021-03-06 11:53:21 -08:00
|
|
|
refresh_interval: Duration,
|
2021-03-04 13:01:11 -08:00
|
|
|
) -> Option<(SocketAddr, SystemTime)> {
|
|
|
|
let mut admin_client = None;
|
|
|
|
loop {
|
|
|
|
if exit.load(Ordering::Relaxed) {
|
|
|
|
return None;
|
|
|
|
}
|
|
|
|
|
|
|
|
if admin_client.is_none() {
|
2021-06-18 06:34:46 -07:00
|
|
|
match admin_rpc_service::connect(ledger_path).await {
|
2021-03-04 13:01:11 -08:00
|
|
|
Ok(new_admin_client) => admin_client = Some(new_admin_client),
|
|
|
|
Err(err) => {
|
2021-06-01 10:38:02 -07:00
|
|
|
progress_bar.set_message(format!("Unable to connect to validator: {}", err));
|
2021-03-06 11:53:21 -08:00
|
|
|
thread::sleep(refresh_interval);
|
2021-03-04 13:01:11 -08:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
match admin_client.as_ref().unwrap().start_progress().await {
|
|
|
|
Ok(start_progress) => {
|
|
|
|
if start_progress == ValidatorStartProgress::Running {
|
|
|
|
let admin_client = admin_client.take().unwrap();
|
|
|
|
|
|
|
|
match async move {
|
|
|
|
let rpc_addr = admin_client.rpc_addr().await?;
|
|
|
|
let start_time = admin_client.start_time().await?;
|
|
|
|
Ok::<_, jsonrpc_core_client::RpcError>((rpc_addr, start_time))
|
|
|
|
}
|
|
|
|
.await
|
|
|
|
{
|
2021-06-01 10:38:02 -07:00
|
|
|
Ok((None, _)) => progress_bar.set_message("RPC service not available"),
|
2021-03-04 13:01:11 -08:00
|
|
|
Ok((Some(rpc_addr), start_time)) => return Some((rpc_addr, start_time)),
|
|
|
|
Err(err) => {
|
|
|
|
progress_bar
|
2021-06-01 10:38:02 -07:00
|
|
|
.set_message(format!("Failed to get validator info: {}", err));
|
2021-03-04 13:01:11 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
2021-06-01 10:38:02 -07:00
|
|
|
progress_bar.set_message(format!("Validator startup: {:?}...", start_progress));
|
2021-03-04 13:01:11 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
Err(err) => {
|
|
|
|
admin_client = None;
|
|
|
|
progress_bar
|
2021-06-01 10:38:02 -07:00
|
|
|
.set_message(format!("Failed to get validator start progress: {}", err));
|
2021-03-04 13:01:11 -08:00
|
|
|
}
|
|
|
|
}
|
2021-03-06 11:53:21 -08:00
|
|
|
thread::sleep(refresh_interval);
|
2021-03-04 13:01:11 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-02-26 21:42:09 -08:00
|
|
|
fn get_contact_info(rpc_client: &RpcClient, identity: &Pubkey) -> Option<RpcContactInfo> {
|
|
|
|
rpc_client
|
|
|
|
.get_cluster_nodes()
|
|
|
|
.ok()
|
|
|
|
.unwrap_or_default()
|
|
|
|
.into_iter()
|
|
|
|
.find(|node| node.pubkey == identity.to_string())
|
|
|
|
}
|
|
|
|
|
|
|
|
fn get_validator_stats(
|
|
|
|
rpc_client: &RpcClient,
|
|
|
|
identity: &Pubkey,
|
2021-03-30 14:23:08 -07:00
|
|
|
) -> client_error::Result<(Slot, Slot, Slot, Slot, u64, Sol, String)> {
|
2021-02-26 21:42:09 -08:00
|
|
|
let finalized_slot = rpc_client.get_slot_with_commitment(CommitmentConfig::finalized())?;
|
2021-03-30 14:23:08 -07:00
|
|
|
let confirmed_slot = rpc_client.get_slot_with_commitment(CommitmentConfig::confirmed())?;
|
|
|
|
let processed_slot = rpc_client.get_slot_with_commitment(CommitmentConfig::processed())?;
|
|
|
|
let max_retransmit_slot = rpc_client.get_max_retransmit_slot()?;
|
2021-02-26 21:42:09 -08:00
|
|
|
let transaction_count =
|
|
|
|
rpc_client.get_transaction_count_with_commitment(CommitmentConfig::processed())?;
|
|
|
|
let identity_balance = rpc_client
|
|
|
|
.get_balance_with_commitment(identity, CommitmentConfig::confirmed())?
|
|
|
|
.value;
|
|
|
|
|
|
|
|
let health = match rpc_client.get_health() {
|
|
|
|
Ok(()) => "ok".to_string(),
|
|
|
|
Err(err) => {
|
|
|
|
if let client_error::ClientErrorKind::RpcError(
|
|
|
|
rpc_request::RpcError::RpcResponseError {
|
|
|
|
code: _,
|
|
|
|
message: _,
|
|
|
|
data:
|
|
|
|
rpc_request::RpcResponseErrorData::NodeUnhealthy {
|
|
|
|
num_slots_behind: Some(num_slots_behind),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
) = &err.kind
|
|
|
|
{
|
|
|
|
format!("{} slots behind", num_slots_behind)
|
|
|
|
} else {
|
2021-03-04 21:18:08 -08:00
|
|
|
"health unknown".to_string()
|
2021-02-26 21:42:09 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
Ok((
|
2021-03-30 14:23:08 -07:00
|
|
|
max_retransmit_slot,
|
2021-02-26 21:42:09 -08:00
|
|
|
processed_slot,
|
|
|
|
confirmed_slot,
|
|
|
|
finalized_slot,
|
|
|
|
transaction_count,
|
|
|
|
Sol(identity_balance),
|
|
|
|
health,
|
|
|
|
))
|
|
|
|
}
|