Report duration of last alarm in the All Clear message (#9766)

automerge

Cargo.lock

@@ -4821,6 +4821,7 @@ name = "solana-watchtower"
 version = "1.2.0"
 dependencies = [
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "humantime 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "reqwest 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)",

watchtower/Cargo.toml

@@ -11,6 +11,7 @@ homepage = "https://solana.com/"
 [dependencies]
 clap = "2.33.0"
 log = "0.4.8"
+humantime = "2.0.0"
 reqwest = { version = "0.10.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
 serde_json = "1.0"
 solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }

watchtower/src/main.rs

@@ -14,7 +14,12 @@ use solana_client::{
 };
 use solana_metrics::{datapoint_error, datapoint_info};
 use solana_sdk::{hash::Hash, native_token::lamports_to_sol, pubkey::Pubkey};
-use std::{error, str::FromStr, thread::sleep, time::Duration};
+use std::{
+    error,
+    str::FromStr,
+    thread::sleep,
+    time::{Duration, Instant},
+};
 
 struct Config {
     interval: Duration,
@@ -153,6 +158,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
     let mut last_transaction_count = 0;
     let mut last_recent_blockhash = Hash::default();
     let mut last_notification_msg = "".into();
+    let mut last_success = Instant::now();
 
     loop {
         let failure = match get_cluster_info(&rpc_client) {
@@ -286,10 +292,18 @@ fn main() -> Result<(), Box<dyn error::Error>> {
             last_notification_msg = notification_msg;
         } else {
             if !last_notification_msg.is_empty() {
-                info!("All clear");
-                notifier.send("solana-watchtower: All clear");
+                let alarm_duration = Instant::now().duration_since(last_success);
+                let alarm_duration = Duration::from_secs(alarm_duration.as_secs()); // Drop milliseconds in message
+
+                let all_clear_msg = format!(
+                    "All clear after {}",
+                    humantime::format_duration(alarm_duration)
+                );
+                info!("{}", all_clear_msg);
+                notifier.send(&format!("solana-watchtower: {}", all_clear_msg));
             }
             last_notification_msg = "".into();
+            last_success = Instant::now();
         }
         sleep(config.interval);
     }