Stop lite-rpc if the rpc is down for long time

lite-rpc/src/main.rs

@@ -245,8 +245,8 @@ pub async fn main() -> anyhow::Result<()> {
 
     tokio::select! {
         err = rpc_tester => {
-            // This should never happen
-            unreachable!("{err:?}")
+            log::error!("{err:?}");
+            Ok(())
         }
         res = main => {
             // This should never happen
@@ -255,7 +255,6 @@ pub async fn main() -> anyhow::Result<()> {
         }
         _ = ctrl_c_signal => {
             log::info!("Received ctrl+c signal");
-
             Ok(())
         }
     }

lite-rpc/src/rpc_tester.rs

@@ -1,5 +1,6 @@
 use std::net::SocketAddr;
 
+use anyhow::bail;
 use lite_rpc::cli::Args;
 use prometheus::{opts, register_gauge, Gauge};
 use solana_rpc_client::nonblocking::rpc_client::RpcClient;
@@ -24,7 +25,8 @@ impl From<&Args> for RpcTester {
 
 impl RpcTester {
     /// Starts a loop that checks if the rpc is responding every 5 seconds
-    pub async fn start(self) -> ! {
+    pub async fn start(self) -> anyhow::Result<()> {
+        let mut error_counter = 0;
         loop {
             // sleep for 5 seconds
             tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
@@ -35,7 +37,10 @@ impl RpcTester {
             };
 
             RPC_RESPONDING.set(0.0);
-            log::error!("Rpc not responding {err:?}");
+            error_counter += 1;
+            if error_counter > 10 {
+                bail!("RPC seems down restarting service error {err:?}");
+            }
         }
     }
 }