Quic handshake timeout (#26306)

* Implement timeout for the quic client and server handshake

client/src/nonblocking/quic_client.rs

@@ -12,12 +12,16 @@ use {
     itertools::Itertools,
     log::*,
     quinn::{
-        ClientConfig, Endpoint, EndpointConfig, IdleTimeout, NewConnection, VarInt, WriteError,
+        ClientConfig, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, NewConnection,
+        VarInt, WriteError,
     },
     solana_measure::measure::Measure,
     solana_net_utils::VALIDATOR_PORT_RANGE,
     solana_sdk::{
-        quic::{QUIC_KEEP_ALIVE_MS, QUIC_MAX_TIMEOUT_MS, QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS},
+        quic::{
+            QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS, QUIC_KEEP_ALIVE_MS, QUIC_MAX_TIMEOUT_MS,
+            QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS,
+        },
         transport::Result as TransportResult,
     },
     std::{
@@ -26,7 +30,7 @@ use {
         thread,
         time::Duration,
     },
-    tokio::sync::RwLock,
+    tokio::{sync::RwLock, time::timeout},
 };
 
 struct SkipServerVerification;
@@ -142,7 +146,12 @@ impl QuicNewConnection {
             .connect(addr, "connect")
             .expect("QuicNewConnection::make_connection endpoint.connect");
         stats.total_connections.fetch_add(1, Ordering::Relaxed);
-        let connecting_result = connecting.await;
+        if let Ok(connecting_result) = timeout(
+            Duration::from_millis(QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS),
+            connecting,
+        )
+        .await
+        {
             if connecting_result.is_err() {
                 stats.connection_errors.fetch_add(1, Ordering::Relaxed);
             }
@@ -157,6 +166,9 @@ impl QuicNewConnection {
                 endpoint,
                 connection: Arc::new(connection),
             })
+        } else {
+            Err(ConnectionError::TimedOut.into())
+        }
     }
 
     fn create_endpoint(config: EndpointConfig, client_socket: UdpSocket) -> Endpoint {
@@ -179,17 +191,35 @@ impl QuicNewConnection {
         stats.total_connections.fetch_add(1, Ordering::Relaxed);
         let connection = match connecting.into_0rtt() {
             Ok((connection, zero_rtt)) => {
-                if zero_rtt.await {
+                if let Ok(zero_rtt) = timeout(
+                    Duration::from_millis(QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS),
+                    zero_rtt,
+                )
+                .await
+                {
+                    if zero_rtt {
                         stats.zero_rtt_accepts.fetch_add(1, Ordering::Relaxed);
                     } else {
                         stats.zero_rtt_rejects.fetch_add(1, Ordering::Relaxed);
                     }
                     connection
+                } else {
+                    return Err(ConnectionError::TimedOut.into());
+                }
             }
             Err(connecting) => {
                 stats.connection_errors.fetch_add(1, Ordering::Relaxed);
-                let connecting = connecting.await;
-                connecting?
+
+                if let Ok(connecting_result) = timeout(
+                    Duration::from_millis(QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS),
+                    connecting,
+                )
+                .await
+                {
+                    connecting_result?
+                } else {
+                    return Err(ConnectionError::TimedOut.into());
+                }
             }
         };
         self.connection = Arc::new(connection);

sdk/src/quic.rs

@@ -6,3 +6,8 @@ pub const QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS: usize = 128;
 
 pub const QUIC_MAX_TIMEOUT_MS: u32 = 2_000;
 pub const QUIC_KEEP_ALIVE_MS: u64 = 1_000;
+
+// Based on commonly-used handshake timeouts for various TCP
+// applications. Different applications vary, but most seem to
+// be in the 30-60 second range
+pub const QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS: u64 = 60_000;

streamer/src/nonblocking/quic.rs

@@ -15,7 +15,7 @@ use {
     solana_perf::packet::PacketBatch,
     solana_sdk::{
         packet::{Packet, PACKET_DATA_SIZE},
-        quic::QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS,
+        quic::{QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS, QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS},
         signature::Keypair,
         timing,
     },
@@ -133,7 +133,7 @@ fn prune_unstaked_connection_table(
 }
 
 async fn setup_connection(
-    connection: Connecting,
+    connecting: Connecting,
     unstaked_connection_table: Arc<Mutex<ConnectionTable>>,
     staked_connection_table: Arc<Mutex<ConnectionTable>>,
     packet_sender: Sender<PacketBatch>,
@@ -143,7 +143,13 @@ async fn setup_connection(
     max_unstaked_connections: usize,
     stats: Arc<StreamStats>,
 ) {
-    if let Ok(new_connection) = connection.await {
+    if let Ok(connecting_result) = timeout(
+        Duration::from_millis(QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS),
+        connecting,
+    )
+    .await
+    {
+        if let Ok(new_connection) = connecting_result {
             stats.total_connections.fetch_add(1, Ordering::Relaxed);
             stats.total_new_connections.fetch_add(1, Ordering::Relaxed);
             let NewConnection {
@@ -212,7 +218,8 @@ async fn setup_connection(
                         let staked_nodes = staked_nodes.read().unwrap();
                         VarInt::from_u64(
                             ((stake as f64 / staked_nodes.total_stake as f64)
-                            * QUIC_TOTAL_STAKED_CONCURRENT_STREAMS) as u64,
+                                * QUIC_TOTAL_STAKED_CONCURRENT_STREAMS)
+                                as u64,
                         )
                     }
                 };
@@ -257,6 +264,9 @@ async fn setup_connection(
                     .connection_add_failed_unstaked_node
                     .fetch_add(1, Ordering::Relaxed);
             }
+        } else {
+            stats.connection_setup_error.fetch_add(1, Ordering::Relaxed);
+        }
     } else {
         stats
             .connection_setup_timeout

streamer/src/quic.rs

@@ -162,6 +162,7 @@ pub struct StreamStats {
     pub(crate) connection_add_failed_unstaked_node: AtomicUsize,
     pub(crate) connection_add_failed_on_pruning: AtomicUsize,
     pub(crate) connection_setup_timeout: AtomicUsize,
+    pub(crate) connection_setup_error: AtomicUsize,
     pub(crate) connection_removed: AtomicUsize,
     pub(crate) connection_remove_failed: AtomicUsize,
 }
@@ -233,6 +234,11 @@ impl StreamStats {
                 self.connection_setup_timeout.swap(0, Ordering::Relaxed),
                 i64
             ),
+            (
+                "connection_setup_error",
+                self.connection_setup_error.swap(0, Ordering::Relaxed),
+                i64
+            ),
             (
                 "invalid_chunk",
                 self.total_invalid_chunks.swap(0, Ordering::Relaxed),