Quic handshake timeout (#26306)
* Implement timeout for the quic client and server handshake
This commit is contained in:
parent
275e47f931
commit
4772a93109
|
@ -12,12 +12,16 @@ use {
|
|||
itertools::Itertools,
|
||||
log::*,
|
||||
quinn::{
|
||||
ClientConfig, Endpoint, EndpointConfig, IdleTimeout, NewConnection, VarInt, WriteError,
|
||||
ClientConfig, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, NewConnection,
|
||||
VarInt, WriteError,
|
||||
},
|
||||
solana_measure::measure::Measure,
|
||||
solana_net_utils::VALIDATOR_PORT_RANGE,
|
||||
solana_sdk::{
|
||||
quic::{QUIC_KEEP_ALIVE_MS, QUIC_MAX_TIMEOUT_MS, QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS},
|
||||
quic::{
|
||||
QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS, QUIC_KEEP_ALIVE_MS, QUIC_MAX_TIMEOUT_MS,
|
||||
QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS,
|
||||
},
|
||||
transport::Result as TransportResult,
|
||||
},
|
||||
std::{
|
||||
|
@ -26,7 +30,7 @@ use {
|
|||
thread,
|
||||
time::Duration,
|
||||
},
|
||||
tokio::sync::RwLock,
|
||||
tokio::{sync::RwLock, time::timeout},
|
||||
};
|
||||
|
||||
struct SkipServerVerification;
|
||||
|
@ -142,7 +146,12 @@ impl QuicNewConnection {
|
|||
.connect(addr, "connect")
|
||||
.expect("QuicNewConnection::make_connection endpoint.connect");
|
||||
stats.total_connections.fetch_add(1, Ordering::Relaxed);
|
||||
let connecting_result = connecting.await;
|
||||
if let Ok(connecting_result) = timeout(
|
||||
Duration::from_millis(QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS),
|
||||
connecting,
|
||||
)
|
||||
.await
|
||||
{
|
||||
if connecting_result.is_err() {
|
||||
stats.connection_errors.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
@ -157,6 +166,9 @@ impl QuicNewConnection {
|
|||
endpoint,
|
||||
connection: Arc::new(connection),
|
||||
})
|
||||
} else {
|
||||
Err(ConnectionError::TimedOut.into())
|
||||
}
|
||||
}
|
||||
|
||||
fn create_endpoint(config: EndpointConfig, client_socket: UdpSocket) -> Endpoint {
|
||||
|
@ -179,17 +191,35 @@ impl QuicNewConnection {
|
|||
stats.total_connections.fetch_add(1, Ordering::Relaxed);
|
||||
let connection = match connecting.into_0rtt() {
|
||||
Ok((connection, zero_rtt)) => {
|
||||
if zero_rtt.await {
|
||||
if let Ok(zero_rtt) = timeout(
|
||||
Duration::from_millis(QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS),
|
||||
zero_rtt,
|
||||
)
|
||||
.await
|
||||
{
|
||||
if zero_rtt {
|
||||
stats.zero_rtt_accepts.fetch_add(1, Ordering::Relaxed);
|
||||
} else {
|
||||
stats.zero_rtt_rejects.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
connection
|
||||
} else {
|
||||
return Err(ConnectionError::TimedOut.into());
|
||||
}
|
||||
}
|
||||
Err(connecting) => {
|
||||
stats.connection_errors.fetch_add(1, Ordering::Relaxed);
|
||||
let connecting = connecting.await;
|
||||
connecting?
|
||||
|
||||
if let Ok(connecting_result) = timeout(
|
||||
Duration::from_millis(QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS),
|
||||
connecting,
|
||||
)
|
||||
.await
|
||||
{
|
||||
connecting_result?
|
||||
} else {
|
||||
return Err(ConnectionError::TimedOut.into());
|
||||
}
|
||||
}
|
||||
};
|
||||
self.connection = Arc::new(connection);
|
||||
|
|
|
@ -6,3 +6,8 @@ pub const QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS: usize = 128;
|
|||
|
||||
pub const QUIC_MAX_TIMEOUT_MS: u32 = 2_000;
|
||||
pub const QUIC_KEEP_ALIVE_MS: u64 = 1_000;
|
||||
|
||||
// Based on commonly-used handshake timeouts for various TCP
|
||||
// applications. Different applications vary, but most seem to
|
||||
// be in the 30-60 second range
|
||||
pub const QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS: u64 = 60_000;
|
||||
|
|
|
@ -15,7 +15,7 @@ use {
|
|||
solana_perf::packet::PacketBatch,
|
||||
solana_sdk::{
|
||||
packet::{Packet, PACKET_DATA_SIZE},
|
||||
quic::QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS,
|
||||
quic::{QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS, QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS},
|
||||
signature::Keypair,
|
||||
timing,
|
||||
},
|
||||
|
@ -133,7 +133,7 @@ fn prune_unstaked_connection_table(
|
|||
}
|
||||
|
||||
async fn setup_connection(
|
||||
connection: Connecting,
|
||||
connecting: Connecting,
|
||||
unstaked_connection_table: Arc<Mutex<ConnectionTable>>,
|
||||
staked_connection_table: Arc<Mutex<ConnectionTable>>,
|
||||
packet_sender: Sender<PacketBatch>,
|
||||
|
@ -143,7 +143,13 @@ async fn setup_connection(
|
|||
max_unstaked_connections: usize,
|
||||
stats: Arc<StreamStats>,
|
||||
) {
|
||||
if let Ok(new_connection) = connection.await {
|
||||
if let Ok(connecting_result) = timeout(
|
||||
Duration::from_millis(QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS),
|
||||
connecting,
|
||||
)
|
||||
.await
|
||||
{
|
||||
if let Ok(new_connection) = connecting_result {
|
||||
stats.total_connections.fetch_add(1, Ordering::Relaxed);
|
||||
stats.total_new_connections.fetch_add(1, Ordering::Relaxed);
|
||||
let NewConnection {
|
||||
|
@ -212,7 +218,8 @@ async fn setup_connection(
|
|||
let staked_nodes = staked_nodes.read().unwrap();
|
||||
VarInt::from_u64(
|
||||
((stake as f64 / staked_nodes.total_stake as f64)
|
||||
* QUIC_TOTAL_STAKED_CONCURRENT_STREAMS) as u64,
|
||||
* QUIC_TOTAL_STAKED_CONCURRENT_STREAMS)
|
||||
as u64,
|
||||
)
|
||||
}
|
||||
};
|
||||
|
@ -257,6 +264,9 @@ async fn setup_connection(
|
|||
.connection_add_failed_unstaked_node
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
} else {
|
||||
stats.connection_setup_error.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
} else {
|
||||
stats
|
||||
.connection_setup_timeout
|
||||
|
|
|
@ -162,6 +162,7 @@ pub struct StreamStats {
|
|||
pub(crate) connection_add_failed_unstaked_node: AtomicUsize,
|
||||
pub(crate) connection_add_failed_on_pruning: AtomicUsize,
|
||||
pub(crate) connection_setup_timeout: AtomicUsize,
|
||||
pub(crate) connection_setup_error: AtomicUsize,
|
||||
pub(crate) connection_removed: AtomicUsize,
|
||||
pub(crate) connection_remove_failed: AtomicUsize,
|
||||
}
|
||||
|
@ -233,6 +234,11 @@ impl StreamStats {
|
|||
self.connection_setup_timeout.swap(0, Ordering::Relaxed),
|
||||
i64
|
||||
),
|
||||
(
|
||||
"connection_setup_error",
|
||||
self.connection_setup_error.swap(0, Ordering::Relaxed),
|
||||
i64
|
||||
),
|
||||
(
|
||||
"invalid_chunk",
|
||||
self.total_invalid_chunks.swap(0, Ordering::Relaxed),
|
||||
|
|
Loading…
Reference in New Issue