blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

separates out quic streamer connection stats from different servers (#31797)

This commit is contained in:
behzad nouri 2023-05-25 16:54:24 +00:00 committed by GitHub
parent 9281ab7d97
commit f1ebc5b5c3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 43 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
client/src/connection_cache.rs
View File

@ -217,8 +217,7 @@ mod tests {
crossbeam_channel::unbounded, crossbeam_channel::unbounded,
solana_sdk::{net::DEFAULT_TPU_COALESCE, signature::Keypair}, solana_sdk::{net::DEFAULT_TPU_COALESCE, signature::Keypair},
solana_streamer::{ solana_streamer::{
nonblocking::quic::DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, quic::StreamStats, nonblocking::quic::DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, streamer::StakedNodes,
streamer::StakedNodes,
}, },
std::{ std::{
net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}, net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},
@ -229,37 +228,25 @@ mod tests {
}, },
}; };
fn server_args() -> ( fn server_args() -> (UdpSocket, Arc<AtomicBool>, Keypair, IpAddr) {
UdpSocket,
Arc<AtomicBool>,
Keypair,
IpAddr,
Arc<StreamStats>,
) {
( (
UdpSocket::bind("127.0.0.1:0").unwrap(), UdpSocket::bind("127.0.0.1:0").unwrap(),
Arc::new(AtomicBool::new(false)), Arc::new(AtomicBool::new(false)),
Keypair::new(), Keypair::new(),
"127.0.0.1".parse().unwrap(), "127.0.0.1".parse().unwrap(),
Arc::new(StreamStats::default()),
) )
} }
#[test] #[test]
fn test_connection_with_specified_client_endpoint() { fn test_connection_with_specified_client_endpoint() {
// Start a response receiver: // Start a response receiver:
let ( let (response_recv_socket, response_recv_exit, keypair2, response_recv_ip) = server_args();
response_recv_socket,
response_recv_exit,
keypair2,
response_recv_ip,
response_recv_stats,
) = server_args();
let (sender2, _receiver2) = unbounded(); let (sender2, _receiver2) = unbounded();
let staked_nodes = Arc::new(RwLock::new(StakedNodes::default())); let staked_nodes = Arc::new(RwLock::new(StakedNodes::default()));
let (response_recv_endpoint, response_recv_thread) = solana_streamer::quic::spawn_server( let (response_recv_endpoint, response_recv_thread) = solana_streamer::quic::spawn_server(
"quic_streamer_test",
response_recv_socket, response_recv_socket,
&keypair2, &keypair2,
response_recv_ip, response_recv_ip,
@ -269,7 +256,6 @@ mod tests {
staked_nodes, staked_nodes,
10, 10,
10, 10,
response_recv_stats,
DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT,
DEFAULT_TPU_COALESCE, DEFAULT_TPU_COALESCE,
) )

7
core/src/tpu.rs
View File

@ -37,7 +37,7 @@ use {
solana_sdk::{pubkey::Pubkey, signature::Keypair}, solana_sdk::{pubkey::Pubkey, signature::Keypair},
solana_streamer::{ solana_streamer::{
nonblocking::quic::DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, nonblocking::quic::DEFAULT_WAIT_FOR_CHUNK_TIMEOUT,
quic::{spawn_server, StreamStats, MAX_STAKED_CONNECTIONS, MAX_UNSTAKED_CONNECTIONS}, quic::{spawn_server, MAX_STAKED_CONNECTIONS, MAX_UNSTAKED_CONNECTIONS},
streamer::StakedNodes, streamer::StakedNodes,
}, },
std::{ std::{
@ -145,8 +145,8 @@ impl Tpu {
let (non_vote_sender, non_vote_receiver) = banking_tracer.create_channel_non_vote(); let (non_vote_sender, non_vote_receiver) = banking_tracer.create_channel_non_vote();
let stats = Arc::new(StreamStats::default());
let (_, tpu_quic_t) = spawn_server( let (_, tpu_quic_t) = spawn_server(
"quic_streamer_tpu",
transactions_quic_sockets, transactions_quic_sockets,
keypair, keypair,
cluster_info cluster_info
@ -160,13 +160,13 @@ impl Tpu {
staked_nodes.clone(), staked_nodes.clone(),
MAX_STAKED_CONNECTIONS, MAX_STAKED_CONNECTIONS,
MAX_UNSTAKED_CONNECTIONS, MAX_UNSTAKED_CONNECTIONS,
stats.clone(),
DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT,
tpu_coalesce, tpu_coalesce,
) )
.unwrap(); .unwrap();
let (_, tpu_forwards_quic_t) = spawn_server( let (_, tpu_forwards_quic_t) = spawn_server(
"quic_streamer_tpu_forwards",
transactions_forwards_quic_sockets, transactions_forwards_quic_sockets,
keypair, keypair,
cluster_info cluster_info
@ -180,7 +180,6 @@ impl Tpu {
staked_nodes.clone(), staked_nodes.clone(),
MAX_STAKED_CONNECTIONS.saturating_add(MAX_UNSTAKED_CONNECTIONS), MAX_STAKED_CONNECTIONS.saturating_add(MAX_UNSTAKED_CONNECTIONS),
0, // Prevent unstaked nodes from forwarding transactions 0, // Prevent unstaked nodes from forwarding transactions
stats,
DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT,
tpu_coalesce, tpu_coalesce,
) )

38
quic-client/tests/quic_client.rs
View File

@ -10,8 +10,8 @@ mod tests {
}, },
solana_sdk::{net::DEFAULT_TPU_COALESCE, packet::PACKET_DATA_SIZE, signature::Keypair}, solana_sdk::{net::DEFAULT_TPU_COALESCE, packet::PACKET_DATA_SIZE, signature::Keypair},
solana_streamer::{ solana_streamer::{
nonblocking::quic::DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, quic::StreamStats, nonblocking::quic::DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, streamer::StakedNodes,
streamer::StakedNodes, tls_certificates::new_self_signed_tls_certificate, tls_certificates::new_self_signed_tls_certificate,
}, },
std::{ std::{
net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}, net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},
@ -49,19 +49,12 @@ mod tests {
assert_eq!(total_packets, num_expected_packets); assert_eq!(total_packets, num_expected_packets);
} }
fn server_args() -> ( fn server_args() -> (UdpSocket, Arc<AtomicBool>, Keypair, IpAddr) {
UdpSocket,
Arc<AtomicBool>,
Keypair,
IpAddr,
Arc<StreamStats>,
) {
( (
UdpSocket::bind("127.0.0.1:0").unwrap(), UdpSocket::bind("127.0.0.1:0").unwrap(),
Arc::new(AtomicBool::new(false)), Arc::new(AtomicBool::new(false)),
Keypair::new(), Keypair::new(),
"127.0.0.1".parse().unwrap(), "127.0.0.1".parse().unwrap(),
Arc::new(StreamStats::default()),
) )
} }
@ -74,8 +67,9 @@ mod tests {
solana_logger::setup(); solana_logger::setup();
let (sender, receiver) = unbounded(); let (sender, receiver) = unbounded();
let staked_nodes = Arc::new(RwLock::new(StakedNodes::default())); let staked_nodes = Arc::new(RwLock::new(StakedNodes::default()));
let (s, exit, keypair, ip, stats) = server_args(); let (s, exit, keypair, ip) = server_args();
let (_, t) = solana_streamer::quic::spawn_server( let (_, t) = solana_streamer::quic::spawn_server(
"quic_streamer_test",
s.try_clone().unwrap(), s.try_clone().unwrap(),
&keypair, &keypair,
ip, ip,
@ -85,7 +79,6 @@ mod tests {
staked_nodes, staked_nodes,
10, 10,
10, 10,
stats,
DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT,
DEFAULT_TPU_COALESCE, DEFAULT_TPU_COALESCE,
) )
@ -154,8 +147,9 @@ mod tests {
solana_logger::setup(); solana_logger::setup();
let (sender, receiver) = unbounded(); let (sender, receiver) = unbounded();
let staked_nodes = Arc::new(RwLock::new(StakedNodes::default())); let staked_nodes = Arc::new(RwLock::new(StakedNodes::default()));
let (s, exit, keypair, ip, stats) = server_args(); let (s, exit, keypair, ip) = server_args();
let (_, t) = solana_streamer::nonblocking::quic::spawn_server( let (_, _, t) = solana_streamer::nonblocking::quic::spawn_server(
"quic_streamer_test",
s.try_clone().unwrap(), s.try_clone().unwrap(),
&keypair, &keypair,
ip, ip,
@ -165,7 +159,6 @@ mod tests {
staked_nodes, staked_nodes,
10, 10,
10, 10,
stats,
Duration::from_secs(1), // wait_for_chunk_timeout Duration::from_secs(1), // wait_for_chunk_timeout
DEFAULT_TPU_COALESCE, DEFAULT_TPU_COALESCE,
) )
@ -210,9 +203,9 @@ mod tests {
// Request Receiver // Request Receiver
let (sender, receiver) = unbounded(); let (sender, receiver) = unbounded();
let staked_nodes = Arc::new(RwLock::new(StakedNodes::default())); let staked_nodes = Arc::new(RwLock::new(StakedNodes::default()));
let (request_recv_socket, request_recv_exit, keypair, request_recv_ip, request_recv_stats) = let (request_recv_socket, request_recv_exit, keypair, request_recv_ip) = server_args();
server_args();
let (request_recv_endpoint, request_recv_thread) = solana_streamer::quic::spawn_server( let (request_recv_endpoint, request_recv_thread) = solana_streamer::quic::spawn_server(
"quic_streamer_test",
request_recv_socket.try_clone().unwrap(), request_recv_socket.try_clone().unwrap(),
&keypair, &keypair,
request_recv_ip, request_recv_ip,
@ -222,7 +215,6 @@ mod tests {
staked_nodes.clone(), staked_nodes.clone(),
10, 10,
10, 10,
request_recv_stats,
DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT,
DEFAULT_TPU_COALESCE, DEFAULT_TPU_COALESCE,
) )
@ -230,19 +222,14 @@ mod tests {
drop(request_recv_endpoint); drop(request_recv_endpoint);
// Response Receiver: // Response Receiver:
let ( let (response_recv_socket, response_recv_exit, keypair2, response_recv_ip) = server_args();
response_recv_socket,
response_recv_exit,
keypair2,
response_recv_ip,
response_recv_stats,
) = server_args();
let (sender2, receiver2) = unbounded(); let (sender2, receiver2) = unbounded();
let addr = response_recv_socket.local_addr().unwrap().ip(); let addr = response_recv_socket.local_addr().unwrap().ip();
let port = response_recv_socket.local_addr().unwrap().port(); let port = response_recv_socket.local_addr().unwrap().port();
let server_addr = SocketAddr::new(addr, port); let server_addr = SocketAddr::new(addr, port);
let (response_recv_endpoint, response_recv_thread) = solana_streamer::quic::spawn_server( let (response_recv_endpoint, response_recv_thread) = solana_streamer::quic::spawn_server(
"quic_streamer_test",
response_recv_socket, response_recv_socket,
&keypair2, &keypair2,
response_recv_ip, response_recv_ip,
@ -252,7 +239,6 @@ mod tests {
staked_nodes, staked_nodes,
10, 10,
10, 10,
response_recv_stats,
DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT,
DEFAULT_TPU_COALESCE, DEFAULT_TPU_COALESCE,
) )

32
streamer/src/nonblocking/quic.rs
View File

@ -85,6 +85,7 @@ struct PacketAccumulator {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn spawn_server( pub fn spawn_server(
name: &'static str,
sock: UdpSocket, sock: UdpSocket,
keypair: &Keypair, keypair: &Keypair,
gossip_host: IpAddr, gossip_host: IpAddr,
@ -94,16 +95,17 @@ pub fn spawn_server(
staked_nodes: Arc<RwLock<StakedNodes>>, staked_nodes: Arc<RwLock<StakedNodes>>,
max_staked_connections: usize, max_staked_connections: usize,
max_unstaked_connections: usize, max_unstaked_connections: usize,
stats: Arc<StreamStats>,
wait_for_chunk_timeout: Duration, wait_for_chunk_timeout: Duration,
coalesce: Duration, coalesce: Duration,
) -> Result<(Endpoint, JoinHandle<()>), QuicServerError> { ) -> Result<(Endpoint, Arc<StreamStats>, JoinHandle<()>), QuicServerError> {
info!("Start quic server on {:?}", sock); info!("Start {name} quic server on {sock:?}");
let (config, _cert) = configure_server(keypair, gossip_host)?; let (config, _cert) = configure_server(keypair, gossip_host)?;
let endpoint = Endpoint::new(EndpointConfig::default(), Some(config), sock, TokioRuntime) let endpoint = Endpoint::new(EndpointConfig::default(), Some(config), sock, TokioRuntime)
.map_err(QuicServerError::EndpointFailed)?; .map_err(QuicServerError::EndpointFailed)?;
let stats = Arc::<StreamStats>::default();
let handle = tokio::spawn(run_server( let handle = tokio::spawn(run_server(
name,
endpoint.clone(), endpoint.clone(),
packet_sender, packet_sender,
exit, exit,
@ -111,15 +113,16 @@ pub fn spawn_server(
staked_nodes, staked_nodes,
max_staked_connections, max_staked_connections,
max_unstaked_connections, max_unstaked_connections,
stats, stats.clone(),
wait_for_chunk_timeout, wait_for_chunk_timeout,
coalesce, coalesce,
)); ));
Ok((endpoint, handle)) Ok((endpoint, stats, handle))
} }
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub async fn run_server( async fn run_server(
name: &'static str,
incoming: Endpoint, incoming: Endpoint,
packet_sender: Sender<PacketBatch>, packet_sender: Sender<PacketBatch>,
exit: Arc<AtomicBool>, exit: Arc<AtomicBool>,
@ -152,7 +155,7 @@ pub async fn run_server(
let timeout_connection = timeout(WAIT_FOR_CONNECTION_TIMEOUT, incoming.accept()).await; let timeout_connection = timeout(WAIT_FOR_CONNECTION_TIMEOUT, incoming.accept()).await;
if last_datapoint.elapsed().as_secs() >= 5 { if last_datapoint.elapsed().as_secs() >= 5 {
stats.report(); stats.report(name);
last_datapoint = Instant::now(); last_datapoint = Instant::now();
} }
@ -1166,8 +1169,8 @@ pub mod test {
let ip = "127.0.0.1".parse().unwrap(); let ip = "127.0.0.1".parse().unwrap();
let server_address = s.local_addr().unwrap(); let server_address = s.local_addr().unwrap();
let staked_nodes = Arc::new(RwLock::new(option_staked_nodes.unwrap_or_default())); let staked_nodes = Arc::new(RwLock::new(option_staked_nodes.unwrap_or_default()));
let stats = Arc::new(StreamStats::default()); let (_, stats, t) = spawn_server(
let (_, t) = spawn_server( "quic_streamer_test",
s, s,
&keypair, &keypair,
ip, ip,
@ -1177,7 +1180,6 @@ pub mod test {
staked_nodes, staked_nodes,
MAX_STAKED_CONNECTIONS, MAX_STAKED_CONNECTIONS,
MAX_UNSTAKED_CONNECTIONS, MAX_UNSTAKED_CONNECTIONS,
stats.clone(),
Duration::from_secs(2), Duration::from_secs(2),
DEFAULT_TPU_COALESCE, DEFAULT_TPU_COALESCE,
) )
@ -1597,8 +1599,8 @@ pub mod test {
let ip = "127.0.0.1".parse().unwrap(); let ip = "127.0.0.1".parse().unwrap();
let server_address = s.local_addr().unwrap(); let server_address = s.local_addr().unwrap();
let staked_nodes = Arc::new(RwLock::new(StakedNodes::default())); let staked_nodes = Arc::new(RwLock::new(StakedNodes::default()));
let stats = Arc::new(StreamStats::default()); let (_, _, t) = spawn_server(
let (_, t) = spawn_server( "quic_streamer_test",
s, s,
&keypair, &keypair,
ip, ip,
@ -1608,7 +1610,6 @@ pub mod test {
staked_nodes, staked_nodes,
MAX_STAKED_CONNECTIONS, MAX_STAKED_CONNECTIONS,
0, // Do not allow any connection from unstaked clients/nodes 0, // Do not allow any connection from unstaked clients/nodes
stats,
DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT,
DEFAULT_TPU_COALESCE, DEFAULT_TPU_COALESCE,
) )
@ -1629,8 +1630,8 @@ pub mod test {
let ip = "127.0.0.1".parse().unwrap(); let ip = "127.0.0.1".parse().unwrap();
let server_address = s.local_addr().unwrap(); let server_address = s.local_addr().unwrap();
let staked_nodes = Arc::new(RwLock::new(StakedNodes::default())); let staked_nodes = Arc::new(RwLock::new(StakedNodes::default()));
let stats = Arc::new(StreamStats::default()); let (_, stats, t) = spawn_server(
let (_, t) = spawn_server( "quic_streamer_test",
s, s,
&keypair, &keypair,
ip, ip,
@ -1640,7 +1641,6 @@ pub mod test {
staked_nodes, staked_nodes,
MAX_STAKED_CONNECTIONS, MAX_STAKED_CONNECTIONS,
MAX_UNSTAKED_CONNECTIONS, MAX_UNSTAKED_CONNECTIONS,
stats.clone(),
DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT,
DEFAULT_TPU_COALESCE, DEFAULT_TPU_COALESCE,
) )

19
streamer/src/quic.rs
View File

@ -158,9 +158,9 @@ pub struct StreamStats {
} }
impl StreamStats { impl StreamStats {
pub fn report(&self) { pub fn report(&self, name: &'static str) {
datapoint_info!( datapoint_info!(
"quic-connections", name,
( (
"active_connections", "active_connections",
self.total_connections.load(Ordering::Relaxed), self.total_connections.load(Ordering::Relaxed),
@ -391,6 +391,7 @@ impl StreamStats {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn spawn_server( pub fn spawn_server(
name: &'static str,
sock: UdpSocket, sock: UdpSocket,
keypair: &Keypair, keypair: &Keypair,
gossip_host: IpAddr, gossip_host: IpAddr,
@ -400,14 +401,14 @@ pub fn spawn_server(
staked_nodes: Arc<RwLock<StakedNodes>>, staked_nodes: Arc<RwLock<StakedNodes>>,
max_staked_connections: usize, max_staked_connections: usize,
max_unstaked_connections: usize, max_unstaked_connections: usize,
stats: Arc<StreamStats>,
wait_for_chunk_timeout: Duration, wait_for_chunk_timeout: Duration,
coalesce: Duration, coalesce: Duration,
) -> Result<(Endpoint, thread::JoinHandle<()>), QuicServerError> { ) -> Result<(Endpoint, thread::JoinHandle<()>), QuicServerError> {
let runtime = rt(); let runtime = rt();
let (endpoint, task) = { let (endpoint, _stats, task) = {
let _guard = runtime.enter(); let _guard = runtime.enter();
crate::nonblocking::quic::spawn_server( crate::nonblocking::quic::spawn_server(
name,
sock, sock,
keypair, keypair,
gossip_host, gossip_host,
@ -417,7 +418,6 @@ pub fn spawn_server(
staked_nodes, staked_nodes,
max_staked_connections, max_staked_connections,
max_unstaked_connections, max_unstaked_connections,
stats,
wait_for_chunk_timeout, wait_for_chunk_timeout,
coalesce, coalesce,
) )
@ -456,8 +456,8 @@ mod test {
let ip = "127.0.0.1".parse().unwrap(); let ip = "127.0.0.1".parse().unwrap();
let server_address = s.local_addr().unwrap(); let server_address = s.local_addr().unwrap();
let staked_nodes = Arc::new(RwLock::new(StakedNodes::default())); let staked_nodes = Arc::new(RwLock::new(StakedNodes::default()));
let stats = Arc::new(StreamStats::default());
let (_, t) = spawn_server( let (_, t) = spawn_server(
"quic_streamer_test",
s, s,
&keypair, &keypair,
ip, ip,
@ -467,7 +467,6 @@ mod test {
staked_nodes, staked_nodes,
MAX_STAKED_CONNECTIONS, MAX_STAKED_CONNECTIONS,
MAX_UNSTAKED_CONNECTIONS, MAX_UNSTAKED_CONNECTIONS,
stats,
DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT,
DEFAULT_TPU_COALESCE, DEFAULT_TPU_COALESCE,
) )
@ -513,8 +512,8 @@ mod test {
let ip = "127.0.0.1".parse().unwrap(); let ip = "127.0.0.1".parse().unwrap();
let server_address = s.local_addr().unwrap(); let server_address = s.local_addr().unwrap();
let staked_nodes = Arc::new(RwLock::new(StakedNodes::default())); let staked_nodes = Arc::new(RwLock::new(StakedNodes::default()));
let stats = Arc::new(StreamStats::default());
let (_, t) = spawn_server( let (_, t) = spawn_server(
"quic_streamer_test",
s, s,
&keypair, &keypair,
ip, ip,
@ -524,7 +523,6 @@ mod test {
staked_nodes, staked_nodes,
MAX_STAKED_CONNECTIONS, MAX_STAKED_CONNECTIONS,
MAX_UNSTAKED_CONNECTIONS, MAX_UNSTAKED_CONNECTIONS,
stats,
DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT,
DEFAULT_TPU_COALESCE, DEFAULT_TPU_COALESCE,
) )
@ -557,8 +555,8 @@ mod test {
let ip = "127.0.0.1".parse().unwrap(); let ip = "127.0.0.1".parse().unwrap();
let server_address = s.local_addr().unwrap(); let server_address = s.local_addr().unwrap();
let staked_nodes = Arc::new(RwLock::new(StakedNodes::default())); let staked_nodes = Arc::new(RwLock::new(StakedNodes::default()));
let stats = Arc::new(StreamStats::default());
let (_, t) = spawn_server( let (_, t) = spawn_server(
"quic_streamer_test",
s, s,
&keypair, &keypair,
ip, ip,
@ -568,7 +566,6 @@ mod test {
staked_nodes, staked_nodes,
MAX_STAKED_CONNECTIONS, MAX_STAKED_CONNECTIONS,
0, // Do not allow any connection from unstaked clients/nodes 0, // Do not allow any connection from unstaked clients/nodes
stats,
DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT,
DEFAULT_TPU_COALESCE, DEFAULT_TPU_COALESCE,
) )