wip works
This commit is contained in:
parent
fd2cf5a0eb
commit
a2207da37a
|
@ -15,7 +15,6 @@ pub mod tls_config_provicer;
|
||||||
pub mod proxy;
|
pub mod proxy;
|
||||||
pub mod proxy_request_format;
|
pub mod proxy_request_format;
|
||||||
pub mod tpu_quic_client;
|
pub mod tpu_quic_client;
|
||||||
pub mod active_connection;
|
|
||||||
pub mod cli;
|
pub mod cli;
|
||||||
pub mod test_client;
|
pub mod test_client;
|
||||||
mod util;
|
mod util;
|
||||||
|
|
|
@ -124,6 +124,7 @@ async fn accept_client_connection(client_connection: Connection, tpu_quic_client
|
||||||
let exit_signal_copy = exit_signal.clone();
|
let exit_signal_copy = exit_signal.clone();
|
||||||
let validator_identity_copy = validator_identity.clone();
|
let validator_identity_copy = validator_identity.clone();
|
||||||
let tpu_quic_client_copy = tpu_quic_client.clone();
|
let tpu_quic_client_copy = tpu_quic_client.clone();
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
|
|
||||||
let raw_request = recv_stream.read_to_end(10_000_000).await // TODO extract to const
|
let raw_request = recv_stream.read_to_end(10_000_000).await // TODO extract to const
|
||||||
|
@ -137,15 +138,16 @@ async fn accept_client_connection(client_connection: Connection, tpu_quic_client
|
||||||
let tpu_address = proxy_request.get_tpu_socket_addr();
|
let tpu_address = proxy_request.get_tpu_socket_addr();
|
||||||
let txs = proxy_request.get_transactions();
|
let txs = proxy_request.get_transactions();
|
||||||
|
|
||||||
|
// TODO join get_or_create_connection future and read_to_end
|
||||||
let tpu_connection = tpu_quic_client_copy.get_or_create_connection(tpu_address).await;
|
let tpu_connection = tpu_quic_client_copy.get_or_create_connection(tpu_address).await;
|
||||||
|
|
||||||
info!("send transaction batch of size {} to address {}", txs.len(), tpu_address);
|
info!("send transaction batch of size {} to address {}", txs.len(), tpu_address);
|
||||||
tpu_quic_client_copy.send_txs_to_tpu(tpu_connection, &txs, exit_signal_copy).await;
|
tpu_quic_client_copy.send_txs_to_tpu(tpu_connection, &txs, exit_signal_copy).await;
|
||||||
|
|
||||||
|
|
||||||
// active_tpu_connection_copy.send_txs_to_tpu(exit_signal_copy, validator_identity_copy, tpu_identity, tpu_address, &txs).await;
|
// active_tpu_connection_copy.send_txs_to_tpu(exit_signal_copy, validator_identity_copy, tpu_identity, tpu_address, &txs).await;
|
||||||
|
|
||||||
}).instrument(debug_span!("send_txs_to_tpu")).await.unwrap();
|
})
|
||||||
|
.await.unwrap();
|
||||||
|
|
||||||
} // -- loop
|
} // -- loop
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,6 +13,7 @@ use std::{
|
||||||
use anyhow::bail;
|
use anyhow::bail;
|
||||||
use tokio::{sync::RwLock, time::timeout};
|
use tokio::{sync::RwLock, time::timeout};
|
||||||
use tokio::time::error::Elapsed;
|
use tokio::time::error::Elapsed;
|
||||||
|
use tracing::instrument;
|
||||||
|
|
||||||
const ALPN_TPU_PROTOCOL_ID: &[u8] = b"solana-tpu";
|
const ALPN_TPU_PROTOCOL_ID: &[u8] = b"solana-tpu";
|
||||||
|
|
||||||
|
@ -185,6 +186,7 @@ impl QuicConnectionUtils {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
#[tracing::instrument(skip_all, level = "debug")]
|
||||||
pub async fn send_transaction_batch(
|
pub async fn send_transaction_batch(
|
||||||
connection: Connection,
|
connection: Connection,
|
||||||
txs: Vec<Vec<u8>>,
|
txs: Vec<Vec<u8>>,
|
||||||
|
|
|
@ -1,7 +1,9 @@
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::io::Write;
|
||||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4};
|
use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4};
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::str::FromStr;
|
use std::str::FromStr;
|
||||||
use std::sync::Arc;
|
use std::sync::{Arc, Mutex};
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU64};
|
use std::sync::atomic::{AtomicBool, AtomicU64};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use anyhow::{anyhow, bail};
|
use anyhow::{anyhow, bail};
|
||||||
|
@ -31,6 +33,7 @@ pub const CONNECTION_RETRY_COUNT: usize = 10;
|
||||||
pub struct TpuQuicClient {
|
pub struct TpuQuicClient {
|
||||||
endpoint: Endpoint,
|
endpoint: Endpoint,
|
||||||
// naive single non-recoverable connection - TODO moke it smarter
|
// naive single non-recoverable connection - TODO moke it smarter
|
||||||
|
// TODO consider using DashMap again
|
||||||
connection_per_tpunode: Arc<DashMap<SocketAddr, Connection>>,
|
connection_per_tpunode: Arc<DashMap<SocketAddr, Connection>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -58,9 +61,19 @@ impl TpuQuicClient {
|
||||||
|
|
||||||
#[tracing::instrument(skip(self), level = "debug")]
|
#[tracing::instrument(skip(self), level = "debug")]
|
||||||
pub async fn get_or_create_connection(&self, tpu_address: SocketAddr) -> Connection {
|
pub async fn get_or_create_connection(&self, tpu_address: SocketAddr) -> Connection {
|
||||||
|
info!("looking up {}", tpu_address);
|
||||||
|
// TODO try 0rff
|
||||||
|
// QuicConnectionUtils::make_connection(
|
||||||
|
// self.endpoint.clone(), tpu_address, QUIC_CONNECTION_TIMEOUT)
|
||||||
|
// .await.unwrap()
|
||||||
|
|
||||||
|
|
||||||
|
{
|
||||||
if let Some(conn) = self.connection_per_tpunode.get(&tpu_address) {
|
if let Some(conn) = self.connection_per_tpunode.get(&tpu_address) {
|
||||||
|
debug!("reusing connection {:?}", conn);
|
||||||
return conn.clone();
|
return conn.clone();
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let connection =
|
let connection =
|
||||||
// TODO try 0rff
|
// TODO try 0rff
|
||||||
|
@ -68,8 +81,8 @@ impl TpuQuicClient {
|
||||||
self.endpoint.clone(), tpu_address, QUIC_CONNECTION_TIMEOUT)
|
self.endpoint.clone(), tpu_address, QUIC_CONNECTION_TIMEOUT)
|
||||||
.await.unwrap();
|
.await.unwrap();
|
||||||
|
|
||||||
|
let old_value = self.connection_per_tpunode.insert(tpu_address, connection.clone());
|
||||||
self.connection_per_tpunode.insert(tpu_address, connection.clone());
|
assert!(old_value.is_none(), "no prev value must be overridden");
|
||||||
|
|
||||||
debug!("Created new Quic connection to TPU node {}, total connections is now {}", tpu_address, self.connection_per_tpunode.len());
|
debug!("Created new Quic connection to TPU node {}, total connections is now {}", tpu_address, self.connection_per_tpunode.len());
|
||||||
return connection;
|
return connection;
|
||||||
|
|
|
@ -11,7 +11,7 @@ use async_trait::async_trait;
|
||||||
use futures::FutureExt;
|
use futures::FutureExt;
|
||||||
use itertools::Itertools;
|
use itertools::Itertools;
|
||||||
use log::{debug, error, info, warn};
|
use log::{debug, error, info, warn};
|
||||||
use quinn::{ClientConfig, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig};
|
use quinn::{ClientConfig, Connection, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig};
|
||||||
use solana_sdk::pubkey::Pubkey;
|
use solana_sdk::pubkey::Pubkey;
|
||||||
use solana_sdk::signer::Signer;
|
use solana_sdk::signer::Signer;
|
||||||
use solana_sdk::signature::Keypair;
|
use solana_sdk::signature::Keypair;
|
||||||
|
@ -65,8 +65,6 @@ impl QuicProxyConnectionManager {
|
||||||
|
|
||||||
|
|
||||||
{
|
{
|
||||||
let mut lock = self.current_tpu_nodes.write().await;
|
|
||||||
|
|
||||||
let list_of_nodes = connections_to_keep.iter().map(|(identity, tpu_address)| {
|
let list_of_nodes = connections_to_keep.iter().map(|(identity, tpu_address)| {
|
||||||
TpuNode {
|
TpuNode {
|
||||||
tpu_identity: identity.clone(),
|
tpu_identity: identity.clone(),
|
||||||
|
@ -74,6 +72,7 @@ impl QuicProxyConnectionManager {
|
||||||
}
|
}
|
||||||
}).collect_vec();
|
}).collect_vec();
|
||||||
|
|
||||||
|
let mut lock = self.current_tpu_nodes.write().await;
|
||||||
*lock = list_of_nodes;
|
*lock = list_of_nodes;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -82,6 +81,7 @@ impl QuicProxyConnectionManager {
|
||||||
// already started
|
// already started
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
self.simple_thread_started.store(true, Relaxed);
|
||||||
|
|
||||||
info!("Starting very simple proxy thread");
|
info!("Starting very simple proxy thread");
|
||||||
|
|
||||||
|
@ -141,6 +141,10 @@ impl QuicProxyConnectionManager {
|
||||||
proxy_addr: SocketAddr,
|
proxy_addr: SocketAddr,
|
||||||
endpoint: Endpoint,
|
endpoint: Endpoint,
|
||||||
) {
|
) {
|
||||||
|
|
||||||
|
let mut connection = endpoint.connect(proxy_addr, "localhost").unwrap()
|
||||||
|
.await.unwrap();
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
// TODO exit signal ???
|
// TODO exit signal ???
|
||||||
|
|
||||||
|
@ -150,7 +154,7 @@ impl QuicProxyConnectionManager {
|
||||||
// exit signal???
|
// exit signal???
|
||||||
|
|
||||||
|
|
||||||
let the_tx: Vec<u8> = match tx {
|
let first_tx: Vec<u8> = match tx {
|
||||||
Ok((sig, tx)) => {
|
Ok((sig, tx)) => {
|
||||||
// if Self::check_for_confirmation(&txs_sent_store, sig) {
|
// if Self::check_for_confirmation(&txs_sent_store, sig) {
|
||||||
// // transaction is already confirmed/ no need to send
|
// // transaction is already confirmed/ no need to send
|
||||||
|
@ -165,20 +169,31 @@ impl QuicProxyConnectionManager {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// TODO read all txs from channel (see "let mut txs = vec![first_tx];")
|
let number_of_transactions_per_unistream = 8; // TODO read from QuicConnectionParameters
|
||||||
let txs = vec![the_tx];
|
|
||||||
|
|
||||||
let tpu_fanout_nodes = current_tpu_nodes.read().await;
|
let mut txs = vec![first_tx];
|
||||||
|
// TODO comment in
|
||||||
|
// for _ in 1..number_of_transactions_per_unistream {
|
||||||
|
// if let Ok((signature, tx)) = transaction_receiver.try_recv() {
|
||||||
|
// // if Self::check_for_confirmation(&txs_sent_store, signature) {
|
||||||
|
// // continue;
|
||||||
|
// // }
|
||||||
|
// txs.push(tx);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
info!("Sending copy of transaction batch of {} to {} tpu nodes via quic proxy",
|
let tpu_fanout_nodes = current_tpu_nodes.read().await.clone();
|
||||||
|
|
||||||
|
info!("Sending copy of transaction batch of {} txs to {} tpu nodes via quic proxy",
|
||||||
txs.len(), tpu_fanout_nodes.len());
|
txs.len(), tpu_fanout_nodes.len());
|
||||||
|
|
||||||
for target_tpu_node in &*tpu_fanout_nodes {
|
for target_tpu_node in tpu_fanout_nodes {
|
||||||
Self::send_copy_of_txs_to_quicproxy(
|
Self::send_copy_of_txs_to_quicproxy(
|
||||||
&txs, endpoint.clone(),
|
&txs, endpoint.clone(),
|
||||||
proxy_addr,
|
proxy_addr,
|
||||||
target_tpu_node.tpu_address,
|
target_tpu_node.tpu_address,
|
||||||
target_tpu_node.tpu_identity).await.unwrap();
|
target_tpu_node.tpu_identity)
|
||||||
|
.await.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
},
|
},
|
||||||
|
@ -190,7 +205,11 @@ impl QuicProxyConnectionManager {
|
||||||
proxy_address: SocketAddr, tpu_target_address: SocketAddr,
|
proxy_address: SocketAddr, tpu_target_address: SocketAddr,
|
||||||
target_tpu_identity: Pubkey) -> anyhow::Result<()> {
|
target_tpu_identity: Pubkey) -> anyhow::Result<()> {
|
||||||
|
|
||||||
info!("sending vecvec: {}", raw_tx_batch.iter().map(|tx| tx.len()).into_iter().join(","));
|
info!("sending vecvec {} to quic proxy for TPU node {}",
|
||||||
|
raw_tx_batch.iter().map(|tx| tx.len()).into_iter().join(","), tpu_target_address);
|
||||||
|
|
||||||
|
// TODO add timeout
|
||||||
|
// let mut send_stream = timeout(Duration::from_millis(500), connection.open_uni()).await??;
|
||||||
|
|
||||||
let raw_tx_batch_copy = raw_tx_batch.clone();
|
let raw_tx_batch_copy = raw_tx_batch.clone();
|
||||||
|
|
||||||
|
@ -207,6 +226,7 @@ impl QuicProxyConnectionManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
let forwarding_request = TpuForwardingRequest::new(tpu_target_address, target_tpu_identity, txs);
|
let forwarding_request = TpuForwardingRequest::new(tpu_target_address, target_tpu_identity, txs);
|
||||||
|
debug!("forwarding_request: {}", forwarding_request);
|
||||||
|
|
||||||
let proxy_request_raw = bincode::serialize(&forwarding_request).expect("Expect to serialize transactions");
|
let proxy_request_raw = bincode::serialize(&forwarding_request).expect("Expect to serialize transactions");
|
||||||
|
|
||||||
|
@ -222,10 +242,10 @@ impl QuicProxyConnectionManager {
|
||||||
bail!("Failed to send data to quic proxy: {:?}", e);
|
bail!("Failed to send data to quic proxy: {:?}", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async fn send_proxy_request(endpoint: Endpoint, proxy_address: SocketAddr, proxy_request_raw: &Vec<u8>) -> anyhow::Result<()> {
|
async fn send_proxy_request(endpoint: Endpoint, proxy_address: SocketAddr, proxy_request_raw: &Vec<u8>) -> anyhow::Result<()> {
|
||||||
info!("sending {} bytes to proxy", proxy_request_raw.len());
|
info!("sending {} bytes to proxy", proxy_request_raw.len());
|
||||||
|
|
||||||
|
@ -240,6 +260,7 @@ impl QuicProxyConnectionManager {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -237,6 +237,7 @@ impl TpuConnectionManager {
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let number_of_clients = fanout * 2;
|
let number_of_clients = fanout * 2;
|
||||||
Self {
|
Self {
|
||||||
|
// TODO
|
||||||
endpoints: RotatingQueue::new(number_of_clients, || {
|
endpoints: RotatingQueue::new(number_of_clients, || {
|
||||||
QuicConnectionUtils::create_endpoint(certificate.clone(), key.clone())
|
QuicConnectionUtils::create_endpoint(certificate.clone(), key.clone())
|
||||||
})
|
})
|
||||||
|
|
Loading…
Reference in New Issue