diff --git a/Cargo.lock b/Cargo.lock index 8cee4506ff..1f71f7c43c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4637,6 +4637,7 @@ dependencies = [ "log", "rand 0.7.3", "rayon", + "solana-client", "solana-core", "solana-gossip", "solana-ledger", @@ -4689,7 +4690,6 @@ dependencies = [ "solana-runtime", "solana-sdk 1.15.0", "solana-send-transaction-service", - "solana-tpu-client", "tarpc", "tokio", "tokio-serde", @@ -4915,6 +4915,7 @@ dependencies = [ "solana-clap-utils", "solana-cli-config", "solana-cli-output", + "solana-client", "solana-config-program", "solana-faucet", "solana-logger 1.15.0", @@ -4983,15 +4984,33 @@ dependencies = [ name = "solana-client" version = "1.15.0" dependencies = [ + "async-trait", + "bincode", + "enum_dispatch", + "futures 0.3.24", + "futures-util", + "indexmap", + "indicatif", "log", + "rand 0.7.3", + "rand_chacha 0.2.2", + "rayon", + "solana-logger 1.15.0", "solana-measure", + "solana-metrics", + "solana-net-utils", "solana-pubsub-client", + "solana-quic-client", "solana-rpc-client", "solana-rpc-client-api", "solana-rpc-client-nonce-utils", "solana-sdk 1.15.0", + "solana-streamer", "solana-thin-client", "solana-tpu-client", + "solana-udp-client", + "thiserror", + "tokio", ] [[package]] @@ -5076,6 +5095,7 @@ dependencies = [ "serial_test", "solana-address-lookup-table-program", "solana-bloom", + "solana-client", "solana-entry", "solana-frozen-abi 1.15.0", "solana-frozen-abi-macro 1.15.0", @@ -5125,6 +5145,7 @@ dependencies = [ "serde", "serial_test", "solana-bench-tps", + "solana-client", "solana-core", "solana-faucet", "solana-gossip", @@ -5387,6 +5408,7 @@ dependencies = [ "serial_test", "solana-bloom", "solana-clap-utils", + "solana-client", "solana-entry", "solana-frozen-abi 1.15.0", "solana-frozen-abi-macro 1.15.0", @@ -5571,6 +5593,7 @@ dependencies = [ "rand 0.7.3", "rayon", "serial_test", + "solana-client", "solana-config-program", "solana-core", "solana-download-utils", @@ -5959,12 +5982,28 @@ dependencies = [ name = "solana-quic-client" version = "1.15.0" dependencies = [ + "async-mutex", + "async-trait", + "crossbeam-channel", + "futures 0.3.24", + "itertools", + "lazy_static", + "log", + "quinn", + "quinn-proto", + "quinn-udp", + "rustls 0.20.6", "solana-logger 1.15.0", + "solana-measure", "solana-metrics", + "solana-net-utils", + "solana-perf", + "solana-rpc-client-api", "solana-sdk 1.15.0", "solana-streamer", "solana-tpu-client", "thiserror", + "tokio", ] [[package]] @@ -6019,6 +6058,7 @@ dependencies = [ "soketto", "solana-account-decoder", "solana-address-lookup-table-program", + "solana-client", "solana-entry", "solana-faucet", "solana-gossip", @@ -6128,6 +6168,7 @@ dependencies = [ "serde", "serde_json", "solana-account-decoder", + "solana-client", "solana-logger 1.15.0", "solana-pubsub-client", "solana-rpc", @@ -6345,6 +6386,7 @@ version = "1.15.0" dependencies = [ "crossbeam-channel", "log", + "solana-client", "solana-logger 1.15.0", "solana-measure", "solana-metrics", @@ -6566,35 +6608,23 @@ dependencies = [ name = "solana-tpu-client" version = "1.15.0" dependencies = [ - "async-mutex", "async-trait", "bincode", - "crossbeam-channel", - "enum_dispatch", - "futures 0.3.24", "futures-util", "indexmap", "indicatif", - "itertools", - "lazy_static", "log", - "quinn", - "quinn-proto", - "quinn-udp", "rand 0.7.3", "rand_chacha 0.2.2", "rayon", - "rustls 0.20.6", "solana-logger 1.15.0", "solana-measure", "solana-metrics", "solana-net-utils", - "solana-perf", "solana-pubsub-client", "solana-rpc-client", "solana-rpc-client-api", "solana-sdk 1.15.0", - "solana-streamer", "thiserror", "tokio", ] @@ -6654,9 +6684,13 @@ dependencies = [ name = "solana-udp-client" version = "1.15.0" dependencies = [ + "async-trait", "solana-net-utils", + "solana-sdk 1.15.0", + "solana-streamer", "solana-tpu-client", "thiserror", + "tokio", ] [[package]] diff --git a/banking-bench/Cargo.toml b/banking-bench/Cargo.toml index 07cbcfee3f..965d89cc9f 100644 --- a/banking-bench/Cargo.toml +++ b/banking-bench/Cargo.toml @@ -14,6 +14,7 @@ crossbeam-channel = "0.5" log = "0.4.17" rand = "0.7.0" rayon = "1.5.3" +solana-client = { path = "../client", version = "=1.15.0" } solana-core = { path = "../core", version = "=1.15.0" } solana-gossip = { path = "../gossip", version = "=1.15.0" } solana-ledger = { path = "../ledger", version = "=1.15.0" } diff --git a/banking-bench/src/main.rs b/banking-bench/src/main.rs index 467493db5e..69110916bf 100644 --- a/banking-bench/src/main.rs +++ b/banking-bench/src/main.rs @@ -5,6 +5,7 @@ use { log::*, rand::{thread_rng, Rng}, rayon::prelude::*, + solana_client::connection_cache::ConnectionCache, solana_core::banking_stage::BankingStage, solana_gossip::cluster_info::{ClusterInfo, Node}, solana_ledger::{ @@ -28,7 +29,7 @@ use { transaction::Transaction, }, solana_streamer::socket::SocketAddrSpace, - solana_tpu_client::connection_cache::{ConnectionCache, DEFAULT_TPU_CONNECTION_POOL_SIZE}, + solana_tpu_client::tpu_connection_cache::DEFAULT_TPU_CONNECTION_POOL_SIZE, std::{ sync::{atomic::Ordering, Arc, RwLock}, thread::sleep, diff --git a/banks-server/Cargo.toml b/banks-server/Cargo.toml index af32367fc2..a9945b8692 100644 --- a/banks-server/Cargo.toml +++ b/banks-server/Cargo.toml @@ -18,7 +18,6 @@ solana-client = { path = "../client", version = "=1.15.0" } solana-runtime = { path = "../runtime", version = "=1.15.0" } solana-sdk = { path = "../sdk", version = "=1.15.0" } solana-send-transaction-service = { path = "../send-transaction-service", version = "=1.15.0" } -solana-tpu-client = { path = "../tpu-client", version = "=1.15.0", default-features = false } tarpc = { version = "0.29.0", features = ["full"] } tokio = { version = "1", features = ["full"] } tokio-serde = { version = "0.8", features = ["bincode"] } diff --git a/banks-server/src/banks_server.rs b/banks-server/src/banks_server.rs index ee0fc26417..3498e7d477 100644 --- a/banks-server/src/banks_server.rs +++ b/banks-server/src/banks_server.rs @@ -7,6 +7,7 @@ use { BanksTransactionResultWithSimulation, TransactionConfirmationStatus, TransactionMetadata, TransactionSimulationDetails, TransactionStatus, }, + solana_client::connection_cache::ConnectionCache, solana_runtime::{ bank::{Bank, TransactionExecutionResult, TransactionSimulationResult}, bank_forks::BankForks, @@ -28,7 +29,6 @@ use { send_transaction_service::{SendTransactionService, TransactionInfo}, tpu_info::NullTpuInfo, }, - solana_tpu_client::connection_cache::ConnectionCache, std::{ convert::TryFrom, io, diff --git a/banks-server/src/rpc_banks_service.rs b/banks-server/src/rpc_banks_service.rs index 85ab418885..8a2f481562 100644 --- a/banks-server/src/rpc_banks_service.rs +++ b/banks-server/src/rpc_banks_service.rs @@ -3,8 +3,8 @@ use { crate::banks_server::start_tcp_server, futures::{future::FutureExt, pin_mut, prelude::stream::StreamExt, select}, + solana_client::connection_cache::ConnectionCache, solana_runtime::{bank_forks::BankForks, commitment::BlockCommitmentCache}, - solana_tpu_client::connection_cache::ConnectionCache, std::{ net::SocketAddr, sync::{ diff --git a/bench-tps/src/bench_tps_client/thin_client.rs b/bench-tps/src/bench_tps_client/thin_client.rs index 7f8c287635..be4770849f 100644 --- a/bench-tps/src/bench_tps_client/thin_client.rs +++ b/bench-tps/src/bench_tps_client/thin_client.rs @@ -1,5 +1,6 @@ use { crate::bench_tps_client::{BenchTpsClient, BenchTpsError, Result}, + solana_client::thin_client::ThinClient, solana_sdk::{ account::Account, client::{AsyncClient, Client, SyncClient}, @@ -11,7 +12,6 @@ use { signature::Signature, transaction::Transaction, }, - solana_thin_client::thin_client::ThinClient, }; impl BenchTpsClient for ThinClient { diff --git a/bench-tps/src/bench_tps_client/tpu_client.rs b/bench-tps/src/bench_tps_client/tpu_client.rs index 4fa6455f4c..b4f06ac6b0 100644 --- a/bench-tps/src/bench_tps_client/tpu_client.rs +++ b/bench-tps/src/bench_tps_client/tpu_client.rs @@ -1,10 +1,10 @@ use { crate::bench_tps_client::{BenchTpsClient, BenchTpsError, Result}, + solana_client::tpu_client::TpuClient, solana_sdk::{ account::Account, commitment_config::CommitmentConfig, epoch_info::EpochInfo, hash::Hash, message::Message, pubkey::Pubkey, signature::Signature, transaction::Transaction, }, - solana_tpu_client::tpu_client::TpuClient, }; impl BenchTpsClient for TpuClient { diff --git a/bench-tps/src/cli.rs b/bench-tps/src/cli.rs index 8f30a6f521..ed4730fe12 100644 --- a/bench-tps/src/cli.rs +++ b/bench-tps/src/cli.rs @@ -8,7 +8,9 @@ use { pubkey::Pubkey, signature::{read_keypair_file, Keypair}, }, - solana_tpu_client::connection_cache::{DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_USE_QUIC}, + solana_tpu_client::tpu_connection_cache::{ + DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_USE_QUIC, + }, std::{net::SocketAddr, process::exit, time::Duration}, }; diff --git a/bench-tps/src/main.rs b/bench-tps/src/main.rs index e4f453a8f7..06488ed935 100644 --- a/bench-tps/src/main.rs +++ b/bench-tps/src/main.rs @@ -10,6 +10,11 @@ use { keypairs::get_keypairs, send_batch::{generate_durable_nonce_accounts, generate_keypairs}, }, + solana_client::{ + connection_cache::ConnectionCache, + thin_client::ThinClient, + tpu_client::{TpuClient, TpuClientConfig}, + }, solana_genesis::Base64Account, solana_gossip::gossip_service::{discover_cluster, get_client, get_multi_client}, solana_rpc_client::rpc_client::RpcClient, @@ -18,11 +23,6 @@ use { system_program, }, solana_streamer::socket::SocketAddrSpace, - solana_thin_client::thin_client::ThinClient, - solana_tpu_client::{ - connection_cache::ConnectionCache, - tpu_client::{TpuClient, TpuClientConfig}, - }, std::{ collections::HashMap, fs::File, io::prelude::*, net::SocketAddr, path::Path, process::exit, sync::Arc, diff --git a/bench-tps/tests/bench_tps.rs b/bench-tps/tests/bench_tps.rs index f66c6137a5..f8098fbe7f 100644 --- a/bench-tps/tests/bench_tps.rs +++ b/bench-tps/tests/bench_tps.rs @@ -8,6 +8,11 @@ use { send_batch::generate_durable_nonce_accounts, spl_convert::FromOtherSolana, }, + solana_client::{ + connection_cache::ConnectionCache, + thin_client::ThinClient, + tpu_client::{TpuClient, TpuClientConfig}, + }, solana_core::validator::ValidatorConfig, solana_faucet::faucet::run_local_faucet, solana_local_cluster::{ @@ -25,11 +30,6 @@ use { }, solana_streamer::socket::SocketAddrSpace, solana_test_validator::TestValidatorGenesis, - solana_thin_client::thin_client::ThinClient, - solana_tpu_client::{ - connection_cache::ConnectionCache, - tpu_client::{TpuClient, TpuClientConfig}, - }, std::{sync::Arc, time::Duration}, }; diff --git a/cli/Cargo.toml b/cli/Cargo.toml index c84d7b7ec4..aa9bd656c0 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -40,6 +40,7 @@ solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.15. solana-clap-utils = { path = "../clap-utils", version = "=1.15.0" } solana-cli-config = { path = "../cli-config", version = "=1.15.0" } solana-cli-output = { path = "../cli-output", version = "=1.15.0" } +solana-client = { path = "../client", version = "=1.15.0" } solana-config-program = { path = "../programs/config", version = "=1.15.0" } solana-faucet = { path = "../faucet", version = "=1.15.0" } solana-logger = { path = "../logger", version = "=1.15.0" } diff --git a/cli/src/cli.rs b/cli/src/cli.rs index d681308369..c63a5650ca 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -31,7 +31,7 @@ use { stake::{instruction::LockupArgs, state::Lockup}, transaction::{TransactionError, VersionedTransaction}, }, - solana_tpu_client::connection_cache::DEFAULT_TPU_ENABLE_UDP, + solana_tpu_client::tpu_connection_cache::DEFAULT_TPU_ENABLE_UDP, solana_vote_program::vote_state::VoteAuthorize, std::{collections::HashMap, error, io::stdout, str::FromStr, sync::Arc, time::Duration}, thiserror::Error, diff --git a/cli/src/program.rs b/cli/src/program.rs index 50f5e732b4..edd4683fe3 100644 --- a/cli/src/program.rs +++ b/cli/src/program.rs @@ -17,6 +17,10 @@ use { CliUpgradeableBuffer, CliUpgradeableBuffers, CliUpgradeableProgram, CliUpgradeableProgramClosed, CliUpgradeablePrograms, }, + solana_client::{ + connection_cache::ConnectionCache, + tpu_client::{TpuClient, TpuClientConfig}, + }, solana_program_runtime::invoke_context::InvokeContext, solana_rbpf::{ elf::Executable, @@ -48,10 +52,6 @@ use { transaction::{Transaction, TransactionError}, transaction_context::TransactionContext, }, - solana_tpu_client::{ - connection_cache::ConnectionCache, - tpu_client::{TpuClient, TpuClientConfig}, - }, std::{ fs::File, io::{Read, Write}, diff --git a/client/Cargo.toml b/client/Cargo.toml index c4234fc09b..f06f61b722 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -10,17 +10,35 @@ license = "Apache-2.0" edition = "2021" [dependencies] +async-trait = "0.1.57" +bincode = "1.3.3" +enum_dispatch = "0.3.8" +futures = "0.3" +futures-util = "0.3.21" +indexmap = "1.9.1" +indicatif = { version = "0.17.1" } log = "0.4.17" +rand = "0.7.0" +rayon = "1.5.3" solana-measure = { path = "../measure", version = "=1.15.0" } +solana-metrics = { path = "../metrics", version = "=1.15.0" } +solana-net-utils = { path = "../net-utils", version = "=1.15.0" } solana-pubsub-client = { path = "../pubsub-client", version = "=1.15.0" } +solana-quic-client = { path = "../quic-client", version = "=1.15.0" } solana-rpc-client = { path = "../rpc-client", version = "=1.15.0" } solana-rpc-client-api = { path = "../rpc-client-api", version = "=1.15.0" } solana-rpc-client-nonce-utils = { path = "../rpc-client-nonce-utils", version = "=1.15.0" } solana-sdk = { path = "../sdk", version = "=1.15.0" } +solana-streamer = { path = "../streamer", version = "=1.15.0" } solana-thin-client = { path = "../thin-client", version = "=1.15.0" } solana-tpu-client = { path = "../tpu-client", version = "=1.15.0" } +solana-udp-client = { path = "../udp-client", version = "=1.15.0" } +thiserror = "1.0" +tokio = { version = "1", features = ["full"] } [dev-dependencies] +rand_chacha = "0.2.2" +solana-logger = { path = "../logger", version = "=1.15.0" } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/tpu-client/src/connection_cache.rs b/client/src/connection_cache.rs similarity index 98% rename from tpu-client/src/connection_cache.rs rename to client/src/connection_cache.rs index 5c5b0b8a02..4ea6148582 100644 --- a/tpu-client/src/connection_cache.rs +++ b/client/src/connection_cache.rs @@ -1,19 +1,16 @@ -pub use crate::tpu_connection_cache::{ +pub use solana_tpu_client::tpu_connection_cache::{ DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_ENABLE_UDP, DEFAULT_TPU_USE_QUIC, }; use { crate::{ - connection_cache_stats::{ConnectionCacheStats, CONNECTION_STAT_SUBMISSION_INTERVAL}, - nonblocking::{ - quic_client::{QuicClient, QuicClientCertificate, QuicLazyInitializedEndpoint}, - tpu_connection::NonblockingConnection, - }, - tpu_connection::BlockingConnection, - tpu_connection_cache::MAX_CONNECTIONS, + nonblocking::tpu_connection::NonblockingConnection, tpu_connection::BlockingConnection, }, indexmap::map::{Entry, IndexMap}, rand::{thread_rng, Rng}, solana_measure::measure::Measure, + solana_quic_client::nonblocking::quic_client::{ + QuicClient, QuicClientCertificate, QuicLazyInitializedEndpoint, + }, solana_sdk::{ pubkey::Pubkey, quic::QUIC_PORT_OFFSET, signature::Keypair, timing::AtomicInterval, }, @@ -22,6 +19,10 @@ use { streamer::StakedNodes, tls_certificates::new_self_signed_tls_certificate_chain, }, + solana_tpu_client::{ + connection_cache_stats::{ConnectionCacheStats, CONNECTION_STAT_SUBMISSION_INTERVAL}, + tpu_connection_cache::MAX_CONNECTIONS, + }, std::{ error::Error, net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}, diff --git a/client/src/lib.rs b/client/src/lib.rs index 748cf76d05..23d4cab220 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -1,6 +1,16 @@ #![allow(clippy::integer_arithmetic)] +pub mod connection_cache; +pub mod nonblocking; +pub mod quic_client; +pub mod thin_client; +pub mod tpu_client; +pub mod tpu_connection; pub mod transaction_executor; +pub mod udp_client; + +#[macro_use] +extern crate solana_metrics; pub use solana_rpc_client::mock_sender_for_cli; @@ -12,50 +22,6 @@ pub mod client_error { reqwest, Error as ClientError, ErrorKind as ClientErrorKind, Result, }; } -pub mod connection_cache { - pub use solana_tpu_client::connection_cache::*; -} -pub mod nonblocking { - pub mod blockhash_query { - pub use solana_rpc_client_nonce_utils::nonblocking::blockhash_query::*; - } - /// Durable transaction nonce helpers. - pub mod nonce_utils { - pub use solana_rpc_client_nonce_utils::nonblocking::*; - } - pub mod pubsub_client { - pub use solana_pubsub_client::nonblocking::pubsub_client::*; - } - /// Simple nonblocking client that connects to a given UDP port with the QUIC protocol - /// and provides an interface for sending transactions which is restricted by the - /// server's flow control. - pub mod quic_client { - pub use solana_tpu_client::nonblocking::quic_client::*; - } - /// Communication with a Solana node over RPC asynchronously . - /// - /// Software that interacts with the Solana blockchain, whether querying its - /// state or submitting transactions, communicates with a Solana node over - /// [JSON-RPC], using the [`RpcClient`] type. - /// - /// [JSON-RPC]: https://www.jsonrpc.org/specification - /// [`RpcClient`]: crate::nonblocking::rpc_client::RpcClient - pub mod rpc_client { - pub use solana_rpc_client::nonblocking::rpc_client::*; - } - pub mod tpu_client { - pub use solana_tpu_client::nonblocking::tpu_client::*; - } - /// Trait defining async send functions, to be used for UDP or QUIC sending - pub mod tpu_connection { - pub use solana_tpu_client::nonblocking::tpu_connection::*; - } - /// Simple UDP client that communicates with the given UDP port with UDP and provides - /// an interface for sending transactions - pub mod udp_client { - pub use solana_tpu_client::nonblocking::udp_client::*; - } -} /// Durable transaction nonce helpers. pub mod nonce_utils { pub use solana_rpc_client_nonce_utils::*; @@ -63,11 +29,6 @@ pub mod nonce_utils { pub mod pubsub_client { pub use solana_pubsub_client::pubsub_client::*; } -/// Simple client that connects to a given UDP port with the QUIC protocol and provides -/// an interface for sending transactions which is restricted by the server's flow control. -pub mod quic_client { - pub use solana_tpu_client::quic_client::*; -} /// Communication with a Solana node over RPC. /// /// Software that interacts with the Solana blockchain, whether querying its @@ -102,21 +63,3 @@ pub mod rpc_response { pub mod rpc_sender { pub use solana_rpc_client::rpc_sender::*; } -/// The `thin_client` module is a client-side object that interfaces with -/// a server-side TPU. Client code should use this object instead of writing -/// messages to the network directly. The binary encoding of its messages are -/// unstable and may change in future releases. -pub mod thin_client { - pub use solana_thin_client::thin_client::*; -} -pub mod tpu_client { - pub use solana_tpu_client::tpu_client::*; -} -pub mod tpu_connection { - pub use solana_tpu_client::tpu_connection::*; -} -/// Simple TPU client that communicates with the given UDP port with UDP and provides -/// an interface for sending transactions -pub mod udp_client { - pub use solana_tpu_client::udp_client::*; -} diff --git a/client/src/nonblocking/mod.rs b/client/src/nonblocking/mod.rs new file mode 100644 index 0000000000..ab11ae5c67 --- /dev/null +++ b/client/src/nonblocking/mod.rs @@ -0,0 +1,26 @@ +pub mod quic_client; +pub mod tpu_client; +pub mod tpu_connection; +pub mod udp_client; + +pub mod blockhash_query { + pub use solana_rpc_client_nonce_utils::nonblocking::blockhash_query::*; +} +/// Durable transaction nonce helpers. +pub mod nonce_utils { + pub use solana_rpc_client_nonce_utils::nonblocking::*; +} +pub mod pubsub_client { + pub use solana_pubsub_client::nonblocking::pubsub_client::*; +} +/// Communication with a Solana node over RPC asynchronously . +/// +/// Software that interacts with the Solana blockchain, whether querying its +/// state or submitting transactions, communicates with a Solana node over +/// [JSON-RPC], using the [`RpcClient`] type. +/// +/// [JSON-RPC]: https://www.jsonrpc.org/specification +/// [`RpcClient`]: crate::nonblocking::rpc_client::RpcClient +pub mod rpc_client { + pub use solana_rpc_client::nonblocking::rpc_client::*; +} diff --git a/client/src/nonblocking/quic_client.rs b/client/src/nonblocking/quic_client.rs new file mode 100644 index 0000000000..2f615557b0 --- /dev/null +++ b/client/src/nonblocking/quic_client.rs @@ -0,0 +1,59 @@ +//! Simple nonblocking client that connects to a given UDP port with the QUIC protocol +//! and provides an interface for sending transactions which is restricted by the +//! server's flow control. +pub use solana_quic_client::nonblocking::quic_client::{ + QuicClient, QuicClientCertificate, QuicError, QuicLazyInitializedEndpoint, QuicTpuConnection, +}; +use { + crate::nonblocking::tpu_connection::TpuConnection, + async_trait::async_trait, + log::*, + solana_sdk::transport::Result as TransportResult, + solana_tpu_client::tpu_connection::ClientStats, + std::{net::SocketAddr, sync::Arc}, +}; + +#[async_trait] +impl TpuConnection for QuicTpuConnection { + fn tpu_addr(&self) -> &SocketAddr { + self.client.tpu_addr() + } + + async fn send_wire_transaction_batch(&self, buffers: &[T]) -> TransportResult<()> + where + T: AsRef<[u8]> + Send + Sync, + { + let stats = ClientStats::default(); + let len = buffers.len(); + let res = self + .client + .send_batch(buffers, &stats, self.connection_stats.clone()) + .await; + self.connection_stats + .add_client_stats(&stats, len, res.is_ok()); + res?; + Ok(()) + } + + async fn send_wire_transaction(&self, wire_transaction: T) -> TransportResult<()> + where + T: AsRef<[u8]> + Send + Sync, + { + let stats = Arc::new(ClientStats::default()); + let send_buffer = + self.client + .send_buffer(wire_transaction, &stats, self.connection_stats.clone()); + if let Err(e) = send_buffer.await { + warn!( + "Failed to send transaction async to {}, error: {:?} ", + self.tpu_addr(), + e + ); + datapoint_warn!("send-wire-async", ("failure", 1, i64),); + self.connection_stats.add_client_stats(&stats, 1, false); + } else { + self.connection_stats.add_client_stats(&stats, 1, true); + } + Ok(()) + } +} diff --git a/client/src/nonblocking/tpu_client.rs b/client/src/nonblocking/tpu_client.rs new file mode 100644 index 0000000000..78cf83fcc6 --- /dev/null +++ b/client/src/nonblocking/tpu_client.rs @@ -0,0 +1,343 @@ +pub use solana_tpu_client::nonblocking::tpu_client::{LeaderTpuService, TpuSenderError}; +use { + crate::{ + connection_cache::ConnectionCache, + nonblocking::tpu_connection::TpuConnection, + tpu_client::{TpuClientConfig, MAX_FANOUT_SLOTS}, + }, + bincode::serialize, + futures_util::future::join_all, + solana_rpc_client::{nonblocking::rpc_client::RpcClient, spinner}, + solana_rpc_client_api::request::MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS, + solana_sdk::{ + message::Message, + signers::Signers, + transaction::{Transaction, TransactionError}, + transport::{Result as TransportResult, TransportError}, + }, + solana_tpu_client::{ + nonblocking::tpu_client::temporary_pub::*, + tpu_client::temporary_pub::{SEND_TRANSACTION_INTERVAL, TRANSACTION_RESEND_INTERVAL}, + }, + std::{ + collections::HashMap, + net::SocketAddr, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + }, + tokio::time::{sleep, Duration, Instant}, +}; + +/// Client which sends transactions directly to the current leader's TPU port over UDP. +/// The client uses RPC to determine the current leader and fetch node contact info +pub struct TpuClient { + fanout_slots: u64, + leader_tpu_service: LeaderTpuService, + exit: Arc, + rpc_client: Arc, + connection_cache: Arc, +} + +async fn send_wire_transaction_to_addr( + connection_cache: &ConnectionCache, + addr: &SocketAddr, + wire_transaction: Vec, +) -> TransportResult<()> { + let conn = connection_cache.get_nonblocking_connection(addr); + conn.send_wire_transaction(wire_transaction.clone()).await +} + +async fn send_wire_transaction_batch_to_addr( + connection_cache: &ConnectionCache, + addr: &SocketAddr, + wire_transactions: &[Vec], +) -> TransportResult<()> { + let conn = connection_cache.get_nonblocking_connection(addr); + conn.send_wire_transaction_batch(wire_transactions).await +} + +impl TpuClient { + /// Serialize and send transaction to the current and upcoming leader TPUs according to fanout + /// size + pub async fn send_transaction(&self, transaction: &Transaction) -> bool { + let wire_transaction = serialize(transaction).expect("serialization should succeed"); + self.send_wire_transaction(wire_transaction).await + } + + /// Send a wire transaction to the current and upcoming leader TPUs according to fanout size + pub async fn send_wire_transaction(&self, wire_transaction: Vec) -> bool { + self.try_send_wire_transaction(wire_transaction) + .await + .is_ok() + } + + /// Serialize and send transaction to the current and upcoming leader TPUs according to fanout + /// size + /// Returns the last error if all sends fail + pub async fn try_send_transaction(&self, transaction: &Transaction) -> TransportResult<()> { + let wire_transaction = serialize(transaction).expect("serialization should succeed"); + self.try_send_wire_transaction(wire_transaction).await + } + + /// Send a wire transaction to the current and upcoming leader TPUs according to fanout size + /// Returns the last error if all sends fail + pub async fn try_send_wire_transaction( + &self, + wire_transaction: Vec, + ) -> TransportResult<()> { + let leaders = self + .leader_tpu_service + .leader_tpu_sockets(self.fanout_slots); + let futures = leaders + .iter() + .map(|addr| { + send_wire_transaction_to_addr( + &self.connection_cache, + addr, + wire_transaction.clone(), + ) + }) + .collect::>(); + let results: Vec> = join_all(futures).await; + + let mut last_error: Option = None; + let mut some_success = false; + for result in results { + if let Err(e) = result { + if last_error.is_none() { + last_error = Some(e); + } + } else { + some_success = true; + } + } + if !some_success { + Err(if let Some(err) = last_error { + err + } else { + std::io::Error::new(std::io::ErrorKind::Other, "No sends attempted").into() + }) + } else { + Ok(()) + } + } + + /// Send a batch of wire transactions to the current and upcoming leader TPUs according to + /// fanout size + /// Returns the last error if all sends fail + pub async fn try_send_wire_transaction_batch( + &self, + wire_transactions: Vec>, + ) -> TransportResult<()> { + let leaders = self + .leader_tpu_service + .leader_tpu_sockets(self.fanout_slots); + let futures = leaders + .iter() + .map(|addr| { + send_wire_transaction_batch_to_addr( + &self.connection_cache, + addr, + &wire_transactions, + ) + }) + .collect::>(); + let results: Vec> = join_all(futures).await; + + let mut last_error: Option = None; + let mut some_success = false; + for result in results { + if let Err(e) = result { + if last_error.is_none() { + last_error = Some(e); + } + } else { + some_success = true; + } + } + if !some_success { + Err(if let Some(err) = last_error { + err + } else { + std::io::Error::new(std::io::ErrorKind::Other, "No sends attempted").into() + }) + } else { + Ok(()) + } + } + + /// Create a new client that disconnects when dropped + pub async fn new( + rpc_client: Arc, + websocket_url: &str, + config: TpuClientConfig, + ) -> Result { + let connection_cache = Arc::new(ConnectionCache::default()); + Self::new_with_connection_cache(rpc_client, websocket_url, config, connection_cache).await + } + + /// Create a new client that disconnects when dropped + pub async fn new_with_connection_cache( + rpc_client: Arc, + websocket_url: &str, + config: TpuClientConfig, + connection_cache: Arc, + ) -> Result { + let exit = Arc::new(AtomicBool::new(false)); + let leader_tpu_service = + LeaderTpuService::new(rpc_client.clone(), websocket_url, exit.clone()).await?; + + Ok(Self { + fanout_slots: config.fanout_slots.clamp(1, MAX_FANOUT_SLOTS), + leader_tpu_service, + exit, + rpc_client, + connection_cache, + }) + } + + pub async fn send_and_confirm_messages_with_spinner( + &self, + messages: &[Message], + signers: &T, + ) -> Result>> { + let mut expired_blockhash_retries = 5; + let progress_bar = spinner::new_progress_bar(); + progress_bar.set_message("Setting up..."); + + let mut transactions = messages + .iter() + .enumerate() + .map(|(i, message)| (i, Transaction::new_unsigned(message.clone()))) + .collect::>(); + let total_transactions = transactions.len(); + let mut transaction_errors = vec![None; transactions.len()]; + let mut confirmed_transactions = 0; + let mut block_height = self.rpc_client.get_block_height().await?; + while expired_blockhash_retries > 0 { + let (blockhash, last_valid_block_height) = self + .rpc_client + .get_latest_blockhash_with_commitment(self.rpc_client.commitment()) + .await?; + + let mut pending_transactions = HashMap::new(); + for (i, mut transaction) in transactions { + transaction.try_sign(signers, blockhash)?; + pending_transactions.insert(transaction.signatures[0], (i, transaction)); + } + + let mut last_resend = Instant::now() - TRANSACTION_RESEND_INTERVAL; + while block_height <= last_valid_block_height { + let num_transactions = pending_transactions.len(); + + // Periodically re-send all pending transactions + if Instant::now().duration_since(last_resend) > TRANSACTION_RESEND_INTERVAL { + for (index, (_i, transaction)) in pending_transactions.values().enumerate() { + if !self.send_transaction(transaction).await { + let _result = self.rpc_client.send_transaction(transaction).await.ok(); + } + set_message_for_confirmed_transactions( + &progress_bar, + confirmed_transactions, + total_transactions, + None, //block_height, + last_valid_block_height, + &format!("Sending {}/{} transactions", index + 1, num_transactions,), + ); + sleep(SEND_TRANSACTION_INTERVAL).await; + } + last_resend = Instant::now(); + } + + // Wait for the next block before checking for transaction statuses + let mut block_height_refreshes = 10; + set_message_for_confirmed_transactions( + &progress_bar, + confirmed_transactions, + total_transactions, + Some(block_height), + last_valid_block_height, + &format!( + "Waiting for next block, {} transactions pending...", + num_transactions + ), + ); + let mut new_block_height = block_height; + while block_height == new_block_height && block_height_refreshes > 0 { + sleep(Duration::from_millis(500)).await; + new_block_height = self.rpc_client.get_block_height().await?; + block_height_refreshes -= 1; + } + block_height = new_block_height; + + // Collect statuses for the transactions, drop those that are confirmed + let pending_signatures = pending_transactions.keys().cloned().collect::>(); + for pending_signatures_chunk in + pending_signatures.chunks(MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS) + { + if let Ok(result) = self + .rpc_client + .get_signature_statuses(pending_signatures_chunk) + .await + { + let statuses = result.value; + for (signature, status) in + pending_signatures_chunk.iter().zip(statuses.into_iter()) + { + if let Some(status) = status { + if status.satisfies_commitment(self.rpc_client.commitment()) { + if let Some((i, _)) = pending_transactions.remove(signature) { + confirmed_transactions += 1; + if status.err.is_some() { + progress_bar.println(format!( + "Failed transaction: {:?}", + status + )); + } + transaction_errors[i] = status.err; + } + } + } + } + } + set_message_for_confirmed_transactions( + &progress_bar, + confirmed_transactions, + total_transactions, + Some(block_height), + last_valid_block_height, + "Checking transaction status...", + ); + } + + if pending_transactions.is_empty() { + return Ok(transaction_errors); + } + } + + transactions = pending_transactions.into_values().collect(); + progress_bar.println(format!( + "Blockhash expired. {} retries remaining", + expired_blockhash_retries + )); + expired_blockhash_retries -= 1; + } + Err(TpuSenderError::Custom("Max retries exceeded".into())) + } + + pub fn rpc_client(&self) -> &RpcClient { + &self.rpc_client + } + + pub async fn shutdown(&mut self) { + self.exit.store(true, Ordering::Relaxed); + self.leader_tpu_service.join().await; + } +} +impl Drop for TpuClient { + fn drop(&mut self) { + self.exit.store(true, Ordering::Relaxed); + } +} diff --git a/client/src/nonblocking/tpu_connection.rs b/client/src/nonblocking/tpu_connection.rs new file mode 100644 index 0000000000..af19da4f5a --- /dev/null +++ b/client/src/nonblocking/tpu_connection.rs @@ -0,0 +1,42 @@ +//! Trait defining async send functions, to be used for UDP or QUIC sending + +use { + async_trait::async_trait, + enum_dispatch::enum_dispatch, + solana_quic_client::nonblocking::quic_client::QuicTpuConnection, + solana_sdk::{transaction::VersionedTransaction, transport::Result as TransportResult}, + solana_udp_client::nonblocking::udp_client::UdpTpuConnection, + std::net::SocketAddr, +}; + +// Due to the existence of `crate::connection_cache::Connection`, if this is named +// `Connection`, enum_dispatch gets confused between the two and throws errors when +// trying to convert later. +#[enum_dispatch] +pub enum NonblockingConnection { + QuicTpuConnection, + UdpTpuConnection, +} + +#[async_trait] +#[enum_dispatch(NonblockingConnection)] +pub trait TpuConnection { + fn tpu_addr(&self) -> &SocketAddr; + + async fn serialize_and_send_transaction( + &self, + transaction: &VersionedTransaction, + ) -> TransportResult<()> { + let wire_transaction = + bincode::serialize(transaction).expect("serialize Transaction in send_batch"); + self.send_wire_transaction(&wire_transaction).await + } + + async fn send_wire_transaction(&self, wire_transaction: T) -> TransportResult<()> + where + T: AsRef<[u8]> + Send + Sync; + + async fn send_wire_transaction_batch(&self, buffers: &[T]) -> TransportResult<()> + where + T: AsRef<[u8]> + Send + Sync; +} diff --git a/client/src/nonblocking/udp_client.rs b/client/src/nonblocking/udp_client.rs new file mode 100644 index 0000000000..4e817e8aa7 --- /dev/null +++ b/client/src/nonblocking/udp_client.rs @@ -0,0 +1,35 @@ +//! Simple UDP client that communicates with the given UDP port with UDP and provides +//! an interface for sending transactions + +pub use solana_udp_client::nonblocking::udp_client::UdpTpuConnection; +use { + crate::nonblocking::tpu_connection::TpuConnection, async_trait::async_trait, + core::iter::repeat, solana_sdk::transport::Result as TransportResult, + solana_streamer::nonblocking::sendmmsg::batch_send, std::net::SocketAddr, +}; + +#[async_trait] +impl TpuConnection for UdpTpuConnection { + fn tpu_addr(&self) -> &SocketAddr { + &self.addr + } + + async fn send_wire_transaction(&self, wire_transaction: T) -> TransportResult<()> + where + T: AsRef<[u8]> + Send + Sync, + { + self.socket + .send_to(wire_transaction.as_ref(), self.addr) + .await?; + Ok(()) + } + + async fn send_wire_transaction_batch(&self, buffers: &[T]) -> TransportResult<()> + where + T: AsRef<[u8]> + Send + Sync, + { + let pkts: Vec<_> = buffers.iter().zip(repeat(self.tpu_addr())).collect(); + batch_send(&self.socket, &pkts).await?; + Ok(()) + } +} diff --git a/client/src/quic_client.rs b/client/src/quic_client.rs new file mode 100644 index 0000000000..3bdd341ae5 --- /dev/null +++ b/client/src/quic_client.rs @@ -0,0 +1,44 @@ +//! Simple client that connects to a given UDP port with the QUIC protocol and provides +//! an interface for sending transactions which is restricted by the server's flow control. + +pub use solana_quic_client::quic_client::QuicTpuConnection; +use { + crate::{ + nonblocking::tpu_connection::TpuConnection as NonblockingTpuConnection, + tpu_connection::TpuConnection, + }, + solana_quic_client::quic_client::temporary_pub::*, + solana_sdk::transport::Result as TransportResult, + std::net::SocketAddr, +}; + +impl TpuConnection for QuicTpuConnection { + fn tpu_addr(&self) -> &SocketAddr { + self.inner.tpu_addr() + } + + fn send_wire_transaction_batch(&self, buffers: &[T]) -> TransportResult<()> + where + T: AsRef<[u8]> + Send + Sync, + { + RUNTIME.block_on(self.inner.send_wire_transaction_batch(buffers))?; + Ok(()) + } + + fn send_wire_transaction_async(&self, wire_transaction: Vec) -> TransportResult<()> { + let _lock = ASYNC_TASK_SEMAPHORE.acquire(); + let inner = self.inner.clone(); + + let _ = RUNTIME + .spawn(async move { send_wire_transaction_async(inner, wire_transaction).await }); + Ok(()) + } + + fn send_wire_transaction_batch_async(&self, buffers: Vec>) -> TransportResult<()> { + let _lock = ASYNC_TASK_SEMAPHORE.acquire(); + let inner = self.inner.clone(); + let _ = + RUNTIME.spawn(async move { send_wire_transaction_batch_async(inner, buffers).await }); + Ok(()) + } +} diff --git a/client/src/thin_client.rs b/client/src/thin_client.rs new file mode 100644 index 0000000000..3e98ef4695 --- /dev/null +++ b/client/src/thin_client.rs @@ -0,0 +1,546 @@ +//! The `thin_client` module is a client-side object that interfaces with +//! a server-side TPU. Client code should use this object instead of writing +//! messages to the network directly. The binary encoding of its messages are +//! unstable and may change in future releases. + +use { + crate::{connection_cache::ConnectionCache, tpu_connection::TpuConnection}, + log::*, + solana_rpc_client::rpc_client::RpcClient, + solana_rpc_client_api::{config::RpcProgramAccountsConfig, response::Response}, + solana_sdk::{ + account::Account, + client::{AsyncClient, Client, SyncClient}, + clock::{Slot, MAX_PROCESSING_AGE}, + commitment_config::CommitmentConfig, + epoch_info::EpochInfo, + fee_calculator::{FeeCalculator, FeeRateGovernor}, + hash::Hash, + instruction::Instruction, + message::Message, + pubkey::Pubkey, + signature::{Keypair, Signature, Signer}, + signers::Signers, + system_instruction, + timing::duration_as_ms, + transaction::{self, Transaction, VersionedTransaction}, + transport::Result as TransportResult, + }, + solana_thin_client::thin_client::temporary_pub::*, + std::{ + io, + net::SocketAddr, + sync::Arc, + time::{Duration, Instant}, + }, +}; + +/// An object for querying and sending transactions to the network. +pub struct ThinClient { + rpc_clients: Vec, + tpu_addrs: Vec, + optimizer: ClientOptimizer, + connection_cache: Arc, +} + +impl ThinClient { + /// Create a new ThinClient that will interface with the Rpc at `rpc_addr` using TCP + /// and the Tpu at `tpu_addr` over `transactions_socket` using Quic or UDP + /// (currently hardcoded to UDP) + pub fn new( + rpc_addr: SocketAddr, + tpu_addr: SocketAddr, + connection_cache: Arc, + ) -> Self { + Self::new_from_client(RpcClient::new_socket(rpc_addr), tpu_addr, connection_cache) + } + + pub fn new_socket_with_timeout( + rpc_addr: SocketAddr, + tpu_addr: SocketAddr, + timeout: Duration, + connection_cache: Arc, + ) -> Self { + let rpc_client = RpcClient::new_socket_with_timeout(rpc_addr, timeout); + Self::new_from_client(rpc_client, tpu_addr, connection_cache) + } + + fn new_from_client( + rpc_client: RpcClient, + tpu_addr: SocketAddr, + connection_cache: Arc, + ) -> Self { + Self { + rpc_clients: vec![rpc_client], + tpu_addrs: vec![tpu_addr], + optimizer: ClientOptimizer::new(0), + connection_cache, + } + } + + pub fn new_from_addrs( + rpc_addrs: Vec, + tpu_addrs: Vec, + connection_cache: Arc, + ) -> Self { + assert!(!rpc_addrs.is_empty()); + assert_eq!(rpc_addrs.len(), tpu_addrs.len()); + + let rpc_clients: Vec<_> = rpc_addrs.into_iter().map(RpcClient::new_socket).collect(); + let optimizer = ClientOptimizer::new(rpc_clients.len()); + Self { + rpc_clients, + tpu_addrs, + optimizer, + connection_cache, + } + } + + fn tpu_addr(&self) -> &SocketAddr { + &self.tpu_addrs[self.optimizer.best()] + } + + pub fn rpc_client(&self) -> &RpcClient { + &self.rpc_clients[self.optimizer.best()] + } + + /// Retry a sending a signed Transaction to the server for processing. + pub fn retry_transfer_until_confirmed( + &self, + keypair: &Keypair, + transaction: &mut Transaction, + tries: usize, + min_confirmed_blocks: usize, + ) -> TransportResult { + self.send_and_confirm_transaction(&[keypair], transaction, tries, min_confirmed_blocks) + } + + /// Retry sending a signed Transaction with one signing Keypair to the server for processing. + pub fn retry_transfer( + &self, + keypair: &Keypair, + transaction: &mut Transaction, + tries: usize, + ) -> TransportResult { + self.send_and_confirm_transaction(&[keypair], transaction, tries, 0) + } + + pub fn send_and_confirm_transaction( + &self, + keypairs: &T, + transaction: &mut Transaction, + tries: usize, + pending_confirmations: usize, + ) -> TransportResult { + for x in 0..tries { + let now = Instant::now(); + let mut num_confirmed = 0; + let mut wait_time = MAX_PROCESSING_AGE; + // resend the same transaction until the transaction has no chance of succeeding + let wire_transaction = + bincode::serialize(&transaction).expect("transaction serialization failed"); + while now.elapsed().as_secs() < wait_time as u64 { + if num_confirmed == 0 { + let conn = self.connection_cache.get_connection(self.tpu_addr()); + // Send the transaction if there has been no confirmation (e.g. the first time) + #[allow(clippy::needless_borrow)] + conn.send_wire_transaction(&wire_transaction)?; + } + + if let Ok(confirmed_blocks) = self.poll_for_signature_confirmation( + &transaction.signatures[0], + pending_confirmations, + ) { + num_confirmed = confirmed_blocks; + if confirmed_blocks >= pending_confirmations { + return Ok(transaction.signatures[0]); + } + // Since network has seen the transaction, wait longer to receive + // all pending confirmations. Resending the transaction could result into + // extra transaction fees + wait_time = wait_time.max( + MAX_PROCESSING_AGE * pending_confirmations.saturating_sub(num_confirmed), + ); + } + } + info!("{} tries failed transfer to {}", x, self.tpu_addr()); + let blockhash = self.get_latest_blockhash()?; + transaction.sign(keypairs, blockhash); + } + Err(io::Error::new( + io::ErrorKind::Other, + format!("retry_transfer failed in {} retries", tries), + ) + .into()) + } + + pub fn poll_get_balance(&self, pubkey: &Pubkey) -> TransportResult { + self.poll_get_balance_with_commitment(pubkey, CommitmentConfig::default()) + } + + pub fn poll_get_balance_with_commitment( + &self, + pubkey: &Pubkey, + commitment_config: CommitmentConfig, + ) -> TransportResult { + self.rpc_client() + .poll_get_balance_with_commitment(pubkey, commitment_config) + .map_err(|e| e.into()) + } + + pub fn wait_for_balance(&self, pubkey: &Pubkey, expected_balance: Option) -> Option { + self.rpc_client().wait_for_balance_with_commitment( + pubkey, + expected_balance, + CommitmentConfig::default(), + ) + } + + pub fn get_program_accounts_with_config( + &self, + pubkey: &Pubkey, + config: RpcProgramAccountsConfig, + ) -> TransportResult> { + self.rpc_client() + .get_program_accounts_with_config(pubkey, config) + .map_err(|e| e.into()) + } + + pub fn wait_for_balance_with_commitment( + &self, + pubkey: &Pubkey, + expected_balance: Option, + commitment_config: CommitmentConfig, + ) -> Option { + self.rpc_client().wait_for_balance_with_commitment( + pubkey, + expected_balance, + commitment_config, + ) + } + + pub fn poll_for_signature_with_commitment( + &self, + signature: &Signature, + commitment_config: CommitmentConfig, + ) -> TransportResult<()> { + self.rpc_client() + .poll_for_signature_with_commitment(signature, commitment_config) + .map_err(|e| e.into()) + } + + pub fn get_num_blocks_since_signature_confirmation( + &mut self, + sig: &Signature, + ) -> TransportResult { + self.rpc_client() + .get_num_blocks_since_signature_confirmation(sig) + .map_err(|e| e.into()) + } +} + +impl Client for ThinClient { + fn tpu_addr(&self) -> String { + self.tpu_addr().to_string() + } +} + +impl SyncClient for ThinClient { + fn send_and_confirm_message( + &self, + keypairs: &T, + message: Message, + ) -> TransportResult { + let blockhash = self.get_latest_blockhash()?; + let mut transaction = Transaction::new(keypairs, message, blockhash); + let signature = self.send_and_confirm_transaction(keypairs, &mut transaction, 5, 0)?; + Ok(signature) + } + + fn send_and_confirm_instruction( + &self, + keypair: &Keypair, + instruction: Instruction, + ) -> TransportResult { + let message = Message::new(&[instruction], Some(&keypair.pubkey())); + self.send_and_confirm_message(&[keypair], message) + } + + fn transfer_and_confirm( + &self, + lamports: u64, + keypair: &Keypair, + pubkey: &Pubkey, + ) -> TransportResult { + let transfer_instruction = + system_instruction::transfer(&keypair.pubkey(), pubkey, lamports); + self.send_and_confirm_instruction(keypair, transfer_instruction) + } + + fn get_account_data(&self, pubkey: &Pubkey) -> TransportResult>> { + Ok(self.rpc_client().get_account_data(pubkey).ok()) + } + + fn get_account(&self, pubkey: &Pubkey) -> TransportResult> { + let account = self.rpc_client().get_account(pubkey); + match account { + Ok(value) => Ok(Some(value)), + Err(_) => Ok(None), + } + } + + fn get_account_with_commitment( + &self, + pubkey: &Pubkey, + commitment_config: CommitmentConfig, + ) -> TransportResult> { + self.rpc_client() + .get_account_with_commitment(pubkey, commitment_config) + .map_err(|e| e.into()) + .map(|r| r.value) + } + + fn get_balance(&self, pubkey: &Pubkey) -> TransportResult { + self.rpc_client().get_balance(pubkey).map_err(|e| e.into()) + } + + fn get_balance_with_commitment( + &self, + pubkey: &Pubkey, + commitment_config: CommitmentConfig, + ) -> TransportResult { + self.rpc_client() + .get_balance_with_commitment(pubkey, commitment_config) + .map_err(|e| e.into()) + .map(|r| r.value) + } + + fn get_minimum_balance_for_rent_exemption(&self, data_len: usize) -> TransportResult { + self.rpc_client() + .get_minimum_balance_for_rent_exemption(data_len) + .map_err(|e| e.into()) + } + + fn get_recent_blockhash(&self) -> TransportResult<(Hash, FeeCalculator)> { + #[allow(deprecated)] + let (blockhash, fee_calculator, _last_valid_slot) = + self.get_recent_blockhash_with_commitment(CommitmentConfig::default())?; + Ok((blockhash, fee_calculator)) + } + + fn get_recent_blockhash_with_commitment( + &self, + commitment_config: CommitmentConfig, + ) -> TransportResult<(Hash, FeeCalculator, Slot)> { + let index = self.optimizer.experiment(); + let now = Instant::now(); + #[allow(deprecated)] + let recent_blockhash = + self.rpc_clients[index].get_recent_blockhash_with_commitment(commitment_config); + match recent_blockhash { + Ok(Response { value, .. }) => { + self.optimizer.report(index, duration_as_ms(&now.elapsed())); + Ok((value.0, value.1, value.2)) + } + Err(e) => { + self.optimizer.report(index, std::u64::MAX); + Err(e.into()) + } + } + } + + fn get_fee_calculator_for_blockhash( + &self, + blockhash: &Hash, + ) -> TransportResult> { + #[allow(deprecated)] + self.rpc_client() + .get_fee_calculator_for_blockhash(blockhash) + .map_err(|e| e.into()) + } + + fn get_fee_rate_governor(&self) -> TransportResult { + #[allow(deprecated)] + self.rpc_client() + .get_fee_rate_governor() + .map_err(|e| e.into()) + .map(|r| r.value) + } + + fn get_signature_status( + &self, + signature: &Signature, + ) -> TransportResult>> { + let status = self + .rpc_client() + .get_signature_status(signature) + .map_err(|err| { + io::Error::new( + io::ErrorKind::Other, + format!("send_transaction failed with error {:?}", err), + ) + })?; + Ok(status) + } + + fn get_signature_status_with_commitment( + &self, + signature: &Signature, + commitment_config: CommitmentConfig, + ) -> TransportResult>> { + let status = self + .rpc_client() + .get_signature_status_with_commitment(signature, commitment_config) + .map_err(|err| { + io::Error::new( + io::ErrorKind::Other, + format!("send_transaction failed with error {:?}", err), + ) + })?; + Ok(status) + } + + fn get_slot(&self) -> TransportResult { + self.get_slot_with_commitment(CommitmentConfig::default()) + } + + fn get_slot_with_commitment( + &self, + commitment_config: CommitmentConfig, + ) -> TransportResult { + let slot = self + .rpc_client() + .get_slot_with_commitment(commitment_config) + .map_err(|err| { + io::Error::new( + io::ErrorKind::Other, + format!("send_transaction failed with error {:?}", err), + ) + })?; + Ok(slot) + } + + fn get_epoch_info(&self) -> TransportResult { + self.rpc_client().get_epoch_info().map_err(|e| e.into()) + } + + fn get_transaction_count(&self) -> TransportResult { + let index = self.optimizer.experiment(); + let now = Instant::now(); + match self.rpc_client().get_transaction_count() { + Ok(transaction_count) => { + self.optimizer.report(index, duration_as_ms(&now.elapsed())); + Ok(transaction_count) + } + Err(e) => { + self.optimizer.report(index, std::u64::MAX); + Err(e.into()) + } + } + } + + fn get_transaction_count_with_commitment( + &self, + commitment_config: CommitmentConfig, + ) -> TransportResult { + let index = self.optimizer.experiment(); + let now = Instant::now(); + match self + .rpc_client() + .get_transaction_count_with_commitment(commitment_config) + { + Ok(transaction_count) => { + self.optimizer.report(index, duration_as_ms(&now.elapsed())); + Ok(transaction_count) + } + Err(e) => { + self.optimizer.report(index, std::u64::MAX); + Err(e.into()) + } + } + } + + /// Poll the server until the signature has been confirmed by at least `min_confirmed_blocks` + fn poll_for_signature_confirmation( + &self, + signature: &Signature, + min_confirmed_blocks: usize, + ) -> TransportResult { + self.rpc_client() + .poll_for_signature_confirmation(signature, min_confirmed_blocks) + .map_err(|e| e.into()) + } + + fn poll_for_signature(&self, signature: &Signature) -> TransportResult<()> { + self.rpc_client() + .poll_for_signature(signature) + .map_err(|e| e.into()) + } + + fn get_new_blockhash(&self, blockhash: &Hash) -> TransportResult<(Hash, FeeCalculator)> { + #[allow(deprecated)] + self.rpc_client() + .get_new_blockhash(blockhash) + .map_err(|e| e.into()) + } + + fn get_latest_blockhash(&self) -> TransportResult { + let (blockhash, _) = + self.get_latest_blockhash_with_commitment(CommitmentConfig::default())?; + Ok(blockhash) + } + + fn get_latest_blockhash_with_commitment( + &self, + commitment_config: CommitmentConfig, + ) -> TransportResult<(Hash, u64)> { + let index = self.optimizer.experiment(); + let now = Instant::now(); + match self.rpc_clients[index].get_latest_blockhash_with_commitment(commitment_config) { + Ok((blockhash, last_valid_block_height)) => { + self.optimizer.report(index, duration_as_ms(&now.elapsed())); + Ok((blockhash, last_valid_block_height)) + } + Err(e) => { + self.optimizer.report(index, std::u64::MAX); + Err(e.into()) + } + } + } + + fn is_blockhash_valid( + &self, + blockhash: &Hash, + commitment_config: CommitmentConfig, + ) -> TransportResult { + self.rpc_client() + .is_blockhash_valid(blockhash, commitment_config) + .map_err(|e| e.into()) + } + + fn get_fee_for_message(&self, message: &Message) -> TransportResult { + self.rpc_client() + .get_fee_for_message(message) + .map_err(|e| e.into()) + } +} + +impl AsyncClient for ThinClient { + fn async_send_versioned_transaction( + &self, + transaction: VersionedTransaction, + ) -> TransportResult { + let conn = self.connection_cache.get_connection(self.tpu_addr()); + conn.serialize_and_send_transaction(&transaction)?; + Ok(transaction.signatures[0]) + } + + fn async_send_versioned_transaction_batch( + &self, + batch: Vec, + ) -> TransportResult<()> { + let conn = self.connection_cache.get_connection(self.tpu_addr()); + conn.par_serialize_and_send_transaction_batch(&batch[..])?; + Ok(()) + } +} diff --git a/client/src/tpu_client.rs b/client/src/tpu_client.rs new file mode 100644 index 0000000000..8beccdca51 --- /dev/null +++ b/client/src/tpu_client.rs @@ -0,0 +1,132 @@ +pub use { + crate::nonblocking::tpu_client::TpuSenderError, + solana_tpu_client::tpu_client::{TpuClientConfig, DEFAULT_FANOUT_SLOTS, MAX_FANOUT_SLOTS}, +}; +use { + crate::{ + connection_cache::ConnectionCache, + nonblocking::tpu_client::TpuClient as NonblockingTpuClient, + }, + rayon::iter::{IntoParallelIterator, ParallelIterator}, + solana_rpc_client::rpc_client::RpcClient, + solana_sdk::{ + message::Message, + signers::Signers, + transaction::{Transaction, TransactionError}, + transport::Result as TransportResult, + }, + solana_tpu_client::tpu_client::temporary_pub::Result, + std::{net::UdpSocket, sync::Arc}, +}; + +/// Client which sends transactions directly to the current leader's TPU port over UDP. +/// The client uses RPC to determine the current leader and fetch node contact info +pub struct TpuClient { + _deprecated: UdpSocket, // TpuClient now uses the connection_cache to choose a send_socket + //todo: get rid of this field + rpc_client: Arc, + tpu_client: Arc, +} + +impl TpuClient { + /// Serialize and send transaction to the current and upcoming leader TPUs according to fanout + /// size + pub fn send_transaction(&self, transaction: &Transaction) -> bool { + self.invoke(self.tpu_client.send_transaction(transaction)) + } + + /// Send a wire transaction to the current and upcoming leader TPUs according to fanout size + pub fn send_wire_transaction(&self, wire_transaction: Vec) -> bool { + self.invoke(self.tpu_client.send_wire_transaction(wire_transaction)) + } + + /// Serialize and send transaction to the current and upcoming leader TPUs according to fanout + /// size + /// Returns the last error if all sends fail + pub fn try_send_transaction(&self, transaction: &Transaction) -> TransportResult<()> { + self.invoke(self.tpu_client.try_send_transaction(transaction)) + } + + /// Serialize and send a batch of transactions to the current and upcoming leader TPUs according + /// to fanout size + /// Returns the last error if all sends fail + pub fn try_send_transaction_batch(&self, transactions: &[Transaction]) -> TransportResult<()> { + let wire_transactions = transactions + .into_par_iter() + .map(|tx| bincode::serialize(&tx).expect("serialize Transaction in send_batch")) + .collect::>(); + self.invoke( + self.tpu_client + .try_send_wire_transaction_batch(wire_transactions), + ) + } + + /// Send a wire transaction to the current and upcoming leader TPUs according to fanout size + /// Returns the last error if all sends fail + pub fn try_send_wire_transaction(&self, wire_transaction: Vec) -> TransportResult<()> { + self.invoke(self.tpu_client.try_send_wire_transaction(wire_transaction)) + } + + /// Create a new client that disconnects when dropped + pub fn new( + rpc_client: Arc, + websocket_url: &str, + config: TpuClientConfig, + ) -> Result { + let create_tpu_client = + NonblockingTpuClient::new(rpc_client.get_inner_client().clone(), websocket_url, config); + let tpu_client = + tokio::task::block_in_place(|| rpc_client.runtime().block_on(create_tpu_client))?; + + Ok(Self { + _deprecated: UdpSocket::bind("0.0.0.0:0").unwrap(), + rpc_client, + tpu_client: Arc::new(tpu_client), + }) + } + + /// Create a new client that disconnects when dropped + pub fn new_with_connection_cache( + rpc_client: Arc, + websocket_url: &str, + config: TpuClientConfig, + connection_cache: Arc, + ) -> Result { + let create_tpu_client = NonblockingTpuClient::new_with_connection_cache( + rpc_client.get_inner_client().clone(), + websocket_url, + config, + connection_cache, + ); + let tpu_client = + tokio::task::block_in_place(|| rpc_client.runtime().block_on(create_tpu_client))?; + + Ok(Self { + _deprecated: UdpSocket::bind("0.0.0.0:0").unwrap(), + rpc_client, + tpu_client: Arc::new(tpu_client), + }) + } + + pub fn send_and_confirm_messages_with_spinner( + &self, + messages: &[Message], + signers: &T, + ) -> Result>> { + self.invoke( + self.tpu_client + .send_and_confirm_messages_with_spinner(messages, signers), + ) + } + + pub fn rpc_client(&self) -> &RpcClient { + &self.rpc_client + } + + fn invoke>(&self, f: F) -> T { + // `block_on()` panics if called within an asynchronous execution context. Whereas + // `block_in_place()` only panics if called from a current_thread runtime, which is the + // lesser evil. + tokio::task::block_in_place(move || self.rpc_client.runtime().block_on(f)) + } +} diff --git a/client/src/tpu_connection.rs b/client/src/tpu_connection.rs new file mode 100644 index 0000000000..81a8b89f66 --- /dev/null +++ b/client/src/tpu_connection.rs @@ -0,0 +1,56 @@ +pub use solana_tpu_client::tpu_connection::ClientStats; +use { + enum_dispatch::enum_dispatch, + rayon::iter::{IntoParallelIterator, ParallelIterator}, + solana_quic_client::quic_client::QuicTpuConnection, + solana_sdk::{transaction::VersionedTransaction, transport::Result as TransportResult}, + solana_udp_client::udp_client::UdpTpuConnection, + std::net::SocketAddr, +}; + +#[enum_dispatch] +pub enum BlockingConnection { + UdpTpuConnection, + QuicTpuConnection, +} + +#[enum_dispatch(BlockingConnection)] +pub trait TpuConnection { + fn tpu_addr(&self) -> &SocketAddr; + + fn serialize_and_send_transaction( + &self, + transaction: &VersionedTransaction, + ) -> TransportResult<()> { + let wire_transaction = + bincode::serialize(transaction).expect("serialize Transaction in send_batch"); + self.send_wire_transaction(wire_transaction) + } + + fn send_wire_transaction(&self, wire_transaction: T) -> TransportResult<()> + where + T: AsRef<[u8]> + Send + Sync, + { + self.send_wire_transaction_batch(&[wire_transaction]) + } + + fn send_wire_transaction_async(&self, wire_transaction: Vec) -> TransportResult<()>; + + fn par_serialize_and_send_transaction_batch( + &self, + transactions: &[VersionedTransaction], + ) -> TransportResult<()> { + let buffers = transactions + .into_par_iter() + .map(|tx| bincode::serialize(&tx).expect("serialize Transaction in send_batch")) + .collect::>(); + + self.send_wire_transaction_batch(&buffers) + } + + fn send_wire_transaction_batch(&self, buffers: &[T]) -> TransportResult<()> + where + T: AsRef<[u8]> + Send + Sync; + + fn send_wire_transaction_batch_async(&self, buffers: Vec>) -> TransportResult<()>; +} diff --git a/tpu-client/src/udp_client.rs b/client/src/udp_client.rs similarity index 54% rename from tpu-client/src/udp_client.rs rename to client/src/udp_client.rs index b7af06c06d..868849ee2f 100644 --- a/tpu-client/src/udp_client.rs +++ b/client/src/udp_client.rs @@ -1,39 +1,13 @@ //! Simple TPU client that communicates with the given UDP port with UDP and provides //! an interface for sending transactions +pub use solana_udp_client::udp_client::UdpTpuConnection; use { - crate::{connection_cache_stats::ConnectionCacheStats, tpu_connection::TpuConnection}, - core::iter::repeat, - solana_sdk::transport::Result as TransportResult, - solana_streamer::sendmmsg::batch_send, - std::{ - net::{SocketAddr, UdpSocket}, - sync::Arc, - }, + crate::tpu_connection::TpuConnection, core::iter::repeat, + solana_sdk::transport::Result as TransportResult, solana_streamer::sendmmsg::batch_send, + std::net::SocketAddr, }; -pub struct UdpTpuConnection { - socket: Arc, - addr: SocketAddr, -} - -impl UdpTpuConnection { - pub fn new_from_addr(local_socket: Arc, tpu_addr: SocketAddr) -> Self { - Self { - socket: local_socket, - addr: tpu_addr, - } - } - - pub fn new( - local_socket: Arc, - tpu_addr: SocketAddr, - _connection_stats: Arc, - ) -> Self { - Self::new_from_addr(local_socket, tpu_addr) - } -} - impl TpuConnection for UdpTpuConnection { fn tpu_addr(&self) -> &SocketAddr { &self.addr diff --git a/core/Cargo.toml b/core/Cargo.toml index b0f6e6ac69..8829d4b950 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -38,6 +38,7 @@ serde = "1.0.144" serde_derive = "1.0.103" solana-address-lookup-table-program = { path = "../programs/address-lookup-table", version = "=1.15.0" } solana-bloom = { path = "../bloom", version = "=1.15.0" } +solana-client = { path = "../client", version = "=1.15.0" } solana-entry = { path = "../entry", version = "=1.15.0" } solana-frozen-abi = { path = "../frozen-abi", version = "=1.15.0" } solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.15.0" } diff --git a/core/benches/banking_stage.rs b/core/benches/banking_stage.rs index bdf7150e71..d7adcfb7f8 100644 --- a/core/benches/banking_stage.rs +++ b/core/benches/banking_stage.rs @@ -8,6 +8,7 @@ use { log::*, rand::{thread_rng, Rng}, rayon::prelude::*, + solana_client::connection_cache::ConnectionCache, solana_core::{ banking_stage::{BankingStage, BankingStageStats}, leader_slot_banking_stage_metrics::LeaderSlotMetricsTracker, @@ -37,7 +38,6 @@ use { transaction::{Transaction, VersionedTransaction}, }, solana_streamer::socket::SocketAddrSpace, - solana_tpu_client::connection_cache::ConnectionCache, solana_vote_program::{ vote_state::VoteStateUpdate, vote_transaction::new_vote_state_update_transaction, }, diff --git a/core/src/banking_stage.rs b/core/src/banking_stage.rs index d7ae55b083..499ee662fa 100644 --- a/core/src/banking_stage.rs +++ b/core/src/banking_stage.rs @@ -26,6 +26,7 @@ use { }, histogram::Histogram, itertools::Itertools, + solana_client::{connection_cache::ConnectionCache, tpu_connection::TpuConnection}, solana_entry::entry::hash_transactions, solana_gossip::{cluster_info::ClusterInfo, contact_info::ContactInfo}, solana_ledger::{ @@ -66,7 +67,6 @@ use { transport::TransportError, }, solana_streamer::sendmmsg::batch_send, - solana_tpu_client::{connection_cache::ConnectionCache, tpu_connection::TpuConnection}, solana_transaction_status::{ token_balances::TransactionTokenBalancesSet, TransactionTokenBalance, }, diff --git a/core/src/fetch_stage.rs b/core/src/fetch_stage.rs index 885f30ea88..9d0b404fc2 100644 --- a/core/src/fetch_stage.rs +++ b/core/src/fetch_stage.rs @@ -13,7 +13,7 @@ use { solana_streamer::streamer::{ self, PacketBatchReceiver, PacketBatchSender, StreamerReceiveStats, }, - solana_tpu_client::connection_cache::DEFAULT_TPU_ENABLE_UDP, + solana_tpu_client::tpu_connection_cache::DEFAULT_TPU_ENABLE_UDP, std::{ net::UdpSocket, sync::{ diff --git a/core/src/tpu.rs b/core/src/tpu.rs index e7e3e5f973..a03195f74e 100644 --- a/core/src/tpu.rs +++ b/core/src/tpu.rs @@ -16,6 +16,7 @@ use { staked_nodes_updater_service::StakedNodesUpdaterService, }, crossbeam_channel::{unbounded, Receiver}, + solana_client::connection_cache::ConnectionCache, solana_gossip::cluster_info::ClusterInfo, solana_ledger::{blockstore::Blockstore, blockstore_processor::TransactionStatusSender}, solana_poh::poh_recorder::{PohRecorder, WorkingBankEntry}, @@ -34,7 +35,6 @@ use { quic::{spawn_server, StreamStats, MAX_STAKED_CONNECTIONS, MAX_UNSTAKED_CONNECTIONS}, streamer::StakedNodes, }, - solana_tpu_client::connection_cache::ConnectionCache, std::{ collections::HashMap, net::UdpSocket, diff --git a/core/src/tvu.rs b/core/src/tvu.rs index fed2eb20a8..e75c174c78 100644 --- a/core/src/tvu.rs +++ b/core/src/tvu.rs @@ -28,6 +28,7 @@ use { window_service::WindowService, }, crossbeam_channel::{unbounded, Receiver}, + solana_client::connection_cache::ConnectionCache, solana_geyser_plugin_manager::block_metadata_notifier_interface::BlockMetadataNotifierLock, solana_gossip::cluster_info::ClusterInfo, solana_ledger::{ @@ -45,7 +46,6 @@ use { prioritization_fee_cache::PrioritizationFeeCache, vote_sender_types::ReplayVoteSender, }, solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Keypair}, - solana_tpu_client::connection_cache::ConnectionCache, std::{ collections::HashSet, net::UdpSocket, diff --git a/core/src/validator.rs b/core/src/validator.rs index c82984e6a6..4f92f51fbc 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -25,6 +25,7 @@ use { }, crossbeam_channel::{bounded, unbounded, Receiver}, rand::{thread_rng, Rng}, + solana_client::connection_cache::ConnectionCache, solana_entry::poh::compute_hash_time_ns, solana_geyser_plugin_manager::geyser_plugin_service::GeyserPluginService, solana_gossip::{ @@ -99,7 +100,6 @@ use { }, solana_send_transaction_service::send_transaction_service, solana_streamer::{socket::SocketAddrSpace, streamer::StakedNodes}, - solana_tpu_client::connection_cache::ConnectionCache, solana_vote_program::vote_state, std::{ collections::{HashMap, HashSet}, @@ -2153,7 +2153,7 @@ mod tests { crossbeam_channel::{bounded, RecvTimeoutError}, solana_ledger::{create_new_tmp_ledger, genesis_utils::create_genesis_config_with_leader}, solana_sdk::{genesis_config::create_genesis_config, poh_config::PohConfig}, - solana_tpu_client::connection_cache::{ + solana_tpu_client::tpu_connection_cache::{ DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_ENABLE_UDP, DEFAULT_TPU_USE_QUIC, }, std::{fs::remove_dir_all, thread, time::Duration}, diff --git a/core/src/warm_quic_cache_service.rs b/core/src/warm_quic_cache_service.rs index 41fbb46b74..d685210dd7 100644 --- a/core/src/warm_quic_cache_service.rs +++ b/core/src/warm_quic_cache_service.rs @@ -3,9 +3,9 @@ use { rand::{thread_rng, Rng}, + solana_client::{connection_cache::ConnectionCache, tpu_connection::TpuConnection}, solana_gossip::cluster_info::ClusterInfo, solana_poh::poh_recorder::PohRecorder, - solana_tpu_client::{connection_cache::ConnectionCache, tpu_connection::TpuConnection}, std::{ sync::{ atomic::{AtomicBool, Ordering}, diff --git a/dos/Cargo.toml b/dos/Cargo.toml index f8b7116f89..3e002b09a6 100644 --- a/dos/Cargo.toml +++ b/dos/Cargo.toml @@ -18,6 +18,7 @@ log = "0.4.17" rand = "0.7.0" serde = "1.0.144" solana-bench-tps = { path = "../bench-tps", version = "=1.15.0" } +solana-client = { path = "../client", version = "=1.15.0" } solana-core = { path = "../core", version = "=1.15.0" } solana-faucet = { path = "../faucet", version = "=1.15.0" } solana-gossip = { path = "../gossip", version = "=1.15.0" } diff --git a/dos/src/main.rs b/dos/src/main.rs index 5db23d1697..ec472fd2d1 100644 --- a/dos/src/main.rs +++ b/dos/src/main.rs @@ -45,6 +45,7 @@ use { log::*, rand::{thread_rng, Rng}, solana_bench_tps::{bench::generate_and_fund_keypairs, bench_tps_client::BenchTpsClient}, + solana_client::{connection_cache::ConnectionCache, tpu_connection::TpuConnection}, solana_core::serve_repair::{RepairProtocol, RepairRequestHeader, ServeRepair}, solana_dos::cli::*, solana_gossip::{ @@ -66,10 +67,7 @@ use { transaction::Transaction, }, solana_streamer::socket::SocketAddrSpace, - solana_tpu_client::{ - connection_cache::{ConnectionCache, DEFAULT_TPU_CONNECTION_POOL_SIZE}, - tpu_connection::TpuConnection, - }, + solana_tpu_client::tpu_connection_cache::DEFAULT_TPU_CONNECTION_POOL_SIZE, std::{ net::{SocketAddr, UdpSocket}, process::exit, @@ -786,6 +784,7 @@ fn main() { pub mod test { use { super::*, + solana_client::thin_client::ThinClient, solana_core::validator::ValidatorConfig, solana_faucet::faucet::run_local_faucet, solana_local_cluster::{ @@ -795,7 +794,6 @@ pub mod test { }, solana_rpc::rpc::JsonRpcConfig, solana_sdk::timing::timestamp, - solana_thin_client::thin_client::ThinClient, }; const TEST_SEND_BATCH_SIZE: usize = 1; diff --git a/gossip/Cargo.toml b/gossip/Cargo.toml index 04c7c84ae9..600770e038 100644 --- a/gossip/Cargo.toml +++ b/gossip/Cargo.toml @@ -29,6 +29,7 @@ serde_bytes = "0.11" serde_derive = "1.0.103" solana-bloom = { path = "../bloom", version = "=1.15.0" } solana-clap-utils = { path = "../clap-utils", version = "=1.15.0" } +solana-client = { path = "../client", version = "=1.15.0" } solana-entry = { path = "../entry", version = "=1.15.0" } solana-frozen-abi = { path = "../frozen-abi", version = "=1.15.0" } solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.15.0" } diff --git a/gossip/src/gossip_service.rs b/gossip/src/gossip_service.rs index bed9b11ec0..d487cf546e 100644 --- a/gossip/src/gossip_service.rs +++ b/gossip/src/gossip_service.rs @@ -4,6 +4,7 @@ use { crate::{cluster_info::ClusterInfo, contact_info::ContactInfo}, crossbeam_channel::{unbounded, Sender}, rand::{thread_rng, Rng}, + solana_client::{connection_cache::ConnectionCache, thin_client::ThinClient}, solana_perf::recycler::Recycler, solana_runtime::bank_forks::BankForks, solana_sdk::{ @@ -14,8 +15,6 @@ use { socket::SocketAddrSpace, streamer::{self, StreamerReceiveStats}, }, - solana_thin_client::thin_client::ThinClient, - solana_tpu_client::connection_cache::ConnectionCache, std::{ collections::HashSet, net::{SocketAddr, TcpListener, UdpSocket}, diff --git a/local-cluster/Cargo.toml b/local-cluster/Cargo.toml index 8f798c31b0..449bb64da7 100644 --- a/local-cluster/Cargo.toml +++ b/local-cluster/Cargo.toml @@ -16,6 +16,7 @@ itertools = "0.10.5" log = "0.4.17" rand = "0.7.0" rayon = "1.5.3" +solana-client = { path = "../client", version = "=1.15.0" } solana-config-program = { path = "../programs/config", version = "=1.15.0" } solana-core = { path = "../core", version = "=1.15.0" } solana-entry = { path = "../entry", version = "=1.15.0" } diff --git a/local-cluster/src/cluster.rs b/local-cluster/src/cluster.rs index 021b1ae5ab..b5bcf658fb 100644 --- a/local-cluster/src/cluster.rs +++ b/local-cluster/src/cluster.rs @@ -1,9 +1,9 @@ use { + solana_client::thin_client::ThinClient, solana_core::validator::{Validator, ValidatorConfig}, solana_gossip::{cluster_info::Node, contact_info::ContactInfo}, solana_sdk::{pubkey::Pubkey, signature::Keypair}, solana_streamer::socket::SocketAddrSpace, - solana_thin_client::thin_client::ThinClient, std::{path::PathBuf, sync::Arc}, }; diff --git a/local-cluster/src/cluster_tests.rs b/local-cluster/src/cluster_tests.rs index c9d786c931..26b0c210b5 100644 --- a/local-cluster/src/cluster_tests.rs +++ b/local-cluster/src/cluster_tests.rs @@ -6,6 +6,7 @@ use log::*; use { rand::{thread_rng, Rng}, rayon::prelude::*, + solana_client::{connection_cache::ConnectionCache, thin_client::ThinClient}, solana_core::consensus::VOTE_THRESHOLD_DEPTH, solana_entry::entry::{Entry, EntrySlice}, solana_gossip::{ @@ -31,8 +32,6 @@ use { transport::TransportError, }, solana_streamer::socket::SocketAddrSpace, - solana_thin_client::thin_client::ThinClient, - solana_tpu_client::connection_cache::ConnectionCache, solana_vote_program::vote_transaction, std::{ collections::{HashMap, HashSet}, diff --git a/local-cluster/src/local_cluster.rs b/local-cluster/src/local_cluster.rs index 6d0deaded7..5f8e3403a5 100644 --- a/local-cluster/src/local_cluster.rs +++ b/local-cluster/src/local_cluster.rs @@ -6,6 +6,7 @@ use { }, itertools::izip, log::*, + solana_client::{connection_cache::ConnectionCache, thin_client::ThinClient}, solana_core::{ tower_storage::FileTowerStorage, validator::{Validator, ValidatorConfig, ValidatorStartProgress}, @@ -41,10 +42,8 @@ use { }, solana_stake_program::{config::create_account as create_stake_config_account, stake_state}, solana_streamer::socket::SocketAddrSpace, - solana_thin_client::thin_client::ThinClient, - solana_tpu_client::connection_cache::{ - ConnectionCache, DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_ENABLE_UDP, - DEFAULT_TPU_USE_QUIC, + solana_tpu_client::tpu_connection_cache::{ + DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_ENABLE_UDP, DEFAULT_TPU_USE_QUIC, }, solana_vote_program::{ vote_instruction, diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index 52c36656b5..59c1fca2cc 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -6,6 +6,7 @@ use { gag::BufferRedirect, log::*, serial_test::serial, + solana_client::thin_client::ThinClient, solana_core::{ broadcast_stage::BroadcastStageType, consensus::{Tower, SWITCH_FORK_THRESHOLD, VOTE_THRESHOLD_DEPTH}, @@ -53,7 +54,6 @@ use { system_program, system_transaction, }, solana_streamer::socket::SocketAddrSpace, - solana_thin_client::thin_client::ThinClient, solana_vote_program::vote_state::MAX_LOCKOUT_HISTORY, std::{ collections::{HashMap, HashSet}, diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index e9befbea83..c0b8e3f375 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -4145,7 +4145,6 @@ dependencies = [ "solana-runtime", "solana-sdk 1.15.0", "solana-send-transaction-service", - "solana-tpu-client", "tarpc", "tokio", "tokio-serde", @@ -4257,15 +4256,31 @@ dependencies = [ name = "solana-client" version = "1.15.0" dependencies = [ + "async-trait", + "bincode", + "enum_dispatch", + "futures 0.3.24", + "futures-util", + "indexmap", + "indicatif", "log", + "rand 0.7.3", + "rayon", "solana-measure", + "solana-metrics", + "solana-net-utils", "solana-pubsub-client", + "solana-quic-client", "solana-rpc-client", "solana-rpc-client-api", "solana-rpc-client-nonce-utils", "solana-sdk 1.15.0", + "solana-streamer", "solana-thin-client", "solana-tpu-client", + "solana-udp-client", + "thiserror", + "tokio", ] [[package]] @@ -4317,6 +4332,7 @@ dependencies = [ "serde_derive", "solana-address-lookup-table-program", "solana-bloom", + "solana-client", "solana-entry", "solana-frozen-abi 1.15.0", "solana-frozen-abi-macro 1.15.0", @@ -4554,6 +4570,7 @@ dependencies = [ "serde_derive", "solana-bloom", "solana-clap-utils", + "solana-client", "solana-entry", "solana-frozen-abi 1.15.0", "solana-frozen-abi-macro 1.15.0", @@ -4913,6 +4930,31 @@ dependencies = [ "url 2.2.2", ] +[[package]] +name = "solana-quic-client" +version = "1.15.0" +dependencies = [ + "async-mutex", + "async-trait", + "futures 0.3.24", + "itertools", + "lazy_static", + "log", + "quinn", + "quinn-proto", + "quinn-udp", + "rustls 0.20.6", + "solana-measure", + "solana-metrics", + "solana-net-utils", + "solana-rpc-client-api", + "solana-sdk 1.15.0", + "solana-streamer", + "solana-tpu-client", + "thiserror", + "tokio", +] + [[package]] name = "solana-rayon-threadlimit" version = "1.15.0" @@ -4962,6 +5004,7 @@ dependencies = [ "serde_json", "soketto", "solana-account-decoder", + "solana-client", "solana-entry", "solana-faucet", "solana-gossip", @@ -5623,6 +5666,7 @@ version = "1.15.0" dependencies = [ "crossbeam-channel", "log", + "solana-client", "solana-measure", "solana-metrics", "solana-runtime", @@ -5776,23 +5820,14 @@ dependencies = [ name = "solana-tpu-client" version = "1.15.0" dependencies = [ - "async-mutex", "async-trait", "bincode", - "enum_dispatch", - "futures 0.3.24", "futures-util", "indexmap", "indicatif", - "itertools", - "lazy_static", "log", - "quinn", - "quinn-proto", - "quinn-udp", "rand 0.7.3", "rayon", - "rustls 0.20.6", "solana-measure", "solana-metrics", "solana-net-utils", @@ -5800,7 +5835,6 @@ dependencies = [ "solana-rpc-client", "solana-rpc-client-api", "solana-sdk 1.15.0", - "solana-streamer", "thiserror", "tokio", ] @@ -5829,6 +5863,19 @@ dependencies = [ "thiserror", ] +[[package]] +name = "solana-udp-client" +version = "1.15.0" +dependencies = [ + "async-trait", + "solana-net-utils", + "solana-sdk 1.15.0", + "solana-streamer", + "solana-tpu-client", + "thiserror", + "tokio", +] + [[package]] name = "solana-validator" version = "1.15.0" diff --git a/quic-client/Cargo.toml b/quic-client/Cargo.toml index 33a39f1c61..95cb1e1960 100644 --- a/quic-client/Cargo.toml +++ b/quic-client/Cargo.toml @@ -10,11 +10,27 @@ documentation = "https://docs.rs/solana-quic-client" edition = "2021" [dependencies] +async-mutex = "1.4.0" +async-trait = "0.1.57" +futures = "0.3" +itertools = "0.10.5" +lazy_static = "1.4.0" +log = "0.4.17" +quinn = "0.8.4" +quinn-proto = "0.8.4" +quinn-udp = "0.1.3" +rustls = { version = "0.20.6", features = ["dangerous_configuration"] } +solana-measure = { path = "../measure", version = "=1.15.0" } solana-metrics = { path = "../metrics", version = "=1.15.0" } +solana-net-utils = { path = "../net-utils", version = "=1.15.0" } +solana-rpc-client-api = { path = "../rpc-client-api", version = "=1.15.0" } solana-sdk = { path = "../sdk", version = "=1.15.0" } solana-streamer = { path = "../streamer", version = "=1.15.0" } solana-tpu-client = { path = "../tpu-client", version = "=1.15.0" } thiserror = "1.0" +tokio = { version = "1", features = ["full"] } [dev-dependencies] +crossbeam-channel = "0.5" solana-logger = { path = "../logger", version = "=1.15.0" } +solana-perf = { path = "../perf", version = "=1.15.0" } diff --git a/quic-client/src/lib.rs b/quic-client/src/lib.rs index a270674fc5..c3a8d16c17 100644 --- a/quic-client/src/lib.rs +++ b/quic-client/src/lib.rs @@ -3,6 +3,9 @@ pub mod nonblocking; pub mod quic_client; +#[macro_use] +extern crate solana_metrics; + use { crate::{ nonblocking::quic_client::{ diff --git a/quic-client/src/nonblocking/quic_client.rs b/quic-client/src/nonblocking/quic_client.rs index d4c5bff911..2def819fcf 100644 --- a/quic-client/src/nonblocking/quic_client.rs +++ b/quic-client/src/nonblocking/quic_client.rs @@ -1,5 +1,598 @@ //! Simple nonblocking client that connects to a given UDP port with the QUIC protocol //! and provides an interface for sending transactions which is restricted by the //! server's flow control. +use { + async_mutex::Mutex, + async_trait::async_trait, + futures::future::join_all, + itertools::Itertools, + log::*, + quinn::{ + ClientConfig, ConnectError, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, + NewConnection, VarInt, WriteError, + }, + solana_measure::measure::Measure, + solana_net_utils::VALIDATOR_PORT_RANGE, + solana_rpc_client_api::client_error::ErrorKind as ClientErrorKind, + solana_sdk::{ + quic::{ + QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS, QUIC_KEEP_ALIVE_MS, QUIC_MAX_TIMEOUT_MS, + QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS, + }, + signature::Keypair, + transport::Result as TransportResult, + }, + solana_streamer::{ + nonblocking::quic::ALPN_TPU_PROTOCOL_ID, + tls_certificates::new_self_signed_tls_certificate_chain, + }, + solana_tpu_client::{ + connection_cache_stats::ConnectionCacheStats, nonblocking::tpu_connection::TpuConnection, + tpu_connection::ClientStats, + }, + std::{ + net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}, + sync::{atomic::Ordering, Arc}, + thread, + time::Duration, + }, + thiserror::Error, + tokio::{sync::RwLock, time::timeout}, +}; -pub use solana_tpu_client::nonblocking::quic_client::*; +struct SkipServerVerification; + +impl SkipServerVerification { + pub fn new() -> Arc { + Arc::new(Self) + } +} + +impl rustls::client::ServerCertVerifier for SkipServerVerification { + fn verify_server_cert( + &self, + _end_entity: &rustls::Certificate, + _intermediates: &[rustls::Certificate], + _server_name: &rustls::ServerName, + _scts: &mut dyn Iterator, + _ocsp_response: &[u8], + _now: std::time::SystemTime, + ) -> Result { + Ok(rustls::client::ServerCertVerified::assertion()) + } +} + +pub struct QuicClientCertificate { + pub certificates: Vec, + pub key: rustls::PrivateKey, +} + +/// A lazy-initialized Quic Endpoint +pub struct QuicLazyInitializedEndpoint { + endpoint: RwLock>>, + client_certificate: Arc, +} + +#[derive(Error, Debug)] +pub enum QuicError { + #[error(transparent)] + WriteError(#[from] WriteError), + #[error(transparent)] + ConnectionError(#[from] ConnectionError), + #[error(transparent)] + ConnectError(#[from] ConnectError), +} + +impl From for ClientErrorKind { + fn from(quic_error: QuicError) -> Self { + Self::Custom(format!("{:?}", quic_error)) + } +} + +impl QuicLazyInitializedEndpoint { + pub fn new(client_certificate: Arc) -> Self { + Self { + endpoint: RwLock::new(None), + client_certificate, + } + } + + fn create_endpoint(&self) -> Endpoint { + let (_, client_socket) = solana_net_utils::bind_in_range( + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + VALIDATOR_PORT_RANGE, + ) + .expect("QuicLazyInitializedEndpoint::create_endpoint bind_in_range"); + + let mut crypto = rustls::ClientConfig::builder() + .with_safe_defaults() + .with_custom_certificate_verifier(SkipServerVerification::new()) + .with_single_cert( + self.client_certificate.certificates.clone(), + self.client_certificate.key.clone(), + ) + .expect("Failed to set QUIC client certificates"); + crypto.enable_early_data = true; + crypto.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec()]; + + let mut endpoint = + QuicNewConnection::create_endpoint(EndpointConfig::default(), client_socket); + + let mut config = ClientConfig::new(Arc::new(crypto)); + let transport_config = Arc::get_mut(&mut config.transport) + .expect("QuicLazyInitializedEndpoint::create_endpoint Arc::get_mut"); + let timeout = IdleTimeout::from(VarInt::from_u32(QUIC_MAX_TIMEOUT_MS)); + transport_config.max_idle_timeout(Some(timeout)); + transport_config.keep_alive_interval(Some(Duration::from_millis(QUIC_KEEP_ALIVE_MS))); + + endpoint.set_default_client_config(config); + endpoint + } + + async fn get_endpoint(&self) -> Arc { + let lock = self.endpoint.read().await; + let endpoint = lock.as_ref(); + + match endpoint { + Some(endpoint) => endpoint.clone(), + None => { + drop(lock); + let mut lock = self.endpoint.write().await; + let endpoint = lock.as_ref(); + + match endpoint { + Some(endpoint) => endpoint.clone(), + None => { + let connection = Arc::new(self.create_endpoint()); + *lock = Some(connection.clone()); + connection + } + } + } + } + } +} + +impl Default for QuicLazyInitializedEndpoint { + fn default() -> Self { + let (certs, priv_key) = new_self_signed_tls_certificate_chain( + &Keypair::new(), + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + ) + .expect("Failed to create QUIC client certificate"); + Self::new(Arc::new(QuicClientCertificate { + certificates: certs, + key: priv_key, + })) + } +} + +/// A wrapper over NewConnection with additional capability to create the endpoint as part +/// of creating a new connection. +#[derive(Clone)] +struct QuicNewConnection { + endpoint: Arc, + connection: Arc, +} + +impl QuicNewConnection { + /// Create a QuicNewConnection given the remote address 'addr'. + async fn make_connection( + endpoint: Arc, + addr: SocketAddr, + stats: &ClientStats, + ) -> Result { + let mut make_connection_measure = Measure::start("make_connection_measure"); + let endpoint = endpoint.get_endpoint().await; + + let connecting = endpoint.connect(addr, "connect")?; + stats.total_connections.fetch_add(1, Ordering::Relaxed); + if let Ok(connecting_result) = timeout( + Duration::from_millis(QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS), + connecting, + ) + .await + { + if connecting_result.is_err() { + stats.connection_errors.fetch_add(1, Ordering::Relaxed); + } + make_connection_measure.stop(); + stats + .make_connection_ms + .fetch_add(make_connection_measure.as_ms(), Ordering::Relaxed); + + let connection = connecting_result?; + + Ok(Self { + endpoint, + connection: Arc::new(connection), + }) + } else { + Err(ConnectionError::TimedOut.into()) + } + } + + fn create_endpoint(config: EndpointConfig, client_socket: UdpSocket) -> Endpoint { + quinn::Endpoint::new(config, None, client_socket) + .expect("QuicNewConnection::create_endpoint quinn::Endpoint::new") + .0 + } + + // Attempts to make a faster connection by taking advantage of pre-existing key material. + // Only works if connection to this endpoint was previously established. + async fn make_connection_0rtt( + &mut self, + addr: SocketAddr, + stats: &ClientStats, + ) -> Result, QuicError> { + let connecting = self.endpoint.connect(addr, "connect")?; + stats.total_connections.fetch_add(1, Ordering::Relaxed); + let connection = match connecting.into_0rtt() { + Ok((connection, zero_rtt)) => { + if let Ok(zero_rtt) = timeout( + Duration::from_millis(QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS), + zero_rtt, + ) + .await + { + if zero_rtt { + stats.zero_rtt_accepts.fetch_add(1, Ordering::Relaxed); + } else { + stats.zero_rtt_rejects.fetch_add(1, Ordering::Relaxed); + } + connection + } else { + return Err(ConnectionError::TimedOut.into()); + } + } + Err(connecting) => { + stats.connection_errors.fetch_add(1, Ordering::Relaxed); + + if let Ok(connecting_result) = timeout( + Duration::from_millis(QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS), + connecting, + ) + .await + { + connecting_result? + } else { + return Err(ConnectionError::TimedOut.into()); + } + } + }; + self.connection = Arc::new(connection); + Ok(self.connection.clone()) + } +} + +pub struct QuicClient { + endpoint: Arc, + connection: Arc>>, + addr: SocketAddr, + stats: Arc, + chunk_size: usize, +} + +impl QuicClient { + pub fn new( + endpoint: Arc, + addr: SocketAddr, + chunk_size: usize, + ) -> Self { + Self { + endpoint, + connection: Arc::new(Mutex::new(None)), + addr, + stats: Arc::new(ClientStats::default()), + chunk_size, + } + } + + async fn _send_buffer_using_conn( + data: &[u8], + connection: &NewConnection, + ) -> Result<(), QuicError> { + let mut send_stream = connection.connection.open_uni().await?; + + send_stream.write_all(data).await?; + send_stream.finish().await?; + Ok(()) + } + + // Attempts to send data, connecting/reconnecting as necessary + // On success, returns the connection used to successfully send the data + async fn _send_buffer( + &self, + data: &[u8], + stats: &ClientStats, + connection_stats: Arc, + ) -> Result, QuicError> { + let mut connection_try_count = 0; + let mut last_connection_id = 0; + let mut last_error = None; + + while connection_try_count < 2 { + let connection = { + let mut conn_guard = self.connection.lock().await; + + let maybe_conn = conn_guard.as_mut(); + match maybe_conn { + Some(conn) => { + if conn.connection.connection.stable_id() == last_connection_id { + // this is the problematic connection we had used before, create a new one + let conn = conn.make_connection_0rtt(self.addr, stats).await; + match conn { + Ok(conn) => { + info!( + "Made 0rtt connection to {} with id {} try_count {}, last_connection_id: {}, last_error: {:?}", + self.addr, + conn.connection.stable_id(), + connection_try_count, + last_connection_id, + last_error, + ); + connection_try_count += 1; + conn + } + Err(err) => { + info!( + "Cannot make 0rtt connection to {}, error {:}", + self.addr, err + ); + return Err(err); + } + } + } else { + stats.connection_reuse.fetch_add(1, Ordering::Relaxed); + conn.connection.clone() + } + } + None => { + let conn = QuicNewConnection::make_connection( + self.endpoint.clone(), + self.addr, + stats, + ) + .await; + match conn { + Ok(conn) => { + *conn_guard = Some(conn.clone()); + info!( + "Made connection to {} id {} try_count {}", + self.addr, + conn.connection.connection.stable_id(), + connection_try_count + ); + connection_try_count += 1; + conn.connection.clone() + } + Err(err) => { + info!("Cannot make connection to {}, error {:}", self.addr, err); + return Err(err); + } + } + } + } + }; + + let new_stats = connection.connection.stats(); + + connection_stats + .total_client_stats + .congestion_events + .update_stat( + &self.stats.congestion_events, + new_stats.path.congestion_events, + ); + + connection_stats + .total_client_stats + .tx_streams_blocked_uni + .update_stat( + &self.stats.tx_streams_blocked_uni, + new_stats.frame_tx.streams_blocked_uni, + ); + + connection_stats + .total_client_stats + .tx_data_blocked + .update_stat(&self.stats.tx_data_blocked, new_stats.frame_tx.data_blocked); + + connection_stats + .total_client_stats + .tx_acks + .update_stat(&self.stats.tx_acks, new_stats.frame_tx.acks); + + last_connection_id = connection.connection.stable_id(); + match Self::_send_buffer_using_conn(data, &connection).await { + Ok(()) => { + return Ok(connection); + } + Err(err) => match err { + QuicError::ConnectionError(_) => { + last_error = Some(err); + } + _ => { + info!( + "Error sending to {} with id {}, error {:?} thread: {:?}", + self.addr, + connection.connection.stable_id(), + err, + thread::current().id(), + ); + return Err(err); + } + }, + } + } + + // if we come here, that means we have exhausted maximum retries, return the error + info!( + "Ran into an error sending transactions {:?}, exhausted retries to {}", + last_error, self.addr + ); + // If we get here but last_error is None, then we have a logic error + // in this function, so panic here with an expect to help debugging + Err(last_error.expect("QuicClient::_send_buffer last_error.expect")) + } + + pub async fn send_buffer( + &self, + data: T, + stats: &ClientStats, + connection_stats: Arc, + ) -> Result<(), ClientErrorKind> + where + T: AsRef<[u8]>, + { + self._send_buffer(data.as_ref(), stats, connection_stats) + .await + .map_err(Into::::into)?; + Ok(()) + } + + pub async fn send_batch( + &self, + buffers: &[T], + stats: &ClientStats, + connection_stats: Arc, + ) -> Result<(), ClientErrorKind> + where + T: AsRef<[u8]>, + { + // Start off by "testing" the connection by sending the first transaction + // This will also connect to the server if not already connected + // and reconnect and retry if the first send attempt failed + // (for example due to a timed out connection), returning an error + // or the connection that was used to successfully send the transaction. + // We will use the returned connection to send the rest of the transactions in the batch + // to avoid touching the mutex in self, and not bother reconnecting if we fail along the way + // since testing even in the ideal GCE environment has found no cases + // where reconnecting and retrying in the middle of a batch send + // (i.e. we encounter a connection error in the middle of a batch send, which presumably cannot + // be due to a timed out connection) has succeeded + if buffers.is_empty() { + return Ok(()); + } + let connection = self + ._send_buffer(buffers[0].as_ref(), stats, connection_stats) + .await + .map_err(Into::::into)?; + + // Used to avoid dereferencing the Arc multiple times below + // by just getting a reference to the NewConnection once + let connection_ref: &NewConnection = &connection; + + let chunks = buffers[1..buffers.len()].iter().chunks(self.chunk_size); + + let futures: Vec<_> = chunks + .into_iter() + .map(|buffs| { + join_all( + buffs + .into_iter() + .map(|buf| Self::_send_buffer_using_conn(buf.as_ref(), connection_ref)), + ) + }) + .collect(); + + for f in futures { + f.await + .into_iter() + .try_for_each(|res| res) + .map_err(Into::::into)?; + } + Ok(()) + } + + pub fn tpu_addr(&self) -> &SocketAddr { + &self.addr + } + + pub fn stats(&self) -> Arc { + self.stats.clone() + } +} + +pub struct QuicTpuConnection { + pub client: Arc, + pub connection_stats: Arc, +} + +impl QuicTpuConnection { + pub fn base_stats(&self) -> Arc { + self.client.stats() + } + + pub fn connection_stats(&self) -> Arc { + self.connection_stats.clone() + } + + pub fn new( + endpoint: Arc, + addr: SocketAddr, + connection_stats: Arc, + ) -> Self { + let client = Arc::new(QuicClient::new( + endpoint, + addr, + QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS, + )); + Self::new_with_client(client, connection_stats) + } + + pub fn new_with_client( + client: Arc, + connection_stats: Arc, + ) -> Self { + Self { + client, + connection_stats, + } + } +} + +#[async_trait] +impl TpuConnection for QuicTpuConnection { + fn tpu_addr(&self) -> &SocketAddr { + self.client.tpu_addr() + } + + async fn send_wire_transaction_batch(&self, buffers: &[T]) -> TransportResult<()> + where + T: AsRef<[u8]> + Send + Sync, + { + let stats = ClientStats::default(); + let len = buffers.len(); + let res = self + .client + .send_batch(buffers, &stats, self.connection_stats.clone()) + .await; + self.connection_stats + .add_client_stats(&stats, len, res.is_ok()); + res?; + Ok(()) + } + + async fn send_wire_transaction(&self, wire_transaction: T) -> TransportResult<()> + where + T: AsRef<[u8]> + Send + Sync, + { + let stats = Arc::new(ClientStats::default()); + let send_buffer = + self.client + .send_buffer(wire_transaction, &stats, self.connection_stats.clone()); + if let Err(e) = send_buffer.await { + warn!( + "Failed to send transaction async to {}, error: {:?} ", + self.tpu_addr(), + e + ); + datapoint_warn!("send-wire-async", ("failure", 1, i64),); + self.connection_stats.add_client_stats(&stats, 1, false); + } else { + self.connection_stats.add_client_stats(&stats, 1, true); + } + Ok(()) + } +} diff --git a/quic-client/src/quic_client.rs b/quic-client/src/quic_client.rs index e1ec971506..316358b0cf 100644 --- a/quic-client/src/quic_client.rs +++ b/quic-client/src/quic_client.rs @@ -1,4 +1,189 @@ //! Simple client that connects to a given UDP port with the QUIC protocol and provides //! an interface for sending transactions which is restricted by the server's flow control. -pub use solana_tpu_client::quic_client::*; +use { + crate::nonblocking::quic_client::{ + QuicClient, QuicLazyInitializedEndpoint, QuicTpuConnection as NonblockingQuicTpuConnection, + }, + lazy_static::lazy_static, + log::*, + solana_sdk::transport::{Result as TransportResult, TransportError}, + solana_tpu_client::{ + connection_cache_stats::ConnectionCacheStats, + nonblocking::tpu_connection::TpuConnection as NonblockingTpuConnection, + tpu_connection::{ClientStats, TpuConnection}, + }, + std::{ + net::SocketAddr, + sync::{atomic::Ordering, Arc, Condvar, Mutex, MutexGuard}, + time::Duration, + }, + tokio::{runtime::Runtime, time::timeout}, +}; + +pub mod temporary_pub { + use super::*; + + pub const MAX_OUTSTANDING_TASK: u64 = 2000; + pub const SEND_TRANSACTION_TIMEOUT_MS: u64 = 10000; + + /// A semaphore used for limiting the number of asynchronous tasks spawn to the + /// runtime. Before spawnning a task, use acquire. After the task is done (be it + /// succsess or failure), call release. + pub struct AsyncTaskSemaphore { + /// Keep the counter info about the usage + counter: Mutex, + /// Conditional variable for signaling when counter is decremented + cond_var: Condvar, + /// The maximum usage allowed by this semaphore. + permits: u64, + } + + impl AsyncTaskSemaphore { + pub fn new(permits: u64) -> Self { + Self { + counter: Mutex::new(0), + cond_var: Condvar::new(), + permits, + } + } + + /// When returned, the lock has been locked and usage count has been + /// incremented. When the returned MutexGuard is dropped the lock is dropped + /// without decrementing the usage count. + pub fn acquire(&self) -> MutexGuard { + let mut count = self.counter.lock().unwrap(); + *count += 1; + while *count > self.permits { + count = self.cond_var.wait(count).unwrap(); + } + count + } + + /// Acquire the lock and decrement the usage count + pub fn release(&self) { + let mut count = self.counter.lock().unwrap(); + *count -= 1; + self.cond_var.notify_one(); + } + } + + lazy_static! { + pub static ref ASYNC_TASK_SEMAPHORE: AsyncTaskSemaphore = + AsyncTaskSemaphore::new(MAX_OUTSTANDING_TASK); + pub static ref RUNTIME: Runtime = tokio::runtime::Builder::new_multi_thread() + .thread_name("quic-client") + .enable_all() + .build() + .unwrap(); + } + + pub async fn send_wire_transaction_async( + connection: Arc, + wire_transaction: Vec, + ) -> TransportResult<()> { + let result = timeout( + Duration::from_millis(SEND_TRANSACTION_TIMEOUT_MS), + connection.send_wire_transaction(wire_transaction), + ) + .await; + ASYNC_TASK_SEMAPHORE.release(); + handle_send_result(result, connection) + } + + pub async fn send_wire_transaction_batch_async( + connection: Arc, + buffers: Vec>, + ) -> TransportResult<()> { + let time_out = SEND_TRANSACTION_TIMEOUT_MS * buffers.len() as u64; + + let result = timeout( + Duration::from_millis(time_out), + connection.send_wire_transaction_batch(&buffers), + ) + .await; + ASYNC_TASK_SEMAPHORE.release(); + handle_send_result(result, connection) + } + + /// Check the send result and update stats if timedout. Returns the checked result. + pub fn handle_send_result( + result: Result, tokio::time::error::Elapsed>, + connection: Arc, + ) -> Result<(), TransportError> { + match result { + Ok(result) => result, + Err(_err) => { + let client_stats = ClientStats::default(); + client_stats.send_timeout.fetch_add(1, Ordering::Relaxed); + let stats = connection.connection_stats(); + stats.add_client_stats(&client_stats, 0, false); + info!("Timedout sending transaction {:?}", connection.tpu_addr()); + Err(TransportError::Custom( + "Timedout sending transaction".to_string(), + )) + } + } + } +} +use temporary_pub::*; + +pub struct QuicTpuConnection { + pub inner: Arc, +} +impl QuicTpuConnection { + pub fn new( + endpoint: Arc, + tpu_addr: SocketAddr, + connection_stats: Arc, + ) -> Self { + let inner = Arc::new(NonblockingQuicTpuConnection::new( + endpoint, + tpu_addr, + connection_stats, + )); + Self { inner } + } + + pub fn new_with_client( + client: Arc, + connection_stats: Arc, + ) -> Self { + let inner = Arc::new(NonblockingQuicTpuConnection::new_with_client( + client, + connection_stats, + )); + Self { inner } + } +} + +impl TpuConnection for QuicTpuConnection { + fn tpu_addr(&self) -> &SocketAddr { + self.inner.tpu_addr() + } + + fn send_wire_transaction_batch(&self, buffers: &[T]) -> TransportResult<()> + where + T: AsRef<[u8]> + Send + Sync, + { + RUNTIME.block_on(self.inner.send_wire_transaction_batch(buffers))?; + Ok(()) + } + + fn send_wire_transaction_async(&self, wire_transaction: Vec) -> TransportResult<()> { + let _lock = ASYNC_TASK_SEMAPHORE.acquire(); + let inner = self.inner.clone(); + + let _ = RUNTIME + .spawn(async move { send_wire_transaction_async(inner, wire_transaction).await }); + Ok(()) + } + + fn send_wire_transaction_batch_async(&self, buffers: Vec>) -> TransportResult<()> { + let _lock = ASYNC_TASK_SEMAPHORE.acquire(); + let inner = self.inner.clone(); + let _ = + RUNTIME.spawn(async move { send_wire_transaction_batch_async(inner, buffers).await }); + Ok(()) + } +} diff --git a/tpu-client/tests/quic_client.rs b/quic-client/tests/quic_client.rs similarity index 90% rename from tpu-client/tests/quic_client.rs rename to quic-client/tests/quic_client.rs index 5f99eb81cd..cb210a88a6 100644 --- a/tpu-client/tests/quic_client.rs +++ b/quic-client/tests/quic_client.rs @@ -3,12 +3,10 @@ mod tests { use { crossbeam_channel::{unbounded, Receiver}, solana_perf::packet::PacketBatch, + solana_quic_client::nonblocking::quic_client::QuicLazyInitializedEndpoint, solana_sdk::{packet::PACKET_DATA_SIZE, signature::Keypair}, solana_streamer::{quic::StreamStats, streamer::StakedNodes}, - solana_tpu_client::{ - connection_cache_stats::ConnectionCacheStats, - nonblocking::quic_client::QuicLazyInitializedEndpoint, - }, + solana_tpu_client::connection_cache_stats::ConnectionCacheStats, std::{ net::{IpAddr, SocketAddr, UdpSocket}, sync::{ @@ -62,7 +60,10 @@ mod tests { #[test] fn test_quic_client_multiple_writes() { - use solana_tpu_client::{quic_client::QuicTpuConnection, tpu_connection::TpuConnection}; + use { + solana_quic_client::quic_client::QuicTpuConnection, + solana_tpu_client::tpu_connection::TpuConnection, + }; solana_logger::setup(); let (sender, receiver) = unbounded(); let staked_nodes = Arc::new(RwLock::new(StakedNodes::default())); @@ -106,8 +107,9 @@ mod tests { #[tokio::test] async fn test_nonblocking_quic_client_multiple_writes() { - use solana_tpu_client::nonblocking::{ - quic_client::QuicTpuConnection, tpu_connection::TpuConnection, + use { + solana_quic_client::nonblocking::quic_client::QuicTpuConnection, + solana_tpu_client::nonblocking::tpu_connection::TpuConnection, }; solana_logger::setup(); let (sender, receiver) = unbounded(); diff --git a/rpc-test/Cargo.toml b/rpc-test/Cargo.toml index fe9fbb997e..b7aae038db 100644 --- a/rpc-test/Cargo.toml +++ b/rpc-test/Cargo.toml @@ -20,6 +20,7 @@ reqwest = { version = "0.11.12", default-features = false, features = ["blocking serde = "1.0.144" serde_json = "1.0.83" solana-account-decoder = { path = "../account-decoder", version = "=1.15.0" } +solana-client = { path = "../client", version = "=1.15.0" } solana-pubsub-client = { path = "../pubsub-client", version = "=1.15.0" } solana-rpc = { path = "../rpc", version = "=1.15.0" } solana-rpc-client = { path = "../rpc-client", version = "=1.15.0", default-features = false } diff --git a/rpc-test/tests/nonblocking.rs b/rpc-test/tests/nonblocking.rs index d75fc7a1cd..ecb9ee5199 100644 --- a/rpc-test/tests/nonblocking.rs +++ b/rpc-test/tests/nonblocking.rs @@ -1,10 +1,10 @@ use { - solana_sdk::{clock::DEFAULT_MS_PER_SLOT, pubkey::Pubkey, system_transaction}, - solana_test_validator::TestValidatorGenesis, - solana_tpu_client::{ + solana_client::{ nonblocking::tpu_client::{LeaderTpuService, TpuClient}, tpu_client::TpuClientConfig, }, + solana_sdk::{clock::DEFAULT_MS_PER_SLOT, pubkey::Pubkey, system_transaction}, + solana_test_validator::TestValidatorGenesis, std::sync::{ atomic::{AtomicBool, Ordering}, Arc, diff --git a/rpc-test/tests/rpc.rs b/rpc-test/tests/rpc.rs index d5eff757eb..0335f7e11f 100644 --- a/rpc-test/tests/rpc.rs +++ b/rpc-test/tests/rpc.rs @@ -6,6 +6,10 @@ use { reqwest::{self, header::CONTENT_TYPE}, serde_json::{json, Value}, solana_account_decoder::UiAccount, + solana_client::{ + connection_cache::{ConnectionCache, DEFAULT_TPU_CONNECTION_POOL_SIZE}, + tpu_client::{TpuClient, TpuClientConfig}, + }, solana_pubsub_client::nonblocking::pubsub_client::PubsubClient, solana_rpc_client::rpc_client::RpcClient, solana_rpc_client_api::{ @@ -25,10 +29,6 @@ use { }, solana_streamer::socket::SocketAddrSpace, solana_test_validator::TestValidator, - solana_tpu_client::{ - connection_cache::{ConnectionCache, DEFAULT_TPU_CONNECTION_POOL_SIZE}, - tpu_client::{TpuClient, TpuClientConfig}, - }, solana_transaction_status::TransactionStatus, std::{ collections::HashSet, diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index 3e0086f95d..15980d3c73 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -30,6 +30,7 @@ serde_derive = "1.0.103" serde_json = "1.0.83" soketto = "0.7" solana-account-decoder = { path = "../account-decoder", version = "=1.15.0" } +solana-client = { path = "../client", version = "=1.15.0" } solana-entry = { path = "../entry", version = "=1.15.0" } solana-faucet = { path = "../faucet", version = "=1.15.0" } solana-gossip = { path = "../gossip", version = "=1.15.0" } diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index 625e9545de..1e931c23ed 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -13,6 +13,7 @@ use { parse_token::{is_known_spl_token_id, token_amount_to_ui_amount, UiTokenAmount}, UiAccount, UiAccountEncoding, UiDataSliceConfig, MAX_BASE58_BYTES, }, + solana_client::connection_cache::ConnectionCache, solana_entry::entry::Entry, solana_faucet::faucet::request_airdrop_transaction, solana_gossip::{cluster_info::ClusterInfo, contact_info::ContactInfo}, @@ -81,7 +82,6 @@ use { solana_stake_program, solana_storage_bigtable::Error as StorageError, solana_streamer::socket::SocketAddrSpace, - solana_tpu_client::connection_cache::ConnectionCache, solana_transaction_status::{ BlockEncodingOptions, ConfirmedBlock, ConfirmedTransactionStatusWithSignature, ConfirmedTransactionWithStatusMeta, EncodedConfirmedTransactionWithStatusMeta, Reward, diff --git a/rpc/src/rpc_service.rs b/rpc/src/rpc_service.rs index 823818b25b..e34f854c9f 100644 --- a/rpc/src/rpc_service.rs +++ b/rpc/src/rpc_service.rs @@ -19,6 +19,7 @@ use { RequestMiddlewareAction, ServerBuilder, }, regex::Regex, + solana_client::connection_cache::ConnectionCache, solana_gossip::cluster_info::ClusterInfo, solana_ledger::{ bigtable_upload::ConfirmedBlockUploadConfig, @@ -40,7 +41,6 @@ use { }, solana_send_transaction_service::send_transaction_service::{self, SendTransactionService}, solana_storage_bigtable::CredentialType, - solana_tpu_client::connection_cache::ConnectionCache, std::{ collections::HashSet, net::SocketAddr, diff --git a/send-transaction-service/Cargo.toml b/send-transaction-service/Cargo.toml index 49928b7674..438fac23dc 100644 --- a/send-transaction-service/Cargo.toml +++ b/send-transaction-service/Cargo.toml @@ -12,6 +12,7 @@ edition = "2021" [dependencies] crossbeam-channel = "0.5" log = "0.4.17" +solana-client = { path = "../client", version = "=1.15.0" } solana-measure = { path = "../measure", version = "=1.15.0" } solana-metrics = { path = "../metrics", version = "=1.15.0" } solana-runtime = { path = "../runtime", version = "=1.15.0" } diff --git a/send-transaction-service/src/send_transaction_service.rs b/send-transaction-service/src/send_transaction_service.rs index 16d2001a25..a1b82e7676 100644 --- a/send-transaction-service/src/send_transaction_service.rs +++ b/send-transaction-service/src/send_transaction_service.rs @@ -2,6 +2,7 @@ use { crate::tpu_info::TpuInfo, crossbeam_channel::{Receiver, RecvTimeoutError}, log::*, + solana_client::{connection_cache::ConnectionCache, tpu_connection::TpuConnection}, solana_measure::measure::Measure, solana_metrics::datapoint_warn, solana_runtime::{bank::Bank, bank_forks::BankForks}, @@ -9,7 +10,6 @@ use { hash::Hash, nonce_account, pubkey::Pubkey, saturating_add_assign, signature::Signature, timing::AtomicInterval, transport::TransportError, }, - solana_tpu_client::{connection_cache::ConnectionCache, tpu_connection::TpuConnection}, std::{ collections::{ hash_map::{Entry, HashMap}, diff --git a/test-validator/src/lib.rs b/test-validator/src/lib.rs index e5f53ac76e..d1d94a0450 100644 --- a/test-validator/src/lib.rs +++ b/test-validator/src/lib.rs @@ -44,7 +44,7 @@ use { signature::{read_keypair_file, write_keypair_file, Keypair, Signer}, }, solana_streamer::socket::SocketAddrSpace, - solana_tpu_client::connection_cache::{ + solana_tpu_client::tpu_connection_cache::{ DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_ENABLE_UDP, DEFAULT_TPU_USE_QUIC, }, std::{ diff --git a/thin-client/src/thin_client.rs b/thin-client/src/thin_client.rs index 8e62cf1e69..48192a3114 100644 --- a/thin-client/src/thin_client.rs +++ b/thin-client/src/thin_client.rs @@ -25,7 +25,10 @@ use { transaction::{self, Transaction, VersionedTransaction}, transport::Result as TransportResult, }, - solana_tpu_client::{connection_cache::ConnectionCache, tpu_connection::TpuConnection}, + solana_tpu_client::{ + tpu_connection::TpuConnection, + tpu_connection_cache::{ConnectionPool, TpuConnectionCache}, + }, std::{ io, net::SocketAddr, @@ -37,101 +40,94 @@ use { }, }; -struct ClientOptimizer { - cur_index: AtomicUsize, - experiment_index: AtomicUsize, - experiment_done: AtomicBool, - times: RwLock>, - num_clients: usize, -} +pub mod temporary_pub { + use super::*; -fn min_index(array: &[u64]) -> (u64, usize) { - let mut min_time = std::u64::MAX; - let mut min_index = 0; - for (i, time) in array.iter().enumerate() { - if *time < min_time { - min_time = *time; - min_index = i; - } - } - (min_time, min_index) -} - -impl ClientOptimizer { - fn new(num_clients: usize) -> Self { - Self { - cur_index: AtomicUsize::new(0), - experiment_index: AtomicUsize::new(0), - experiment_done: AtomicBool::new(false), - times: RwLock::new(vec![std::u64::MAX; num_clients]), - num_clients, - } + pub struct ClientOptimizer { + cur_index: AtomicUsize, + experiment_index: AtomicUsize, + experiment_done: AtomicBool, + times: RwLock>, + num_clients: usize, } - fn experiment(&self) -> usize { - if self.experiment_index.load(Ordering::Relaxed) < self.num_clients { - let old = self.experiment_index.fetch_add(1, Ordering::Relaxed); - if old < self.num_clients { - old + impl ClientOptimizer { + pub fn new(num_clients: usize) -> Self { + Self { + cur_index: AtomicUsize::new(0), + experiment_index: AtomicUsize::new(0), + experiment_done: AtomicBool::new(false), + times: RwLock::new(vec![std::u64::MAX; num_clients]), + num_clients, + } + } + + pub fn experiment(&self) -> usize { + if self.experiment_index.load(Ordering::Relaxed) < self.num_clients { + let old = self.experiment_index.fetch_add(1, Ordering::Relaxed); + if old < self.num_clients { + old + } else { + self.best() + } } else { self.best() } - } else { - self.best() } - } - fn report(&self, index: usize, time_ms: u64) { - if self.num_clients > 1 - && (!self.experiment_done.load(Ordering::Relaxed) || time_ms == std::u64::MAX) - { - trace!( - "report {} with {} exp: {}", - index, - time_ms, - self.experiment_index.load(Ordering::Relaxed) - ); - - self.times.write().unwrap()[index] = time_ms; - - if index == (self.num_clients - 1) || time_ms == std::u64::MAX { - let times = self.times.read().unwrap(); - let (min_time, min_index) = min_index(×); + pub fn report(&self, index: usize, time_ms: u64) { + if self.num_clients > 1 + && (!self.experiment_done.load(Ordering::Relaxed) || time_ms == std::u64::MAX) + { trace!( - "done experimenting min: {} time: {} times: {:?}", - min_index, - min_time, - times + "report {} with {} exp: {}", + index, + time_ms, + self.experiment_index.load(Ordering::Relaxed) ); - // Only 1 thread should grab the num_clients-1 index, so this should be ok. - self.cur_index.store(min_index, Ordering::Relaxed); - self.experiment_done.store(true, Ordering::Relaxed); + self.times.write().unwrap()[index] = time_ms; + + if index == (self.num_clients - 1) || time_ms == std::u64::MAX { + let times = self.times.read().unwrap(); + let (min_time, min_index) = min_index(×); + trace!( + "done experimenting min: {} time: {} times: {:?}", + min_index, + min_time, + times + ); + + // Only 1 thread should grab the num_clients-1 index, so this should be ok. + self.cur_index.store(min_index, Ordering::Relaxed); + self.experiment_done.store(true, Ordering::Relaxed); + } } } - } - fn best(&self) -> usize { - self.cur_index.load(Ordering::Relaxed) + pub fn best(&self) -> usize { + self.cur_index.load(Ordering::Relaxed) + } } } +use temporary_pub::*; /// An object for querying and sending transactions to the network. -pub struct ThinClient { +pub struct ThinClient { rpc_clients: Vec, tpu_addrs: Vec, optimizer: ClientOptimizer, - connection_cache: Arc, + connection_cache: Arc>, } -impl ThinClient { +impl ThinClient

{ /// Create a new ThinClient that will interface with the Rpc at `rpc_addr` using TCP /// and the Tpu at `tpu_addr` over `transactions_socket` using Quic or UDP /// (currently hardcoded to UDP) pub fn new( rpc_addr: SocketAddr, tpu_addr: SocketAddr, - connection_cache: Arc, + connection_cache: Arc>, ) -> Self { Self::new_from_client(RpcClient::new_socket(rpc_addr), tpu_addr, connection_cache) } @@ -140,7 +136,7 @@ impl ThinClient { rpc_addr: SocketAddr, tpu_addr: SocketAddr, timeout: Duration, - connection_cache: Arc, + connection_cache: Arc>, ) -> Self { let rpc_client = RpcClient::new_socket_with_timeout(rpc_addr, timeout); Self::new_from_client(rpc_client, tpu_addr, connection_cache) @@ -149,7 +145,7 @@ impl ThinClient { fn new_from_client( rpc_client: RpcClient, tpu_addr: SocketAddr, - connection_cache: Arc, + connection_cache: Arc>, ) -> Self { Self { rpc_clients: vec![rpc_client], @@ -162,7 +158,7 @@ impl ThinClient { pub fn new_from_addrs( rpc_addrs: Vec, tpu_addrs: Vec, - connection_cache: Arc, + connection_cache: Arc>, ) -> Self { assert!(!rpc_addrs.is_empty()); assert_eq!(rpc_addrs.len(), tpu_addrs.len()); @@ -320,13 +316,13 @@ impl ThinClient { } } -impl Client for ThinClient { +impl Client for ThinClient

{ fn tpu_addr(&self) -> String { self.tpu_addr().to_string() } } -impl SyncClient for ThinClient { +impl SyncClient for ThinClient

{ fn send_and_confirm_message( &self, keypairs: &T, @@ -606,7 +602,7 @@ impl SyncClient for ThinClient { } } -impl AsyncClient for ThinClient { +impl AsyncClient for ThinClient

{ fn async_send_versioned_transaction( &self, transaction: VersionedTransaction, @@ -626,6 +622,18 @@ impl AsyncClient for ThinClient { } } +fn min_index(array: &[u64]) -> (u64, usize) { + let mut min_time = std::u64::MAX; + let mut min_index = 0; + for (i, time) in array.iter().enumerate() { + if *time < min_time { + min_time = *time; + min_index = i; + } + } + (min_time, min_index) +} + #[cfg(test)] mod tests { use {super::*, rayon::prelude::*}; diff --git a/tpu-client/Cargo.toml b/tpu-client/Cargo.toml index d2dba4af8d..dca7a401b3 100644 --- a/tpu-client/Cargo.toml +++ b/tpu-client/Cargo.toml @@ -10,23 +10,14 @@ documentation = "https://docs.rs/solana-tpu-client" edition = "2021" [dependencies] -async-mutex = "1.4.0" async-trait = "0.1.57" bincode = "1.3.3" -enum_dispatch = "0.3.8" -futures = "0.3" futures-util = "0.3.21" indexmap = "1.9.1" indicatif = { version = "0.17.1", optional = true } -itertools = "0.10.5" -lazy_static = "1.4.0" log = "0.4.17" -quinn = "0.8.4" -quinn-proto = "0.8.4" -quinn-udp = "0.1.3" rand = "0.7.0" rayon = "1.5.3" -rustls = { version = "0.20.6", features = ["dangerous_configuration"] } solana-measure = { path = "../measure", version = "=1.15.0" } solana-metrics = { path = "../metrics", version = "=1.15.0" } solana-net-utils = { path = "../net-utils", version = "=1.15.0" } @@ -34,15 +25,12 @@ solana-pubsub-client = { path = "../pubsub-client", version = "=1.15.0" } solana-rpc-client = { path = "../rpc-client", version = "=1.15.0", default-features = false } solana-rpc-client-api = { path = "../rpc-client-api", version = "=1.15.0" } solana-sdk = { path = "../sdk", version = "=1.15.0" } -solana-streamer = { path = "../streamer", version = "=1.15.0" } thiserror = "1.0" tokio = { version = "1", features = ["full"] } [dev-dependencies] -crossbeam-channel = "0.5" rand_chacha = "0.2.2" solana-logger = { path = "../logger", version = "=1.15.0" } -solana-perf = { path = "../perf", version = "=1.15.0" } [features] default = ["spinner"] diff --git a/tpu-client/src/connection_cache_stats.rs b/tpu-client/src/connection_cache_stats.rs index 235827cb6d..9827cc125f 100644 --- a/tpu-client/src/connection_cache_stats.rs +++ b/tpu-client/src/connection_cache_stats.rs @@ -5,25 +5,25 @@ use { #[derive(Default)] pub struct ConnectionCacheStats { - pub(crate) cache_hits: AtomicU64, - pub(crate) cache_misses: AtomicU64, - pub(crate) cache_evictions: AtomicU64, - pub(crate) eviction_time_ms: AtomicU64, - pub(crate) sent_packets: AtomicU64, - pub(crate) total_batches: AtomicU64, - pub(crate) batch_success: AtomicU64, - pub(crate) batch_failure: AtomicU64, - pub(crate) get_connection_ms: AtomicU64, - pub(crate) get_connection_lock_ms: AtomicU64, - pub(crate) get_connection_hit_ms: AtomicU64, - pub(crate) get_connection_miss_ms: AtomicU64, + pub cache_hits: AtomicU64, + pub cache_misses: AtomicU64, + pub cache_evictions: AtomicU64, + pub eviction_time_ms: AtomicU64, + pub sent_packets: AtomicU64, + pub total_batches: AtomicU64, + pub batch_success: AtomicU64, + pub batch_failure: AtomicU64, + pub get_connection_ms: AtomicU64, + pub get_connection_lock_ms: AtomicU64, + pub get_connection_hit_ms: AtomicU64, + pub get_connection_miss_ms: AtomicU64, // Need to track these separately per-connection // because we need to track the base stat value from quinn pub total_client_stats: ClientStats, } -pub(crate) const CONNECTION_STAT_SUBMISSION_INTERVAL: u64 = 2000; +pub const CONNECTION_STAT_SUBMISSION_INTERVAL: u64 = 2000; impl ConnectionCacheStats { pub fn add_client_stats( @@ -70,7 +70,7 @@ impl ConnectionCacheStats { } } - pub(crate) fn report(&self) { + pub fn report(&self) { datapoint_info!( "quic-client-connection-stats", ( diff --git a/tpu-client/src/lib.rs b/tpu-client/src/lib.rs index cb834b9297..5fec3e4eda 100644 --- a/tpu-client/src/lib.rs +++ b/tpu-client/src/lib.rs @@ -1,13 +1,10 @@ #![allow(clippy::integer_arithmetic)] -pub mod connection_cache; pub mod connection_cache_stats; pub mod nonblocking; -pub mod quic_client; pub mod tpu_client; pub mod tpu_connection; pub mod tpu_connection_cache; -pub mod udp_client; #[macro_use] extern crate solana_metrics; diff --git a/tpu-client/src/nonblocking/mod.rs b/tpu-client/src/nonblocking/mod.rs index 69860d7bff..fe2549410a 100644 --- a/tpu-client/src/nonblocking/mod.rs +++ b/tpu-client/src/nonblocking/mod.rs @@ -1,4 +1,2 @@ -pub mod quic_client; pub mod tpu_client; pub mod tpu_connection; -pub mod udp_client; diff --git a/tpu-client/src/nonblocking/quic_client.rs b/tpu-client/src/nonblocking/quic_client.rs deleted file mode 100644 index cfd8159cb8..0000000000 --- a/tpu-client/src/nonblocking/quic_client.rs +++ /dev/null @@ -1,598 +0,0 @@ -//! Simple nonblocking client that connects to a given UDP port with the QUIC protocol -//! and provides an interface for sending transactions which is restricted by the -//! server's flow control. -use { - crate::{ - connection_cache_stats::ConnectionCacheStats, nonblocking::tpu_connection::TpuConnection, - tpu_connection::ClientStats, - }, - async_mutex::Mutex, - async_trait::async_trait, - futures::future::join_all, - itertools::Itertools, - log::*, - quinn::{ - ClientConfig, ConnectError, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, - NewConnection, VarInt, WriteError, - }, - solana_measure::measure::Measure, - solana_net_utils::VALIDATOR_PORT_RANGE, - solana_rpc_client_api::client_error::ErrorKind as ClientErrorKind, - solana_sdk::{ - quic::{ - QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS, QUIC_KEEP_ALIVE_MS, QUIC_MAX_TIMEOUT_MS, - QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS, - }, - signature::Keypair, - transport::Result as TransportResult, - }, - solana_streamer::{ - nonblocking::quic::ALPN_TPU_PROTOCOL_ID, - tls_certificates::new_self_signed_tls_certificate_chain, - }, - std::{ - net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}, - sync::{atomic::Ordering, Arc}, - thread, - time::Duration, - }, - thiserror::Error, - tokio::{sync::RwLock, time::timeout}, -}; - -struct SkipServerVerification; - -impl SkipServerVerification { - pub fn new() -> Arc { - Arc::new(Self) - } -} - -impl rustls::client::ServerCertVerifier for SkipServerVerification { - fn verify_server_cert( - &self, - _end_entity: &rustls::Certificate, - _intermediates: &[rustls::Certificate], - _server_name: &rustls::ServerName, - _scts: &mut dyn Iterator, - _ocsp_response: &[u8], - _now: std::time::SystemTime, - ) -> Result { - Ok(rustls::client::ServerCertVerified::assertion()) - } -} - -pub struct QuicClientCertificate { - pub certificates: Vec, - pub key: rustls::PrivateKey, -} - -/// A lazy-initialized Quic Endpoint -pub struct QuicLazyInitializedEndpoint { - endpoint: RwLock>>, - client_certificate: Arc, -} - -#[derive(Error, Debug)] -pub enum QuicError { - #[error(transparent)] - WriteError(#[from] WriteError), - #[error(transparent)] - ConnectionError(#[from] ConnectionError), - #[error(transparent)] - ConnectError(#[from] ConnectError), -} - -impl From for ClientErrorKind { - fn from(quic_error: QuicError) -> Self { - Self::Custom(format!("{:?}", quic_error)) - } -} - -impl QuicLazyInitializedEndpoint { - pub fn new(client_certificate: Arc) -> Self { - Self { - endpoint: RwLock::new(None), - client_certificate, - } - } - - fn create_endpoint(&self) -> Endpoint { - let (_, client_socket) = solana_net_utils::bind_in_range( - IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), - VALIDATOR_PORT_RANGE, - ) - .expect("QuicLazyInitializedEndpoint::create_endpoint bind_in_range"); - - let mut crypto = rustls::ClientConfig::builder() - .with_safe_defaults() - .with_custom_certificate_verifier(SkipServerVerification::new()) - .with_single_cert( - self.client_certificate.certificates.clone(), - self.client_certificate.key.clone(), - ) - .expect("Failed to set QUIC client certificates"); - crypto.enable_early_data = true; - crypto.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec()]; - - let mut endpoint = - QuicNewConnection::create_endpoint(EndpointConfig::default(), client_socket); - - let mut config = ClientConfig::new(Arc::new(crypto)); - let transport_config = Arc::get_mut(&mut config.transport) - .expect("QuicLazyInitializedEndpoint::create_endpoint Arc::get_mut"); - let timeout = IdleTimeout::from(VarInt::from_u32(QUIC_MAX_TIMEOUT_MS)); - transport_config.max_idle_timeout(Some(timeout)); - transport_config.keep_alive_interval(Some(Duration::from_millis(QUIC_KEEP_ALIVE_MS))); - - endpoint.set_default_client_config(config); - endpoint - } - - async fn get_endpoint(&self) -> Arc { - let lock = self.endpoint.read().await; - let endpoint = lock.as_ref(); - - match endpoint { - Some(endpoint) => endpoint.clone(), - None => { - drop(lock); - let mut lock = self.endpoint.write().await; - let endpoint = lock.as_ref(); - - match endpoint { - Some(endpoint) => endpoint.clone(), - None => { - let connection = Arc::new(self.create_endpoint()); - *lock = Some(connection.clone()); - connection - } - } - } - } - } -} - -impl Default for QuicLazyInitializedEndpoint { - fn default() -> Self { - let (certs, priv_key) = new_self_signed_tls_certificate_chain( - &Keypair::new(), - IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), - ) - .expect("Failed to create QUIC client certificate"); - Self::new(Arc::new(QuicClientCertificate { - certificates: certs, - key: priv_key, - })) - } -} - -/// A wrapper over NewConnection with additional capability to create the endpoint as part -/// of creating a new connection. -#[derive(Clone)] -struct QuicNewConnection { - endpoint: Arc, - connection: Arc, -} - -impl QuicNewConnection { - /// Create a QuicNewConnection given the remote address 'addr'. - async fn make_connection( - endpoint: Arc, - addr: SocketAddr, - stats: &ClientStats, - ) -> Result { - let mut make_connection_measure = Measure::start("make_connection_measure"); - let endpoint = endpoint.get_endpoint().await; - - let connecting = endpoint.connect(addr, "connect")?; - stats.total_connections.fetch_add(1, Ordering::Relaxed); - if let Ok(connecting_result) = timeout( - Duration::from_millis(QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS), - connecting, - ) - .await - { - if connecting_result.is_err() { - stats.connection_errors.fetch_add(1, Ordering::Relaxed); - } - make_connection_measure.stop(); - stats - .make_connection_ms - .fetch_add(make_connection_measure.as_ms(), Ordering::Relaxed); - - let connection = connecting_result?; - - Ok(Self { - endpoint, - connection: Arc::new(connection), - }) - } else { - Err(ConnectionError::TimedOut.into()) - } - } - - fn create_endpoint(config: EndpointConfig, client_socket: UdpSocket) -> Endpoint { - quinn::Endpoint::new(config, None, client_socket) - .expect("QuicNewConnection::create_endpoint quinn::Endpoint::new") - .0 - } - - // Attempts to make a faster connection by taking advantage of pre-existing key material. - // Only works if connection to this endpoint was previously established. - async fn make_connection_0rtt( - &mut self, - addr: SocketAddr, - stats: &ClientStats, - ) -> Result, QuicError> { - let connecting = self.endpoint.connect(addr, "connect")?; - stats.total_connections.fetch_add(1, Ordering::Relaxed); - let connection = match connecting.into_0rtt() { - Ok((connection, zero_rtt)) => { - if let Ok(zero_rtt) = timeout( - Duration::from_millis(QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS), - zero_rtt, - ) - .await - { - if zero_rtt { - stats.zero_rtt_accepts.fetch_add(1, Ordering::Relaxed); - } else { - stats.zero_rtt_rejects.fetch_add(1, Ordering::Relaxed); - } - connection - } else { - return Err(ConnectionError::TimedOut.into()); - } - } - Err(connecting) => { - stats.connection_errors.fetch_add(1, Ordering::Relaxed); - - if let Ok(connecting_result) = timeout( - Duration::from_millis(QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS), - connecting, - ) - .await - { - connecting_result? - } else { - return Err(ConnectionError::TimedOut.into()); - } - } - }; - self.connection = Arc::new(connection); - Ok(self.connection.clone()) - } -} - -pub struct QuicClient { - endpoint: Arc, - connection: Arc>>, - addr: SocketAddr, - stats: Arc, - chunk_size: usize, -} - -impl QuicClient { - pub fn new( - endpoint: Arc, - addr: SocketAddr, - chunk_size: usize, - ) -> Self { - Self { - endpoint, - connection: Arc::new(Mutex::new(None)), - addr, - stats: Arc::new(ClientStats::default()), - chunk_size, - } - } - - async fn _send_buffer_using_conn( - data: &[u8], - connection: &NewConnection, - ) -> Result<(), QuicError> { - let mut send_stream = connection.connection.open_uni().await?; - - send_stream.write_all(data).await?; - send_stream.finish().await?; - Ok(()) - } - - // Attempts to send data, connecting/reconnecting as necessary - // On success, returns the connection used to successfully send the data - async fn _send_buffer( - &self, - data: &[u8], - stats: &ClientStats, - connection_stats: Arc, - ) -> Result, QuicError> { - let mut connection_try_count = 0; - let mut last_connection_id = 0; - let mut last_error = None; - - while connection_try_count < 2 { - let connection = { - let mut conn_guard = self.connection.lock().await; - - let maybe_conn = conn_guard.as_mut(); - match maybe_conn { - Some(conn) => { - if conn.connection.connection.stable_id() == last_connection_id { - // this is the problematic connection we had used before, create a new one - let conn = conn.make_connection_0rtt(self.addr, stats).await; - match conn { - Ok(conn) => { - info!( - "Made 0rtt connection to {} with id {} try_count {}, last_connection_id: {}, last_error: {:?}", - self.addr, - conn.connection.stable_id(), - connection_try_count, - last_connection_id, - last_error, - ); - connection_try_count += 1; - conn - } - Err(err) => { - info!( - "Cannot make 0rtt connection to {}, error {:}", - self.addr, err - ); - return Err(err); - } - } - } else { - stats.connection_reuse.fetch_add(1, Ordering::Relaxed); - conn.connection.clone() - } - } - None => { - let conn = QuicNewConnection::make_connection( - self.endpoint.clone(), - self.addr, - stats, - ) - .await; - match conn { - Ok(conn) => { - *conn_guard = Some(conn.clone()); - info!( - "Made connection to {} id {} try_count {}", - self.addr, - conn.connection.connection.stable_id(), - connection_try_count - ); - connection_try_count += 1; - conn.connection.clone() - } - Err(err) => { - info!("Cannot make connection to {}, error {:}", self.addr, err); - return Err(err); - } - } - } - } - }; - - let new_stats = connection.connection.stats(); - - connection_stats - .total_client_stats - .congestion_events - .update_stat( - &self.stats.congestion_events, - new_stats.path.congestion_events, - ); - - connection_stats - .total_client_stats - .tx_streams_blocked_uni - .update_stat( - &self.stats.tx_streams_blocked_uni, - new_stats.frame_tx.streams_blocked_uni, - ); - - connection_stats - .total_client_stats - .tx_data_blocked - .update_stat(&self.stats.tx_data_blocked, new_stats.frame_tx.data_blocked); - - connection_stats - .total_client_stats - .tx_acks - .update_stat(&self.stats.tx_acks, new_stats.frame_tx.acks); - - last_connection_id = connection.connection.stable_id(); - match Self::_send_buffer_using_conn(data, &connection).await { - Ok(()) => { - return Ok(connection); - } - Err(err) => match err { - QuicError::ConnectionError(_) => { - last_error = Some(err); - } - _ => { - info!( - "Error sending to {} with id {}, error {:?} thread: {:?}", - self.addr, - connection.connection.stable_id(), - err, - thread::current().id(), - ); - return Err(err); - } - }, - } - } - - // if we come here, that means we have exhausted maximum retries, return the error - info!( - "Ran into an error sending transactions {:?}, exhausted retries to {}", - last_error, self.addr - ); - // If we get here but last_error is None, then we have a logic error - // in this function, so panic here with an expect to help debugging - Err(last_error.expect("QuicClient::_send_buffer last_error.expect")) - } - - pub async fn send_buffer( - &self, - data: T, - stats: &ClientStats, - connection_stats: Arc, - ) -> Result<(), ClientErrorKind> - where - T: AsRef<[u8]>, - { - self._send_buffer(data.as_ref(), stats, connection_stats) - .await - .map_err(Into::::into)?; - Ok(()) - } - - pub async fn send_batch( - &self, - buffers: &[T], - stats: &ClientStats, - connection_stats: Arc, - ) -> Result<(), ClientErrorKind> - where - T: AsRef<[u8]>, - { - // Start off by "testing" the connection by sending the first transaction - // This will also connect to the server if not already connected - // and reconnect and retry if the first send attempt failed - // (for example due to a timed out connection), returning an error - // or the connection that was used to successfully send the transaction. - // We will use the returned connection to send the rest of the transactions in the batch - // to avoid touching the mutex in self, and not bother reconnecting if we fail along the way - // since testing even in the ideal GCE environment has found no cases - // where reconnecting and retrying in the middle of a batch send - // (i.e. we encounter a connection error in the middle of a batch send, which presumably cannot - // be due to a timed out connection) has succeeded - if buffers.is_empty() { - return Ok(()); - } - let connection = self - ._send_buffer(buffers[0].as_ref(), stats, connection_stats) - .await - .map_err(Into::::into)?; - - // Used to avoid dereferencing the Arc multiple times below - // by just getting a reference to the NewConnection once - let connection_ref: &NewConnection = &connection; - - let chunks = buffers[1..buffers.len()].iter().chunks(self.chunk_size); - - let futures: Vec<_> = chunks - .into_iter() - .map(|buffs| { - join_all( - buffs - .into_iter() - .map(|buf| Self::_send_buffer_using_conn(buf.as_ref(), connection_ref)), - ) - }) - .collect(); - - for f in futures { - f.await - .into_iter() - .try_for_each(|res| res) - .map_err(Into::::into)?; - } - Ok(()) - } - - pub fn tpu_addr(&self) -> &SocketAddr { - &self.addr - } - - pub fn stats(&self) -> Arc { - self.stats.clone() - } -} - -pub struct QuicTpuConnection { - client: Arc, - connection_stats: Arc, -} - -impl QuicTpuConnection { - pub fn base_stats(&self) -> Arc { - self.client.stats() - } - - pub fn connection_stats(&self) -> Arc { - self.connection_stats.clone() - } - - pub fn new( - endpoint: Arc, - addr: SocketAddr, - connection_stats: Arc, - ) -> Self { - let client = Arc::new(QuicClient::new( - endpoint, - addr, - QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS, - )); - Self::new_with_client(client, connection_stats) - } - - pub fn new_with_client( - client: Arc, - connection_stats: Arc, - ) -> Self { - Self { - client, - connection_stats, - } - } -} - -#[async_trait] -impl TpuConnection for QuicTpuConnection { - fn tpu_addr(&self) -> &SocketAddr { - self.client.tpu_addr() - } - - async fn send_wire_transaction_batch(&self, buffers: &[T]) -> TransportResult<()> - where - T: AsRef<[u8]> + Send + Sync, - { - let stats = ClientStats::default(); - let len = buffers.len(); - let res = self - .client - .send_batch(buffers, &stats, self.connection_stats.clone()) - .await; - self.connection_stats - .add_client_stats(&stats, len, res.is_ok()); - res?; - Ok(()) - } - - async fn send_wire_transaction(&self, wire_transaction: T) -> TransportResult<()> - where - T: AsRef<[u8]> + Send + Sync, - { - let stats = Arc::new(ClientStats::default()); - let send_buffer = - self.client - .send_buffer(wire_transaction, &stats, self.connection_stats.clone()); - if let Err(e) = send_buffer.await { - warn!( - "Failed to send transaction async to {}, error: {:?} ", - self.tpu_addr(), - e - ); - datapoint_warn!("send-wire-async", ("failure", 1, i64),); - self.connection_stats.add_client_stats(&stats, 1, false); - } else { - self.connection_stats.add_client_stats(&stats, 1, true); - } - Ok(()) - } -} diff --git a/tpu-client/src/nonblocking/tpu_client.rs b/tpu-client/src/nonblocking/tpu_client.rs index 7a49334f6f..c5e01fc0b8 100644 --- a/tpu-client/src/nonblocking/tpu_client.rs +++ b/tpu-client/src/nonblocking/tpu_client.rs @@ -1,6 +1,6 @@ #[cfg(feature = "spinner")] use { - crate::tpu_client::{SEND_TRANSACTION_INTERVAL, TRANSACTION_RESEND_INTERVAL}, + crate::tpu_client::temporary_pub::{SEND_TRANSACTION_INTERVAL, TRANSACTION_RESEND_INTERVAL}, indicatif::ProgressBar, solana_rpc_client::spinner, solana_rpc_client_api::request::MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS, @@ -8,9 +8,11 @@ use { }; use { crate::{ - connection_cache::ConnectionCache, nonblocking::tpu_connection::TpuConnection, tpu_client::{RecentLeaderSlots, TpuClientConfig, MAX_FANOUT_SLOTS}, + tpu_connection_cache::{ + ConnectionPool, TpuConnectionCache, DEFAULT_TPU_CONNECTION_POOL_SIZE, + }, }, bincode::serialize, futures_util::{future::join_all, stream::StreamExt}, @@ -46,6 +48,37 @@ use { }, }; +pub mod temporary_pub { + use super::*; + + pub type Result = std::result::Result; + + #[cfg(feature = "spinner")] + pub fn set_message_for_confirmed_transactions( + progress_bar: &ProgressBar, + confirmed_transactions: u32, + total_transactions: usize, + block_height: Option, + last_valid_block_height: u64, + status: &str, + ) { + progress_bar.set_message(format!( + "{:>5.1}% | {:<40}{}", + confirmed_transactions as f64 * 100. / total_transactions as f64, + status, + match block_height { + Some(block_height) => format!( + " [block height {}; re-sign in {} blocks]", + block_height, + last_valid_block_height.saturating_sub(block_height), + ), + None => String::new(), + }, + )); + } +} +use temporary_pub::*; + #[derive(Error, Debug)] pub enum TpuSenderError { #[error("Pubsub error: {0:?}")] @@ -61,9 +94,9 @@ pub enum TpuSenderError { } struct LeaderTpuCacheUpdateInfo { - maybe_cluster_nodes: Option>>, - maybe_epoch_info: Option>, - maybe_slot_leaders: Option>>, + pub(super) maybe_cluster_nodes: Option>>, + pub(super) maybe_epoch_info: Option>, + pub(super) maybe_slot_leaders: Option>>, } impl LeaderTpuCacheUpdateInfo { pub fn has_some(&self) -> bool { @@ -210,20 +243,18 @@ impl LeaderTpuCache { } } -type Result = std::result::Result; - /// Client which sends transactions directly to the current leader's TPU port over UDP. /// The client uses RPC to determine the current leader and fetch node contact info -pub struct TpuClient { +pub struct TpuClient { fanout_slots: u64, leader_tpu_service: LeaderTpuService, exit: Arc, rpc_client: Arc, - connection_cache: Arc, + connection_cache: Arc>, } -async fn send_wire_transaction_to_addr( - connection_cache: &ConnectionCache, +async fn send_wire_transaction_to_addr( + connection_cache: &TpuConnectionCache

, addr: &SocketAddr, wire_transaction: Vec, ) -> TransportResult<()> { @@ -231,8 +262,8 @@ async fn send_wire_transaction_to_addr( conn.send_wire_transaction(wire_transaction.clone()).await } -async fn send_wire_transaction_batch_to_addr( - connection_cache: &ConnectionCache, +async fn send_wire_transaction_batch_to_addr( + connection_cache: &TpuConnectionCache

, addr: &SocketAddr, wire_transactions: &[Vec], ) -> TransportResult<()> { @@ -240,7 +271,7 @@ async fn send_wire_transaction_batch_to_addr( conn.send_wire_transaction_batch(wire_transactions).await } -impl TpuClient { +impl TpuClient

{ /// Serialize and send transaction to the current and upcoming leader TPUs according to fanout /// size pub async fn send_transaction(&self, transaction: &Transaction) -> bool { @@ -356,7 +387,8 @@ impl TpuClient { websocket_url: &str, config: TpuClientConfig, ) -> Result { - let connection_cache = Arc::new(ConnectionCache::default()); + let connection_cache = + Arc::new(TpuConnectionCache::new(DEFAULT_TPU_CONNECTION_POOL_SIZE).unwrap()); // TODO: Handle error properly, as the TpuConnectionCache ctor is now fallible. Self::new_with_connection_cache(rpc_client, websocket_url, config, connection_cache).await } @@ -365,7 +397,7 @@ impl TpuClient { rpc_client: Arc, websocket_url: &str, config: TpuClientConfig, - connection_cache: Arc, + connection_cache: Arc>, ) -> Result { let exit = Arc::new(AtomicBool::new(false)); let leader_tpu_service = @@ -519,7 +551,8 @@ impl TpuClient { self.leader_tpu_service.join().await; } } -impl Drop for TpuClient { + +impl Drop for TpuClient

{ fn drop(&mut self) { self.exit.store(true, Ordering::Relaxed); } @@ -594,7 +627,7 @@ impl LeaderTpuService { self.recent_slots.estimated_current_slot() } - fn leader_tpu_sockets(&self, fanout_slots: u64) -> Vec { + pub fn leader_tpu_sockets(&self, fanout_slots: u64) -> Vec { let current_slot = self.recent_slots.estimated_current_slot(); self.leader_tpu_cache .read() @@ -721,27 +754,3 @@ async fn maybe_fetch_cache_info( maybe_slot_leaders, } } - -#[cfg(feature = "spinner")] -fn set_message_for_confirmed_transactions( - progress_bar: &ProgressBar, - confirmed_transactions: u32, - total_transactions: usize, - block_height: Option, - last_valid_block_height: u64, - status: &str, -) { - progress_bar.set_message(format!( - "{:>5.1}% | {:<40}{}", - confirmed_transactions as f64 * 100. / total_transactions as f64, - status, - match block_height { - Some(block_height) => format!( - " [block height {}; re-sign in {} blocks]", - block_height, - last_valid_block_height.saturating_sub(block_height), - ), - None => String::new(), - }, - )); -} diff --git a/tpu-client/src/nonblocking/tpu_connection.rs b/tpu-client/src/nonblocking/tpu_connection.rs index 9e819070bc..4a711ac7b8 100644 --- a/tpu-client/src/nonblocking/tpu_connection.rs +++ b/tpu-client/src/nonblocking/tpu_connection.rs @@ -1,24 +1,12 @@ //! Trait defining async send functions, to be used for UDP or QUIC sending use { - crate::nonblocking::{quic_client::QuicTpuConnection, udp_client::UdpTpuConnection}, async_trait::async_trait, - enum_dispatch::enum_dispatch, solana_sdk::{transaction::VersionedTransaction, transport::Result as TransportResult}, std::net::SocketAddr, }; -// Due to the existence of `crate::connection_cache::Connection`, if this is named -// `Connection`, enum_dispatch gets confused between the two and throws errors when -// trying to convert later. -#[enum_dispatch] -pub enum NonblockingConnection { - QuicTpuConnection, - UdpTpuConnection, -} - #[async_trait] -#[enum_dispatch(NonblockingConnection)] pub trait TpuConnection { fn tpu_addr(&self) -> &SocketAddr; diff --git a/tpu-client/src/nonblocking/udp_client.rs b/tpu-client/src/nonblocking/udp_client.rs deleted file mode 100644 index 1a41876504..0000000000 --- a/tpu-client/src/nonblocking/udp_client.rs +++ /dev/null @@ -1,93 +0,0 @@ -//! Simple UDP client that communicates with the given UDP port with UDP and provides -//! an interface for sending transactions - -use { - crate::nonblocking::tpu_connection::TpuConnection, async_trait::async_trait, - core::iter::repeat, solana_sdk::transport::Result as TransportResult, - solana_streamer::nonblocking::sendmmsg::batch_send, std::net::SocketAddr, - tokio::net::UdpSocket, -}; - -pub struct UdpTpuConnection { - socket: UdpSocket, - addr: SocketAddr, -} - -impl UdpTpuConnection { - pub fn new_from_addr(socket: std::net::UdpSocket, tpu_addr: SocketAddr) -> Self { - socket.set_nonblocking(true).unwrap(); - let socket = UdpSocket::from_std(socket).unwrap(); - Self { - socket, - addr: tpu_addr, - } - } -} - -#[async_trait] -impl TpuConnection for UdpTpuConnection { - fn tpu_addr(&self) -> &SocketAddr { - &self.addr - } - - async fn send_wire_transaction(&self, wire_transaction: T) -> TransportResult<()> - where - T: AsRef<[u8]> + Send + Sync, - { - self.socket - .send_to(wire_transaction.as_ref(), self.addr) - .await?; - Ok(()) - } - - async fn send_wire_transaction_batch(&self, buffers: &[T]) -> TransportResult<()> - where - T: AsRef<[u8]> + Send + Sync, - { - let pkts: Vec<_> = buffers.iter().zip(repeat(self.tpu_addr())).collect(); - batch_send(&self.socket, &pkts).await?; - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use { - crate::nonblocking::{tpu_connection::TpuConnection, udp_client::UdpTpuConnection}, - solana_sdk::packet::{Packet, PACKET_DATA_SIZE}, - solana_streamer::nonblocking::recvmmsg::recv_mmsg, - std::net::{IpAddr, Ipv4Addr}, - tokio::net::UdpSocket, - }; - - async fn check_send_one(connection: &UdpTpuConnection, reader: &UdpSocket) { - let packet = vec![111u8; PACKET_DATA_SIZE]; - connection.send_wire_transaction(&packet).await.unwrap(); - let mut packets = vec![Packet::default(); 32]; - let recv = recv_mmsg(reader, &mut packets[..]).await.unwrap(); - assert_eq!(1, recv); - } - - async fn check_send_batch(connection: &UdpTpuConnection, reader: &UdpSocket) { - let packets: Vec<_> = (0..32).map(|_| vec![0u8; PACKET_DATA_SIZE]).collect(); - connection - .send_wire_transaction_batch(&packets) - .await - .unwrap(); - let mut packets = vec![Packet::default(); 32]; - let recv = recv_mmsg(reader, &mut packets[..]).await.unwrap(); - assert_eq!(32, recv); - } - - #[tokio::test] - async fn test_send_from_addr() { - let addr_str = "0.0.0.0:50100"; - let addr = addr_str.parse().unwrap(); - let socket = - solana_net_utils::bind_with_any_port(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))).unwrap(); - let connection = UdpTpuConnection::new_from_addr(socket, addr); - let reader = UdpSocket::bind(addr_str).await.expect("bind"); - check_send_one(&connection, &reader).await; - check_send_batch(&connection, &reader).await; - } -} diff --git a/tpu-client/src/quic_client.rs b/tpu-client/src/quic_client.rs deleted file mode 100644 index c34dddfb77..0000000000 --- a/tpu-client/src/quic_client.rs +++ /dev/null @@ -1,187 +0,0 @@ -//! Simple client that connects to a given UDP port with the QUIC protocol and provides -//! an interface for sending transactions which is restricted by the server's flow control. - -use { - crate::{ - connection_cache_stats::ConnectionCacheStats, - nonblocking::{ - quic_client::{ - QuicClient, QuicLazyInitializedEndpoint, - QuicTpuConnection as NonblockingQuicTpuConnection, - }, - tpu_connection::TpuConnection as NonblockingTpuConnection, - }, - tpu_connection::{ClientStats, TpuConnection}, - }, - lazy_static::lazy_static, - log::*, - solana_sdk::transport::{Result as TransportResult, TransportError}, - std::{ - net::SocketAddr, - sync::{atomic::Ordering, Arc, Condvar, Mutex, MutexGuard}, - time::Duration, - }, - tokio::{runtime::Runtime, time::timeout}, -}; - -const MAX_OUTSTANDING_TASK: u64 = 2000; -const SEND_TRANSACTION_TIMEOUT_MS: u64 = 10000; - -/// A semaphore used for limiting the number of asynchronous tasks spawn to the -/// runtime. Before spawnning a task, use acquire. After the task is done (be it -/// succsess or failure), call release. -struct AsyncTaskSemaphore { - /// Keep the counter info about the usage - counter: Mutex, - /// Conditional variable for signaling when counter is decremented - cond_var: Condvar, - /// The maximum usage allowed by this semaphore. - permits: u64, -} - -impl AsyncTaskSemaphore { - fn new(permits: u64) -> Self { - Self { - counter: Mutex::new(0), - cond_var: Condvar::new(), - permits, - } - } - - /// When returned, the lock has been locked and usage count has been - /// incremented. When the returned MutexGuard is dropped the lock is dropped - /// without decrementing the usage count. - fn acquire(&self) -> MutexGuard { - let mut count = self.counter.lock().unwrap(); - *count += 1; - while *count > self.permits { - count = self.cond_var.wait(count).unwrap(); - } - count - } - - /// Acquire the lock and decrement the usage count - fn release(&self) { - let mut count = self.counter.lock().unwrap(); - *count -= 1; - self.cond_var.notify_one(); - } -} - -lazy_static! { - static ref ASYNC_TASK_SEMAPHORE: AsyncTaskSemaphore = - AsyncTaskSemaphore::new(MAX_OUTSTANDING_TASK); - static ref RUNTIME: Runtime = tokio::runtime::Builder::new_multi_thread() - .thread_name("quic-client") - .enable_all() - .build() - .unwrap(); -} - -pub struct QuicTpuConnection { - inner: Arc, -} -impl QuicTpuConnection { - pub fn new( - endpoint: Arc, - tpu_addr: SocketAddr, - connection_stats: Arc, - ) -> Self { - let inner = Arc::new(NonblockingQuicTpuConnection::new( - endpoint, - tpu_addr, - connection_stats, - )); - Self { inner } - } - - pub fn new_with_client( - client: Arc, - connection_stats: Arc, - ) -> Self { - let inner = Arc::new(NonblockingQuicTpuConnection::new_with_client( - client, - connection_stats, - )); - Self { inner } - } -} - -async fn send_wire_transaction_async( - connection: Arc, - wire_transaction: Vec, -) -> TransportResult<()> { - let result = timeout( - Duration::from_millis(SEND_TRANSACTION_TIMEOUT_MS), - connection.send_wire_transaction(wire_transaction), - ) - .await; - ASYNC_TASK_SEMAPHORE.release(); - handle_send_result(result, connection) -} - -async fn send_wire_transaction_batch_async( - connection: Arc, - buffers: Vec>, -) -> TransportResult<()> { - let time_out = SEND_TRANSACTION_TIMEOUT_MS * buffers.len() as u64; - - let result = timeout( - Duration::from_millis(time_out), - connection.send_wire_transaction_batch(&buffers), - ) - .await; - ASYNC_TASK_SEMAPHORE.release(); - handle_send_result(result, connection) -} - -/// Check the send result and update stats if timedout. Returns the checked result. -fn handle_send_result( - result: Result, tokio::time::error::Elapsed>, - connection: Arc, -) -> Result<(), TransportError> { - match result { - Ok(result) => result, - Err(_err) => { - let client_stats = ClientStats::default(); - client_stats.send_timeout.fetch_add(1, Ordering::Relaxed); - let stats = connection.connection_stats(); - stats.add_client_stats(&client_stats, 0, false); - info!("Timedout sending transaction {:?}", connection.tpu_addr()); - Err(TransportError::Custom( - "Timedout sending transaction".to_string(), - )) - } - } -} - -impl TpuConnection for QuicTpuConnection { - fn tpu_addr(&self) -> &SocketAddr { - self.inner.tpu_addr() - } - - fn send_wire_transaction_batch(&self, buffers: &[T]) -> TransportResult<()> - where - T: AsRef<[u8]> + Send + Sync, - { - RUNTIME.block_on(self.inner.send_wire_transaction_batch(buffers))?; - Ok(()) - } - - fn send_wire_transaction_async(&self, wire_transaction: Vec) -> TransportResult<()> { - let _lock = ASYNC_TASK_SEMAPHORE.acquire(); - let inner = self.inner.clone(); - - let _ = RUNTIME - .spawn(async move { send_wire_transaction_async(inner, wire_transaction).await }); - Ok(()) - } - - fn send_wire_transaction_batch_async(&self, buffers: Vec>) -> TransportResult<()> { - let _lock = ASYNC_TASK_SEMAPHORE.acquire(); - let inner = self.inner.clone(); - let _ = - RUNTIME.spawn(async move { send_wire_transaction_batch_async(inner, buffers).await }); - Ok(()) - } -} diff --git a/tpu-client/src/tpu_client.rs b/tpu-client/src/tpu_client.rs index 7ce01f671d..3a343fb301 100644 --- a/tpu-client/src/tpu_client.rs +++ b/tpu-client/src/tpu_client.rs @@ -1,8 +1,8 @@ pub use crate::nonblocking::tpu_client::TpuSenderError; use { crate::{ - connection_cache::ConnectionCache, nonblocking::tpu_client::TpuClient as NonblockingTpuClient, + tpu_connection_cache::{ConnectionPool, TpuConnectionCache}, }, rayon::iter::{IntoParallelIterator, ParallelIterator}, solana_rpc_client::rpc_client::RpcClient, @@ -19,7 +19,19 @@ use { tokio::time::Duration, }; -type Result = std::result::Result; +pub mod temporary_pub { + use super::*; + + pub type Result = std::result::Result; + + /// Send at ~100 TPS + #[cfg(feature = "spinner")] + pub const SEND_TRANSACTION_INTERVAL: Duration = Duration::from_millis(10); + /// Retry batch send after 4 seconds + #[cfg(feature = "spinner")] + pub const TRANSACTION_RESEND_INTERVAL: Duration = Duration::from_secs(4); +} +use temporary_pub::*; /// Default number of slots used to build TPU socket fanout set pub const DEFAULT_FANOUT_SLOTS: u64 = 12; @@ -27,13 +39,6 @@ pub const DEFAULT_FANOUT_SLOTS: u64 = 12; /// Maximum number of slots used to build TPU socket fanout set pub const MAX_FANOUT_SLOTS: u64 = 100; -/// Send at ~100 TPS -#[cfg(feature = "spinner")] -pub(crate) const SEND_TRANSACTION_INTERVAL: Duration = Duration::from_millis(10); -/// Retry batch send after 4 seconds -#[cfg(feature = "spinner")] -pub(crate) const TRANSACTION_RESEND_INTERVAL: Duration = Duration::from_secs(4); - /// Config params for `TpuClient` #[derive(Clone, Debug)] pub struct TpuClientConfig { @@ -52,14 +57,14 @@ impl Default for TpuClientConfig { /// Client which sends transactions directly to the current leader's TPU port over UDP. /// The client uses RPC to determine the current leader and fetch node contact info -pub struct TpuClient { +pub struct TpuClient { _deprecated: UdpSocket, // TpuClient now uses the connection_cache to choose a send_socket //todo: get rid of this field rpc_client: Arc, - tpu_client: Arc, + tpu_client: Arc>, } -impl TpuClient { +impl TpuClient

{ /// Serialize and send transaction to the current and upcoming leader TPUs according to fanout /// size pub fn send_transaction(&self, transaction: &Transaction) -> bool { @@ -121,7 +126,7 @@ impl TpuClient { rpc_client: Arc, websocket_url: &str, config: TpuClientConfig, - connection_cache: Arc, + connection_cache: Arc>, ) -> Result { let create_tpu_client = NonblockingTpuClient::new_with_connection_cache( rpc_client.get_inner_client().clone(), diff --git a/tpu-client/src/tpu_connection.rs b/tpu-client/src/tpu_connection.rs index c5eb3b0da2..44a35c899c 100644 --- a/tpu-client/src/tpu_connection.rs +++ b/tpu-client/src/tpu_connection.rs @@ -1,6 +1,4 @@ use { - crate::{quic_client::QuicTpuConnection, udp_client::UdpTpuConnection}, - enum_dispatch::enum_dispatch, rayon::iter::{IntoParallelIterator, ParallelIterator}, solana_metrics::MovingStat, solana_sdk::{transaction::VersionedTransaction, transport::Result as TransportResult}, @@ -24,13 +22,6 @@ pub struct ClientStats { pub send_timeout: AtomicU64, } -#[enum_dispatch] -pub enum BlockingConnection { - UdpTpuConnection, - QuicTpuConnection, -} - -#[enum_dispatch(BlockingConnection)] pub trait TpuConnection { fn tpu_addr(&self) -> &SocketAddr; diff --git a/tpu-client/src/tpu_connection_cache.rs b/tpu-client/src/tpu_connection_cache.rs index 8b40377917..aa687634ba 100644 --- a/tpu-client/src/tpu_connection_cache.rs +++ b/tpu-client/src/tpu_connection_cache.rs @@ -16,7 +16,7 @@ use { }; // Should be non-zero -pub(crate) static MAX_CONNECTIONS: usize = 1024; +pub static MAX_CONNECTIONS: usize = 1024; /// Used to decide whether the TPU and underlying connection cache should use /// QUIC connections. @@ -262,7 +262,7 @@ pub enum ConnectionPoolError { } pub trait NewTpuConfig { - type ClientError; + type ClientError: std::fmt::Debug; fn new() -> Result where Self: Sized; diff --git a/udp-client/Cargo.toml b/udp-client/Cargo.toml index 8d324dff77..ff92cdd5f7 100644 --- a/udp-client/Cargo.toml +++ b/udp-client/Cargo.toml @@ -10,6 +10,10 @@ documentation = "https://docs.rs/solana-udp-client" edition = "2021" [dependencies] +async-trait = "0.1.57" solana-net-utils = { path = "../net-utils", version = "=1.15.0" } +solana-sdk = { path = "../sdk", version = "=1.15.0" } +solana-streamer = { path = "../streamer", version = "=1.15.0" } solana-tpu-client = { path = "../tpu-client", version = "=1.15.0" } thiserror = "1.0" +tokio = { version = "1", features = ["full"] } diff --git a/udp-client/src/nonblocking/udp_client.rs b/udp-client/src/nonblocking/udp_client.rs index 65d6fad2a1..9e8a318799 100644 --- a/udp-client/src/nonblocking/udp_client.rs +++ b/udp-client/src/nonblocking/udp_client.rs @@ -1,4 +1,93 @@ //! Simple UDP client that communicates with the given UDP port with UDP and provides //! an interface for sending transactions -pub use solana_tpu_client::nonblocking::udp_client::*; +use { + async_trait::async_trait, core::iter::repeat, solana_sdk::transport::Result as TransportResult, + solana_streamer::nonblocking::sendmmsg::batch_send, + solana_tpu_client::nonblocking::tpu_connection::TpuConnection, std::net::SocketAddr, + tokio::net::UdpSocket, +}; + +pub struct UdpTpuConnection { + pub socket: UdpSocket, + pub addr: SocketAddr, +} + +impl UdpTpuConnection { + pub fn new_from_addr(socket: std::net::UdpSocket, tpu_addr: SocketAddr) -> Self { + socket.set_nonblocking(true).unwrap(); + let socket = UdpSocket::from_std(socket).unwrap(); + Self { + socket, + addr: tpu_addr, + } + } +} + +#[async_trait] +impl TpuConnection for UdpTpuConnection { + fn tpu_addr(&self) -> &SocketAddr { + &self.addr + } + + async fn send_wire_transaction(&self, wire_transaction: T) -> TransportResult<()> + where + T: AsRef<[u8]> + Send + Sync, + { + self.socket + .send_to(wire_transaction.as_ref(), self.addr) + .await?; + Ok(()) + } + + async fn send_wire_transaction_batch(&self, buffers: &[T]) -> TransportResult<()> + where + T: AsRef<[u8]> + Send + Sync, + { + let pkts: Vec<_> = buffers.iter().zip(repeat(self.tpu_addr())).collect(); + batch_send(&self.socket, &pkts).await?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use { + super::*, + solana_sdk::packet::{Packet, PACKET_DATA_SIZE}, + solana_streamer::nonblocking::recvmmsg::recv_mmsg, + std::net::{IpAddr, Ipv4Addr}, + tokio::net::UdpSocket, + }; + + async fn check_send_one(connection: &UdpTpuConnection, reader: &UdpSocket) { + let packet = vec![111u8; PACKET_DATA_SIZE]; + connection.send_wire_transaction(&packet).await.unwrap(); + let mut packets = vec![Packet::default(); 32]; + let recv = recv_mmsg(reader, &mut packets[..]).await.unwrap(); + assert_eq!(1, recv); + } + + async fn check_send_batch(connection: &UdpTpuConnection, reader: &UdpSocket) { + let packets: Vec<_> = (0..32).map(|_| vec![0u8; PACKET_DATA_SIZE]).collect(); + connection + .send_wire_transaction_batch(&packets) + .await + .unwrap(); + let mut packets = vec![Packet::default(); 32]; + let recv = recv_mmsg(reader, &mut packets[..]).await.unwrap(); + assert_eq!(32, recv); + } + + #[tokio::test] + async fn test_send_from_addr() { + let addr_str = "0.0.0.0:50100"; + let addr = addr_str.parse().unwrap(); + let socket = + solana_net_utils::bind_with_any_port(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))).unwrap(); + let connection = UdpTpuConnection::new_from_addr(socket, addr); + let reader = UdpSocket::bind(addr_str).await.expect("bind"); + check_send_one(&connection, &reader).await; + check_send_batch(&connection, &reader).await; + } +} diff --git a/udp-client/src/udp_client.rs b/udp-client/src/udp_client.rs index 8689aa1c71..e090b056d0 100644 --- a/udp-client/src/udp_client.rs +++ b/udp-client/src/udp_client.rs @@ -1,4 +1,63 @@ //! Simple TPU client that communicates with the given UDP port with UDP and provides //! an interface for sending transactions -pub use solana_tpu_client::udp_client::*; +use { + core::iter::repeat, + solana_sdk::transport::Result as TransportResult, + solana_streamer::sendmmsg::batch_send, + solana_tpu_client::{ + connection_cache_stats::ConnectionCacheStats, tpu_connection::TpuConnection, + }, + std::{ + net::{SocketAddr, UdpSocket}, + sync::Arc, + }, +}; + +pub struct UdpTpuConnection { + pub socket: Arc, + pub addr: SocketAddr, +} + +impl UdpTpuConnection { + pub fn new_from_addr(local_socket: Arc, tpu_addr: SocketAddr) -> Self { + Self { + socket: local_socket, + addr: tpu_addr, + } + } + + pub fn new( + local_socket: Arc, + tpu_addr: SocketAddr, + _connection_stats: Arc, + ) -> Self { + Self::new_from_addr(local_socket, tpu_addr) + } +} + +impl TpuConnection for UdpTpuConnection { + fn tpu_addr(&self) -> &SocketAddr { + &self.addr + } + + fn send_wire_transaction_async(&self, wire_transaction: Vec) -> TransportResult<()> { + self.socket.send_to(wire_transaction.as_ref(), self.addr)?; + Ok(()) + } + + fn send_wire_transaction_batch(&self, buffers: &[T]) -> TransportResult<()> + where + T: AsRef<[u8]> + Send + Sync, + { + let pkts: Vec<_> = buffers.iter().zip(repeat(self.tpu_addr())).collect(); + batch_send(&self.socket, &pkts)?; + Ok(()) + } + + fn send_wire_transaction_batch_async(&self, buffers: Vec>) -> TransportResult<()> { + let pkts: Vec<_> = buffers.into_iter().zip(repeat(self.tpu_addr())).collect(); + batch_send(&self.socket, &pkts)?; + Ok(()) + } +} diff --git a/validator/src/main.rs b/validator/src/main.rs index f02bb9c882..7e572ed0bf 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -70,7 +70,7 @@ use { self, MAX_BATCH_SEND_RATE_MS, MAX_TRANSACTION_BATCH_SIZE, }, solana_streamer::socket::SocketAddrSpace, - solana_tpu_client::connection_cache::{ + solana_tpu_client::tpu_connection_cache::{ DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_ENABLE_UDP, }, solana_validator::{