2018-05-14 16:36:19 -07:00
|
|
|
//! The `tpu` module implements the Transaction Processing Unit, a
|
2019-01-08 08:21:39 -08:00
|
|
|
//! multi-stage transaction processing pipeline in software.
|
2018-05-14 16:36:19 -07:00
|
|
|
|
2023-03-31 08:42:49 -07:00
|
|
|
pub use solana_sdk::net::DEFAULT_TPU_COALESCE;
|
2021-12-03 09:00:31 -08:00
|
|
|
use {
|
|
|
|
crate::{
|
|
|
|
banking_stage::BankingStage,
|
2023-01-25 04:54:38 -08:00
|
|
|
banking_trace::{BankingTracer, TracerThread},
|
2021-12-03 09:00:31 -08:00
|
|
|
cluster_info_vote_listener::{
|
|
|
|
ClusterInfoVoteListener, GossipDuplicateConfirmedSlotsSender,
|
|
|
|
GossipVerifiedVoteHashSender, VerifiedVoteSender, VoteTracker,
|
|
|
|
},
|
|
|
|
fetch_stage::FetchStage,
|
|
|
|
sigverify::TransactionSigVerifier,
|
|
|
|
sigverify_stage::SigVerifyStage,
|
2022-04-16 01:54:22 -07:00
|
|
|
staked_nodes_updater_service::StakedNodesUpdaterService,
|
2023-06-05 17:19:17 -07:00
|
|
|
tpu_entry_notifier::TpuEntryNotifier,
|
2023-05-16 13:08:39 -07:00
|
|
|
validator::GeneratorConfig,
|
2021-03-24 23:41:52 -07:00
|
|
|
},
|
2023-07-12 07:15:28 -07:00
|
|
|
bytes::Bytes,
|
2022-06-22 08:57:46 -07:00
|
|
|
crossbeam_channel::{unbounded, Receiver},
|
2023-05-12 09:16:20 -07:00
|
|
|
solana_client::connection_cache::{ConnectionCache, Protocol},
|
2021-12-03 09:00:31 -08:00
|
|
|
solana_gossip::cluster_info::ClusterInfo,
|
2023-05-10 16:20:51 -07:00
|
|
|
solana_ledger::{
|
|
|
|
blockstore::Blockstore, blockstore_processor::TransactionStatusSender,
|
|
|
|
entry_notifier_service::EntryNotifierSender,
|
|
|
|
},
|
2021-12-03 09:00:31 -08:00
|
|
|
solana_poh::poh_recorder::{PohRecorder, WorkingBankEntry},
|
|
|
|
solana_rpc::{
|
|
|
|
optimistically_confirmed_bank_tracker::BankNotificationSender,
|
|
|
|
rpc_subscriptions::RpcSubscriptions,
|
|
|
|
},
|
|
|
|
solana_runtime::{
|
|
|
|
bank_forks::BankForks,
|
2023-03-23 17:05:54 -07:00
|
|
|
prioritization_fee_cache::PrioritizationFeeCache,
|
2021-12-03 09:00:31 -08:00
|
|
|
vote_sender_types::{ReplayVoteReceiver, ReplayVoteSender},
|
|
|
|
},
|
2023-06-22 09:22:11 -07:00
|
|
|
solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Keypair},
|
2022-06-21 12:06:44 -07:00
|
|
|
solana_streamer::{
|
2023-03-31 08:42:49 -07:00
|
|
|
nonblocking::quic::DEFAULT_WAIT_FOR_CHUNK_TIMEOUT,
|
2023-05-25 09:54:24 -07:00
|
|
|
quic::{spawn_server, MAX_STAKED_CONNECTIONS, MAX_UNSTAKED_CONNECTIONS},
|
2022-06-21 12:06:44 -07:00
|
|
|
streamer::StakedNodes,
|
2022-06-02 11:14:58 -07:00
|
|
|
},
|
2023-06-22 09:22:11 -07:00
|
|
|
solana_turbine::broadcast_stage::{BroadcastStage, BroadcastStageType},
|
2021-12-03 09:00:31 -08:00
|
|
|
std::{
|
2022-08-11 14:34:04 -07:00
|
|
|
collections::HashMap,
|
2023-07-12 07:15:28 -07:00
|
|
|
net::{SocketAddr, UdpSocket},
|
2022-07-05 07:29:44 -07:00
|
|
|
sync::{atomic::AtomicBool, Arc, RwLock},
|
2021-12-03 09:00:31 -08:00
|
|
|
thread,
|
2023-03-31 08:42:49 -07:00
|
|
|
time::Duration,
|
2019-11-20 15:43:10 -08:00
|
|
|
},
|
2023-07-12 07:15:28 -07:00
|
|
|
tokio::sync::mpsc::Sender as AsyncSender,
|
2019-11-20 15:43:10 -08:00
|
|
|
};
|
2018-09-14 00:17:40 -07:00
|
|
|
|
2022-03-09 01:52:31 -08:00
|
|
|
// allow multiple connections for NAT and any open/close overlap
|
2022-07-13 09:59:01 -07:00
|
|
|
pub const MAX_QUIC_CONNECTIONS_PER_PEER: usize = 8;
|
2022-03-09 01:52:31 -08:00
|
|
|
|
2022-01-10 09:29:48 -08:00
|
|
|
pub struct TpuSockets {
|
|
|
|
pub transactions: Vec<UdpSocket>,
|
|
|
|
pub transaction_forwards: Vec<UdpSocket>,
|
|
|
|
pub vote: Vec<UdpSocket>,
|
|
|
|
pub broadcast: Vec<UdpSocket>,
|
2022-02-04 06:27:09 -08:00
|
|
|
pub transactions_quic: UdpSocket,
|
2022-06-02 11:14:58 -07:00
|
|
|
pub transactions_forwards_quic: UdpSocket,
|
2022-01-10 09:29:48 -08:00
|
|
|
}
|
|
|
|
|
2019-03-04 20:08:21 -08:00
|
|
|
pub struct Tpu {
|
2019-02-16 18:03:55 -08:00
|
|
|
fetch_stage: FetchStage,
|
|
|
|
sigverify_stage: SigVerifyStage,
|
2021-10-07 02:38:23 -07:00
|
|
|
vote_sigverify_stage: SigVerifyStage,
|
2019-02-16 18:03:55 -08:00
|
|
|
banking_stage: BankingStage,
|
|
|
|
cluster_info_vote_listener: ClusterInfoVoteListener,
|
2019-03-01 19:43:30 -08:00
|
|
|
broadcast_stage: BroadcastStage,
|
2022-08-19 07:15:15 -07:00
|
|
|
tpu_quic_t: thread::JoinHandle<()>,
|
|
|
|
tpu_forwards_quic_t: thread::JoinHandle<()>,
|
2023-06-05 17:19:17 -07:00
|
|
|
tpu_entry_notifier: Option<TpuEntryNotifier>,
|
2022-04-16 01:54:22 -07:00
|
|
|
staked_nodes_updater_service: StakedNodesUpdaterService,
|
2023-01-25 04:54:38 -08:00
|
|
|
tracer_thread_hdl: TracerThread,
|
2019-01-26 00:28:08 -08:00
|
|
|
}
|
|
|
|
|
2018-05-14 16:36:19 -07:00
|
|
|
impl Tpu {
|
2019-03-09 02:47:41 -08:00
|
|
|
#[allow(clippy::too_many_arguments)]
|
2019-03-03 16:44:06 -08:00
|
|
|
pub fn new(
|
2020-04-21 12:54:45 -07:00
|
|
|
cluster_info: &Arc<ClusterInfo>,
|
2022-07-05 07:29:44 -07:00
|
|
|
poh_recorder: &Arc<RwLock<PohRecorder>>,
|
2019-09-18 12:16:22 -07:00
|
|
|
entry_receiver: Receiver<WorkingBankEntry>,
|
2023-06-22 09:22:11 -07:00
|
|
|
retransmit_slots_receiver: Receiver<Slot>,
|
2022-01-10 09:29:48 -08:00
|
|
|
sockets: TpuSockets,
|
2020-05-17 14:01:08 -07:00
|
|
|
subscriptions: &Arc<RpcSubscriptions>,
|
2019-11-20 15:43:10 -08:00
|
|
|
transaction_status_sender: Option<TransactionStatusSender>,
|
2023-06-05 17:19:17 -07:00
|
|
|
entry_notification_sender: Option<EntryNotifierSender>,
|
2023-08-03 14:10:25 -07:00
|
|
|
blockstore: Arc<Blockstore>,
|
2019-06-19 00:13:19 -07:00
|
|
|
broadcast_type: &BroadcastStageType,
|
2023-06-01 17:25:48 -07:00
|
|
|
exit: Arc<AtomicBool>,
|
2019-11-18 18:05:02 -08:00
|
|
|
shred_version: u16,
|
2020-03-09 22:03:09 -07:00
|
|
|
vote_tracker: Arc<VoteTracker>,
|
|
|
|
bank_forks: Arc<RwLock<BankForks>>,
|
2020-07-09 22:52:54 -07:00
|
|
|
verified_vote_sender: VerifiedVoteSender,
|
2021-04-10 17:34:45 -07:00
|
|
|
gossip_verified_vote_hash_sender: GossipVerifiedVoteHashSender,
|
2020-08-07 11:21:35 -07:00
|
|
|
replay_vote_receiver: ReplayVoteReceiver,
|
|
|
|
replay_vote_sender: ReplayVoteSender,
|
2020-09-28 19:43:05 -07:00
|
|
|
bank_notification_sender: Option<BankNotificationSender>,
|
2023-03-31 08:42:49 -07:00
|
|
|
tpu_coalesce: Duration,
|
2021-03-24 23:41:52 -07:00
|
|
|
cluster_confirmed_slot_sender: GossipDuplicateConfirmedSlotsSender,
|
2022-06-08 04:57:12 -07:00
|
|
|
connection_cache: &Arc<ConnectionCache>,
|
2023-07-12 07:15:28 -07:00
|
|
|
turbine_quic_endpoint_sender: AsyncSender<(SocketAddr, Bytes)>,
|
2022-02-04 06:27:09 -08:00
|
|
|
keypair: &Keypair,
|
2022-07-11 08:53:18 -07:00
|
|
|
log_messages_bytes_limit: Option<usize>,
|
2022-07-29 08:44:24 -07:00
|
|
|
staked_nodes: &Arc<RwLock<StakedNodes>>,
|
2022-08-11 14:34:04 -07:00
|
|
|
shared_staked_nodes_overrides: Arc<RwLock<HashMap<Pubkey, u64>>>,
|
2023-01-25 04:54:38 -08:00
|
|
|
banking_tracer: Arc<BankingTracer>,
|
|
|
|
tracer_thread_hdl: TracerThread,
|
2022-09-07 13:19:14 -07:00
|
|
|
tpu_enable_udp: bool,
|
2023-03-23 17:05:54 -07:00
|
|
|
prioritization_fee_cache: &Arc<PrioritizationFeeCache>,
|
2023-05-16 13:08:39 -07:00
|
|
|
_generator_config: Option<GeneratorConfig>, /* vestigial code for replay invalidator */
|
2019-03-03 16:44:06 -08:00
|
|
|
) -> Self {
|
2022-01-10 09:29:48 -08:00
|
|
|
let TpuSockets {
|
|
|
|
transactions: transactions_sockets,
|
|
|
|
transaction_forwards: tpu_forwards_sockets,
|
|
|
|
vote: tpu_vote_sockets,
|
|
|
|
broadcast: broadcast_sockets,
|
2022-02-04 06:27:09 -08:00
|
|
|
transactions_quic: transactions_quic_sockets,
|
2022-06-02 11:14:58 -07:00
|
|
|
transactions_forwards_quic: transactions_forwards_quic_sockets,
|
2022-01-10 09:29:48 -08:00
|
|
|
} = sockets;
|
|
|
|
|
2022-01-11 02:44:46 -08:00
|
|
|
let (packet_sender, packet_receiver) = unbounded();
|
|
|
|
let (vote_packet_sender, vote_packet_receiver) = unbounded();
|
2022-06-02 11:14:58 -07:00
|
|
|
let (forwarded_packet_sender, forwarded_packet_receiver) = unbounded();
|
2019-03-08 14:59:11 -08:00
|
|
|
let fetch_stage = FetchStage::new_with_sender(
|
|
|
|
transactions_sockets,
|
2019-07-30 14:50:02 -07:00
|
|
|
tpu_forwards_sockets,
|
2021-10-07 02:38:23 -07:00
|
|
|
tpu_vote_sockets,
|
2023-06-01 17:25:48 -07:00
|
|
|
exit.clone(),
|
2019-04-09 12:57:12 -07:00
|
|
|
&packet_sender,
|
2021-10-07 02:38:23 -07:00
|
|
|
&vote_packet_sender,
|
2022-06-02 11:14:58 -07:00
|
|
|
&forwarded_packet_sender,
|
|
|
|
forwarded_packet_receiver,
|
2021-06-18 06:34:46 -07:00
|
|
|
poh_recorder,
|
2023-03-31 08:42:49 -07:00
|
|
|
tpu_coalesce,
|
2022-05-14 08:53:37 -07:00
|
|
|
Some(bank_forks.read().unwrap().get_vote_only_mode_signal()),
|
2022-09-07 13:19:14 -07:00
|
|
|
tpu_enable_udp,
|
2019-03-08 14:59:11 -08:00
|
|
|
);
|
2021-12-16 06:47:55 -08:00
|
|
|
|
2022-05-20 12:51:07 -07:00
|
|
|
let staked_nodes_updater_service = StakedNodesUpdaterService::new(
|
|
|
|
exit.clone(),
|
|
|
|
bank_forks.clone(),
|
|
|
|
staked_nodes.clone(),
|
2022-08-11 14:34:04 -07:00
|
|
|
shared_staked_nodes_overrides,
|
2022-05-20 12:51:07 -07:00
|
|
|
);
|
|
|
|
|
2023-01-25 04:54:38 -08:00
|
|
|
let (non_vote_sender, non_vote_receiver) = banking_tracer.create_channel_non_vote();
|
2018-05-14 16:36:19 -07:00
|
|
|
|
2022-12-09 10:59:43 -08:00
|
|
|
let (_, tpu_quic_t) = spawn_server(
|
2023-05-25 09:54:24 -07:00
|
|
|
"quic_streamer_tpu",
|
2022-08-19 07:15:15 -07:00
|
|
|
transactions_quic_sockets,
|
|
|
|
keypair,
|
2023-02-10 12:07:45 -08:00
|
|
|
cluster_info
|
|
|
|
.my_contact_info()
|
2023-05-12 09:16:20 -07:00
|
|
|
.tpu(Protocol::QUIC)
|
2023-02-10 12:07:45 -08:00
|
|
|
.expect("Operator must spin up node with valid (QUIC) TPU address")
|
|
|
|
.ip(),
|
2022-08-19 07:15:15 -07:00
|
|
|
packet_sender,
|
|
|
|
exit.clone(),
|
|
|
|
MAX_QUIC_CONNECTIONS_PER_PEER,
|
|
|
|
staked_nodes.clone(),
|
|
|
|
MAX_STAKED_CONNECTIONS,
|
|
|
|
MAX_UNSTAKED_CONNECTIONS,
|
2023-03-31 08:42:49 -07:00
|
|
|
DEFAULT_WAIT_FOR_CHUNK_TIMEOUT,
|
|
|
|
tpu_coalesce,
|
2022-08-19 07:15:15 -07:00
|
|
|
)
|
|
|
|
.unwrap();
|
2022-06-02 11:14:58 -07:00
|
|
|
|
2022-12-09 10:59:43 -08:00
|
|
|
let (_, tpu_forwards_quic_t) = spawn_server(
|
2023-05-25 09:54:24 -07:00
|
|
|
"quic_streamer_tpu_forwards",
|
2022-08-19 07:15:15 -07:00
|
|
|
transactions_forwards_quic_sockets,
|
|
|
|
keypair,
|
2023-02-10 12:07:45 -08:00
|
|
|
cluster_info
|
|
|
|
.my_contact_info()
|
2023-05-12 09:16:20 -07:00
|
|
|
.tpu_forwards(Protocol::QUIC)
|
2023-02-10 12:07:45 -08:00
|
|
|
.expect("Operator must spin up node with valid (QUIC) TPU-forwards address")
|
|
|
|
.ip(),
|
2022-08-19 07:15:15 -07:00
|
|
|
forwarded_packet_sender,
|
|
|
|
exit.clone(),
|
|
|
|
MAX_QUIC_CONNECTIONS_PER_PEER,
|
|
|
|
staked_nodes.clone(),
|
|
|
|
MAX_STAKED_CONNECTIONS.saturating_add(MAX_UNSTAKED_CONNECTIONS),
|
|
|
|
0, // Prevent unstaked nodes from forwarding transactions
|
2023-03-31 08:42:49 -07:00
|
|
|
DEFAULT_WAIT_FOR_CHUNK_TIMEOUT,
|
|
|
|
tpu_coalesce,
|
2022-08-19 07:15:15 -07:00
|
|
|
)
|
|
|
|
.unwrap();
|
2022-02-04 06:27:09 -08:00
|
|
|
|
2020-05-08 10:00:23 -07:00
|
|
|
let sigverify_stage = {
|
2023-01-18 17:04:55 -08:00
|
|
|
let verifier = TransactionSigVerifier::new(non_vote_sender);
|
2023-04-06 14:33:43 -07:00
|
|
|
SigVerifyStage::new(packet_receiver, verifier, "tpu-verifier")
|
2019-10-28 16:07:51 -07:00
|
|
|
};
|
2019-04-09 12:57:12 -07:00
|
|
|
|
2023-01-25 04:54:38 -08:00
|
|
|
let (tpu_vote_sender, tpu_vote_receiver) = banking_tracer.create_channel_tpu_vote();
|
2021-10-07 02:38:23 -07:00
|
|
|
|
|
|
|
let vote_sigverify_stage = {
|
2023-01-18 17:04:55 -08:00
|
|
|
let verifier = TransactionSigVerifier::new_reject_non_vote(tpu_vote_sender);
|
2023-04-06 14:33:43 -07:00
|
|
|
SigVerifyStage::new(vote_packet_receiver, verifier, "tpu-vote-verifier")
|
2021-10-07 02:38:23 -07:00
|
|
|
};
|
|
|
|
|
2023-01-25 04:54:38 -08:00
|
|
|
let (gossip_vote_sender, gossip_vote_receiver) =
|
|
|
|
banking_tracer.create_channel_gossip_vote();
|
2019-04-09 12:57:12 -07:00
|
|
|
let cluster_info_vote_listener = ClusterInfoVoteListener::new(
|
2021-12-30 07:03:14 -08:00
|
|
|
exit.clone(),
|
2019-04-09 12:57:12 -07:00
|
|
|
cluster_info.clone(),
|
2023-01-18 17:04:55 -08:00
|
|
|
gossip_vote_sender,
|
2021-12-30 07:03:14 -08:00
|
|
|
poh_recorder.clone(),
|
2020-03-09 22:03:09 -07:00
|
|
|
vote_tracker,
|
2021-07-07 08:21:12 -07:00
|
|
|
bank_forks.clone(),
|
2020-05-17 14:01:08 -07:00
|
|
|
subscriptions.clone(),
|
2020-07-09 22:52:54 -07:00
|
|
|
verified_vote_sender,
|
2021-04-10 17:34:45 -07:00
|
|
|
gossip_verified_vote_hash_sender,
|
2020-08-07 11:21:35 -07:00
|
|
|
replay_vote_receiver,
|
2020-07-28 02:33:27 -07:00
|
|
|
blockstore.clone(),
|
2020-09-28 19:43:05 -07:00
|
|
|
bank_notification_sender,
|
2021-03-24 23:41:52 -07:00
|
|
|
cluster_confirmed_slot_sender,
|
2019-04-09 12:57:12 -07:00
|
|
|
);
|
2018-05-14 16:36:19 -07:00
|
|
|
|
2019-04-17 21:07:45 -07:00
|
|
|
let banking_stage = BankingStage::new(
|
2021-06-18 06:34:46 -07:00
|
|
|
cluster_info,
|
2019-04-17 21:07:45 -07:00
|
|
|
poh_recorder,
|
2023-01-18 17:04:55 -08:00
|
|
|
non_vote_receiver,
|
|
|
|
tpu_vote_receiver,
|
|
|
|
gossip_vote_receiver,
|
2019-11-20 15:43:10 -08:00
|
|
|
transaction_status_sender,
|
2020-08-07 11:21:35 -07:00
|
|
|
replay_vote_sender,
|
2022-07-11 08:53:18 -07:00
|
|
|
log_messages_bytes_limit,
|
2022-06-08 04:57:12 -07:00
|
|
|
connection_cache.clone(),
|
2022-07-05 21:24:58 -07:00
|
|
|
bank_forks.clone(),
|
2023-03-23 17:05:54 -07:00
|
|
|
prioritization_fee_cache,
|
2019-04-17 21:07:45 -07:00
|
|
|
);
|
2019-01-26 00:28:08 -08:00
|
|
|
|
2023-06-05 17:19:17 -07:00
|
|
|
let (entry_receiver, tpu_entry_notifier) =
|
|
|
|
if let Some(entry_notification_sender) = entry_notification_sender {
|
|
|
|
let (broadcast_entry_sender, broadcast_entry_receiver) = unbounded();
|
|
|
|
let tpu_entry_notifier = TpuEntryNotifier::new(
|
|
|
|
entry_receiver,
|
|
|
|
entry_notification_sender,
|
|
|
|
broadcast_entry_sender,
|
|
|
|
exit.clone(),
|
|
|
|
);
|
|
|
|
(broadcast_entry_receiver, Some(tpu_entry_notifier))
|
|
|
|
} else {
|
|
|
|
(entry_receiver, None)
|
|
|
|
};
|
|
|
|
|
2019-06-19 00:13:19 -07:00
|
|
|
let broadcast_stage = broadcast_type.new_broadcast_stage(
|
2019-12-16 17:11:18 -08:00
|
|
|
broadcast_sockets,
|
2019-03-03 16:44:06 -08:00
|
|
|
cluster_info.clone(),
|
2019-01-26 00:28:08 -08:00
|
|
|
entry_receiver,
|
2020-03-19 23:35:01 -07:00
|
|
|
retransmit_slots_receiver,
|
2023-06-01 17:25:48 -07:00
|
|
|
exit,
|
2023-08-03 14:10:25 -07:00
|
|
|
blockstore,
|
2022-06-15 06:44:24 -07:00
|
|
|
bank_forks,
|
2019-11-18 18:05:02 -08:00
|
|
|
shred_version,
|
2023-07-12 07:15:28 -07:00
|
|
|
turbine_quic_endpoint_sender,
|
2018-10-18 22:57:48 -07:00
|
|
|
);
|
2018-05-14 16:36:19 -07:00
|
|
|
|
2019-03-04 20:08:21 -08:00
|
|
|
Self {
|
2018-07-03 21:14:08 -07:00
|
|
|
fetch_stage,
|
|
|
|
sigverify_stage,
|
2021-10-07 02:38:23 -07:00
|
|
|
vote_sigverify_stage,
|
2018-07-03 21:14:08 -07:00
|
|
|
banking_stage,
|
2019-01-31 15:51:29 -08:00
|
|
|
cluster_info_vote_listener,
|
2019-03-01 19:43:30 -08:00
|
|
|
broadcast_stage,
|
2022-02-04 06:27:09 -08:00
|
|
|
tpu_quic_t,
|
2022-06-02 11:14:58 -07:00
|
|
|
tpu_forwards_quic_t,
|
2023-06-05 17:19:17 -07:00
|
|
|
tpu_entry_notifier,
|
2022-04-16 01:54:22 -07:00
|
|
|
staked_nodes_updater_service,
|
2023-01-25 04:54:38 -08:00
|
|
|
tracer_thread_hdl,
|
2019-03-03 16:44:06 -08:00
|
|
|
}
|
2019-01-26 00:28:08 -08:00
|
|
|
}
|
2018-07-03 21:14:08 -07:00
|
|
|
|
2019-11-13 10:12:09 -08:00
|
|
|
pub fn join(self) -> thread::Result<()> {
|
2021-01-23 11:55:15 -08:00
|
|
|
let results = vec![
|
|
|
|
self.fetch_stage.join(),
|
|
|
|
self.sigverify_stage.join(),
|
2021-10-07 02:38:23 -07:00
|
|
|
self.vote_sigverify_stage.join(),
|
2021-01-23 11:55:15 -08:00
|
|
|
self.cluster_info_vote_listener.join(),
|
|
|
|
self.banking_stage.join(),
|
2022-04-16 01:54:22 -07:00
|
|
|
self.staked_nodes_updater_service.join(),
|
2022-08-19 07:15:15 -07:00
|
|
|
self.tpu_quic_t.join(),
|
|
|
|
self.tpu_forwards_quic_t.join(),
|
2021-01-23 11:55:15 -08:00
|
|
|
];
|
2019-03-04 20:08:21 -08:00
|
|
|
let broadcast_result = self.broadcast_stage.join();
|
|
|
|
for result in results {
|
|
|
|
result?;
|
|
|
|
}
|
2023-06-05 17:19:17 -07:00
|
|
|
if let Some(tpu_entry_notifier) = self.tpu_entry_notifier {
|
|
|
|
tpu_entry_notifier.join()?;
|
|
|
|
}
|
2019-03-04 20:08:21 -08:00
|
|
|
let _ = broadcast_result?;
|
2023-01-25 04:54:38 -08:00
|
|
|
if let Some(tracer_thread_hdl) = self.tracer_thread_hdl {
|
|
|
|
if let Err(tracer_result) = tracer_thread_hdl.join()? {
|
|
|
|
error!(
|
|
|
|
"banking tracer thread returned error after successful thread join: {:?}",
|
|
|
|
tracer_result
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
2019-03-04 20:08:21 -08:00
|
|
|
Ok(())
|
2018-05-14 16:36:19 -07:00
|
|
|
}
|
|
|
|
}
|