Send messages to EntryNotifierService in Tpu, simpler (#31962)
* Add TpuEntryNotifier to send EntryNotifications from Tpu * Optionally run TpuEntryNotifier to send out EntrySummarys alongside BroadcastStage messages * Track entry index in TpuEntryNotifier * Allow for leader slots that switch forks * Exit if broadcast send fails
This commit is contained in:
parent
6371240746
commit
841609959c
|
@ -74,6 +74,7 @@ mod tower1_14_11;
|
||||||
mod tower1_7_14;
|
mod tower1_7_14;
|
||||||
pub mod tower_storage;
|
pub mod tower_storage;
|
||||||
pub mod tpu;
|
pub mod tpu;
|
||||||
|
mod tpu_entry_notifier;
|
||||||
pub mod tracer_packet_stats;
|
pub mod tracer_packet_stats;
|
||||||
pub mod tree_diff;
|
pub mod tree_diff;
|
||||||
pub mod tvu;
|
pub mod tvu;
|
||||||
|
|
|
@ -15,6 +15,7 @@ use {
|
||||||
sigverify::TransactionSigVerifier,
|
sigverify::TransactionSigVerifier,
|
||||||
sigverify_stage::SigVerifyStage,
|
sigverify_stage::SigVerifyStage,
|
||||||
staked_nodes_updater_service::StakedNodesUpdaterService,
|
staked_nodes_updater_service::StakedNodesUpdaterService,
|
||||||
|
tpu_entry_notifier::TpuEntryNotifier,
|
||||||
validator::GeneratorConfig,
|
validator::GeneratorConfig,
|
||||||
},
|
},
|
||||||
crossbeam_channel::{unbounded, Receiver},
|
crossbeam_channel::{unbounded, Receiver},
|
||||||
|
@ -70,6 +71,7 @@ pub struct Tpu {
|
||||||
broadcast_stage: BroadcastStage,
|
broadcast_stage: BroadcastStage,
|
||||||
tpu_quic_t: thread::JoinHandle<()>,
|
tpu_quic_t: thread::JoinHandle<()>,
|
||||||
tpu_forwards_quic_t: thread::JoinHandle<()>,
|
tpu_forwards_quic_t: thread::JoinHandle<()>,
|
||||||
|
tpu_entry_notifier: Option<TpuEntryNotifier>,
|
||||||
staked_nodes_updater_service: StakedNodesUpdaterService,
|
staked_nodes_updater_service: StakedNodesUpdaterService,
|
||||||
tracer_thread_hdl: TracerThread,
|
tracer_thread_hdl: TracerThread,
|
||||||
}
|
}
|
||||||
|
@ -84,7 +86,7 @@ impl Tpu {
|
||||||
sockets: TpuSockets,
|
sockets: TpuSockets,
|
||||||
subscriptions: &Arc<RpcSubscriptions>,
|
subscriptions: &Arc<RpcSubscriptions>,
|
||||||
transaction_status_sender: Option<TransactionStatusSender>,
|
transaction_status_sender: Option<TransactionStatusSender>,
|
||||||
_entry_notification_sender: Option<EntryNotifierSender>,
|
entry_notification_sender: Option<EntryNotifierSender>,
|
||||||
blockstore: &Arc<Blockstore>,
|
blockstore: &Arc<Blockstore>,
|
||||||
broadcast_type: &BroadcastStageType,
|
broadcast_type: &BroadcastStageType,
|
||||||
exit: Arc<AtomicBool>,
|
exit: Arc<AtomicBool>,
|
||||||
|
@ -229,6 +231,20 @@ impl Tpu {
|
||||||
prioritization_fee_cache,
|
prioritization_fee_cache,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
let (entry_receiver, tpu_entry_notifier) =
|
||||||
|
if let Some(entry_notification_sender) = entry_notification_sender {
|
||||||
|
let (broadcast_entry_sender, broadcast_entry_receiver) = unbounded();
|
||||||
|
let tpu_entry_notifier = TpuEntryNotifier::new(
|
||||||
|
entry_receiver,
|
||||||
|
entry_notification_sender,
|
||||||
|
broadcast_entry_sender,
|
||||||
|
exit.clone(),
|
||||||
|
);
|
||||||
|
(broadcast_entry_receiver, Some(tpu_entry_notifier))
|
||||||
|
} else {
|
||||||
|
(entry_receiver, None)
|
||||||
|
};
|
||||||
|
|
||||||
let broadcast_stage = broadcast_type.new_broadcast_stage(
|
let broadcast_stage = broadcast_type.new_broadcast_stage(
|
||||||
broadcast_sockets,
|
broadcast_sockets,
|
||||||
cluster_info.clone(),
|
cluster_info.clone(),
|
||||||
|
@ -249,6 +265,7 @@ impl Tpu {
|
||||||
broadcast_stage,
|
broadcast_stage,
|
||||||
tpu_quic_t,
|
tpu_quic_t,
|
||||||
tpu_forwards_quic_t,
|
tpu_forwards_quic_t,
|
||||||
|
tpu_entry_notifier,
|
||||||
staked_nodes_updater_service,
|
staked_nodes_updater_service,
|
||||||
tracer_thread_hdl,
|
tracer_thread_hdl,
|
||||||
}
|
}
|
||||||
|
@ -269,6 +286,9 @@ impl Tpu {
|
||||||
for result in results {
|
for result in results {
|
||||||
result?;
|
result?;
|
||||||
}
|
}
|
||||||
|
if let Some(tpu_entry_notifier) = self.tpu_entry_notifier {
|
||||||
|
tpu_entry_notifier.join()?;
|
||||||
|
}
|
||||||
let _ = broadcast_result?;
|
let _ = broadcast_result?;
|
||||||
if let Some(tracer_thread_hdl) = self.tracer_thread_hdl {
|
if let Some(tracer_thread_hdl) = self.tracer_thread_hdl {
|
||||||
if let Err(tracer_result) = tracer_thread_hdl.join()? {
|
if let Err(tracer_result) = tracer_thread_hdl.join()? {
|
||||||
|
|
|
@ -0,0 +1,101 @@
|
||||||
|
use {
|
||||||
|
crossbeam_channel::{Receiver, RecvTimeoutError, Sender},
|
||||||
|
solana_entry::entry::EntrySummary,
|
||||||
|
solana_ledger::entry_notifier_service::{EntryNotification, EntryNotifierSender},
|
||||||
|
solana_poh::poh_recorder::WorkingBankEntry,
|
||||||
|
std::{
|
||||||
|
sync::{
|
||||||
|
atomic::{AtomicBool, Ordering},
|
||||||
|
Arc,
|
||||||
|
},
|
||||||
|
thread::{self, Builder, JoinHandle},
|
||||||
|
time::Duration,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
pub(crate) struct TpuEntryNotifier {
|
||||||
|
thread_hdl: JoinHandle<()>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TpuEntryNotifier {
|
||||||
|
pub(crate) fn new(
|
||||||
|
entry_receiver: Receiver<WorkingBankEntry>,
|
||||||
|
entry_notification_sender: EntryNotifierSender,
|
||||||
|
broadcast_entry_sender: Sender<WorkingBankEntry>,
|
||||||
|
exit: Arc<AtomicBool>,
|
||||||
|
) -> Self {
|
||||||
|
let thread_hdl = Builder::new()
|
||||||
|
.name("solTpuEntry".to_string())
|
||||||
|
.spawn(move || {
|
||||||
|
let mut current_slot = 0;
|
||||||
|
let mut current_index = 0;
|
||||||
|
loop {
|
||||||
|
if exit.load(Ordering::Relaxed) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Err(RecvTimeoutError::Disconnected) = Self::send_entry_notification(
|
||||||
|
exit.clone(),
|
||||||
|
&entry_receiver,
|
||||||
|
&entry_notification_sender,
|
||||||
|
&broadcast_entry_sender,
|
||||||
|
&mut current_slot,
|
||||||
|
&mut current_index,
|
||||||
|
) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
Self { thread_hdl }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn send_entry_notification(
|
||||||
|
exit: Arc<AtomicBool>,
|
||||||
|
entry_receiver: &Receiver<WorkingBankEntry>,
|
||||||
|
entry_notification_sender: &EntryNotifierSender,
|
||||||
|
broadcast_entry_sender: &Sender<WorkingBankEntry>,
|
||||||
|
current_slot: &mut u64,
|
||||||
|
current_index: &mut usize,
|
||||||
|
) -> Result<(), RecvTimeoutError> {
|
||||||
|
let (bank, (entry, tick_height)) = entry_receiver.recv_timeout(Duration::from_secs(1))?;
|
||||||
|
let slot = bank.slot();
|
||||||
|
let index = if slot != *current_slot {
|
||||||
|
*current_index = 0;
|
||||||
|
*current_slot = slot;
|
||||||
|
0
|
||||||
|
} else {
|
||||||
|
*current_index += 1;
|
||||||
|
*current_index
|
||||||
|
};
|
||||||
|
|
||||||
|
let entry_summary = EntrySummary {
|
||||||
|
num_hashes: entry.num_hashes,
|
||||||
|
hash: entry.hash,
|
||||||
|
num_transactions: entry.transactions.len() as u64,
|
||||||
|
};
|
||||||
|
if let Err(err) = entry_notification_sender.send(EntryNotification {
|
||||||
|
slot,
|
||||||
|
index,
|
||||||
|
entry: entry_summary,
|
||||||
|
}) {
|
||||||
|
warn!(
|
||||||
|
"Failed to send slot {slot:?} entry {index:?} from Tpu to EntryNotifierService, error {err:?}",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Err(err) = broadcast_entry_sender.send((bank, (entry, tick_height))) {
|
||||||
|
warn!(
|
||||||
|
"Failed to send slot {slot:?} entry {index:?} from Tpu to BroadcastStage, error {err:?}",
|
||||||
|
);
|
||||||
|
// If the BroadcastStage channel is closed, the validator has halted. Try to exit
|
||||||
|
// gracefully.
|
||||||
|
exit.store(true, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn join(self) -> thread::Result<()> {
|
||||||
|
self.thread_hdl.join()
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue