2021-12-03 09:00:31 -08:00
|
|
|
use {
|
2022-07-07 04:13:13 -07:00
|
|
|
crossbeam_channel::{Receiver, RecvTimeoutError, SendError, Sender},
|
2023-03-22 06:19:16 -07:00
|
|
|
rayon::{prelude::*, ThreadPool, ThreadPoolBuilder},
|
2023-01-21 12:07:41 -08:00
|
|
|
solana_gossip::cluster_info::ClusterInfo,
|
2021-12-03 09:00:31 -08:00
|
|
|
solana_ledger::{
|
2022-05-26 06:06:27 -07:00
|
|
|
leader_schedule_cache::LeaderScheduleCache, shred, sigverify_shreds::verify_shreds_gpu,
|
2021-12-03 09:00:31 -08:00
|
|
|
},
|
2023-03-22 06:19:16 -07:00
|
|
|
solana_perf::{self, deduper::Deduper, packet::PacketBatch, recycler_cache::RecyclerCache},
|
2023-03-20 13:33:22 -07:00
|
|
|
solana_rayon_threadlimit::get_thread_count,
|
2022-06-27 13:12:23 -07:00
|
|
|
solana_runtime::{bank::Bank, bank_forks::BankForks},
|
2023-01-24 08:57:55 -08:00
|
|
|
solana_sdk::{clock::Slot, pubkey::Pubkey},
|
2021-12-03 09:00:31 -08:00
|
|
|
std::{
|
2022-06-27 13:12:23 -07:00
|
|
|
collections::HashMap,
|
2023-03-20 13:34:41 -07:00
|
|
|
sync::{Arc, RwLock},
|
2022-07-07 04:13:13 -07:00
|
|
|
thread::{Builder, JoinHandle},
|
|
|
|
time::{Duration, Instant},
|
2021-12-03 09:00:31 -08:00
|
|
|
},
|
|
|
|
};
|
2019-10-28 10:29:38 -07:00
|
|
|
|
2023-03-22 06:19:16 -07:00
|
|
|
const DEDUPER_FALSE_POSITIVE_RATE: f64 = 0.001;
|
|
|
|
const DEDUPER_NUM_BITS: u64 = 637_534_199; // 76MB
|
|
|
|
const DEDUPER_RESET_CYCLE: Duration = Duration::from_secs(5 * 60);
|
|
|
|
|
2022-07-07 04:13:13 -07:00
|
|
|
#[allow(clippy::enum_variant_names)]
|
|
|
|
enum Error {
|
|
|
|
RecvDisconnected,
|
|
|
|
RecvTimeout,
|
|
|
|
SendError,
|
|
|
|
}
|
|
|
|
|
2023-06-22 09:22:11 -07:00
|
|
|
pub fn spawn_shred_sigverify(
|
2023-01-21 12:07:41 -08:00
|
|
|
cluster_info: Arc<ClusterInfo>,
|
2019-10-28 16:07:51 -07:00
|
|
|
bank_forks: Arc<RwLock<BankForks>>,
|
|
|
|
leader_schedule_cache: Arc<LeaderScheduleCache>,
|
2022-07-07 04:13:13 -07:00
|
|
|
shred_fetch_receiver: Receiver<PacketBatch>,
|
2022-07-06 04:49:58 -07:00
|
|
|
retransmit_sender: Sender<Vec</*shred:*/ Vec<u8>>>,
|
2022-07-07 04:13:13 -07:00
|
|
|
verified_sender: Sender<Vec<PacketBatch>>,
|
|
|
|
) -> JoinHandle<()> {
|
|
|
|
let recycler_cache = RecyclerCache::warmed();
|
|
|
|
let mut stats = ShredSigVerifyStats::new(Instant::now());
|
2023-03-20 13:33:22 -07:00
|
|
|
let thread_pool = ThreadPoolBuilder::new()
|
|
|
|
.num_threads(get_thread_count())
|
|
|
|
.thread_name(|i| format!("solSvrfyShred{i:02}"))
|
|
|
|
.build()
|
|
|
|
.unwrap();
|
|
|
|
let run_shred_sigverify = move || {
|
2023-03-22 06:19:16 -07:00
|
|
|
let mut rng = rand::thread_rng();
|
|
|
|
let mut deduper = Deduper::<2, [u8]>::new(&mut rng, DEDUPER_NUM_BITS);
|
2023-03-20 13:33:22 -07:00
|
|
|
loop {
|
2023-03-22 06:19:16 -07:00
|
|
|
if deduper.maybe_reset(&mut rng, DEDUPER_FALSE_POSITIVE_RATE, DEDUPER_RESET_CYCLE) {
|
|
|
|
stats.num_deduper_saturations += 1;
|
|
|
|
}
|
2022-07-07 04:13:13 -07:00
|
|
|
match run_shred_sigverify(
|
2023-03-20 13:33:22 -07:00
|
|
|
&thread_pool,
|
2023-01-24 08:57:55 -08:00
|
|
|
// We can't store the pubkey outside the loop
|
|
|
|
// because the identity might be hot swapped.
|
|
|
|
&cluster_info.id(),
|
2022-07-07 04:13:13 -07:00
|
|
|
&bank_forks,
|
|
|
|
&leader_schedule_cache,
|
|
|
|
&recycler_cache,
|
2023-03-22 06:19:16 -07:00
|
|
|
&deduper,
|
2022-07-07 04:13:13 -07:00
|
|
|
&shred_fetch_receiver,
|
|
|
|
&retransmit_sender,
|
|
|
|
&verified_sender,
|
|
|
|
&mut stats,
|
|
|
|
) {
|
|
|
|
Ok(()) => (),
|
|
|
|
Err(Error::RecvTimeout) => (),
|
|
|
|
Err(Error::RecvDisconnected) => break,
|
|
|
|
Err(Error::SendError) => break,
|
|
|
|
}
|
|
|
|
stats.maybe_submit();
|
2023-03-20 13:33:22 -07:00
|
|
|
}
|
|
|
|
};
|
|
|
|
Builder::new()
|
|
|
|
.name("solShredVerifr".to_string())
|
|
|
|
.spawn(run_shred_sigverify)
|
2022-07-07 04:13:13 -07:00
|
|
|
.unwrap()
|
2019-10-28 16:07:51 -07:00
|
|
|
}
|
|
|
|
|
2023-03-20 13:33:22 -07:00
|
|
|
#[allow(clippy::too_many_arguments)]
|
2023-03-22 06:19:16 -07:00
|
|
|
fn run_shred_sigverify<const K: usize>(
|
2023-03-20 13:33:22 -07:00
|
|
|
thread_pool: &ThreadPool,
|
2022-07-07 04:13:13 -07:00
|
|
|
self_pubkey: &Pubkey,
|
|
|
|
bank_forks: &RwLock<BankForks>,
|
|
|
|
leader_schedule_cache: &LeaderScheduleCache,
|
|
|
|
recycler_cache: &RecyclerCache,
|
2023-03-22 06:19:16 -07:00
|
|
|
deduper: &Deduper<K, [u8]>,
|
2022-07-07 04:13:13 -07:00
|
|
|
shred_fetch_receiver: &Receiver<PacketBatch>,
|
|
|
|
retransmit_sender: &Sender<Vec</*shred:*/ Vec<u8>>>,
|
|
|
|
verified_sender: &Sender<Vec<PacketBatch>>,
|
|
|
|
stats: &mut ShredSigVerifyStats,
|
|
|
|
) -> Result<(), Error> {
|
|
|
|
const RECV_TIMEOUT: Duration = Duration::from_secs(1);
|
|
|
|
let packets = shred_fetch_receiver.recv_timeout(RECV_TIMEOUT)?;
|
|
|
|
let mut packets: Vec<_> = std::iter::once(packets)
|
|
|
|
.chain(shred_fetch_receiver.try_iter())
|
|
|
|
.collect();
|
|
|
|
let now = Instant::now();
|
|
|
|
stats.num_iters += 1;
|
|
|
|
stats.num_packets += packets.iter().map(PacketBatch::len).sum::<usize>();
|
|
|
|
stats.num_discards_pre += count_discards(&packets);
|
2023-03-22 06:19:16 -07:00
|
|
|
stats.num_duplicates += thread_pool.install(|| {
|
|
|
|
packets
|
|
|
|
.par_iter_mut()
|
|
|
|
.flatten()
|
|
|
|
.filter(|packet| {
|
|
|
|
!packet.meta().discard()
|
|
|
|
&& packet
|
|
|
|
.data(..)
|
|
|
|
.map(|data| deduper.dedup(data))
|
|
|
|
.unwrap_or(true)
|
|
|
|
})
|
|
|
|
.map(|packet| packet.meta_mut().set_discard(true))
|
|
|
|
.count()
|
|
|
|
});
|
2022-07-07 04:13:13 -07:00
|
|
|
verify_packets(
|
2023-03-20 13:33:22 -07:00
|
|
|
thread_pool,
|
2022-07-07 04:13:13 -07:00
|
|
|
self_pubkey,
|
|
|
|
bank_forks,
|
|
|
|
leader_schedule_cache,
|
|
|
|
recycler_cache,
|
|
|
|
&mut packets,
|
|
|
|
);
|
|
|
|
stats.num_discards_post += count_discards(&packets);
|
|
|
|
// Exclude repair packets from retransmit.
|
|
|
|
let shreds: Vec<_> = packets
|
|
|
|
.iter()
|
|
|
|
.flat_map(PacketBatch::iter)
|
2022-12-06 03:54:49 -08:00
|
|
|
.filter(|packet| !packet.meta().discard() && !packet.meta().repair())
|
2022-07-07 04:13:13 -07:00
|
|
|
.filter_map(shred::layout::get_shred)
|
|
|
|
.map(<[u8]>::to_vec)
|
|
|
|
.collect();
|
|
|
|
stats.num_retransmit_shreds += shreds.len();
|
2023-03-20 13:34:41 -07:00
|
|
|
retransmit_sender.send(shreds)?;
|
|
|
|
verified_sender.send(packets)?;
|
2022-07-07 04:13:13 -07:00
|
|
|
stats.elapsed_micros += now.elapsed().as_micros() as u64;
|
|
|
|
Ok(())
|
2019-10-28 16:07:51 -07:00
|
|
|
}
|
|
|
|
|
2022-07-07 04:13:13 -07:00
|
|
|
fn verify_packets(
|
2023-03-20 13:33:22 -07:00
|
|
|
thread_pool: &ThreadPool,
|
2022-07-07 04:13:13 -07:00
|
|
|
self_pubkey: &Pubkey,
|
|
|
|
bank_forks: &RwLock<BankForks>,
|
|
|
|
leader_schedule_cache: &LeaderScheduleCache,
|
|
|
|
recycler_cache: &RecyclerCache,
|
|
|
|
packets: &mut [PacketBatch],
|
|
|
|
) {
|
|
|
|
let working_bank = bank_forks.read().unwrap().working_bank();
|
2023-08-18 16:28:08 -07:00
|
|
|
let leader_slots: HashMap<Slot, Pubkey> =
|
2022-07-07 04:13:13 -07:00
|
|
|
get_slot_leaders(self_pubkey, packets, leader_schedule_cache, &working_bank)
|
|
|
|
.into_iter()
|
2023-08-18 16:28:08 -07:00
|
|
|
.filter_map(|(slot, pubkey)| Some((slot, pubkey?)))
|
|
|
|
.chain(std::iter::once((Slot::MAX, Pubkey::default())))
|
2022-07-07 04:13:13 -07:00
|
|
|
.collect();
|
2023-03-20 13:33:22 -07:00
|
|
|
let out = verify_shreds_gpu(thread_pool, packets, &leader_slots, recycler_cache);
|
2022-07-07 04:13:13 -07:00
|
|
|
solana_perf::sigverify::mark_disabled(packets, &out);
|
2019-10-28 16:07:51 -07:00
|
|
|
}
|
|
|
|
|
2022-06-27 13:12:23 -07:00
|
|
|
// Returns pubkey of leaders for shred slots refrenced in the packets.
|
|
|
|
// Marks packets as discard if:
|
|
|
|
// - fails to deserialize the shred slot.
|
|
|
|
// - slot leader is unknown.
|
|
|
|
// - slot leader is the node itself (circular transmission).
|
|
|
|
fn get_slot_leaders(
|
|
|
|
self_pubkey: &Pubkey,
|
|
|
|
batches: &mut [PacketBatch],
|
|
|
|
leader_schedule_cache: &LeaderScheduleCache,
|
|
|
|
bank: &Bank,
|
|
|
|
) -> HashMap<Slot, Option<Pubkey>> {
|
|
|
|
let mut leaders = HashMap::<Slot, Option<Pubkey>>::new();
|
|
|
|
for batch in batches {
|
|
|
|
for packet in batch.iter_mut() {
|
2022-12-06 03:54:49 -08:00
|
|
|
if packet.meta().discard() {
|
2022-06-27 13:12:23 -07:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
let shred = shred::layout::get_shred(packet);
|
2023-08-18 16:28:08 -07:00
|
|
|
let Some(slot) = shred.and_then(shred::layout::get_slot) else {
|
|
|
|
packet.meta_mut().set_discard(true);
|
|
|
|
continue;
|
2022-06-27 13:12:23 -07:00
|
|
|
};
|
|
|
|
let leader = leaders.entry(slot).or_insert_with(|| {
|
|
|
|
let leader = leader_schedule_cache.slot_leader_at(slot, Some(bank))?;
|
|
|
|
// Discard the shred if the slot leader is the node itself.
|
2022-08-22 18:01:03 -07:00
|
|
|
(&leader != self_pubkey).then_some(leader)
|
2022-06-27 13:12:23 -07:00
|
|
|
});
|
|
|
|
if leader.is_none() {
|
2022-12-06 03:54:49 -08:00
|
|
|
packet.meta_mut().set_discard(true);
|
2022-06-27 13:12:23 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
leaders
|
|
|
|
}
|
|
|
|
|
2022-07-07 04:13:13 -07:00
|
|
|
fn count_discards(packets: &[PacketBatch]) -> usize {
|
|
|
|
packets
|
|
|
|
.iter()
|
|
|
|
.flat_map(PacketBatch::iter)
|
2022-12-06 03:54:49 -08:00
|
|
|
.filter(|packet| packet.meta().discard())
|
2022-07-07 04:13:13 -07:00
|
|
|
.count()
|
|
|
|
}
|
|
|
|
|
|
|
|
impl From<RecvTimeoutError> for Error {
|
|
|
|
fn from(err: RecvTimeoutError) -> Self {
|
|
|
|
match err {
|
|
|
|
RecvTimeoutError::Timeout => Self::RecvTimeout,
|
|
|
|
RecvTimeoutError::Disconnected => Self::RecvDisconnected,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl<T> From<SendError<T>> for Error {
|
|
|
|
fn from(_: SendError<T>) -> Self {
|
|
|
|
Self::SendError
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
struct ShredSigVerifyStats {
|
|
|
|
since: Instant,
|
|
|
|
num_iters: usize,
|
|
|
|
num_packets: usize,
|
2023-03-22 06:19:16 -07:00
|
|
|
num_deduper_saturations: usize,
|
2022-07-07 04:13:13 -07:00
|
|
|
num_discards_post: usize,
|
2023-03-22 06:19:16 -07:00
|
|
|
num_discards_pre: usize,
|
|
|
|
num_duplicates: usize,
|
2022-07-07 04:13:13 -07:00
|
|
|
num_retransmit_shreds: usize,
|
|
|
|
elapsed_micros: u64,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl ShredSigVerifyStats {
|
|
|
|
const METRICS_SUBMIT_CADENCE: Duration = Duration::from_secs(2);
|
|
|
|
|
|
|
|
fn new(now: Instant) -> Self {
|
|
|
|
Self {
|
|
|
|
since: now,
|
|
|
|
num_iters: 0usize,
|
|
|
|
num_packets: 0usize,
|
|
|
|
num_discards_pre: 0usize,
|
2023-03-22 06:19:16 -07:00
|
|
|
num_deduper_saturations: 0usize,
|
2022-07-07 04:13:13 -07:00
|
|
|
num_discards_post: 0usize,
|
2023-03-22 06:19:16 -07:00
|
|
|
num_duplicates: 0usize,
|
2022-07-07 04:13:13 -07:00
|
|
|
num_retransmit_shreds: 0usize,
|
|
|
|
elapsed_micros: 0u64,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn maybe_submit(&mut self) {
|
|
|
|
if self.since.elapsed() <= Self::METRICS_SUBMIT_CADENCE {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
datapoint_info!(
|
|
|
|
"shred_sigverify",
|
|
|
|
("num_iters", self.num_iters, i64),
|
|
|
|
("num_packets", self.num_packets, i64),
|
|
|
|
("num_discards_pre", self.num_discards_pre, i64),
|
2023-03-22 06:19:16 -07:00
|
|
|
("num_deduper_saturations", self.num_deduper_saturations, i64),
|
2022-07-07 04:13:13 -07:00
|
|
|
("num_discards_post", self.num_discards_post, i64),
|
2023-03-22 06:19:16 -07:00
|
|
|
("num_duplicates", self.num_duplicates, i64),
|
2022-07-07 04:13:13 -07:00
|
|
|
("num_retransmit_shreds", self.num_retransmit_shreds, i64),
|
|
|
|
("elapsed_micros", self.elapsed_micros, i64),
|
|
|
|
);
|
|
|
|
*self = Self::new(Instant::now());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-10-28 10:29:38 -07:00
|
|
|
#[cfg(test)]
|
2022-07-07 04:13:13 -07:00
|
|
|
mod tests {
|
2021-12-03 09:00:31 -08:00
|
|
|
use {
|
|
|
|
super::*,
|
2022-05-02 16:33:53 -07:00
|
|
|
solana_ledger::{
|
|
|
|
genesis_utils::create_genesis_config_with_leader,
|
|
|
|
shred::{Shred, ShredFlags},
|
|
|
|
},
|
2021-12-03 09:00:31 -08:00
|
|
|
solana_perf::packet::Packet,
|
|
|
|
solana_runtime::bank::Bank,
|
|
|
|
solana_sdk::signature::{Keypair, Signer},
|
|
|
|
};
|
2019-11-02 06:23:14 -07:00
|
|
|
|
2019-10-28 16:07:51 -07:00
|
|
|
#[test]
|
2021-12-11 06:44:15 -08:00
|
|
|
fn test_sigverify_shreds_verify_batches() {
|
2019-10-28 16:07:51 -07:00
|
|
|
let leader_keypair = Arc::new(Keypair::new());
|
|
|
|
let leader_pubkey = leader_keypair.pubkey();
|
2021-08-05 06:42:38 -07:00
|
|
|
let bank = Bank::new_for_tests(
|
|
|
|
&create_genesis_config_with_leader(100, &leader_pubkey, 10).genesis_config,
|
|
|
|
);
|
2022-07-07 04:13:13 -07:00
|
|
|
let leader_schedule_cache = LeaderScheduleCache::new_from_bank(&bank);
|
|
|
|
let bank_forks = RwLock::new(BankForks::new(bank));
|
2022-05-23 13:30:15 -07:00
|
|
|
let batch_size = 2;
|
|
|
|
let mut batch = PacketBatch::with_capacity(batch_size);
|
|
|
|
batch.resize(batch_size, Packet::default());
|
|
|
|
let mut batches = vec![batch];
|
2019-10-28 16:07:51 -07:00
|
|
|
|
2022-05-02 16:33:53 -07:00
|
|
|
let mut shred = Shred::new_from_data(
|
|
|
|
0,
|
|
|
|
0xc0de,
|
|
|
|
0xdead,
|
|
|
|
&[1, 2, 3, 4],
|
|
|
|
ShredFlags::LAST_SHRED_IN_SLOT,
|
|
|
|
0,
|
|
|
|
0,
|
|
|
|
0xc0de,
|
|
|
|
);
|
2022-04-19 13:00:05 -07:00
|
|
|
shred.sign(&leader_keypair);
|
2022-05-25 09:52:54 -07:00
|
|
|
batches[0][0].buffer_mut()[..shred.payload().len()].copy_from_slice(shred.payload());
|
2022-12-06 03:54:49 -08:00
|
|
|
batches[0][0].meta_mut().size = shred.payload().len();
|
2019-10-28 16:07:51 -07:00
|
|
|
|
2022-05-02 16:33:53 -07:00
|
|
|
let mut shred = Shred::new_from_data(
|
|
|
|
0,
|
|
|
|
0xbeef,
|
|
|
|
0xc0de,
|
|
|
|
&[1, 2, 3, 4],
|
|
|
|
ShredFlags::LAST_SHRED_IN_SLOT,
|
|
|
|
0,
|
|
|
|
0,
|
|
|
|
0xc0de,
|
|
|
|
);
|
2019-10-28 16:07:51 -07:00
|
|
|
let wrong_keypair = Keypair::new();
|
2022-04-19 13:00:05 -07:00
|
|
|
shred.sign(&wrong_keypair);
|
2022-05-25 09:52:54 -07:00
|
|
|
batches[0][1].buffer_mut()[..shred.payload().len()].copy_from_slice(shred.payload());
|
2022-12-06 03:54:49 -08:00
|
|
|
batches[0][1].meta_mut().size = shred.payload().len();
|
2019-10-28 16:07:51 -07:00
|
|
|
|
2023-03-20 13:33:22 -07:00
|
|
|
let thread_pool = ThreadPoolBuilder::new().num_threads(3).build().unwrap();
|
2022-07-07 04:13:13 -07:00
|
|
|
verify_packets(
|
2023-03-20 13:33:22 -07:00
|
|
|
&thread_pool,
|
2022-07-07 04:13:13 -07:00
|
|
|
&Pubkey::new_unique(), // self_pubkey
|
|
|
|
&bank_forks,
|
|
|
|
&leader_schedule_cache,
|
|
|
|
&RecyclerCache::warmed(),
|
|
|
|
&mut batches,
|
|
|
|
);
|
2022-12-06 03:54:49 -08:00
|
|
|
assert!(!batches[0][0].meta().discard());
|
|
|
|
assert!(batches[0][1].meta().discard());
|
2019-10-28 16:07:51 -07:00
|
|
|
}
|
2019-10-28 10:29:38 -07:00
|
|
|
}
|