2018-10-08 19:55:54 -07:00
|
|
|
//! The `cluster_info` module defines a data structure that is shared by all the nodes in the network over
|
2018-04-28 00:31:20 -07:00
|
|
|
//! a gossip control plane. The goal is to share small bits of off-chain information and detect and
|
2018-04-21 11:02:49 -07:00
|
|
|
//! repair partitions.
|
|
|
|
//!
|
2018-08-09 08:13:57 -07:00
|
|
|
//! This CRDT only supports a very limited set of types. A map of Pubkey -> Versioned Struct.
|
2018-05-15 04:35:41 -07:00
|
|
|
//! The last version is always picked during an update.
|
2018-04-28 00:31:20 -07:00
|
|
|
//!
|
|
|
|
//! The network is arranged in layers:
|
|
|
|
//!
|
|
|
|
//! * layer 0 - Leader.
|
|
|
|
//! * layer 1 - As many nodes as we can fit
|
|
|
|
//! * layer 2 - Everyone else, if layer 1 is `2^10`, layer 2 should be able to fit `2^20` number of nodes.
|
|
|
|
//!
|
2018-05-14 14:33:11 -07:00
|
|
|
//! Bank needs to provide an interface for us to query the stake weight
|
2019-02-20 21:36:08 -08:00
|
|
|
use crate::bank_forks::BankForks;
|
2019-02-07 20:52:39 -08:00
|
|
|
use crate::blocktree::Blocktree;
|
2018-12-07 19:16:27 -08:00
|
|
|
use crate::contact_info::ContactInfo;
|
|
|
|
use crate::crds_gossip::CrdsGossip;
|
|
|
|
use crate::crds_gossip_error::CrdsGossipError;
|
2019-08-13 18:04:14 -07:00
|
|
|
use crate::crds_gossip_pull::{CrdsFilter, CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS};
|
2019-05-08 13:50:32 -07:00
|
|
|
use crate::crds_value::{CrdsValue, CrdsValueLabel, EpochSlots, Vote};
|
2019-09-04 12:47:09 -07:00
|
|
|
use crate::packet::{to_shared_blob, Blob, Packet, SharedBlob};
|
2019-04-06 19:41:22 -07:00
|
|
|
use crate::repair_service::RepairType;
|
2018-12-07 19:16:27 -08:00
|
|
|
use crate::result::Result;
|
2019-02-28 13:15:25 -08:00
|
|
|
use crate::staking_utils;
|
2018-12-07 19:16:27 -08:00
|
|
|
use crate::streamer::{BlobReceiver, BlobSender};
|
2019-06-01 07:55:43 -07:00
|
|
|
use crate::weighted_shuffle::weighted_shuffle;
|
2019-08-13 18:04:14 -07:00
|
|
|
use bincode::{deserialize, serialize, serialized_size};
|
2019-02-18 08:51:00 -08:00
|
|
|
use core::cmp;
|
2019-06-01 07:55:43 -07:00
|
|
|
use itertools::Itertools;
|
2019-06-03 20:38:05 -07:00
|
|
|
use rand::SeedableRng;
|
2018-08-28 16:32:40 -07:00
|
|
|
use rand::{thread_rng, Rng};
|
2019-06-01 07:55:43 -07:00
|
|
|
use rand_chacha::ChaChaRng;
|
2018-04-28 00:31:20 -07:00
|
|
|
use rayon::prelude::*;
|
2019-07-30 13:18:33 -07:00
|
|
|
use solana_metrics::{datapoint_debug, inc_new_counter_debug, inc_new_counter_error};
|
2019-04-12 18:17:34 -07:00
|
|
|
use solana_netutil::{
|
|
|
|
bind_in_range, bind_to, find_available_port_in_range, multi_bind_in_range, PortRange,
|
|
|
|
};
|
2019-08-13 18:04:14 -07:00
|
|
|
use solana_sdk::packet::PACKET_DATA_SIZE;
|
2018-10-25 11:13:08 -07:00
|
|
|
use solana_sdk::pubkey::Pubkey;
|
2018-12-03 10:26:28 -08:00
|
|
|
use solana_sdk::signature::{Keypair, KeypairUtil, Signable, Signature};
|
2018-11-16 08:45:59 -08:00
|
|
|
use solana_sdk::timing::{duration_as_ms, timestamp};
|
2019-01-31 15:51:29 -08:00
|
|
|
use solana_sdk::transaction::Transaction;
|
2019-06-19 00:13:19 -07:00
|
|
|
use std::borrow::Borrow;
|
2019-06-12 16:43:05 -07:00
|
|
|
use std::borrow::Cow;
|
2019-01-02 00:46:15 -08:00
|
|
|
use std::cmp::min;
|
2019-06-26 00:30:16 -07:00
|
|
|
use std::collections::{BTreeSet, HashMap, HashSet};
|
2019-02-11 16:20:31 -08:00
|
|
|
use std::fmt;
|
2018-08-25 10:24:16 -07:00
|
|
|
use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket};
|
2019-02-13 20:04:20 -08:00
|
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
2018-04-21 11:02:49 -07:00
|
|
|
use std::sync::{Arc, RwLock};
|
2018-05-30 13:25:32 -07:00
|
|
|
use std::thread::{sleep, Builder, JoinHandle};
|
2018-08-08 11:15:36 -07:00
|
|
|
use std::time::{Duration, Instant};
|
2018-06-14 12:31:52 -07:00
|
|
|
|
2019-04-12 18:17:34 -07:00
|
|
|
pub const FULLNODE_PORT_RANGE: PortRange = (8000, 10_000);
|
2018-09-11 18:40:38 -07:00
|
|
|
|
2019-05-07 13:24:58 -07:00
|
|
|
/// The Data plane fanout size, also used as the neighborhood size
|
|
|
|
pub const DATA_PLANE_FANOUT: usize = 200;
|
2018-06-14 22:03:49 -07:00
|
|
|
/// milliseconds we sleep for between gossip requests
|
2019-01-31 15:51:29 -08:00
|
|
|
pub const GOSSIP_SLEEP_MILLIS: u64 = 100;
|
2018-08-30 12:07:54 -07:00
|
|
|
|
2019-04-06 19:41:22 -07:00
|
|
|
/// the number of slots to respond with when responding to `Orphan` requests
|
|
|
|
pub const MAX_ORPHAN_REPAIR_RESPONSES: usize = 10;
|
|
|
|
|
2019-08-13 18:04:14 -07:00
|
|
|
/// The maximum size of a protocol payload
|
|
|
|
const MAX_PROTOCOL_PAYLOAD_SIZE: u64 = PACKET_DATA_SIZE as u64;
|
2019-07-30 15:43:17 -07:00
|
|
|
|
2018-07-05 13:37:13 -07:00
|
|
|
#[derive(Debug, PartialEq, Eq)]
|
2018-10-08 19:55:54 -07:00
|
|
|
pub enum ClusterInfoError {
|
2018-07-17 09:44:48 -07:00
|
|
|
NoPeers,
|
2018-07-09 15:53:49 -07:00
|
|
|
NoLeader,
|
2018-07-16 19:31:52 -07:00
|
|
|
BadContactInfo,
|
2018-07-17 16:27:46 -07:00
|
|
|
BadGossipAddress,
|
2018-07-05 13:37:13 -07:00
|
|
|
}
|
2019-02-11 16:20:31 -08:00
|
|
|
#[derive(Clone)]
|
2018-10-08 19:55:54 -07:00
|
|
|
pub struct ClusterInfo {
|
2018-11-15 13:23:26 -08:00
|
|
|
/// The network
|
|
|
|
pub gossip: CrdsGossip,
|
2018-12-01 12:00:30 -08:00
|
|
|
/// set the keypair that will be used to sign crds values generated. It is unset only in tests.
|
2018-12-12 18:57:48 -08:00
|
|
|
pub(crate) keypair: Arc<Keypair>,
|
2019-03-08 18:08:24 -08:00
|
|
|
/// The network entrypoint
|
2019-03-08 17:23:07 -08:00
|
|
|
entrypoint: Option<ContactInfo>,
|
2018-04-21 11:02:49 -07:00
|
|
|
}
|
2018-09-17 15:43:23 -07:00
|
|
|
|
2019-01-02 00:46:15 -08:00
|
|
|
#[derive(Default, Clone)]
|
|
|
|
pub struct Locality {
|
|
|
|
/// The bounds of the neighborhood represented by this locality
|
|
|
|
pub neighbor_bounds: (usize, usize),
|
2019-06-06 12:48:40 -07:00
|
|
|
/// The `turbine` layer this locality is in
|
2019-01-02 00:46:15 -08:00
|
|
|
pub layer_ix: usize,
|
|
|
|
/// The bounds of the current layer
|
|
|
|
pub layer_bounds: (usize, usize),
|
|
|
|
/// The bounds of the next layer
|
2019-05-07 13:24:58 -07:00
|
|
|
pub next_layer_bounds: Option<(usize, usize)>,
|
2019-01-02 00:46:15 -08:00
|
|
|
/// The indices of the nodes that should be contacted in next layer
|
2019-05-07 13:24:58 -07:00
|
|
|
pub next_layer_peers: Vec<usize>,
|
2019-01-02 00:46:15 -08:00
|
|
|
}
|
|
|
|
|
2019-02-11 16:20:31 -08:00
|
|
|
impl fmt::Debug for Locality {
|
|
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
|
|
write!(
|
|
|
|
f,
|
2019-05-07 13:24:58 -07:00
|
|
|
"Locality {{ neighborhood_bounds: {:?}, current_layer: {:?}, child_layer_bounds: {:?} child_layer_peers: {:?} }}",
|
|
|
|
self.neighbor_bounds, self.layer_ix, self.next_layer_bounds, self.next_layer_peers
|
2019-02-11 16:20:31 -08:00
|
|
|
)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-12-01 12:00:30 -08:00
|
|
|
#[derive(Debug, Deserialize, Serialize)]
|
|
|
|
pub struct PruneData {
|
|
|
|
/// Pubkey of the node that sent this prune data
|
|
|
|
pub pubkey: Pubkey,
|
|
|
|
/// Pubkeys of nodes that should be pruned
|
|
|
|
pub prunes: Vec<Pubkey>,
|
|
|
|
/// Signature of this Prune Message
|
|
|
|
pub signature: Signature,
|
|
|
|
/// The Pubkey of the intended node/destination for this message
|
|
|
|
pub destination: Pubkey,
|
|
|
|
/// Wallclock of the node that generated this message
|
|
|
|
pub wallclock: u64,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Signable for PruneData {
|
|
|
|
fn pubkey(&self) -> Pubkey {
|
|
|
|
self.pubkey
|
|
|
|
}
|
|
|
|
|
2019-06-12 16:43:05 -07:00
|
|
|
fn signable_data(&self) -> Cow<[u8]> {
|
2018-12-01 12:00:30 -08:00
|
|
|
#[derive(Serialize)]
|
|
|
|
struct SignData {
|
|
|
|
pubkey: Pubkey,
|
|
|
|
prunes: Vec<Pubkey>,
|
|
|
|
destination: Pubkey,
|
|
|
|
wallclock: u64,
|
|
|
|
}
|
|
|
|
let data = SignData {
|
|
|
|
pubkey: self.pubkey,
|
|
|
|
prunes: self.prunes.clone(),
|
|
|
|
destination: self.destination,
|
|
|
|
wallclock: self.wallclock,
|
|
|
|
};
|
2019-06-12 16:43:05 -07:00
|
|
|
Cow::Owned(serialize(&data).expect("serialize PruneData"))
|
2018-12-01 12:00:30 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
fn get_signature(&self) -> Signature {
|
|
|
|
self.signature
|
|
|
|
}
|
|
|
|
|
|
|
|
fn set_signature(&mut self, signature: Signature) {
|
|
|
|
self.signature = signature
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-08-15 17:04:45 -07:00
|
|
|
struct PullData {
|
|
|
|
pub from_addr: SocketAddr,
|
|
|
|
pub caller: CrdsValue,
|
|
|
|
pub filter: CrdsFilter,
|
|
|
|
}
|
|
|
|
|
2018-12-01 12:00:30 -08:00
|
|
|
// TODO These messages should go through the gpu pipeline for spam filtering
|
2018-06-03 19:59:17 -07:00
|
|
|
#[derive(Serialize, Deserialize, Debug)]
|
2018-12-07 19:01:28 -08:00
|
|
|
#[allow(clippy::large_enum_variant)]
|
2018-04-21 11:02:49 -07:00
|
|
|
enum Protocol {
|
2018-12-01 12:00:30 -08:00
|
|
|
/// Gossip protocol messages
|
2019-08-13 18:04:14 -07:00
|
|
|
PullRequest(CrdsFilter, CrdsValue),
|
2018-11-15 13:23:26 -08:00
|
|
|
PullResponse(Pubkey, Vec<CrdsValue>),
|
|
|
|
PushMessage(Pubkey, Vec<CrdsValue>),
|
2018-12-01 12:00:30 -08:00
|
|
|
PruneMessage(Pubkey, PruneData),
|
2018-11-15 13:23:26 -08:00
|
|
|
|
|
|
|
/// Window protocol messages
|
|
|
|
/// TODO: move this message to a different module
|
2019-03-08 17:23:07 -08:00
|
|
|
RequestWindowIndex(ContactInfo, u64, u64),
|
|
|
|
RequestHighestWindowIndex(ContactInfo, u64, u64),
|
2019-04-06 19:41:22 -07:00
|
|
|
RequestOrphan(ContactInfo, u64),
|
2018-04-21 11:02:49 -07:00
|
|
|
}
|
|
|
|
|
2018-10-08 19:55:54 -07:00
|
|
|
impl ClusterInfo {
|
2019-03-06 13:47:18 -08:00
|
|
|
/// Without a valid keypair gossip will not function. Only useful for tests.
|
2019-03-08 17:23:07 -08:00
|
|
|
pub fn new_with_invalid_keypair(contact_info: ContactInfo) -> Self {
|
|
|
|
Self::new(contact_info, Arc::new(Keypair::new()))
|
2018-12-01 12:00:30 -08:00
|
|
|
}
|
2019-03-11 14:42:24 -07:00
|
|
|
|
2019-03-08 17:23:07 -08:00
|
|
|
pub fn new(contact_info: ContactInfo, keypair: Arc<Keypair>) -> Self {
|
2019-03-07 08:56:53 -08:00
|
|
|
let mut me = Self {
|
2018-11-15 13:23:26 -08:00
|
|
|
gossip: CrdsGossip::default(),
|
2018-12-01 12:00:30 -08:00
|
|
|
keypair,
|
2019-03-08 18:08:24 -08:00
|
|
|
entrypoint: None,
|
2018-04-21 11:02:49 -07:00
|
|
|
};
|
2019-03-08 17:23:07 -08:00
|
|
|
let id = contact_info.id;
|
2019-03-09 19:28:43 -08:00
|
|
|
me.gossip.set_self(&id);
|
2019-03-08 17:23:07 -08:00
|
|
|
me.insert_self(contact_info);
|
2019-02-20 20:02:47 -08:00
|
|
|
me.push_self(&HashMap::new());
|
2018-11-19 11:25:14 -08:00
|
|
|
me
|
2018-04-21 11:02:49 -07:00
|
|
|
}
|
2019-03-11 14:42:24 -07:00
|
|
|
|
2019-03-08 17:23:07 -08:00
|
|
|
pub fn insert_self(&mut self, contact_info: ContactInfo) {
|
|
|
|
if self.id() == contact_info.id {
|
|
|
|
let mut value = CrdsValue::ContactInfo(contact_info.clone());
|
2019-03-08 18:08:24 -08:00
|
|
|
value.sign(&self.keypair);
|
|
|
|
let _ = self.gossip.crds.insert(value, timestamp());
|
|
|
|
}
|
|
|
|
}
|
2019-03-11 14:42:24 -07:00
|
|
|
|
|
|
|
fn push_self(&mut self, stakes: &HashMap<Pubkey, u64>) {
|
2018-11-15 13:23:26 -08:00
|
|
|
let mut my_data = self.my_data();
|
|
|
|
let now = timestamp();
|
|
|
|
my_data.wallclock = now;
|
2018-12-01 12:00:30 -08:00
|
|
|
let mut entry = CrdsValue::ContactInfo(my_data);
|
|
|
|
entry.sign(&self.keypair);
|
2019-02-20 20:02:47 -08:00
|
|
|
self.gossip.refresh_push_active_set(stakes);
|
2019-06-26 00:30:16 -07:00
|
|
|
self.gossip
|
|
|
|
.process_push_message(&self.id(), vec![entry], now);
|
2018-11-15 13:23:26 -08:00
|
|
|
}
|
2019-03-11 14:42:24 -07:00
|
|
|
|
2019-03-08 18:08:24 -08:00
|
|
|
// TODO kill insert_info, only used by tests
|
2019-03-08 17:23:07 -08:00
|
|
|
pub fn insert_info(&mut self, contact_info: ContactInfo) {
|
|
|
|
let mut value = CrdsValue::ContactInfo(contact_info);
|
2018-12-01 12:00:30 -08:00
|
|
|
value.sign(&self.keypair);
|
2018-11-15 13:23:26 -08:00
|
|
|
let _ = self.gossip.crds.insert(value, timestamp());
|
|
|
|
}
|
2019-03-11 14:42:24 -07:00
|
|
|
|
2019-03-08 17:23:07 -08:00
|
|
|
pub fn set_entrypoint(&mut self, entrypoint: ContactInfo) {
|
2019-03-08 18:08:24 -08:00
|
|
|
self.entrypoint = Some(entrypoint)
|
|
|
|
}
|
2019-03-11 14:42:24 -07:00
|
|
|
|
2018-11-15 13:23:26 -08:00
|
|
|
pub fn id(&self) -> Pubkey {
|
|
|
|
self.gossip.id
|
|
|
|
}
|
2019-03-11 14:42:24 -07:00
|
|
|
|
2019-03-09 19:28:43 -08:00
|
|
|
pub fn lookup(&self, id: &Pubkey) -> Option<&ContactInfo> {
|
|
|
|
let entry = CrdsValueLabel::ContactInfo(*id);
|
2018-11-15 13:23:26 -08:00
|
|
|
self.gossip
|
|
|
|
.crds
|
|
|
|
.lookup(&entry)
|
2019-04-15 14:56:08 -07:00
|
|
|
.and_then(CrdsValue::contact_info)
|
2018-11-15 13:23:26 -08:00
|
|
|
}
|
2019-03-11 14:42:24 -07:00
|
|
|
|
2019-03-08 17:23:07 -08:00
|
|
|
pub fn my_data(&self) -> ContactInfo {
|
2019-03-09 19:28:43 -08:00
|
|
|
self.lookup(&self.id()).cloned().unwrap()
|
2018-11-15 13:23:26 -08:00
|
|
|
}
|
2019-03-11 14:42:24 -07:00
|
|
|
|
2019-03-08 17:23:07 -08:00
|
|
|
pub fn contact_info_trace(&self) -> String {
|
2019-04-18 09:48:21 -07:00
|
|
|
let now = timestamp();
|
|
|
|
let mut spy_nodes = 0;
|
2019-05-16 07:14:58 -07:00
|
|
|
let mut replicators = 0;
|
2019-05-23 23:20:04 -07:00
|
|
|
let my_pubkey = self.my_data().id;
|
2018-09-10 12:06:14 -07:00
|
|
|
let nodes: Vec<_> = self
|
2019-04-18 09:48:21 -07:00
|
|
|
.all_peers()
|
2018-11-15 13:23:26 -08:00
|
|
|
.into_iter()
|
2019-04-29 13:19:24 -07:00
|
|
|
.map(|(node, last_updated)| {
|
2019-04-30 16:42:56 -07:00
|
|
|
if Self::is_spy_node(&node) {
|
2019-04-18 09:48:21 -07:00
|
|
|
spy_nodes += 1;
|
2019-05-16 07:14:58 -07:00
|
|
|
} else if Self::is_replicator(&node) {
|
|
|
|
replicators += 1;
|
2019-03-07 10:13:54 -08:00
|
|
|
}
|
2019-04-18 09:48:21 -07:00
|
|
|
fn addr_to_string(addr: &SocketAddr) -> String {
|
|
|
|
if ContactInfo::is_valid_address(addr) {
|
|
|
|
addr.to_string()
|
2019-04-15 13:25:09 -07:00
|
|
|
} else {
|
|
|
|
"none".to_string()
|
|
|
|
}
|
2019-04-18 09:48:21 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
format!(
|
2019-04-23 14:46:41 -07:00
|
|
|
"- gossip: {:20} | {:5}ms | {} {}\n \
|
2019-04-18 09:48:21 -07:00
|
|
|
tpu: {:20} | |\n \
|
|
|
|
rpc: {:20} | |\n",
|
|
|
|
addr_to_string(&node.gossip),
|
2019-04-29 13:19:24 -07:00
|
|
|
now.saturating_sub(last_updated),
|
2019-04-18 09:48:21 -07:00
|
|
|
node.id,
|
2019-05-23 23:20:04 -07:00
|
|
|
if node.id == my_pubkey { "(me)" } else { "" }.to_string(),
|
2019-04-18 09:48:21 -07:00
|
|
|
addr_to_string(&node.tpu),
|
|
|
|
addr_to_string(&node.rpc),
|
2018-09-10 12:06:14 -07:00
|
|
|
)
|
2018-12-07 19:01:28 -08:00
|
|
|
})
|
|
|
|
.collect();
|
2018-09-10 12:06:14 -07:00
|
|
|
|
|
|
|
format!(
|
2019-04-18 09:48:21 -07:00
|
|
|
" Node contact info | Age | Node identifier \n\
|
|
|
|
-------------------------------+---------+-----------------------------------\n\
|
2019-03-07 10:13:54 -08:00
|
|
|
{}\
|
2019-05-16 07:14:58 -07:00
|
|
|
Nodes: {}{}{}",
|
2018-09-10 12:06:14 -07:00
|
|
|
nodes.join(""),
|
2019-05-16 07:14:58 -07:00
|
|
|
nodes.len() - spy_nodes - replicators,
|
|
|
|
if replicators > 0 {
|
|
|
|
format!("\nReplicators: {}", replicators)
|
|
|
|
} else {
|
|
|
|
"".to_string()
|
|
|
|
},
|
2019-04-18 09:48:21 -07:00
|
|
|
if spy_nodes > 0 {
|
|
|
|
format!("\nSpies: {}", spy_nodes)
|
|
|
|
} else {
|
|
|
|
"".to_string()
|
|
|
|
}
|
2018-09-10 12:06:14 -07:00
|
|
|
)
|
|
|
|
}
|
|
|
|
|
2019-05-23 03:50:41 -07:00
|
|
|
pub fn push_epoch_slots(&mut self, id: Pubkey, root: u64, slots: BTreeSet<u64>) {
|
2019-05-08 13:50:32 -07:00
|
|
|
let now = timestamp();
|
|
|
|
let mut entry = CrdsValue::EpochSlots(EpochSlots::new(id, root, slots, now));
|
|
|
|
entry.sign(&self.keypair);
|
2019-06-26 00:30:16 -07:00
|
|
|
self.gossip
|
|
|
|
.process_push_message(&self.id(), vec![entry], now);
|
2019-05-08 13:50:32 -07:00
|
|
|
}
|
|
|
|
|
2019-01-31 15:51:29 -08:00
|
|
|
pub fn push_vote(&mut self, vote: Transaction) {
|
|
|
|
let now = timestamp();
|
2019-03-11 16:43:30 -07:00
|
|
|
let vote = Vote::new(&self.id(), vote, now);
|
2019-01-31 15:51:29 -08:00
|
|
|
let mut entry = CrdsValue::Vote(vote);
|
|
|
|
entry.sign(&self.keypair);
|
2019-06-26 00:30:16 -07:00
|
|
|
self.gossip
|
|
|
|
.process_push_message(&self.id(), vec![entry], now);
|
2019-01-31 15:51:29 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
/// Get votes in the crds
|
2019-04-10 17:16:08 -07:00
|
|
|
/// * since - The timestamp of when the vote inserted must be greater than
|
2019-01-31 15:51:29 -08:00
|
|
|
/// since. This allows the bank to query for new votes only.
|
|
|
|
///
|
2019-04-10 17:16:08 -07:00
|
|
|
/// * return - The votes, and the max timestamp from the new set.
|
2019-01-31 15:51:29 -08:00
|
|
|
pub fn get_votes(&self, since: u64) -> (Vec<Transaction>, u64) {
|
|
|
|
let votes: Vec<_> = self
|
|
|
|
.gossip
|
|
|
|
.crds
|
|
|
|
.table
|
|
|
|
.values()
|
2019-04-10 17:16:08 -07:00
|
|
|
.filter(|x| x.insert_timestamp > since)
|
2019-01-31 15:51:29 -08:00
|
|
|
.filter_map(|x| {
|
|
|
|
x.value
|
|
|
|
.vote()
|
2019-04-10 17:16:08 -07:00
|
|
|
.map(|v| (x.insert_timestamp, v.transaction.clone()))
|
2019-01-31 15:51:29 -08:00
|
|
|
})
|
|
|
|
.collect();
|
|
|
|
let max_ts = votes.iter().map(|x| x.0).max().unwrap_or(since);
|
|
|
|
let txs: Vec<Transaction> = votes.into_iter().map(|x| x.1).collect();
|
|
|
|
(txs, max_ts)
|
|
|
|
}
|
|
|
|
|
2019-05-23 03:10:16 -07:00
|
|
|
pub fn get_epoch_state_for_node(
|
|
|
|
&self,
|
|
|
|
pubkey: &Pubkey,
|
|
|
|
since: Option<u64>,
|
|
|
|
) -> Option<(&EpochSlots, u64)> {
|
|
|
|
self.gossip
|
|
|
|
.crds
|
|
|
|
.table
|
|
|
|
.get(&CrdsValueLabel::EpochSlots(*pubkey))
|
|
|
|
.filter(|x| {
|
|
|
|
since
|
|
|
|
.map(|since| x.insert_timestamp > since)
|
|
|
|
.unwrap_or(true)
|
|
|
|
})
|
|
|
|
.map(|x| (x.value.epoch_slots().unwrap(), x.insert_timestamp))
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn get_gossiped_root_for_node(&self, pubkey: &Pubkey, since: Option<u64>) -> Option<u64> {
|
|
|
|
self.gossip
|
|
|
|
.crds
|
|
|
|
.table
|
|
|
|
.get(&CrdsValueLabel::EpochSlots(*pubkey))
|
|
|
|
.filter(|x| {
|
|
|
|
since
|
|
|
|
.map(|since| x.insert_timestamp > since)
|
|
|
|
.unwrap_or(true)
|
|
|
|
})
|
|
|
|
.map(|x| x.value.epoch_slots().unwrap().root)
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn get_contact_info_for_node(&self, pubkey: &Pubkey) -> Option<&ContactInfo> {
|
|
|
|
self.gossip
|
|
|
|
.crds
|
|
|
|
.table
|
|
|
|
.get(&CrdsValueLabel::ContactInfo(*pubkey))
|
|
|
|
.map(|x| x.value.contact_info().unwrap())
|
|
|
|
}
|
|
|
|
|
2018-11-15 13:23:26 -08:00
|
|
|
pub fn purge(&mut self, now: u64) {
|
|
|
|
self.gossip.purge(now);
|
|
|
|
}
|
2019-03-07 12:24:30 -08:00
|
|
|
|
2019-03-08 17:23:07 -08:00
|
|
|
pub fn rpc_peers(&self) -> Vec<ContactInfo> {
|
2018-09-15 23:46:16 -07:00
|
|
|
let me = self.my_data().id;
|
2018-11-15 13:23:26 -08:00
|
|
|
self.gossip
|
|
|
|
.crds
|
|
|
|
.table
|
2018-09-15 23:46:16 -07:00
|
|
|
.values()
|
2018-11-15 13:23:26 -08:00
|
|
|
.filter_map(|x| x.value.contact_info())
|
2018-09-15 23:46:16 -07:00
|
|
|
.filter(|x| x.id != me)
|
2018-11-17 19:57:28 -08:00
|
|
|
.filter(|x| ContactInfo::is_valid_address(&x.rpc))
|
2018-09-15 23:46:16 -07:00
|
|
|
.cloned()
|
|
|
|
.collect()
|
|
|
|
}
|
|
|
|
|
2019-04-29 13:19:24 -07:00
|
|
|
// All nodes in gossip (including spy nodes) and the last time we heard about them
|
|
|
|
pub(crate) fn all_peers(&self) -> Vec<(ContactInfo, u64)> {
|
2019-04-18 09:48:21 -07:00
|
|
|
self.gossip
|
|
|
|
.crds
|
|
|
|
.table
|
|
|
|
.values()
|
2019-04-29 13:19:24 -07:00
|
|
|
.filter_map(|x| {
|
|
|
|
x.value
|
|
|
|
.contact_info()
|
|
|
|
.map(|ci| (ci.clone(), x.local_timestamp))
|
|
|
|
})
|
2019-04-18 09:48:21 -07:00
|
|
|
.collect()
|
|
|
|
}
|
|
|
|
|
2019-03-08 17:23:07 -08:00
|
|
|
pub fn gossip_peers(&self) -> Vec<ContactInfo> {
|
2018-11-15 13:23:26 -08:00
|
|
|
let me = self.my_data().id;
|
|
|
|
self.gossip
|
|
|
|
.crds
|
|
|
|
.table
|
|
|
|
.values()
|
|
|
|
.filter_map(|x| x.value.contact_info())
|
|
|
|
.filter(|x| x.id != me)
|
2018-12-06 12:52:47 -08:00
|
|
|
.filter(|x| ContactInfo::is_valid_address(&x.gossip))
|
2018-11-15 13:23:26 -08:00
|
|
|
.cloned()
|
|
|
|
.collect()
|
2018-06-14 12:31:52 -07:00
|
|
|
}
|
2018-06-18 23:50:41 -07:00
|
|
|
|
2019-05-16 07:14:58 -07:00
|
|
|
/// all peers that have a valid tvu port.
|
2019-03-08 17:23:07 -08:00
|
|
|
pub fn tvu_peers(&self) -> Vec<ContactInfo> {
|
2019-02-01 17:13:15 -08:00
|
|
|
let me = self.my_data().id;
|
2018-12-11 15:51:47 -08:00
|
|
|
self.gossip
|
|
|
|
.crds
|
|
|
|
.table
|
|
|
|
.values()
|
|
|
|
.filter_map(|x| x.value.contact_info())
|
|
|
|
.filter(|x| ContactInfo::is_valid_address(&x.tvu))
|
2019-02-01 17:13:15 -08:00
|
|
|
.filter(|x| x.id != me)
|
2018-12-11 15:51:47 -08:00
|
|
|
.cloned()
|
|
|
|
.collect()
|
|
|
|
}
|
|
|
|
|
2019-05-16 07:14:58 -07:00
|
|
|
/// all peers that have a valid storage addr
|
|
|
|
pub fn storage_peers(&self) -> Vec<ContactInfo> {
|
|
|
|
let me = self.my_data().id;
|
|
|
|
self.gossip
|
|
|
|
.crds
|
|
|
|
.table
|
|
|
|
.values()
|
|
|
|
.filter_map(|x| x.value.contact_info())
|
|
|
|
.filter(|x| ContactInfo::is_valid_address(&x.storage_addr))
|
|
|
|
.filter(|x| x.id != me)
|
|
|
|
.cloned()
|
|
|
|
.collect()
|
|
|
|
}
|
|
|
|
|
2019-01-26 00:28:08 -08:00
|
|
|
/// all peers that have a valid tvu
|
2019-03-08 17:23:07 -08:00
|
|
|
pub fn retransmit_peers(&self) -> Vec<ContactInfo> {
|
2018-11-15 13:23:26 -08:00
|
|
|
let me = self.my_data().id;
|
|
|
|
self.gossip
|
|
|
|
.crds
|
|
|
|
.table
|
|
|
|
.values()
|
|
|
|
.filter_map(|x| x.value.contact_info())
|
2019-01-26 00:28:08 -08:00
|
|
|
.filter(|x| x.id != me)
|
2018-11-17 19:57:28 -08:00
|
|
|
.filter(|x| ContactInfo::is_valid_address(&x.tvu))
|
2018-11-15 13:23:26 -08:00
|
|
|
.cloned()
|
|
|
|
.collect()
|
2018-04-21 11:02:49 -07:00
|
|
|
}
|
2018-04-28 00:31:20 -07:00
|
|
|
|
2018-12-11 15:51:47 -08:00
|
|
|
/// all tvu peers with valid gossip addrs
|
2019-03-11 14:42:24 -07:00
|
|
|
fn repair_peers(&self) -> Vec<ContactInfo> {
|
2019-01-26 00:28:08 -08:00
|
|
|
let me = self.my_data().id;
|
2018-12-11 15:51:47 -08:00
|
|
|
ClusterInfo::tvu_peers(self)
|
|
|
|
.into_iter()
|
2019-01-26 00:28:08 -08:00
|
|
|
.filter(|x| x.id != me)
|
2018-12-11 15:51:47 -08:00
|
|
|
.filter(|x| ContactInfo::is_valid_address(&x.gossip))
|
|
|
|
.collect()
|
|
|
|
}
|
|
|
|
|
2019-04-30 16:42:56 -07:00
|
|
|
fn is_spy_node(contact_info: &ContactInfo) -> bool {
|
2019-05-16 07:14:58 -07:00
|
|
|
(!ContactInfo::is_valid_address(&contact_info.tpu)
|
2019-04-30 16:42:56 -07:00
|
|
|
|| !ContactInfo::is_valid_address(&contact_info.gossip)
|
2019-05-16 07:14:58 -07:00
|
|
|
|| !ContactInfo::is_valid_address(&contact_info.tvu))
|
|
|
|
&& !ContactInfo::is_valid_address(&contact_info.storage_addr)
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn is_replicator(contact_info: &ContactInfo) -> bool {
|
|
|
|
ContactInfo::is_valid_address(&contact_info.storage_addr)
|
|
|
|
&& !ContactInfo::is_valid_address(&contact_info.tpu)
|
2019-04-30 16:42:56 -07:00
|
|
|
}
|
|
|
|
|
2019-06-01 07:55:43 -07:00
|
|
|
fn stake_weighted_shuffle<S: std::hash::BuildHasher>(
|
2019-03-08 17:23:07 -08:00
|
|
|
peers: &[ContactInfo],
|
2019-05-14 16:15:51 -07:00
|
|
|
stakes: Option<&HashMap<Pubkey, u64, S>>,
|
2019-06-01 07:55:43 -07:00
|
|
|
rng: ChaChaRng,
|
2019-03-08 17:23:07 -08:00
|
|
|
) -> Vec<(u64, ContactInfo)> {
|
2019-06-01 07:55:43 -07:00
|
|
|
let (stake_weights, peers_with_stakes): (Vec<_>, Vec<_>) = peers
|
2019-01-02 00:46:15 -08:00
|
|
|
.iter()
|
2019-05-14 16:15:51 -07:00
|
|
|
.map(|c| {
|
2019-06-01 07:55:43 -07:00
|
|
|
let stake = stakes.map_or(0, |stakes| *stakes.get(&c.id).unwrap_or(&0));
|
|
|
|
// For stake weighted shuffle a valid weight is atleast 1. Weight 0 is
|
|
|
|
// assumed to be missing entry. So let's make sure stake weights are atleast 1
|
|
|
|
(cmp::max(1, stake), (stake, c.clone()))
|
|
|
|
})
|
|
|
|
.sorted_by(|(_, (l_stake, l_info)), (_, (r_stake, r_info))| {
|
|
|
|
if r_stake == l_stake {
|
|
|
|
r_info.id.cmp(&l_info.id)
|
|
|
|
} else {
|
|
|
|
r_stake.cmp(&l_stake)
|
|
|
|
}
|
2019-05-14 16:15:51 -07:00
|
|
|
})
|
2019-06-01 07:55:43 -07:00
|
|
|
.unzip();
|
|
|
|
|
|
|
|
let shuffle = weighted_shuffle(stake_weights, rng);
|
|
|
|
|
|
|
|
let mut out: Vec<(u64, ContactInfo)> = shuffle
|
|
|
|
.iter()
|
|
|
|
.map(|x| peers_with_stakes[*x].clone())
|
2019-01-02 00:46:15 -08:00
|
|
|
.collect();
|
2019-06-01 07:55:43 -07:00
|
|
|
|
|
|
|
out.dedup();
|
|
|
|
out
|
2019-01-02 00:46:15 -08:00
|
|
|
}
|
|
|
|
|
2019-04-19 21:07:21 -07:00
|
|
|
/// Return sorted Retransmit peers and index of `Self.id()` as if it were in that list
|
2019-06-03 20:38:05 -07:00
|
|
|
pub fn shuffle_peers_and_index<S: std::hash::BuildHasher>(
|
2019-02-20 21:38:16 -08:00
|
|
|
&self,
|
2019-05-14 16:15:51 -07:00
|
|
|
stakes: Option<&HashMap<Pubkey, u64, S>>,
|
2019-06-01 07:55:43 -07:00
|
|
|
rng: ChaChaRng,
|
2019-04-19 21:07:21 -07:00
|
|
|
) -> (usize, Vec<ContactInfo>) {
|
|
|
|
let mut peers = self.retransmit_peers();
|
|
|
|
peers.push(self.lookup(&self.id()).unwrap().clone());
|
2019-06-01 07:55:43 -07:00
|
|
|
let contacts_and_stakes: Vec<_> = ClusterInfo::stake_weighted_shuffle(&peers, stakes, rng);
|
2019-04-19 21:07:21 -07:00
|
|
|
let mut index = 0;
|
|
|
|
let peers: Vec<_> = contacts_and_stakes
|
|
|
|
.into_iter()
|
|
|
|
.enumerate()
|
2019-06-03 20:38:05 -07:00
|
|
|
.map(|(i, (_, peer))| {
|
2019-04-19 21:07:21 -07:00
|
|
|
if peer.id == self.id() {
|
|
|
|
index = i;
|
|
|
|
}
|
2019-06-03 20:38:05 -07:00
|
|
|
peer
|
2019-04-19 21:07:21 -07:00
|
|
|
})
|
|
|
|
.collect();
|
|
|
|
(index, peers)
|
2019-01-02 00:46:15 -08:00
|
|
|
}
|
|
|
|
|
2019-06-01 07:55:43 -07:00
|
|
|
pub fn sorted_tvu_peers(
|
|
|
|
&self,
|
|
|
|
stakes: Option<&HashMap<Pubkey, u64>>,
|
|
|
|
rng: ChaChaRng,
|
|
|
|
) -> Vec<ContactInfo> {
|
2019-01-02 00:46:15 -08:00
|
|
|
let peers = self.tvu_peers();
|
2019-06-01 07:55:43 -07:00
|
|
|
let peers_with_stakes: Vec<_> = ClusterInfo::stake_weighted_shuffle(&peers, stakes, rng);
|
2019-01-02 00:46:15 -08:00
|
|
|
peers_with_stakes
|
|
|
|
.iter()
|
|
|
|
.map(|(_, peer)| (*peer).clone())
|
|
|
|
.collect()
|
|
|
|
}
|
|
|
|
|
2018-07-09 15:53:49 -07:00
|
|
|
/// compute broadcast table
|
2019-03-08 17:23:07 -08:00
|
|
|
pub fn tpu_peers(&self) -> Vec<ContactInfo> {
|
2018-11-15 13:23:26 -08:00
|
|
|
let me = self.my_data().id;
|
|
|
|
self.gossip
|
|
|
|
.crds
|
|
|
|
.table
|
|
|
|
.values()
|
|
|
|
.filter_map(|x| x.value.contact_info())
|
|
|
|
.filter(|x| x.id != me)
|
2018-11-17 19:57:28 -08:00
|
|
|
.filter(|x| ContactInfo::is_valid_address(&x.tpu))
|
2018-11-15 13:23:26 -08:00
|
|
|
.cloned()
|
|
|
|
.collect()
|
2018-07-09 15:53:49 -07:00
|
|
|
}
|
|
|
|
|
2019-05-07 13:24:58 -07:00
|
|
|
/// Given a node count and fanout, it calculates how many layers are needed and at what index each layer begins.
|
|
|
|
pub fn describe_data_plane(nodes: usize, fanout: usize) -> (usize, Vec<usize>) {
|
2019-01-02 00:46:15 -08:00
|
|
|
let mut layer_indices: Vec<usize> = vec![0];
|
|
|
|
if nodes == 0 {
|
|
|
|
(0, vec![])
|
|
|
|
} else if nodes <= fanout {
|
|
|
|
// single layer data plane
|
|
|
|
(1, layer_indices)
|
|
|
|
} else {
|
|
|
|
//layer 1 is going to be the first num fanout nodes, so exclude those
|
|
|
|
let mut remaining_nodes = nodes - fanout;
|
|
|
|
layer_indices.push(fanout);
|
|
|
|
let mut num_layers = 2;
|
2019-05-07 13:24:58 -07:00
|
|
|
// fanout * num_nodes in a neighborhood, which is also fanout.
|
|
|
|
let mut layer_capacity = fanout * fanout;
|
2019-01-02 00:46:15 -08:00
|
|
|
while remaining_nodes > 0 {
|
|
|
|
if remaining_nodes > layer_capacity {
|
|
|
|
// Needs more layers.
|
|
|
|
num_layers += 1;
|
|
|
|
remaining_nodes -= layer_capacity;
|
|
|
|
let end = *layer_indices.last().unwrap();
|
|
|
|
layer_indices.push(layer_capacity + end);
|
|
|
|
|
2019-05-07 13:24:58 -07:00
|
|
|
// Next layer's capacity
|
|
|
|
layer_capacity *= fanout;
|
2019-01-02 00:46:15 -08:00
|
|
|
} else {
|
|
|
|
//everything will now fit in the layers we have
|
|
|
|
let end = *layer_indices.last().unwrap();
|
|
|
|
layer_indices.push(layer_capacity + end);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
assert_eq!(num_layers, layer_indices.len() - 1);
|
|
|
|
(num_layers, layer_indices)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn localize_item(
|
|
|
|
layer_indices: &[usize],
|
2019-05-07 13:24:58 -07:00
|
|
|
fanout: usize,
|
2019-01-02 00:46:15 -08:00
|
|
|
select_index: usize,
|
|
|
|
curr_index: usize,
|
|
|
|
) -> Option<(Locality)> {
|
|
|
|
let end = layer_indices.len() - 1;
|
|
|
|
let next = min(end, curr_index + 1);
|
2019-05-07 13:24:58 -07:00
|
|
|
let layer_start = layer_indices[curr_index];
|
|
|
|
// localized if selected index lies within the current layer's bounds
|
|
|
|
let localized = select_index >= layer_start && select_index < layer_indices[next];
|
2019-01-02 00:46:15 -08:00
|
|
|
if localized {
|
2019-05-07 13:24:58 -07:00
|
|
|
let mut locality = Locality::default();
|
|
|
|
let hood_ix = (select_index - layer_start) / fanout;
|
2019-01-02 00:46:15 -08:00
|
|
|
match curr_index {
|
|
|
|
_ if curr_index == 0 => {
|
|
|
|
locality.layer_ix = 0;
|
2019-05-07 13:24:58 -07:00
|
|
|
locality.layer_bounds = (0, fanout);
|
2019-01-02 00:46:15 -08:00
|
|
|
locality.neighbor_bounds = locality.layer_bounds;
|
2019-05-07 13:24:58 -07:00
|
|
|
|
2019-01-02 00:46:15 -08:00
|
|
|
if next == end {
|
2019-05-07 13:24:58 -07:00
|
|
|
locality.next_layer_bounds = None;
|
|
|
|
locality.next_layer_peers = vec![];
|
2019-01-02 00:46:15 -08:00
|
|
|
} else {
|
2019-05-07 13:24:58 -07:00
|
|
|
locality.next_layer_bounds =
|
2019-01-02 00:46:15 -08:00
|
|
|
Some((layer_indices[next], layer_indices[next + 1]));
|
2019-05-07 13:24:58 -07:00
|
|
|
locality.next_layer_peers = ClusterInfo::next_layer_peers(
|
2019-01-02 00:46:15 -08:00
|
|
|
select_index,
|
2019-05-07 13:24:58 -07:00
|
|
|
hood_ix,
|
2019-01-02 00:46:15 -08:00
|
|
|
layer_indices[next],
|
2019-05-07 13:24:58 -07:00
|
|
|
fanout,
|
2019-01-02 00:46:15 -08:00
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
_ if curr_index == end => {
|
|
|
|
locality.layer_ix = end;
|
2019-05-07 13:24:58 -07:00
|
|
|
locality.layer_bounds = (end - fanout, end);
|
2019-01-02 00:46:15 -08:00
|
|
|
locality.neighbor_bounds = locality.layer_bounds;
|
2019-05-07 13:24:58 -07:00
|
|
|
locality.next_layer_bounds = None;
|
|
|
|
locality.next_layer_peers = vec![];
|
2019-01-02 00:46:15 -08:00
|
|
|
}
|
|
|
|
ix => {
|
|
|
|
locality.layer_ix = ix;
|
2019-05-07 13:24:58 -07:00
|
|
|
locality.layer_bounds = (layer_start, layer_indices[next]);
|
2019-01-02 00:46:15 -08:00
|
|
|
locality.neighbor_bounds = (
|
2019-05-07 13:24:58 -07:00
|
|
|
((hood_ix * fanout) + layer_start),
|
|
|
|
((hood_ix + 1) * fanout + layer_start),
|
2019-01-02 00:46:15 -08:00
|
|
|
);
|
2019-05-07 13:24:58 -07:00
|
|
|
|
2019-01-02 00:46:15 -08:00
|
|
|
if next == end {
|
2019-05-07 13:24:58 -07:00
|
|
|
locality.next_layer_bounds = None;
|
|
|
|
locality.next_layer_peers = vec![];
|
2019-01-02 00:46:15 -08:00
|
|
|
} else {
|
2019-05-07 13:24:58 -07:00
|
|
|
locality.next_layer_bounds =
|
2019-01-02 00:46:15 -08:00
|
|
|
Some((layer_indices[next], layer_indices[next + 1]));
|
2019-05-07 13:24:58 -07:00
|
|
|
locality.next_layer_peers = ClusterInfo::next_layer_peers(
|
2019-01-02 00:46:15 -08:00
|
|
|
select_index,
|
2019-05-07 13:24:58 -07:00
|
|
|
hood_ix,
|
2019-01-02 00:46:15 -08:00
|
|
|
layer_indices[next],
|
2019-05-07 13:24:58 -07:00
|
|
|
fanout,
|
2019-01-02 00:46:15 -08:00
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
Some(locality)
|
|
|
|
} else {
|
|
|
|
None
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-05-07 13:24:58 -07:00
|
|
|
/// Given a array of layer indices and an index of interest, returns (as a `Locality`) the layer,
|
|
|
|
/// layer-bounds, and neighborhood-bounds in which the index resides
|
|
|
|
fn localize(layer_indices: &[usize], fanout: usize, select_index: usize) -> Locality {
|
2019-01-02 00:46:15 -08:00
|
|
|
(0..layer_indices.len())
|
2019-05-07 13:24:58 -07:00
|
|
|
.find_map(|i| ClusterInfo::localize_item(layer_indices, fanout, select_index, i))
|
2019-01-02 00:46:15 -08:00
|
|
|
.or_else(|| Some(Locality::default()))
|
|
|
|
.unwrap()
|
|
|
|
}
|
|
|
|
|
2019-05-07 13:24:58 -07:00
|
|
|
/// Selects a range in the next layer and chooses nodes from that range as peers for the given index
|
|
|
|
fn next_layer_peers(index: usize, hood_ix: usize, start: usize, fanout: usize) -> Vec<usize> {
|
|
|
|
// Each neighborhood is only tasked with pushing to `fanout` neighborhoods where each neighborhood contains `fanout` nodes.
|
|
|
|
let fanout_nodes = fanout * fanout;
|
|
|
|
// Skip first N nodes, where N is hood_ix * (fanout_nodes)
|
|
|
|
let start = start + (hood_ix * fanout_nodes);
|
|
|
|
let end = start + fanout_nodes;
|
2019-01-02 00:46:15 -08:00
|
|
|
(start..end)
|
2019-05-07 13:24:58 -07:00
|
|
|
.step_by(fanout)
|
|
|
|
.map(|x| x + index % fanout)
|
2019-01-02 00:46:15 -08:00
|
|
|
.collect()
|
|
|
|
}
|
|
|
|
|
2018-07-09 15:53:49 -07:00
|
|
|
/// broadcast messages from the leader to layer 1 nodes
|
|
|
|
/// # Remarks
|
2019-06-19 00:13:19 -07:00
|
|
|
pub fn broadcast<I>(
|
2019-06-03 20:38:05 -07:00
|
|
|
&self,
|
2018-07-09 15:53:49 -07:00
|
|
|
s: &UdpSocket,
|
2019-06-19 00:13:19 -07:00
|
|
|
blobs: I,
|
2019-06-03 20:38:05 -07:00
|
|
|
stakes: Option<&HashMap<Pubkey, u64>>,
|
2019-06-19 00:13:19 -07:00
|
|
|
) -> Result<()>
|
|
|
|
where
|
|
|
|
I: IntoIterator,
|
|
|
|
I::Item: Borrow<SharedBlob>,
|
|
|
|
{
|
2019-06-03 20:38:05 -07:00
|
|
|
let mut last_err = Ok(());
|
|
|
|
let mut broadcast_table_len = 0;
|
2019-06-19 00:13:19 -07:00
|
|
|
let mut blobs_len = 0;
|
|
|
|
blobs.into_iter().for_each(|b| {
|
|
|
|
blobs_len += 1;
|
|
|
|
let blob = b.borrow().read().unwrap();
|
2019-06-03 20:38:05 -07:00
|
|
|
let broadcast_table = self.sorted_tvu_peers(stakes, ChaChaRng::from_seed(blob.seed()));
|
|
|
|
broadcast_table_len = cmp::max(broadcast_table_len, broadcast_table.len());
|
|
|
|
|
|
|
|
if !broadcast_table.is_empty() {
|
|
|
|
if let Err(e) = s.send_to(&blob.data[..blob.meta.size], &broadcast_table[0].tvu) {
|
|
|
|
trace!("{}: broadcast result {:?}", self.id(), e);
|
|
|
|
last_err = Err(e);
|
|
|
|
}
|
2018-04-28 00:31:20 -07:00
|
|
|
}
|
2019-06-03 20:38:05 -07:00
|
|
|
});
|
2019-01-15 10:51:53 -08:00
|
|
|
|
2019-06-03 20:38:05 -07:00
|
|
|
last_err?;
|
2018-07-25 15:02:39 -07:00
|
|
|
|
2019-06-19 00:13:19 -07:00
|
|
|
inc_new_counter_debug!("cluster_info-broadcast-max_idx", blobs_len);
|
2019-07-30 13:18:33 -07:00
|
|
|
datapoint_info!(
|
|
|
|
"cluster_info-num_nodes",
|
|
|
|
("count", broadcast_table_len + 1, i64)
|
|
|
|
);
|
2018-04-28 00:31:20 -07:00
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2019-08-20 17:16:06 -07:00
|
|
|
pub fn broadcast_shreds(
|
|
|
|
&self,
|
|
|
|
s: &UdpSocket,
|
|
|
|
shreds: &[Vec<u8>],
|
|
|
|
seeds: &[[u8; 32]],
|
|
|
|
stakes: Option<&HashMap<Pubkey, u64>>,
|
|
|
|
) -> Result<()> {
|
|
|
|
let mut last_err = Ok(());
|
|
|
|
let mut broadcast_table_len = 0;
|
|
|
|
shreds.iter().zip(seeds).for_each(|(shred, seed)| {
|
|
|
|
let broadcast_table = self.sorted_tvu_peers(stakes, ChaChaRng::from_seed(*seed));
|
|
|
|
broadcast_table_len = cmp::max(broadcast_table_len, broadcast_table.len());
|
|
|
|
|
|
|
|
if !broadcast_table.is_empty() {
|
|
|
|
if let Err(e) = s.send_to(shred, &broadcast_table[0].tvu) {
|
|
|
|
trace!("{}: broadcast result {:?}", self.id(), e);
|
|
|
|
last_err = Err(e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
});
|
|
|
|
|
|
|
|
last_err?;
|
|
|
|
Ok(())
|
|
|
|
}
|
2019-01-02 00:46:15 -08:00
|
|
|
/// retransmit messages to a list of nodes
|
2018-04-28 00:31:20 -07:00
|
|
|
/// # Remarks
|
2019-07-01 16:49:05 -07:00
|
|
|
/// We need to avoid having obj locked while doing a io, such as the `send_to`
|
2019-01-02 00:46:15 -08:00
|
|
|
pub fn retransmit_to(
|
|
|
|
obj: &Arc<RwLock<Self>>,
|
2019-03-08 17:23:07 -08:00
|
|
|
peers: &[ContactInfo],
|
2019-08-20 17:16:06 -07:00
|
|
|
packet: &Packet,
|
2019-05-23 23:20:04 -07:00
|
|
|
slot_leader_pubkey: Option<Pubkey>,
|
2019-01-02 00:46:15 -08:00
|
|
|
s: &UdpSocket,
|
2019-04-20 16:44:06 -07:00
|
|
|
forwarded: bool,
|
2019-01-02 00:46:15 -08:00
|
|
|
) -> Result<()> {
|
2019-03-08 17:23:07 -08:00
|
|
|
let (me, orders): (ContactInfo, &[ContactInfo]) = {
|
2018-05-15 04:35:41 -07:00
|
|
|
// copy to avoid locking during IO
|
2019-01-02 00:46:15 -08:00
|
|
|
let s = obj.read().unwrap();
|
|
|
|
(s.my_data().clone(), peers)
|
2018-04-28 00:31:20 -07:00
|
|
|
};
|
2018-07-02 10:07:32 -07:00
|
|
|
trace!("retransmit orders {}", orders.len());
|
2018-05-04 11:11:39 -07:00
|
|
|
let errs: Vec<_> = orders
|
|
|
|
.par_iter()
|
2019-05-23 23:20:04 -07:00
|
|
|
.filter(|v| v.id != slot_leader_pubkey.unwrap_or_default())
|
2018-05-04 11:11:39 -07:00
|
|
|
.map(|v| {
|
2019-08-20 17:16:06 -07:00
|
|
|
let dest = if forwarded { &v.tvu_forwards } else { &v.tvu };
|
|
|
|
debug!("{}: retransmit packet to {} {}", me.id, v.id, *dest,);
|
|
|
|
s.send_to(&packet.data, dest)
|
2018-12-07 19:01:28 -08:00
|
|
|
})
|
|
|
|
.collect();
|
2018-04-28 00:31:20 -07:00
|
|
|
for e in errs {
|
2018-07-05 13:37:13 -07:00
|
|
|
if let Err(e) = &e {
|
2019-05-17 07:00:06 -07:00
|
|
|
inc_new_counter_error!("cluster_info-retransmit-send_to_error", 1, 1);
|
2018-07-17 08:20:35 -07:00
|
|
|
error!("retransmit result {:?}", e);
|
2018-04-28 00:31:20 -07:00
|
|
|
}
|
2018-07-05 13:37:13 -07:00
|
|
|
e?;
|
2018-04-28 00:31:20 -07:00
|
|
|
}
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2019-03-05 14:18:29 -08:00
|
|
|
pub fn window_index_request_bytes(&self, slot: u64, blob_index: u64) -> Result<Vec<u8>> {
|
|
|
|
let req = Protocol::RequestWindowIndex(self.my_data().clone(), slot, blob_index);
|
2019-01-09 11:19:04 -08:00
|
|
|
let out = serialize(&req)?;
|
|
|
|
Ok(out)
|
|
|
|
}
|
|
|
|
|
2019-03-11 14:42:24 -07:00
|
|
|
fn window_highest_index_request_bytes(&self, slot: u64, blob_index: u64) -> Result<Vec<u8>> {
|
2019-03-05 14:18:29 -08:00
|
|
|
let req = Protocol::RequestHighestWindowIndex(self.my_data().clone(), slot, blob_index);
|
2019-02-14 12:47:21 -08:00
|
|
|
let out = serialize(&req)?;
|
|
|
|
Ok(out)
|
|
|
|
}
|
|
|
|
|
2019-04-06 19:41:22 -07:00
|
|
|
fn orphan_bytes(&self, slot: u64) -> Result<Vec<u8>> {
|
|
|
|
let req = Protocol::RequestOrphan(self.my_data().clone(), slot);
|
|
|
|
let out = serialize(&req)?;
|
|
|
|
Ok(out)
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn repair_request(&self, repair_request: &RepairType) -> Result<(SocketAddr, Vec<u8>)> {
|
2018-08-30 12:07:54 -07:00
|
|
|
// find a peer that appears to be accepting replication, as indicated
|
|
|
|
// by a valid tvu port location
|
2018-12-11 15:51:47 -08:00
|
|
|
let valid: Vec<_> = self.repair_peers();
|
2018-05-24 23:18:41 -07:00
|
|
|
if valid.is_empty() {
|
2018-10-08 19:55:54 -07:00
|
|
|
Err(ClusterInfoError::NoPeers)?;
|
2018-05-12 19:00:22 -07:00
|
|
|
}
|
2018-08-28 16:32:40 -07:00
|
|
|
let n = thread_rng().gen::<usize>() % valid.len();
|
2018-12-06 12:52:47 -08:00
|
|
|
let addr = valid[n].gossip; // send the request to the peer's gossip port
|
2019-06-17 18:12:13 -07:00
|
|
|
let out = self.map_repair_request(repair_request)?;
|
2018-12-11 15:43:41 -08:00
|
|
|
|
2018-05-12 19:00:22 -07:00
|
|
|
Ok((addr, out))
|
|
|
|
}
|
2019-06-17 18:12:13 -07:00
|
|
|
pub fn map_repair_request(&self, repair_request: &RepairType) -> Result<Vec<u8>> {
|
|
|
|
match repair_request {
|
|
|
|
RepairType::Blob(slot, blob_index) => {
|
|
|
|
datapoint_debug!(
|
|
|
|
"cluster_info-repair",
|
|
|
|
("repair-slot", *slot, i64),
|
|
|
|
("repair-ix", *blob_index, i64)
|
|
|
|
);
|
|
|
|
Ok(self.window_index_request_bytes(*slot, *blob_index)?)
|
|
|
|
}
|
|
|
|
RepairType::HighestBlob(slot, blob_index) => {
|
|
|
|
datapoint_debug!(
|
|
|
|
"cluster_info-repair_highest",
|
|
|
|
("repair-highest-slot", *slot, i64),
|
|
|
|
("repair-highest-ix", *blob_index, i64)
|
|
|
|
);
|
|
|
|
Ok(self.window_highest_index_request_bytes(*slot, *blob_index)?)
|
|
|
|
}
|
|
|
|
RepairType::Orphan(slot) => {
|
|
|
|
datapoint_debug!("cluster_info-repair_orphan", ("repair-orphan", *slot, i64));
|
|
|
|
Ok(self.orphan_bytes(*slot)?)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2019-03-08 18:08:24 -08:00
|
|
|
// If the network entrypoint hasn't been discovered yet, add it to the crds table
|
2019-08-13 18:04:14 -07:00
|
|
|
fn add_entrypoint(&mut self, pulls: &mut Vec<(Pubkey, CrdsFilter, SocketAddr, CrdsValue)>) {
|
2019-03-08 18:08:24 -08:00
|
|
|
match &self.entrypoint {
|
|
|
|
Some(entrypoint) => {
|
|
|
|
let self_info = self
|
|
|
|
.gossip
|
|
|
|
.crds
|
|
|
|
.lookup(&CrdsValueLabel::ContactInfo(self.id()))
|
|
|
|
.unwrap_or_else(|| panic!("self_id invalid {}", self.id()));
|
|
|
|
|
2019-08-13 18:04:14 -07:00
|
|
|
self.gossip
|
|
|
|
.pull
|
|
|
|
.build_crds_filters(&self.gossip.crds, Self::max_bloom_size())
|
|
|
|
.into_iter()
|
|
|
|
.for_each(|filter| {
|
|
|
|
pulls.push((entrypoint.id, filter, entrypoint.gossip, self_info.clone()))
|
|
|
|
})
|
2019-03-08 18:08:24 -08:00
|
|
|
}
|
|
|
|
None => (),
|
|
|
|
}
|
|
|
|
}
|
2019-02-14 12:47:21 -08:00
|
|
|
|
2019-07-30 15:43:17 -07:00
|
|
|
/// Splits a Vec of CrdsValues into a nested Vec, trying to make sure that
|
|
|
|
/// each Vec is no larger than `PROTOCOL_PAYLOAD_SIZE`
|
|
|
|
/// Note: some messages cannot be contained within that size so in the worst case this returns
|
|
|
|
/// N nested Vecs with 1 item each.
|
|
|
|
fn split_gossip_messages(mut msgs: Vec<CrdsValue>) -> Vec<Vec<CrdsValue>> {
|
|
|
|
let mut messages = vec![];
|
|
|
|
while !msgs.is_empty() {
|
|
|
|
let mut size = 0;
|
|
|
|
let mut payload = vec![];
|
|
|
|
while let Some(msg) = msgs.pop() {
|
|
|
|
let msg_size = msg.size();
|
2019-08-13 18:04:14 -07:00
|
|
|
if size + msg_size > MAX_PROTOCOL_PAYLOAD_SIZE as u64 {
|
|
|
|
if msg_size < MAX_PROTOCOL_PAYLOAD_SIZE as u64 {
|
|
|
|
msgs.push(msg);
|
|
|
|
} else {
|
2019-08-15 19:42:27 -07:00
|
|
|
debug!(
|
2019-08-13 18:04:14 -07:00
|
|
|
"dropping message larger than the maximum payload size {:?}",
|
|
|
|
msg
|
|
|
|
);
|
|
|
|
}
|
2019-07-30 15:43:17 -07:00
|
|
|
break;
|
|
|
|
}
|
2019-08-13 18:04:14 -07:00
|
|
|
size += msg_size;
|
|
|
|
payload.push(msg);
|
2019-07-30 15:43:17 -07:00
|
|
|
}
|
|
|
|
messages.push(payload);
|
|
|
|
}
|
|
|
|
messages
|
|
|
|
}
|
|
|
|
|
2019-08-13 18:04:14 -07:00
|
|
|
// computes the maximum size for pull request blooms
|
|
|
|
pub fn max_bloom_size() -> usize {
|
|
|
|
let filter_size = serialized_size(&CrdsFilter::default())
|
|
|
|
.expect("unable to serialize default filter") as usize;
|
|
|
|
let protocol = Protocol::PullRequest(
|
|
|
|
CrdsFilter::default(),
|
|
|
|
CrdsValue::ContactInfo(ContactInfo::default()),
|
|
|
|
);
|
|
|
|
let protocol_size =
|
|
|
|
serialized_size(&protocol).expect("unable to serialize gossip protocol") as usize;
|
|
|
|
PACKET_DATA_SIZE - (protocol_size - filter_size)
|
|
|
|
}
|
|
|
|
|
2019-02-20 20:02:47 -08:00
|
|
|
fn new_pull_requests(&mut self, stakes: &HashMap<Pubkey, u64>) -> Vec<(SocketAddr, Protocol)> {
|
2018-11-15 13:23:26 -08:00
|
|
|
let now = timestamp();
|
2019-08-13 18:04:14 -07:00
|
|
|
let mut pulls: Vec<_> = self
|
2019-02-20 17:08:56 -08:00
|
|
|
.gossip
|
2019-08-13 18:04:14 -07:00
|
|
|
.new_pull_request(now, stakes, Self::max_bloom_size())
|
2019-02-20 17:08:56 -08:00
|
|
|
.ok()
|
|
|
|
.into_iter()
|
2019-08-13 18:04:14 -07:00
|
|
|
.filter_map(|(peer, filters, me)| {
|
2018-11-15 13:23:26 -08:00
|
|
|
let peer_label = CrdsValueLabel::ContactInfo(peer);
|
|
|
|
self.gossip
|
|
|
|
.crds
|
|
|
|
.lookup(&peer_label)
|
2019-04-15 14:56:08 -07:00
|
|
|
.and_then(CrdsValue::contact_info)
|
2019-08-13 18:04:14 -07:00
|
|
|
.map(move |peer_info| {
|
|
|
|
filters
|
|
|
|
.into_iter()
|
|
|
|
.map(move |f| (peer, f, peer_info.gossip, me.clone()))
|
|
|
|
})
|
2018-12-07 19:01:28 -08:00
|
|
|
})
|
2019-08-13 18:04:14 -07:00
|
|
|
.flatten()
|
2018-12-07 19:01:28 -08:00
|
|
|
.collect();
|
2019-08-13 18:04:14 -07:00
|
|
|
if pulls.is_empty() {
|
|
|
|
self.add_entrypoint(&mut pulls);
|
2019-03-08 18:08:24 -08:00
|
|
|
}
|
2019-08-13 18:04:14 -07:00
|
|
|
pulls
|
|
|
|
.into_iter()
|
2018-12-06 12:52:47 -08:00
|
|
|
.map(|(peer, filter, gossip, self_info)| {
|
2019-03-09 19:28:43 -08:00
|
|
|
self.gossip.mark_pull_request_creation_time(&peer, now);
|
2018-12-06 12:52:47 -08:00
|
|
|
(gossip, Protocol::PullRequest(filter, self_info))
|
2018-12-07 19:01:28 -08:00
|
|
|
})
|
|
|
|
.collect()
|
2018-11-15 13:23:26 -08:00
|
|
|
}
|
|
|
|
fn new_push_requests(&mut self) -> Vec<(SocketAddr, Protocol)> {
|
|
|
|
let self_id = self.gossip.id;
|
2019-05-28 18:39:40 -07:00
|
|
|
let (_, push_messages) = self.gossip.new_push_messages(timestamp());
|
|
|
|
push_messages
|
2018-11-15 13:23:26 -08:00
|
|
|
.into_iter()
|
2019-05-28 18:39:40 -07:00
|
|
|
.filter_map(|(peer, messages)| {
|
|
|
|
let peer_label = CrdsValueLabel::ContactInfo(peer);
|
2018-11-15 13:23:26 -08:00
|
|
|
self.gossip
|
|
|
|
.crds
|
|
|
|
.lookup(&peer_label)
|
2019-04-15 14:56:08 -07:00
|
|
|
.and_then(CrdsValue::contact_info)
|
2019-05-28 18:39:40 -07:00
|
|
|
.map(|p| (p.gossip, messages))
|
2018-12-07 19:01:28 -08:00
|
|
|
})
|
2019-07-30 15:43:17 -07:00
|
|
|
.map(|(peer, msgs)| {
|
|
|
|
Self::split_gossip_messages(msgs)
|
|
|
|
.into_iter()
|
|
|
|
.map(move |payload| (peer, Protocol::PushMessage(self_id, payload)))
|
|
|
|
})
|
|
|
|
.flatten()
|
2018-11-15 13:23:26 -08:00
|
|
|
.collect()
|
|
|
|
}
|
2018-06-18 23:50:41 -07:00
|
|
|
|
2019-02-20 20:02:47 -08:00
|
|
|
fn gossip_request(&mut self, stakes: &HashMap<Pubkey, u64>) -> Vec<(SocketAddr, Protocol)> {
|
|
|
|
let pulls: Vec<_> = self.new_pull_requests(stakes);
|
2018-11-15 13:23:26 -08:00
|
|
|
let pushes: Vec<_> = self.new_push_requests();
|
|
|
|
vec![pulls, pushes].into_iter().flat_map(|x| x).collect()
|
2018-04-21 11:02:49 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/// At random pick a node and try to get updated changes from them
|
2019-02-20 17:08:56 -08:00
|
|
|
fn run_gossip(
|
|
|
|
obj: &Arc<RwLock<Self>>,
|
2019-02-20 20:02:47 -08:00
|
|
|
stakes: &HashMap<Pubkey, u64>,
|
2019-02-20 17:08:56 -08:00
|
|
|
blob_sender: &BlobSender,
|
|
|
|
) -> Result<()> {
|
2019-02-20 20:02:47 -08:00
|
|
|
let reqs = obj.write().unwrap().gossip_request(&stakes);
|
2018-11-15 13:23:26 -08:00
|
|
|
let blobs = reqs
|
|
|
|
.into_iter()
|
2018-12-22 19:30:30 -08:00
|
|
|
.filter_map(|(remote_gossip_addr, req)| to_shared_blob(req, remote_gossip_addr).ok())
|
2018-11-15 13:23:26 -08:00
|
|
|
.collect();
|
|
|
|
blob_sender.send(blobs)?;
|
2018-04-21 11:02:49 -07:00
|
|
|
Ok(())
|
|
|
|
}
|
2018-10-10 16:49:41 -07:00
|
|
|
|
2018-04-21 11:02:49 -07:00
|
|
|
/// randomly pick a node and ask them for updates asynchronously
|
2018-05-27 18:21:39 -07:00
|
|
|
pub fn gossip(
|
|
|
|
obj: Arc<RwLock<Self>>,
|
2019-02-20 21:36:08 -08:00
|
|
|
bank_forks: Option<Arc<RwLock<BankForks>>>,
|
2018-05-27 18:21:39 -07:00
|
|
|
blob_sender: BlobSender,
|
2019-03-04 20:50:02 -08:00
|
|
|
exit: &Arc<AtomicBool>,
|
2018-05-27 18:21:39 -07:00
|
|
|
) -> JoinHandle<()> {
|
2019-03-04 20:50:02 -08:00
|
|
|
let exit = exit.clone();
|
2018-05-30 13:25:32 -07:00
|
|
|
Builder::new()
|
|
|
|
.name("solana-gossip".to_string())
|
2018-11-15 13:23:26 -08:00
|
|
|
.spawn(move || {
|
|
|
|
let mut last_push = timestamp();
|
2019-08-26 18:31:14 -07:00
|
|
|
let mut last_contact_info_trace = timestamp();
|
2018-11-15 13:23:26 -08:00
|
|
|
loop {
|
|
|
|
let start = timestamp();
|
2019-08-26 18:31:14 -07:00
|
|
|
if start - last_contact_info_trace > 10000 {
|
|
|
|
// Log contact info every 10 seconds
|
2019-08-27 08:33:48 -07:00
|
|
|
info!("\n{}", obj.read().unwrap().contact_info_trace());
|
2019-08-26 18:31:14 -07:00
|
|
|
last_contact_info_trace = start;
|
|
|
|
}
|
|
|
|
|
2019-02-20 21:36:08 -08:00
|
|
|
let stakes: HashMap<_, _> = match bank_forks {
|
2019-05-14 16:15:51 -07:00
|
|
|
Some(ref bank_forks) => {
|
|
|
|
staking_utils::staked_nodes(&bank_forks.read().unwrap().working_bank())
|
|
|
|
}
|
2019-02-20 20:02:47 -08:00
|
|
|
None => HashMap::new(),
|
|
|
|
};
|
|
|
|
let _ = Self::run_gossip(&obj, &stakes, &blob_sender);
|
2018-11-15 13:23:26 -08:00
|
|
|
if exit.load(Ordering::Relaxed) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
obj.write().unwrap().purge(timestamp());
|
|
|
|
//TODO: possibly tune this parameter
|
|
|
|
//we saw a deadlock passing an obj.read().unwrap().timeout into sleep
|
|
|
|
if start - last_push > CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS / 2 {
|
2019-02-20 20:02:47 -08:00
|
|
|
obj.write().unwrap().push_self(&stakes);
|
2018-11-15 13:23:26 -08:00
|
|
|
last_push = timestamp();
|
|
|
|
}
|
|
|
|
let elapsed = timestamp() - start;
|
|
|
|
if GOSSIP_SLEEP_MILLIS > elapsed {
|
|
|
|
let time_left = GOSSIP_SLEEP_MILLIS - elapsed;
|
|
|
|
sleep(Duration::from_millis(time_left));
|
|
|
|
}
|
2018-06-23 16:08:53 -07:00
|
|
|
}
|
2018-12-07 19:01:28 -08:00
|
|
|
})
|
|
|
|
.unwrap()
|
2018-04-21 11:02:49 -07:00
|
|
|
}
|
2019-02-07 15:10:54 -08:00
|
|
|
|
2019-09-04 12:47:09 -07:00
|
|
|
fn get_data_shred_as_blob(
|
|
|
|
blocktree: &Arc<Blocktree>,
|
|
|
|
slot: u64,
|
|
|
|
shred_index: u64,
|
|
|
|
) -> Result<Option<Blob>> {
|
|
|
|
let bytes = blocktree.get_data_shred(slot, shred_index)?;
|
|
|
|
Ok(bytes.map(|bytes| Blob::new(&bytes)))
|
|
|
|
}
|
|
|
|
|
2018-05-12 19:00:22 -07:00
|
|
|
fn run_window_request(
|
2019-03-08 17:23:07 -08:00
|
|
|
from: &ContactInfo,
|
2018-08-30 12:07:54 -07:00
|
|
|
from_addr: &SocketAddr,
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree: Option<&Arc<Blocktree>>,
|
2019-03-08 17:23:07 -08:00
|
|
|
me: &ContactInfo,
|
2019-03-05 14:18:29 -08:00
|
|
|
slot: u64,
|
2019-02-07 15:10:54 -08:00
|
|
|
blob_index: u64,
|
2018-11-15 13:23:26 -08:00
|
|
|
) -> Vec<SharedBlob> {
|
2019-02-07 20:52:39 -08:00
|
|
|
if let Some(blocktree) = blocktree {
|
2019-02-07 15:10:54 -08:00
|
|
|
// Try to find the requested index in one of the slots
|
2019-09-04 12:47:09 -07:00
|
|
|
let blob = Self::get_data_shred_as_blob(blocktree, slot, blob_index);
|
2018-08-06 12:35:38 -07:00
|
|
|
|
2019-02-07 15:10:54 -08:00
|
|
|
if let Ok(Some(mut blob)) = blob {
|
2019-05-17 07:00:06 -07:00
|
|
|
inc_new_counter_debug!("cluster_info-window-request-ledger", 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
blob.meta.set_addr(from_addr);
|
2018-08-06 12:35:38 -07:00
|
|
|
|
2019-02-07 15:10:54 -08:00
|
|
|
return vec![Arc::new(RwLock::new(blob))];
|
2018-08-06 12:35:38 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-05-17 07:00:06 -07:00
|
|
|
inc_new_counter_debug!("cluster_info-window-request-fail", 1);
|
2019-02-07 15:10:54 -08:00
|
|
|
trace!(
|
|
|
|
"{}: failed RequestWindowIndex {} {} {}",
|
|
|
|
me.id,
|
|
|
|
from.id,
|
2019-03-05 14:18:29 -08:00
|
|
|
slot,
|
2019-02-07 15:10:54 -08:00
|
|
|
blob_index,
|
|
|
|
);
|
2018-08-06 12:35:38 -07:00
|
|
|
|
2018-11-15 13:23:26 -08:00
|
|
|
vec![]
|
2018-05-12 19:00:22 -07:00
|
|
|
}
|
2018-05-27 18:21:39 -07:00
|
|
|
|
2019-02-14 12:47:21 -08:00
|
|
|
fn run_highest_window_request(
|
|
|
|
from_addr: &SocketAddr,
|
|
|
|
blocktree: Option<&Arc<Blocktree>>,
|
2019-03-05 14:18:29 -08:00
|
|
|
slot: u64,
|
2019-02-14 12:47:21 -08:00
|
|
|
highest_index: u64,
|
|
|
|
) -> Vec<SharedBlob> {
|
|
|
|
if let Some(blocktree) = blocktree {
|
|
|
|
// Try to find the requested index in one of the slots
|
2019-03-05 14:18:29 -08:00
|
|
|
let meta = blocktree.meta(slot);
|
2019-02-14 12:47:21 -08:00
|
|
|
|
|
|
|
if let Ok(Some(meta)) = meta {
|
|
|
|
if meta.received > highest_index {
|
|
|
|
// meta.received must be at least 1 by this point
|
2019-09-04 12:47:09 -07:00
|
|
|
let blob = Self::get_data_shred_as_blob(blocktree, slot, meta.received - 1);
|
2019-02-14 12:47:21 -08:00
|
|
|
|
|
|
|
if let Ok(Some(mut blob)) = blob {
|
|
|
|
blob.meta.set_addr(from_addr);
|
|
|
|
return vec![Arc::new(RwLock::new(blob))];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
vec![]
|
|
|
|
}
|
|
|
|
|
2019-04-06 19:41:22 -07:00
|
|
|
fn run_orphan(
|
|
|
|
from_addr: &SocketAddr,
|
|
|
|
blocktree: Option<&Arc<Blocktree>>,
|
|
|
|
mut slot: u64,
|
|
|
|
max_responses: usize,
|
|
|
|
) -> Vec<SharedBlob> {
|
|
|
|
let mut res = vec![];
|
|
|
|
if let Some(blocktree) = blocktree {
|
|
|
|
// Try to find the next "n" parent slots of the input slot
|
|
|
|
while let Ok(Some(meta)) = blocktree.meta(slot) {
|
|
|
|
if meta.received == 0 {
|
|
|
|
break;
|
|
|
|
}
|
2019-09-04 12:47:09 -07:00
|
|
|
let blob = Self::get_data_shred_as_blob(blocktree, slot, meta.received - 1);
|
2019-04-06 19:41:22 -07:00
|
|
|
if let Ok(Some(mut blob)) = blob {
|
|
|
|
blob.meta.set_addr(from_addr);
|
|
|
|
res.push(Arc::new(RwLock::new(blob)));
|
|
|
|
}
|
|
|
|
if meta.is_parent_set() && res.len() <= max_responses {
|
|
|
|
slot = meta.parent_slot;
|
|
|
|
} else {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
res
|
|
|
|
}
|
|
|
|
|
2019-08-15 17:04:45 -07:00
|
|
|
fn handle_blobs(
|
|
|
|
me: &Arc<RwLock<Self>>,
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree: Option<&Arc<Blocktree>>,
|
2019-06-26 00:30:16 -07:00
|
|
|
stakes: &HashMap<Pubkey, u64>,
|
2019-08-15 17:04:45 -07:00
|
|
|
blobs: &[SharedBlob],
|
|
|
|
response_sender: &BlobSender,
|
|
|
|
) {
|
|
|
|
// iter over the blobs, collect pulls separately and process everything else
|
|
|
|
let mut gossip_pull_data: Vec<PullData> = vec![];
|
|
|
|
blobs.iter().for_each(|blob| {
|
|
|
|
let blob = blob.read().unwrap();
|
|
|
|
let from_addr = blob.meta.addr();
|
|
|
|
deserialize(&blob.data[..blob.meta.size])
|
|
|
|
.into_iter()
|
|
|
|
.for_each(|request| match request {
|
|
|
|
Protocol::PullRequest(filter, caller) => {
|
|
|
|
if !caller.verify() {
|
|
|
|
inc_new_counter_error!(
|
|
|
|
"cluster_info-gossip_pull_request_verify_fail",
|
|
|
|
1
|
|
|
|
);
|
|
|
|
} else if caller.contact_info().is_some() {
|
|
|
|
if caller.contact_info().unwrap().pubkey()
|
|
|
|
== me.read().unwrap().gossip.id
|
|
|
|
{
|
|
|
|
warn!("PullRequest ignored, I'm talking to myself");
|
|
|
|
inc_new_counter_debug!("cluster_info-window-request-loopback", 1);
|
|
|
|
} else {
|
|
|
|
gossip_pull_data.push(PullData {
|
|
|
|
from_addr,
|
|
|
|
caller,
|
|
|
|
filter,
|
|
|
|
});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
Protocol::PullResponse(from, mut data) => {
|
|
|
|
data.retain(|v| {
|
|
|
|
let ret = v.verify();
|
|
|
|
if !ret {
|
|
|
|
inc_new_counter_error!(
|
|
|
|
"cluster_info-gossip_pull_response_verify_fail",
|
|
|
|
1
|
|
|
|
);
|
|
|
|
}
|
|
|
|
ret
|
|
|
|
});
|
|
|
|
Self::handle_pull_response(me, &from, data);
|
|
|
|
}
|
|
|
|
Protocol::PushMessage(from, mut data) => {
|
|
|
|
data.retain(|v| {
|
|
|
|
let ret = v.verify();
|
|
|
|
if !ret {
|
|
|
|
inc_new_counter_error!(
|
|
|
|
"cluster_info-gossip_push_msg_verify_fail",
|
|
|
|
1
|
|
|
|
);
|
|
|
|
}
|
|
|
|
ret
|
|
|
|
});
|
|
|
|
let _ignore_disconnect = response_sender
|
|
|
|
.send(Self::handle_push_message(me, &from, data, stakes));
|
|
|
|
}
|
|
|
|
Protocol::PruneMessage(from, data) => {
|
|
|
|
if data.verify() {
|
|
|
|
inc_new_counter_debug!("cluster_info-prune_message", 1);
|
|
|
|
inc_new_counter_debug!(
|
|
|
|
"cluster_info-prune_message-size",
|
|
|
|
data.prunes.len()
|
|
|
|
);
|
|
|
|
match me.write().unwrap().gossip.process_prune_msg(
|
|
|
|
&from,
|
|
|
|
&data.destination,
|
|
|
|
&data.prunes,
|
|
|
|
data.wallclock,
|
|
|
|
timestamp(),
|
|
|
|
) {
|
|
|
|
Err(CrdsGossipError::PruneMessageTimeout) => {
|
|
|
|
inc_new_counter_debug!("cluster_info-prune_message_timeout", 1)
|
|
|
|
}
|
|
|
|
Err(CrdsGossipError::BadPruneDestination) => {
|
|
|
|
inc_new_counter_debug!("cluster_info-bad_prune_destination", 1)
|
|
|
|
}
|
|
|
|
_ => (),
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
inc_new_counter_debug!("cluster_info-gossip_prune_msg_verify_fail", 1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
_ => {
|
|
|
|
let _ignore_disconnect = response_sender
|
|
|
|
.send(Self::handle_repair(me, &from_addr, blocktree, request));
|
|
|
|
}
|
|
|
|
})
|
|
|
|
});
|
|
|
|
// process the collected pulls together
|
|
|
|
let _ignore_disconnect =
|
|
|
|
response_sender.send(Self::handle_pull_requests(me, gossip_pull_data));
|
|
|
|
}
|
|
|
|
|
|
|
|
fn handle_pull_requests(me: &Arc<RwLock<Self>>, requests: Vec<PullData>) -> Vec<SharedBlob> {
|
|
|
|
// split the requests into addrs and filters
|
|
|
|
let mut caller_and_filters = vec![];
|
|
|
|
let mut addrs = vec![];
|
|
|
|
for pull_data in requests {
|
|
|
|
caller_and_filters.push((pull_data.caller, pull_data.filter));
|
|
|
|
addrs.push(pull_data.from_addr);
|
2018-11-15 13:23:26 -08:00
|
|
|
}
|
|
|
|
let now = timestamp();
|
2019-08-15 17:04:45 -07:00
|
|
|
let self_id = me.read().unwrap().id();
|
|
|
|
let pull_responses = me
|
2018-11-15 13:23:26 -08:00
|
|
|
.write()
|
|
|
|
.unwrap()
|
|
|
|
.gossip
|
2019-08-15 17:04:45 -07:00
|
|
|
.process_pull_requests(caller_and_filters, now);
|
|
|
|
pull_responses
|
2019-07-30 15:43:17 -07:00
|
|
|
.into_iter()
|
2019-08-15 17:04:45 -07:00
|
|
|
.zip(addrs.into_iter())
|
|
|
|
.flat_map(|(response, from_addr)| {
|
|
|
|
let len = response.len();
|
|
|
|
trace!("get updates since response {}", len);
|
|
|
|
inc_new_counter_debug!("cluster_info-pull_request-rsp", len);
|
|
|
|
Self::split_gossip_messages(response)
|
|
|
|
.into_iter()
|
|
|
|
.filter_map(move |payload| {
|
|
|
|
let protocol = Protocol::PullResponse(self_id, payload);
|
|
|
|
// The remote node may not know its public IP:PORT. Instead of responding to the caller's
|
|
|
|
// gossip addr, respond to the origin addr. The last origin addr is picked from the list of
|
|
|
|
// addrs.
|
|
|
|
to_shared_blob(protocol, from_addr).ok()
|
|
|
|
})
|
|
|
|
})
|
2019-07-30 15:43:17 -07:00
|
|
|
.collect()
|
2018-07-18 17:07:01 -07:00
|
|
|
}
|
2019-06-26 00:30:16 -07:00
|
|
|
|
2019-03-09 19:28:43 -08:00
|
|
|
fn handle_pull_response(me: &Arc<RwLock<Self>>, from: &Pubkey, data: Vec<CrdsValue>) {
|
2018-11-15 13:23:26 -08:00
|
|
|
let len = data.len();
|
|
|
|
let now = Instant::now();
|
|
|
|
let self_id = me.read().unwrap().gossip.id;
|
2019-03-01 10:36:52 -08:00
|
|
|
trace!("PullResponse me: {} from: {} len={}", self_id, from, len);
|
2018-11-15 13:23:26 -08:00
|
|
|
me.write()
|
|
|
|
.unwrap()
|
|
|
|
.gossip
|
|
|
|
.process_pull_response(from, data, timestamp());
|
2019-05-17 07:00:06 -07:00
|
|
|
inc_new_counter_debug!("cluster_info-pull_request_response", 1);
|
|
|
|
inc_new_counter_debug!("cluster_info-pull_request_response-size", len);
|
2018-07-18 17:07:01 -07:00
|
|
|
|
2018-11-15 13:23:26 -08:00
|
|
|
report_time_spent("ReceiveUpdates", &now.elapsed(), &format!(" len: {}", len));
|
|
|
|
}
|
2019-06-26 00:30:16 -07:00
|
|
|
|
2018-11-15 13:23:26 -08:00
|
|
|
fn handle_push_message(
|
|
|
|
me: &Arc<RwLock<Self>>,
|
2019-03-09 19:28:43 -08:00
|
|
|
from: &Pubkey,
|
2019-05-08 13:50:32 -07:00
|
|
|
data: Vec<CrdsValue>,
|
2019-06-26 00:30:16 -07:00
|
|
|
stakes: &HashMap<Pubkey, u64>,
|
2018-11-15 13:23:26 -08:00
|
|
|
) -> Vec<SharedBlob> {
|
|
|
|
let self_id = me.read().unwrap().gossip.id;
|
2019-08-12 15:15:34 -07:00
|
|
|
inc_new_counter_debug!("cluster_info-push_message", 1);
|
2019-05-23 03:50:41 -07:00
|
|
|
|
2019-06-26 00:30:16 -07:00
|
|
|
let updated: Vec<_> =
|
|
|
|
me.write()
|
|
|
|
.unwrap()
|
|
|
|
.gossip
|
|
|
|
.process_push_message(from, data, timestamp());
|
|
|
|
|
|
|
|
let updated_labels: Vec<_> = updated.into_iter().map(|u| u.value.label()).collect();
|
|
|
|
let prunes_map: HashMap<Pubkey, HashSet<Pubkey>> = me
|
2018-11-15 13:23:26 -08:00
|
|
|
.write()
|
|
|
|
.unwrap()
|
|
|
|
.gossip
|
2019-06-26 00:30:16 -07:00
|
|
|
.prune_received_cache(updated_labels, stakes);
|
2019-05-23 03:50:41 -07:00
|
|
|
|
2019-06-26 00:30:16 -07:00
|
|
|
let mut rsp: Vec<_> = prunes_map
|
|
|
|
.into_iter()
|
|
|
|
.map(|(from, prune_set)| {
|
|
|
|
inc_new_counter_debug!("cluster_info-push_message-prunes", prune_set.len());
|
|
|
|
me.read().unwrap().lookup(&from).cloned().and_then(|ci| {
|
2018-12-01 12:00:30 -08:00
|
|
|
let mut prune_msg = PruneData {
|
|
|
|
pubkey: self_id,
|
2019-06-26 00:30:16 -07:00
|
|
|
prunes: prune_set.into_iter().collect(),
|
2018-12-01 12:00:30 -08:00
|
|
|
signature: Signature::default(),
|
2019-06-26 00:30:16 -07:00
|
|
|
destination: from,
|
2018-12-01 12:00:30 -08:00
|
|
|
wallclock: timestamp(),
|
|
|
|
};
|
|
|
|
prune_msg.sign(&me.read().unwrap().keypair);
|
|
|
|
let rsp = Protocol::PruneMessage(self_id, prune_msg);
|
2018-12-22 19:30:30 -08:00
|
|
|
to_shared_blob(rsp, ci.gossip).ok()
|
2018-12-07 19:01:28 -08:00
|
|
|
})
|
2019-06-26 00:30:16 -07:00
|
|
|
})
|
|
|
|
.flatten()
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
if !rsp.is_empty() {
|
|
|
|
let pushes: Vec<_> = me.write().unwrap().new_push_requests();
|
|
|
|
inc_new_counter_debug!("cluster_info-push_message-pushes", pushes.len());
|
2018-11-15 13:23:26 -08:00
|
|
|
let mut blobs: Vec<_> = pushes
|
|
|
|
.into_iter()
|
2018-12-22 19:30:30 -08:00
|
|
|
.filter_map(|(remote_gossip_addr, req)| {
|
|
|
|
to_shared_blob(req, remote_gossip_addr).ok()
|
|
|
|
})
|
2018-11-15 13:23:26 -08:00
|
|
|
.collect();
|
|
|
|
rsp.append(&mut blobs);
|
|
|
|
rsp
|
|
|
|
} else {
|
|
|
|
vec![]
|
|
|
|
}
|
|
|
|
}
|
2019-04-06 19:41:22 -07:00
|
|
|
|
|
|
|
fn get_repair_sender(request: &Protocol) -> &ContactInfo {
|
|
|
|
match request {
|
|
|
|
Protocol::RequestWindowIndex(ref from, _, _) => from,
|
|
|
|
Protocol::RequestHighestWindowIndex(ref from, _, _) => from,
|
|
|
|
Protocol::RequestOrphan(ref from, _) => from,
|
|
|
|
_ => panic!("Not a repair request"),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn handle_repair(
|
2018-11-15 13:23:26 -08:00
|
|
|
me: &Arc<RwLock<Self>>,
|
|
|
|
from_addr: &SocketAddr,
|
2019-04-06 19:41:22 -07:00
|
|
|
blocktree: Option<&Arc<Blocktree>>,
|
|
|
|
request: Protocol,
|
2018-11-15 13:23:26 -08:00
|
|
|
) -> Vec<SharedBlob> {
|
|
|
|
let now = Instant::now();
|
|
|
|
|
|
|
|
//TODO this doesn't depend on cluster_info module, could be moved
|
|
|
|
//but we are using the listen thread to service these request
|
|
|
|
//TODO verify from is signed
|
|
|
|
|
|
|
|
let self_id = me.read().unwrap().gossip.id;
|
2019-04-06 19:41:22 -07:00
|
|
|
let from = Self::get_repair_sender(&request);
|
2018-11-15 13:23:26 -08:00
|
|
|
if from.id == me.read().unwrap().gossip.id {
|
|
|
|
warn!(
|
2019-04-06 19:41:22 -07:00
|
|
|
"{}: Ignored received repair request from ME {}",
|
|
|
|
self_id, from.id,
|
2018-11-15 13:23:26 -08:00
|
|
|
);
|
2019-05-17 07:00:06 -07:00
|
|
|
inc_new_counter_debug!("cluster_info-handle-repair--eq", 1);
|
2018-11-15 13:23:26 -08:00
|
|
|
return vec![];
|
|
|
|
}
|
|
|
|
|
2019-03-08 18:08:24 -08:00
|
|
|
me.write()
|
|
|
|
.unwrap()
|
|
|
|
.gossip
|
|
|
|
.crds
|
2019-03-09 19:28:43 -08:00
|
|
|
.update_record_timestamp(&from.id, timestamp());
|
2018-11-15 13:23:26 -08:00
|
|
|
let my_info = me.read().unwrap().my_data().clone();
|
2019-04-06 19:41:22 -07:00
|
|
|
|
|
|
|
let (res, label) = {
|
|
|
|
match &request {
|
|
|
|
Protocol::RequestWindowIndex(from, slot, blob_index) => {
|
2019-05-17 07:00:06 -07:00
|
|
|
inc_new_counter_debug!("cluster_info-request-window-index", 1);
|
2019-04-06 19:41:22 -07:00
|
|
|
(
|
|
|
|
Self::run_window_request(
|
|
|
|
from,
|
|
|
|
&from_addr,
|
|
|
|
blocktree,
|
|
|
|
&my_info,
|
|
|
|
*slot,
|
|
|
|
*blob_index,
|
|
|
|
),
|
|
|
|
"RequestWindowIndex",
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
|
|
|
Protocol::RequestHighestWindowIndex(_, slot, highest_index) => {
|
2019-05-17 07:00:06 -07:00
|
|
|
inc_new_counter_debug!("cluster_info-request-highest-window-index", 1);
|
2019-04-06 19:41:22 -07:00
|
|
|
(
|
|
|
|
Self::run_highest_window_request(
|
|
|
|
&from_addr,
|
|
|
|
blocktree,
|
|
|
|
*slot,
|
|
|
|
*highest_index,
|
|
|
|
),
|
|
|
|
"RequestHighestWindowIndex",
|
|
|
|
)
|
|
|
|
}
|
|
|
|
Protocol::RequestOrphan(_, slot) => {
|
2019-05-17 07:00:06 -07:00
|
|
|
inc_new_counter_debug!("cluster_info-request-orphan", 1);
|
2019-04-06 19:41:22 -07:00
|
|
|
(
|
|
|
|
Self::run_orphan(&from_addr, blocktree, *slot, MAX_ORPHAN_REPAIR_RESPONSES),
|
|
|
|
"RequestOrphan",
|
|
|
|
)
|
|
|
|
}
|
|
|
|
_ => panic!("Not a repair request"),
|
2019-02-14 12:47:21 -08:00
|
|
|
}
|
|
|
|
};
|
2019-04-06 19:41:22 -07:00
|
|
|
|
|
|
|
trace!("{}: received repair request: {:?}", self_id, request);
|
|
|
|
report_time_spent(label, &now.elapsed(), "");
|
2018-11-15 13:23:26 -08:00
|
|
|
res
|
|
|
|
}
|
2019-02-14 12:47:21 -08:00
|
|
|
|
2018-05-27 18:21:39 -07:00
|
|
|
/// Process messages from the network
|
|
|
|
fn run_listen(
|
|
|
|
obj: &Arc<RwLock<Self>>,
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree: Option<&Arc<Blocktree>>,
|
2019-06-26 00:30:16 -07:00
|
|
|
bank_forks: Option<&Arc<RwLock<BankForks>>>,
|
2018-05-27 18:21:39 -07:00
|
|
|
requests_receiver: &BlobReceiver,
|
|
|
|
response_sender: &BlobSender,
|
|
|
|
) -> Result<()> {
|
|
|
|
//TODO cache connections
|
|
|
|
let timeout = Duration::new(1, 0);
|
|
|
|
let mut reqs = requests_receiver.recv_timeout(timeout)?;
|
|
|
|
while let Ok(mut more) = requests_receiver.try_recv() {
|
|
|
|
reqs.append(&mut more);
|
|
|
|
}
|
2019-06-26 00:30:16 -07:00
|
|
|
|
|
|
|
let stakes: HashMap<_, _> = match bank_forks {
|
|
|
|
Some(ref bank_forks) => {
|
|
|
|
staking_utils::staked_nodes(&bank_forks.read().unwrap().working_bank())
|
|
|
|
}
|
|
|
|
None => HashMap::new(),
|
|
|
|
};
|
|
|
|
|
2019-08-15 17:04:45 -07:00
|
|
|
Self::handle_blobs(obj, blocktree, &stakes, &reqs, response_sender);
|
2018-04-21 11:02:49 -07:00
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
pub fn listen(
|
2018-08-30 12:07:54 -07:00
|
|
|
me: Arc<RwLock<Self>>,
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree: Option<Arc<Blocktree>>,
|
2019-06-26 00:30:16 -07:00
|
|
|
bank_forks: Option<Arc<RwLock<BankForks>>>,
|
2018-05-27 18:21:39 -07:00
|
|
|
requests_receiver: BlobReceiver,
|
|
|
|
response_sender: BlobSender,
|
2019-03-04 20:50:02 -08:00
|
|
|
exit: &Arc<AtomicBool>,
|
2018-04-21 11:02:49 -07:00
|
|
|
) -> JoinHandle<()> {
|
2019-03-04 20:50:02 -08:00
|
|
|
let exit = exit.clone();
|
2018-05-30 13:25:32 -07:00
|
|
|
Builder::new()
|
|
|
|
.name("solana-listen".to_string())
|
|
|
|
.spawn(move || loop {
|
|
|
|
let e = Self::run_listen(
|
2018-08-30 12:07:54 -07:00
|
|
|
&me,
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree.as_ref(),
|
2019-06-26 00:30:16 -07:00
|
|
|
bank_forks.as_ref(),
|
2018-05-30 13:25:32 -07:00
|
|
|
&requests_receiver,
|
|
|
|
&response_sender,
|
2018-05-23 21:45:40 -07:00
|
|
|
);
|
2018-07-09 15:53:49 -07:00
|
|
|
if exit.load(Ordering::Relaxed) {
|
|
|
|
return;
|
|
|
|
}
|
2018-05-30 13:25:32 -07:00
|
|
|
if e.is_err() {
|
2018-08-30 12:07:54 -07:00
|
|
|
let me = me.read().unwrap();
|
2018-07-31 22:07:53 -07:00
|
|
|
debug!(
|
2018-09-05 21:36:59 -07:00
|
|
|
"{}: run_listen timeout, table size: {}",
|
2018-11-15 13:23:26 -08:00
|
|
|
me.gossip.id,
|
|
|
|
me.gossip.crds.table.len()
|
2018-05-30 13:25:32 -07:00
|
|
|
);
|
|
|
|
}
|
2018-12-07 19:01:28 -08:00
|
|
|
})
|
|
|
|
.unwrap()
|
2018-04-21 11:02:49 -07:00
|
|
|
}
|
2018-08-26 11:36:27 -07:00
|
|
|
|
2019-04-30 16:42:56 -07:00
|
|
|
/// An alternative to Spy Node that has a valid gossip address and fully participate in Gossip.
|
|
|
|
pub fn gossip_node(id: &Pubkey, gossip_addr: &SocketAddr) -> (ContactInfo, UdpSocket) {
|
|
|
|
let (port, gossip_socket) = Node::get_gossip_port(gossip_addr, FULLNODE_PORT_RANGE);
|
|
|
|
let daddr = socketaddr_any!();
|
|
|
|
|
|
|
|
let node = ContactInfo::new(
|
|
|
|
id,
|
|
|
|
SocketAddr::new(gossip_addr.ip(), port),
|
|
|
|
daddr,
|
|
|
|
daddr,
|
|
|
|
daddr,
|
|
|
|
daddr,
|
|
|
|
daddr,
|
|
|
|
daddr,
|
2019-08-20 17:16:06 -07:00
|
|
|
daddr,
|
2019-04-30 16:42:56 -07:00
|
|
|
timestamp(),
|
|
|
|
);
|
|
|
|
(node, gossip_socket)
|
|
|
|
}
|
|
|
|
|
|
|
|
/// A Node with invalid ports to spy on gossip via pull requests
|
2019-03-08 17:23:07 -08:00
|
|
|
pub fn spy_node(id: &Pubkey) -> (ContactInfo, UdpSocket) {
|
2018-09-08 12:50:43 -07:00
|
|
|
let (_, gossip_socket) = bind_in_range(FULLNODE_PORT_RANGE).unwrap();
|
2018-08-30 12:07:54 -07:00
|
|
|
let daddr = socketaddr_any!();
|
|
|
|
|
2019-03-08 14:59:11 -08:00
|
|
|
let node = ContactInfo::new(
|
2019-03-11 03:06:22 -07:00
|
|
|
id,
|
2019-03-08 14:59:11 -08:00
|
|
|
daddr,
|
|
|
|
daddr,
|
|
|
|
daddr,
|
|
|
|
daddr,
|
|
|
|
daddr,
|
|
|
|
daddr,
|
|
|
|
daddr,
|
2019-08-20 17:16:06 -07:00
|
|
|
daddr,
|
2019-03-08 14:59:11 -08:00
|
|
|
timestamp(),
|
|
|
|
);
|
2018-08-28 16:32:40 -07:00
|
|
|
(node, gossip_socket)
|
2018-07-18 01:07:43 -07:00
|
|
|
}
|
2018-04-21 11:02:49 -07:00
|
|
|
}
|
|
|
|
|
2019-06-06 12:48:40 -07:00
|
|
|
/// Turbine logic
|
2019-02-18 08:51:00 -08:00
|
|
|
/// 1 - For the current node find out if it is in layer 1
|
|
|
|
/// 1.1 - If yes, then broadcast to all layer 1 nodes
|
|
|
|
/// 1 - using the layer 1 index, broadcast to all layer 2 nodes assuming you know neighborhood size
|
|
|
|
/// 1.2 - If no, then figure out what layer the node is in and who the neighbors are and only broadcast to them
|
2019-05-07 13:24:58 -07:00
|
|
|
/// 1 - also check if there are nodes in the next layer and repeat the layer 1 to layer 2 logic
|
2019-02-18 08:51:00 -08:00
|
|
|
|
|
|
|
/// Returns Neighbor Nodes and Children Nodes `(neighbors, children)` for a given node based on its stake (Bank Balance)
|
2019-06-03 20:38:05 -07:00
|
|
|
pub fn compute_retransmit_peers(
|
2019-02-18 08:51:00 -08:00
|
|
|
fanout: usize,
|
2019-06-03 20:38:05 -07:00
|
|
|
my_index: usize,
|
|
|
|
peers: Vec<ContactInfo>,
|
2019-03-08 17:23:07 -08:00
|
|
|
) -> (Vec<ContactInfo>, Vec<ContactInfo>) {
|
2019-02-18 08:51:00 -08:00
|
|
|
//calc num_layers and num_neighborhoods using the total number of nodes
|
2019-05-07 13:24:58 -07:00
|
|
|
let (num_layers, layer_indices) = ClusterInfo::describe_data_plane(peers.len(), fanout);
|
2019-02-18 08:51:00 -08:00
|
|
|
|
|
|
|
if num_layers <= 1 {
|
|
|
|
/* single layer data plane */
|
|
|
|
(peers, vec![])
|
|
|
|
} else {
|
|
|
|
//find my layer
|
2019-05-07 13:24:58 -07:00
|
|
|
let locality = ClusterInfo::localize(&layer_indices, fanout, my_index);
|
2019-02-18 08:51:00 -08:00
|
|
|
let upper_bound = cmp::min(locality.neighbor_bounds.1, peers.len());
|
|
|
|
let neighbors = peers[locality.neighbor_bounds.0..upper_bound].to_vec();
|
|
|
|
let mut children = Vec::new();
|
2019-05-07 13:24:58 -07:00
|
|
|
for ix in locality.next_layer_peers {
|
2019-02-18 08:51:00 -08:00
|
|
|
if let Some(peer) = peers.get(ix) {
|
|
|
|
children.push(peer.clone());
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
(neighbors, children)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-09-21 15:32:15 -07:00
|
|
|
#[derive(Debug)]
|
2018-05-27 18:21:39 -07:00
|
|
|
pub struct Sockets {
|
|
|
|
pub gossip: UdpSocket,
|
2018-12-07 14:09:29 -08:00
|
|
|
pub tvu: Vec<UdpSocket>,
|
2019-08-20 17:16:06 -07:00
|
|
|
pub tvu_forwards: Vec<UdpSocket>,
|
2018-12-07 14:09:29 -08:00
|
|
|
pub tpu: Vec<UdpSocket>,
|
2019-07-30 14:50:02 -07:00
|
|
|
pub tpu_forwards: Vec<UdpSocket>,
|
2018-05-27 18:21:39 -07:00
|
|
|
pub broadcast: UdpSocket,
|
2018-06-02 08:32:51 -07:00
|
|
|
pub repair: UdpSocket,
|
2018-06-13 21:52:23 -07:00
|
|
|
pub retransmit: UdpSocket,
|
2019-03-18 11:02:32 -07:00
|
|
|
pub storage: Option<UdpSocket>,
|
2018-05-27 18:21:39 -07:00
|
|
|
}
|
|
|
|
|
2018-09-21 15:32:15 -07:00
|
|
|
#[derive(Debug)]
|
2018-08-28 16:32:40 -07:00
|
|
|
pub struct Node {
|
2019-03-08 17:23:07 -08:00
|
|
|
pub info: ContactInfo,
|
2018-05-27 18:21:39 -07:00
|
|
|
pub sockets: Sockets,
|
|
|
|
}
|
2018-04-21 11:02:49 -07:00
|
|
|
|
2018-08-28 16:32:40 -07:00
|
|
|
impl Node {
|
2018-07-16 19:31:52 -07:00
|
|
|
pub fn new_localhost() -> Self {
|
2019-03-30 20:37:33 -07:00
|
|
|
let pubkey = Pubkey::new_rand();
|
2019-03-09 19:28:43 -08:00
|
|
|
Self::new_localhost_with_pubkey(&pubkey)
|
2018-07-09 15:53:49 -07:00
|
|
|
}
|
2019-03-18 11:02:32 -07:00
|
|
|
pub fn new_localhost_replicator(pubkey: &Pubkey) -> Self {
|
2019-03-18 11:12:19 -07:00
|
|
|
let gossip = UdpSocket::bind("127.0.0.1:0").unwrap();
|
|
|
|
let tvu = UdpSocket::bind("127.0.0.1:0").unwrap();
|
2019-08-20 17:16:06 -07:00
|
|
|
let tvu_forwards = UdpSocket::bind("127.0.0.1:0").unwrap();
|
2019-03-18 11:02:32 -07:00
|
|
|
let storage = UdpSocket::bind("127.0.0.1:0").unwrap();
|
2019-03-18 11:12:19 -07:00
|
|
|
let empty = "0.0.0.0:0".parse().unwrap();
|
|
|
|
let repair = UdpSocket::bind("127.0.0.1:0").unwrap();
|
|
|
|
|
|
|
|
let broadcast = UdpSocket::bind("0.0.0.0:0").unwrap();
|
|
|
|
let retransmit = UdpSocket::bind("0.0.0.0:0").unwrap();
|
|
|
|
let info = ContactInfo::new(
|
|
|
|
pubkey,
|
|
|
|
gossip.local_addr().unwrap(),
|
|
|
|
tvu.local_addr().unwrap(),
|
2019-08-20 17:16:06 -07:00
|
|
|
tvu_forwards.local_addr().unwrap(),
|
2019-03-18 11:12:19 -07:00
|
|
|
empty,
|
|
|
|
empty,
|
|
|
|
storage.local_addr().unwrap(),
|
|
|
|
empty,
|
|
|
|
empty,
|
|
|
|
timestamp(),
|
|
|
|
);
|
|
|
|
|
|
|
|
Node {
|
|
|
|
info,
|
|
|
|
sockets: Sockets {
|
|
|
|
gossip,
|
|
|
|
tvu: vec![tvu],
|
2019-08-20 17:16:06 -07:00
|
|
|
tvu_forwards: vec![],
|
2019-03-18 11:12:19 -07:00
|
|
|
tpu: vec![],
|
2019-07-30 14:50:02 -07:00
|
|
|
tpu_forwards: vec![],
|
2019-03-18 11:12:19 -07:00
|
|
|
broadcast,
|
|
|
|
repair,
|
|
|
|
retransmit,
|
|
|
|
storage: Some(storage),
|
|
|
|
},
|
|
|
|
}
|
2019-03-18 11:02:32 -07:00
|
|
|
}
|
2019-03-09 19:28:43 -08:00
|
|
|
pub fn new_localhost_with_pubkey(pubkey: &Pubkey) -> Self {
|
2018-12-07 14:09:29 -08:00
|
|
|
let tpu = UdpSocket::bind("127.0.0.1:0").unwrap();
|
2018-07-16 19:31:52 -07:00
|
|
|
let gossip = UdpSocket::bind("127.0.0.1:0").unwrap();
|
2018-12-07 14:09:29 -08:00
|
|
|
let tvu = UdpSocket::bind("127.0.0.1:0").unwrap();
|
2019-08-20 17:16:06 -07:00
|
|
|
let tvu_forwards = UdpSocket::bind("127.0.0.1:0").unwrap();
|
2019-07-30 14:50:02 -07:00
|
|
|
let tpu_forwards = UdpSocket::bind("127.0.0.1:0").unwrap();
|
2018-07-16 19:31:52 -07:00
|
|
|
let repair = UdpSocket::bind("127.0.0.1:0").unwrap();
|
2018-11-05 09:50:58 -08:00
|
|
|
let rpc_port = find_available_port_in_range((1024, 65535)).unwrap();
|
|
|
|
let rpc_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), rpc_port);
|
|
|
|
let rpc_pubsub_port = find_available_port_in_range((1024, 65535)).unwrap();
|
|
|
|
let rpc_pubsub_addr =
|
|
|
|
SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), rpc_pubsub_port);
|
2018-07-16 19:31:52 -07:00
|
|
|
|
2018-05-27 18:21:39 -07:00
|
|
|
let broadcast = UdpSocket::bind("0.0.0.0:0").unwrap();
|
2018-06-13 21:52:23 -07:00
|
|
|
let retransmit = UdpSocket::bind("0.0.0.0:0").unwrap();
|
2018-09-21 15:32:15 -07:00
|
|
|
let storage = UdpSocket::bind("0.0.0.0:0").unwrap();
|
2019-03-08 17:23:07 -08:00
|
|
|
let info = ContactInfo::new(
|
2018-04-28 00:31:20 -07:00
|
|
|
pubkey,
|
|
|
|
gossip.local_addr().unwrap(),
|
2018-12-07 14:09:29 -08:00
|
|
|
tvu.local_addr().unwrap(),
|
2019-08-20 17:16:06 -07:00
|
|
|
tvu_forwards.local_addr().unwrap(),
|
2018-12-07 14:09:29 -08:00
|
|
|
tpu.local_addr().unwrap(),
|
2019-07-30 14:50:02 -07:00
|
|
|
tpu_forwards.local_addr().unwrap(),
|
2018-09-21 15:32:15 -07:00
|
|
|
storage.local_addr().unwrap(),
|
2018-11-05 09:50:58 -08:00
|
|
|
rpc_addr,
|
|
|
|
rpc_pubsub_addr,
|
2018-11-15 13:23:26 -08:00
|
|
|
timestamp(),
|
2018-04-28 00:31:20 -07:00
|
|
|
);
|
2018-08-28 16:32:40 -07:00
|
|
|
Node {
|
2018-08-31 00:10:39 -07:00
|
|
|
info,
|
2018-05-27 18:21:39 -07:00
|
|
|
sockets: Sockets {
|
|
|
|
gossip,
|
2018-12-07 14:09:29 -08:00
|
|
|
tvu: vec![tvu],
|
2019-08-20 17:16:06 -07:00
|
|
|
tvu_forwards: vec![tvu_forwards],
|
2018-12-07 14:09:29 -08:00
|
|
|
tpu: vec![tpu],
|
2019-07-30 14:50:02 -07:00
|
|
|
tpu_forwards: vec![tpu_forwards],
|
2018-05-27 18:21:39 -07:00
|
|
|
broadcast,
|
2018-06-02 08:32:51 -07:00
|
|
|
repair,
|
2018-06-13 21:52:23 -07:00
|
|
|
retransmit,
|
2019-03-18 11:02:32 -07:00
|
|
|
storage: None,
|
2018-05-27 18:21:39 -07:00
|
|
|
},
|
2018-04-21 11:02:49 -07:00
|
|
|
}
|
2018-04-26 13:48:42 -07:00
|
|
|
}
|
2019-04-12 18:17:34 -07:00
|
|
|
fn get_gossip_port(gossip_addr: &SocketAddr, port_range: PortRange) -> (u16, UdpSocket) {
|
2019-03-18 11:12:19 -07:00
|
|
|
if gossip_addr.port() != 0 {
|
2018-12-06 12:52:47 -08:00
|
|
|
(
|
|
|
|
gossip_addr.port(),
|
2018-12-17 08:40:56 -08:00
|
|
|
bind_to(gossip_addr.port(), false).unwrap_or_else(|e| {
|
|
|
|
panic!("gossip_addr bind_to port {}: {}", gossip_addr.port(), e)
|
|
|
|
}),
|
2018-12-06 12:52:47 -08:00
|
|
|
)
|
2018-08-25 10:24:16 -07:00
|
|
|
} else {
|
2019-04-12 18:17:34 -07:00
|
|
|
Self::bind(port_range)
|
2019-03-18 11:12:19 -07:00
|
|
|
}
|
|
|
|
}
|
2019-04-12 18:17:34 -07:00
|
|
|
fn bind(port_range: PortRange) -> (u16, UdpSocket) {
|
|
|
|
bind_in_range(port_range).expect("Failed to bind")
|
2019-03-18 11:12:19 -07:00
|
|
|
}
|
2019-04-12 18:17:34 -07:00
|
|
|
pub fn new_with_external_ip(
|
|
|
|
pubkey: &Pubkey,
|
|
|
|
gossip_addr: &SocketAddr,
|
|
|
|
port_range: PortRange,
|
|
|
|
) -> Node {
|
|
|
|
let (gossip_port, gossip) = Self::get_gossip_port(gossip_addr, port_range);
|
2018-08-30 12:07:54 -07:00
|
|
|
|
2019-04-12 18:17:34 -07:00
|
|
|
let (tvu_port, tvu_sockets) = multi_bind_in_range(port_range, 8).expect("tvu multi_bind");
|
2018-09-14 16:56:06 -07:00
|
|
|
|
2019-08-20 17:16:06 -07:00
|
|
|
let (tvu_forwards_port, tvu_forwards_sockets) =
|
|
|
|
multi_bind_in_range(port_range, 8).expect("tpu multi_bind");
|
|
|
|
|
2019-04-12 18:17:34 -07:00
|
|
|
let (tpu_port, tpu_sockets) = multi_bind_in_range(port_range, 32).expect("tpu multi_bind");
|
2018-09-06 14:13:40 -07:00
|
|
|
|
2019-07-30 14:50:02 -07:00
|
|
|
let (tpu_forwards_port, tpu_forwards_sockets) =
|
2019-04-12 18:17:34 -07:00
|
|
|
multi_bind_in_range(port_range, 8).expect("tpu multi_bind");
|
2019-03-08 14:59:11 -08:00
|
|
|
|
2019-04-12 18:17:34 -07:00
|
|
|
let (_, repair) = Self::bind(port_range);
|
|
|
|
let (_, broadcast) = Self::bind(port_range);
|
|
|
|
let (_, retransmit) = Self::bind(port_range);
|
2018-08-25 10:24:16 -07:00
|
|
|
|
2019-03-08 17:23:07 -08:00
|
|
|
let info = ContactInfo::new(
|
2018-08-25 10:24:16 -07:00
|
|
|
pubkey,
|
2018-12-06 12:52:47 -08:00
|
|
|
SocketAddr::new(gossip_addr.ip(), gossip_port),
|
2018-12-07 14:09:29 -08:00
|
|
|
SocketAddr::new(gossip_addr.ip(), tvu_port),
|
2019-08-20 17:16:06 -07:00
|
|
|
SocketAddr::new(gossip_addr.ip(), tvu_forwards_port),
|
2018-12-07 14:09:29 -08:00
|
|
|
SocketAddr::new(gossip_addr.ip(), tpu_port),
|
2019-07-30 14:50:02 -07:00
|
|
|
SocketAddr::new(gossip_addr.ip(), tpu_forwards_port),
|
2019-04-12 18:17:34 -07:00
|
|
|
socketaddr_any!(),
|
|
|
|
socketaddr_any!(),
|
|
|
|
socketaddr_any!(),
|
2018-11-15 13:23:26 -08:00
|
|
|
0,
|
2018-08-25 10:24:16 -07:00
|
|
|
);
|
2019-03-08 17:23:07 -08:00
|
|
|
trace!("new ContactInfo: {:?}", info);
|
2018-08-25 10:24:16 -07:00
|
|
|
|
2018-08-28 16:32:40 -07:00
|
|
|
Node {
|
2018-08-31 00:10:39 -07:00
|
|
|
info,
|
2018-08-25 10:24:16 -07:00
|
|
|
sockets: Sockets {
|
|
|
|
gossip,
|
2018-12-07 14:09:29 -08:00
|
|
|
tvu: tvu_sockets,
|
2019-08-20 17:16:06 -07:00
|
|
|
tvu_forwards: tvu_forwards_sockets,
|
2018-12-07 14:09:29 -08:00
|
|
|
tpu: tpu_sockets,
|
2019-07-30 14:50:02 -07:00
|
|
|
tpu_forwards: tpu_forwards_sockets,
|
2018-08-25 10:24:16 -07:00
|
|
|
broadcast,
|
|
|
|
repair,
|
|
|
|
retransmit,
|
2019-03-18 11:02:32 -07:00
|
|
|
storage: None,
|
2018-08-25 10:24:16 -07:00
|
|
|
},
|
|
|
|
}
|
|
|
|
}
|
2019-04-12 18:17:34 -07:00
|
|
|
pub fn new_replicator_with_external_ip(
|
|
|
|
pubkey: &Pubkey,
|
|
|
|
gossip_addr: &SocketAddr,
|
|
|
|
port_range: PortRange,
|
|
|
|
) -> Node {
|
|
|
|
let mut new = Self::new_with_external_ip(pubkey, gossip_addr, port_range);
|
|
|
|
let (storage_port, storage_socket) = Self::bind(port_range);
|
2019-03-18 11:12:19 -07:00
|
|
|
|
|
|
|
new.info.storage_addr = SocketAddr::new(gossip_addr.ip(), storage_port);
|
|
|
|
new.sockets.storage = Some(storage_socket);
|
|
|
|
|
2019-04-12 18:17:34 -07:00
|
|
|
let empty = socketaddr_any!();
|
2019-03-18 11:12:19 -07:00
|
|
|
new.info.tpu = empty;
|
2019-07-30 14:50:02 -07:00
|
|
|
new.info.tpu_forwards = empty;
|
2019-03-18 11:12:19 -07:00
|
|
|
new.sockets.tpu = vec![];
|
2019-07-30 14:50:02 -07:00
|
|
|
new.sockets.tpu_forwards = vec![];
|
2019-03-18 11:12:19 -07:00
|
|
|
|
|
|
|
new
|
|
|
|
}
|
2018-05-27 18:21:39 -07:00
|
|
|
}
|
2018-04-26 13:48:42 -07:00
|
|
|
|
2018-08-08 11:15:36 -07:00
|
|
|
fn report_time_spent(label: &str, time: &Duration, extra: &str) {
|
|
|
|
let count = duration_as_ms(time);
|
|
|
|
if count > 5 {
|
|
|
|
info!("{} took: {} ms {}", label, count, extra);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-05-27 18:21:39 -07:00
|
|
|
#[cfg(test)]
|
|
|
|
mod tests {
|
2018-11-15 13:23:26 -08:00
|
|
|
use super::*;
|
2019-02-07 20:52:39 -08:00
|
|
|
use crate::blocktree::get_tmp_ledger_path;
|
2019-09-03 21:32:51 -07:00
|
|
|
use crate::blocktree::tests::make_many_slot_entries;
|
2019-02-07 20:52:39 -08:00
|
|
|
use crate::blocktree::Blocktree;
|
2019-08-20 17:16:06 -07:00
|
|
|
use crate::blocktree_processor::tests::fill_blocktree_slot_with_ticks;
|
2018-12-07 19:16:27 -08:00
|
|
|
use crate::crds_value::CrdsValueLabel;
|
2019-04-06 19:41:22 -07:00
|
|
|
use crate::repair_service::RepairType;
|
2018-12-07 19:16:27 -08:00
|
|
|
use crate::result::Error;
|
2019-08-20 17:16:06 -07:00
|
|
|
use crate::shred::{FirstDataShred, Shred};
|
2019-01-31 15:51:29 -08:00
|
|
|
use crate::test_tx::test_tx;
|
2019-08-20 17:16:06 -07:00
|
|
|
use solana_sdk::hash::Hash;
|
2018-12-03 10:26:28 -08:00
|
|
|
use solana_sdk::signature::{Keypair, KeypairUtil};
|
2019-08-20 17:16:06 -07:00
|
|
|
use solana_sdk::timing::DEFAULT_TICKS_PER_SLOT;
|
2019-01-02 00:46:15 -08:00
|
|
|
use std::collections::HashSet;
|
2019-03-08 19:28:19 -08:00
|
|
|
use std::net::{IpAddr, Ipv4Addr};
|
2018-06-03 19:59:17 -07:00
|
|
|
use std::sync::{Arc, RwLock};
|
2018-04-26 13:48:42 -07:00
|
|
|
|
2019-04-30 16:42:56 -07:00
|
|
|
#[test]
|
|
|
|
fn test_gossip_node() {
|
|
|
|
//check that a gossip nodes always show up as spies
|
|
|
|
let (node, _) = ClusterInfo::spy_node(&Pubkey::new_rand());
|
|
|
|
assert!(ClusterInfo::is_spy_node(&node));
|
|
|
|
let (node, _) =
|
|
|
|
ClusterInfo::gossip_node(&Pubkey::new_rand(), &"1.1.1.1:1111".parse().unwrap());
|
|
|
|
assert!(ClusterInfo::is_spy_node(&node));
|
|
|
|
}
|
|
|
|
|
2018-11-17 19:57:28 -08:00
|
|
|
#[test]
|
|
|
|
fn test_cluster_spy_gossip() {
|
|
|
|
//check that gossip doesn't try to push to invalid addresses
|
|
|
|
let node = Node::new_localhost();
|
2019-03-30 20:37:33 -07:00
|
|
|
let (spy, _) = ClusterInfo::spy_node(&Pubkey::new_rand());
|
2019-03-06 13:47:18 -08:00
|
|
|
let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(
|
|
|
|
node.info,
|
|
|
|
)));
|
2018-11-17 19:57:28 -08:00
|
|
|
cluster_info.write().unwrap().insert_info(spy);
|
|
|
|
cluster_info
|
|
|
|
.write()
|
|
|
|
.unwrap()
|
|
|
|
.gossip
|
2019-02-20 20:02:47 -08:00
|
|
|
.refresh_push_active_set(&HashMap::new());
|
|
|
|
let reqs = cluster_info
|
|
|
|
.write()
|
|
|
|
.unwrap()
|
|
|
|
.gossip_request(&HashMap::new());
|
2018-11-17 19:57:28 -08:00
|
|
|
//assert none of the addrs are invalid.
|
|
|
|
reqs.iter().all(|(addr, _)| {
|
|
|
|
let res = ContactInfo::is_valid_address(addr);
|
|
|
|
assert!(res);
|
|
|
|
res
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2018-04-21 11:02:49 -07:00
|
|
|
#[test]
|
2018-11-15 13:23:26 -08:00
|
|
|
fn test_cluster_info_new() {
|
2019-03-30 20:37:33 -07:00
|
|
|
let d = ContactInfo::new_localhost(&Pubkey::new_rand(), timestamp());
|
2019-03-06 13:47:18 -08:00
|
|
|
let cluster_info = ClusterInfo::new_with_invalid_keypair(d.clone());
|
2018-11-15 13:23:26 -08:00
|
|
|
assert_eq!(d.id, cluster_info.my_data().id);
|
2018-04-21 11:02:49 -07:00
|
|
|
}
|
2018-10-14 06:45:02 -07:00
|
|
|
|
2018-11-15 13:23:26 -08:00
|
|
|
#[test]
|
|
|
|
fn insert_info_test() {
|
2019-03-30 20:37:33 -07:00
|
|
|
let d = ContactInfo::new_localhost(&Pubkey::new_rand(), timestamp());
|
2019-03-06 13:47:18 -08:00
|
|
|
let mut cluster_info = ClusterInfo::new_with_invalid_keypair(d);
|
2019-03-30 20:37:33 -07:00
|
|
|
let d = ContactInfo::new_localhost(&Pubkey::new_rand(), timestamp());
|
2018-11-15 13:23:26 -08:00
|
|
|
let label = CrdsValueLabel::ContactInfo(d.id);
|
|
|
|
cluster_info.insert_info(d);
|
|
|
|
assert!(cluster_info.gossip.crds.lookup(&label).is_some());
|
2018-04-28 00:31:20 -07:00
|
|
|
}
|
2018-06-02 08:32:51 -07:00
|
|
|
#[test]
|
2019-03-08 18:08:24 -08:00
|
|
|
fn test_insert_self() {
|
2019-03-30 20:37:33 -07:00
|
|
|
let d = ContactInfo::new_localhost(&Pubkey::new_rand(), timestamp());
|
2019-03-08 18:08:24 -08:00
|
|
|
let mut cluster_info = ClusterInfo::new_with_invalid_keypair(d.clone());
|
|
|
|
let entry_label = CrdsValueLabel::ContactInfo(cluster_info.id());
|
|
|
|
assert!(cluster_info.gossip.crds.lookup(&entry_label).is_some());
|
|
|
|
|
|
|
|
// inserting something else shouldn't work
|
2019-03-30 20:37:33 -07:00
|
|
|
let d = ContactInfo::new_localhost(&Pubkey::new_rand(), timestamp());
|
2019-03-08 18:08:24 -08:00
|
|
|
cluster_info.insert_self(d.clone());
|
|
|
|
let label = CrdsValueLabel::ContactInfo(d.id);
|
|
|
|
assert!(cluster_info.gossip.crds.lookup(&label).is_none());
|
|
|
|
}
|
|
|
|
#[test]
|
2018-06-02 08:32:51 -07:00
|
|
|
fn window_index_request() {
|
2019-03-30 20:37:33 -07:00
|
|
|
let me = ContactInfo::new_localhost(&Pubkey::new_rand(), timestamp());
|
2019-03-06 13:47:18 -08:00
|
|
|
let mut cluster_info = ClusterInfo::new_with_invalid_keypair(me);
|
2019-04-06 19:41:22 -07:00
|
|
|
let rv = cluster_info.repair_request(&RepairType::Blob(0, 0));
|
2018-10-08 19:55:54 -07:00
|
|
|
assert_matches!(rv, Err(Error::ClusterInfoError(ClusterInfoError::NoPeers)));
|
2018-08-30 12:07:54 -07:00
|
|
|
|
2018-12-06 12:52:47 -08:00
|
|
|
let gossip_addr = socketaddr!([127, 0, 0, 1], 1234);
|
2019-03-08 17:23:07 -08:00
|
|
|
let nxt = ContactInfo::new(
|
2019-03-30 20:37:33 -07:00
|
|
|
&Pubkey::new_rand(),
|
2018-12-06 12:52:47 -08:00
|
|
|
gossip_addr,
|
2018-08-30 12:07:54 -07:00
|
|
|
socketaddr!([127, 0, 0, 1], 1235),
|
|
|
|
socketaddr!([127, 0, 0, 1], 1236),
|
|
|
|
socketaddr!([127, 0, 0, 1], 1237),
|
2018-09-21 15:32:15 -07:00
|
|
|
socketaddr!([127, 0, 0, 1], 1238),
|
2018-11-05 09:50:58 -08:00
|
|
|
socketaddr!([127, 0, 0, 1], 1239),
|
2019-03-08 14:59:11 -08:00
|
|
|
socketaddr!([127, 0, 0, 1], 1240),
|
2019-08-20 17:16:06 -07:00
|
|
|
socketaddr!([127, 0, 0, 1], 1241),
|
2018-11-15 13:23:26 -08:00
|
|
|
0,
|
2018-06-02 08:32:51 -07:00
|
|
|
);
|
2018-11-15 13:23:26 -08:00
|
|
|
cluster_info.insert_info(nxt.clone());
|
2019-04-06 19:41:22 -07:00
|
|
|
let rv = cluster_info
|
|
|
|
.repair_request(&RepairType::Blob(0, 0))
|
|
|
|
.unwrap();
|
2018-12-06 12:52:47 -08:00
|
|
|
assert_eq!(nxt.gossip, gossip_addr);
|
|
|
|
assert_eq!(rv.0, nxt.gossip);
|
2018-05-27 18:21:39 -07:00
|
|
|
|
2018-12-06 12:52:47 -08:00
|
|
|
let gossip_addr2 = socketaddr!([127, 0, 0, 2], 1234);
|
2019-03-08 17:23:07 -08:00
|
|
|
let nxt = ContactInfo::new(
|
2019-03-30 20:37:33 -07:00
|
|
|
&Pubkey::new_rand(),
|
2018-12-06 12:52:47 -08:00
|
|
|
gossip_addr2,
|
2018-08-30 12:07:54 -07:00
|
|
|
socketaddr!([127, 0, 0, 1], 1235),
|
|
|
|
socketaddr!([127, 0, 0, 1], 1236),
|
|
|
|
socketaddr!([127, 0, 0, 1], 1237),
|
2018-09-21 15:32:15 -07:00
|
|
|
socketaddr!([127, 0, 0, 1], 1238),
|
2018-11-05 09:50:58 -08:00
|
|
|
socketaddr!([127, 0, 0, 1], 1239),
|
2019-03-08 14:59:11 -08:00
|
|
|
socketaddr!([127, 0, 0, 1], 1240),
|
2019-08-20 17:16:06 -07:00
|
|
|
socketaddr!([127, 0, 0, 1], 1241),
|
2018-11-15 13:23:26 -08:00
|
|
|
0,
|
2018-06-02 08:32:51 -07:00
|
|
|
);
|
2018-11-15 13:23:26 -08:00
|
|
|
cluster_info.insert_info(nxt);
|
2018-06-02 08:32:51 -07:00
|
|
|
let mut one = false;
|
|
|
|
let mut two = false;
|
|
|
|
while !one || !two {
|
|
|
|
//this randomly picks an option, so eventually it should pick both
|
2019-04-06 19:41:22 -07:00
|
|
|
let rv = cluster_info
|
|
|
|
.repair_request(&RepairType::Blob(0, 0))
|
|
|
|
.unwrap();
|
2018-12-06 12:52:47 -08:00
|
|
|
if rv.0 == gossip_addr {
|
2018-06-02 08:32:51 -07:00
|
|
|
one = true;
|
|
|
|
}
|
2018-12-06 12:52:47 -08:00
|
|
|
if rv.0 == gossip_addr2 {
|
2018-06-02 08:32:51 -07:00
|
|
|
two = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
assert!(one && two);
|
|
|
|
}
|
2018-06-03 20:36:27 -07:00
|
|
|
|
|
|
|
/// test window requests respond with the right blob, and do not overrun
|
2018-06-03 20:31:09 -07:00
|
|
|
#[test]
|
|
|
|
fn run_window_request() {
|
2018-12-14 12:36:50 -08:00
|
|
|
solana_logger::setup();
|
2019-02-26 17:11:26 -08:00
|
|
|
let ledger_path = get_tmp_ledger_path!();
|
2018-12-10 01:24:41 -08:00
|
|
|
{
|
2019-02-07 20:52:39 -08:00
|
|
|
let blocktree = Arc::new(Blocktree::open(&ledger_path).unwrap());
|
2019-03-08 17:23:07 -08:00
|
|
|
let me = ContactInfo::new(
|
2019-03-30 20:37:33 -07:00
|
|
|
&Pubkey::new_rand(),
|
2018-12-10 01:24:41 -08:00
|
|
|
socketaddr!("127.0.0.1:1234"),
|
|
|
|
socketaddr!("127.0.0.1:1235"),
|
|
|
|
socketaddr!("127.0.0.1:1236"),
|
|
|
|
socketaddr!("127.0.0.1:1237"),
|
|
|
|
socketaddr!("127.0.0.1:1238"),
|
|
|
|
socketaddr!("127.0.0.1:1239"),
|
2019-03-08 14:59:11 -08:00
|
|
|
socketaddr!("127.0.0.1:1240"),
|
2019-08-20 17:16:06 -07:00
|
|
|
socketaddr!("127.0.0.1:1241"),
|
2018-12-10 01:24:41 -08:00
|
|
|
0,
|
|
|
|
);
|
2019-02-07 15:10:54 -08:00
|
|
|
let rv = ClusterInfo::run_window_request(
|
|
|
|
&me,
|
|
|
|
&socketaddr_any!(),
|
2019-02-07 20:52:39 -08:00
|
|
|
Some(&blocktree),
|
2019-02-07 15:10:54 -08:00
|
|
|
&me,
|
|
|
|
0,
|
|
|
|
0,
|
|
|
|
);
|
2018-12-10 01:24:41 -08:00
|
|
|
assert!(rv.is_empty());
|
2019-08-20 17:16:06 -07:00
|
|
|
let mut shred = Shred::FirstInSlot(FirstDataShred::default());
|
|
|
|
shred.set_slot(2);
|
|
|
|
shred.set_index(1);
|
2018-08-06 12:35:38 -07:00
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree
|
2019-08-26 18:27:45 -07:00
|
|
|
.insert_shreds(vec![shred])
|
2018-12-18 15:18:57 -08:00
|
|
|
.expect("Expect successful ledger write");
|
2018-06-17 04:33:24 -07:00
|
|
|
|
2019-02-07 15:10:54 -08:00
|
|
|
let rv = ClusterInfo::run_window_request(
|
|
|
|
&me,
|
|
|
|
&socketaddr_any!(),
|
2019-02-07 20:52:39 -08:00
|
|
|
Some(&blocktree),
|
2019-02-07 15:10:54 -08:00
|
|
|
&me,
|
|
|
|
2,
|
|
|
|
1,
|
|
|
|
);
|
2018-12-10 01:24:41 -08:00
|
|
|
assert!(!rv.is_empty());
|
2019-08-20 17:16:06 -07:00
|
|
|
let rv: Vec<Shred> = rv
|
|
|
|
.into_iter()
|
|
|
|
.map(|b| bincode::deserialize(&b.read().unwrap().data).unwrap())
|
|
|
|
.collect();
|
|
|
|
assert_eq!(rv[0].index(), 1);
|
|
|
|
assert_eq!(rv[0].slot(), 2);
|
2018-06-17 04:33:24 -07:00
|
|
|
}
|
2018-12-10 01:24:41 -08:00
|
|
|
|
2019-02-07 20:52:39 -08:00
|
|
|
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
2018-06-03 20:31:09 -07:00
|
|
|
}
|
2018-07-18 17:07:01 -07:00
|
|
|
|
2019-02-14 12:47:21 -08:00
|
|
|
/// test run_window_requestwindow requests respond with the right blob, and do not overrun
|
|
|
|
#[test]
|
|
|
|
fn run_highest_window_request() {
|
|
|
|
solana_logger::setup();
|
2019-02-26 17:11:26 -08:00
|
|
|
let ledger_path = get_tmp_ledger_path!();
|
2019-02-14 12:47:21 -08:00
|
|
|
{
|
|
|
|
let blocktree = Arc::new(Blocktree::open(&ledger_path).unwrap());
|
|
|
|
let rv =
|
|
|
|
ClusterInfo::run_highest_window_request(&socketaddr_any!(), Some(&blocktree), 0, 0);
|
|
|
|
assert!(rv.is_empty());
|
|
|
|
|
2019-08-20 17:16:06 -07:00
|
|
|
let _ = fill_blocktree_slot_with_ticks(
|
|
|
|
&blocktree,
|
|
|
|
DEFAULT_TICKS_PER_SLOT,
|
|
|
|
2,
|
|
|
|
1,
|
|
|
|
Hash::default(),
|
|
|
|
);
|
2019-02-14 12:47:21 -08:00
|
|
|
|
|
|
|
let rv =
|
|
|
|
ClusterInfo::run_highest_window_request(&socketaddr_any!(), Some(&blocktree), 2, 1);
|
2019-08-20 17:16:06 -07:00
|
|
|
let rv: Vec<Shred> = rv
|
|
|
|
.into_iter()
|
|
|
|
.map(|b| bincode::deserialize(&b.read().unwrap().data).unwrap())
|
|
|
|
.collect();
|
2019-02-14 12:47:21 -08:00
|
|
|
assert!(!rv.is_empty());
|
2019-08-20 17:16:06 -07:00
|
|
|
let index = blocktree.meta(2).unwrap().unwrap().received - 1;
|
|
|
|
assert_eq!(rv[0].index(), index as u32);
|
|
|
|
assert_eq!(rv[0].slot(), 2);
|
2019-02-14 12:47:21 -08:00
|
|
|
|
|
|
|
let rv = ClusterInfo::run_highest_window_request(
|
|
|
|
&socketaddr_any!(),
|
|
|
|
Some(&blocktree),
|
|
|
|
2,
|
2019-08-20 17:16:06 -07:00
|
|
|
index + 1,
|
2019-02-14 12:47:21 -08:00
|
|
|
);
|
|
|
|
assert!(rv.is_empty());
|
|
|
|
}
|
|
|
|
|
2019-04-06 19:41:22 -07:00
|
|
|
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn run_orphan() {
|
|
|
|
solana_logger::setup();
|
|
|
|
let ledger_path = get_tmp_ledger_path!();
|
|
|
|
{
|
|
|
|
let blocktree = Arc::new(Blocktree::open(&ledger_path).unwrap());
|
|
|
|
let rv = ClusterInfo::run_orphan(&socketaddr_any!(), Some(&blocktree), 2, 0);
|
|
|
|
assert!(rv.is_empty());
|
|
|
|
|
|
|
|
// Create slots 1, 2, 3 with 5 blobs apiece
|
2019-09-03 21:32:51 -07:00
|
|
|
let (blobs, _) = make_many_slot_entries(1, 3, 5);
|
2019-04-06 19:41:22 -07:00
|
|
|
|
|
|
|
blocktree
|
2019-08-26 18:27:45 -07:00
|
|
|
.insert_shreds(blobs)
|
2019-04-06 19:41:22 -07:00
|
|
|
.expect("Expect successful ledger write");
|
|
|
|
|
|
|
|
// We don't have slot 4, so we don't know how to service this requeset
|
|
|
|
let rv = ClusterInfo::run_orphan(&socketaddr_any!(), Some(&blocktree), 4, 5);
|
|
|
|
assert!(rv.is_empty());
|
|
|
|
|
|
|
|
// For slot 3, we should return the highest blobs from slots 3, 2, 1 respectively
|
|
|
|
// for this request
|
|
|
|
let rv: Vec<_> = ClusterInfo::run_orphan(&socketaddr_any!(), Some(&blocktree), 3, 5)
|
|
|
|
.iter()
|
|
|
|
.map(|b| b.read().unwrap().clone())
|
|
|
|
.collect();
|
|
|
|
let expected: Vec<_> = (1..=3)
|
|
|
|
.rev()
|
2019-08-20 17:16:06 -07:00
|
|
|
.map(|slot| {
|
|
|
|
let index = blocktree.meta(slot).unwrap().unwrap().received - 1;
|
2019-09-04 12:47:09 -07:00
|
|
|
ClusterInfo::get_data_shred_as_blob(&blocktree, slot, index)
|
2019-08-20 17:16:06 -07:00
|
|
|
.unwrap()
|
|
|
|
.unwrap()
|
|
|
|
})
|
2019-04-06 19:41:22 -07:00
|
|
|
.collect();
|
|
|
|
assert_eq!(rv, expected)
|
|
|
|
}
|
|
|
|
|
2019-02-14 12:47:21 -08:00
|
|
|
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
|
|
|
}
|
|
|
|
|
2019-03-18 11:02:32 -07:00
|
|
|
fn assert_in_range(x: u16, range: (u16, u16)) {
|
|
|
|
assert!(x >= range.0);
|
|
|
|
assert!(x < range.1);
|
|
|
|
}
|
|
|
|
|
|
|
|
fn check_sockets(sockets: &Vec<UdpSocket>, ip: IpAddr, range: (u16, u16)) {
|
|
|
|
assert!(sockets.len() > 1);
|
|
|
|
let port = sockets[0].local_addr().unwrap().port();
|
|
|
|
for socket in sockets.iter() {
|
|
|
|
check_socket(socket, ip, range);
|
|
|
|
assert_eq!(socket.local_addr().unwrap().port(), port);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn check_socket(socket: &UdpSocket, ip: IpAddr, range: (u16, u16)) {
|
|
|
|
let local_addr = socket.local_addr().unwrap();
|
|
|
|
assert_eq!(local_addr.ip(), ip);
|
|
|
|
assert_in_range(local_addr.port(), range);
|
|
|
|
}
|
|
|
|
|
|
|
|
fn check_node_sockets(node: &Node, ip: IpAddr, range: (u16, u16)) {
|
|
|
|
check_socket(&node.sockets.gossip, ip, range);
|
|
|
|
check_socket(&node.sockets.repair, ip, range);
|
|
|
|
|
|
|
|
check_sockets(&node.sockets.tvu, ip, range);
|
|
|
|
check_sockets(&node.sockets.tpu, ip, range);
|
|
|
|
}
|
|
|
|
|
2018-08-25 10:24:16 -07:00
|
|
|
#[test]
|
|
|
|
fn new_with_external_ip_test_random() {
|
2018-08-31 00:10:39 -07:00
|
|
|
let ip = Ipv4Addr::from(0);
|
2019-04-12 18:17:34 -07:00
|
|
|
let node = Node::new_with_external_ip(
|
|
|
|
&Pubkey::new_rand(),
|
|
|
|
&socketaddr!(ip, 0),
|
|
|
|
FULLNODE_PORT_RANGE,
|
|
|
|
);
|
2019-03-18 11:02:32 -07:00
|
|
|
|
|
|
|
check_node_sockets(&node, IpAddr::V4(ip), FULLNODE_PORT_RANGE);
|
2018-08-25 10:24:16 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn new_with_external_ip_test_gossip() {
|
2018-08-30 12:07:54 -07:00
|
|
|
let ip = IpAddr::V4(Ipv4Addr::from(0));
|
2019-03-20 12:24:40 -07:00
|
|
|
let port = {
|
|
|
|
bind_in_range(FULLNODE_PORT_RANGE)
|
|
|
|
.expect("Failed to bind")
|
|
|
|
.0
|
|
|
|
};
|
2019-04-12 18:17:34 -07:00
|
|
|
let node = Node::new_with_external_ip(
|
|
|
|
&Pubkey::new_rand(),
|
|
|
|
&socketaddr!(0, port),
|
|
|
|
FULLNODE_PORT_RANGE,
|
|
|
|
);
|
2019-03-18 11:02:32 -07:00
|
|
|
|
|
|
|
check_node_sockets(&node, ip, FULLNODE_PORT_RANGE);
|
2018-08-25 10:24:16 -07:00
|
|
|
|
2019-03-20 12:24:40 -07:00
|
|
|
assert_eq!(node.sockets.gossip.local_addr().unwrap().port(), port);
|
2018-08-25 10:24:16 -07:00
|
|
|
}
|
2018-12-01 12:00:30 -08:00
|
|
|
|
2019-03-18 11:12:19 -07:00
|
|
|
#[test]
|
|
|
|
fn new_replicator_external_ip_test() {
|
2019-03-20 11:28:55 -07:00
|
|
|
let ip = Ipv4Addr::from(0);
|
2019-04-12 18:17:34 -07:00
|
|
|
let node = Node::new_replicator_with_external_ip(
|
|
|
|
&Pubkey::new_rand(),
|
|
|
|
&socketaddr!(ip, 0),
|
|
|
|
FULLNODE_PORT_RANGE,
|
|
|
|
);
|
2019-03-18 11:12:19 -07:00
|
|
|
|
2019-03-20 11:28:55 -07:00
|
|
|
let ip = IpAddr::V4(ip);
|
2019-03-18 11:12:19 -07:00
|
|
|
check_socket(&node.sockets.storage.unwrap(), ip, FULLNODE_PORT_RANGE);
|
|
|
|
check_socket(&node.sockets.gossip, ip, FULLNODE_PORT_RANGE);
|
|
|
|
check_socket(&node.sockets.repair, ip, FULLNODE_PORT_RANGE);
|
|
|
|
|
|
|
|
check_sockets(&node.sockets.tvu, ip, FULLNODE_PORT_RANGE);
|
|
|
|
}
|
|
|
|
|
2018-12-01 12:00:30 -08:00
|
|
|
//test that all cluster_info objects only generate signed messages
|
|
|
|
//when constructed with keypairs
|
|
|
|
#[test]
|
|
|
|
fn test_gossip_signature_verification() {
|
|
|
|
//create new cluster info, leader, and peer
|
|
|
|
let keypair = Keypair::new();
|
|
|
|
let peer_keypair = Keypair::new();
|
2019-03-09 19:28:43 -08:00
|
|
|
let contact_info = ContactInfo::new_localhost(&keypair.pubkey(), 0);
|
|
|
|
let peer = ContactInfo::new_localhost(&peer_keypair.pubkey(), 0);
|
2019-03-08 17:23:07 -08:00
|
|
|
let mut cluster_info = ClusterInfo::new(contact_info.clone(), Arc::new(keypair));
|
2018-12-01 12:00:30 -08:00
|
|
|
cluster_info.insert_info(peer.clone());
|
2019-05-28 18:39:40 -07:00
|
|
|
cluster_info.gossip.refresh_push_active_set(&HashMap::new());
|
2018-12-01 12:00:30 -08:00
|
|
|
//check that all types of gossip messages are signed correctly
|
2019-05-28 18:39:40 -07:00
|
|
|
let (_, push_messages) = cluster_info.gossip.new_push_messages(timestamp());
|
2018-12-01 12:00:30 -08:00
|
|
|
// there should be some pushes ready
|
2019-05-28 18:39:40 -07:00
|
|
|
assert_eq!(push_messages.len() > 0, true);
|
|
|
|
push_messages
|
|
|
|
.values()
|
|
|
|
.for_each(|v| v.par_iter().for_each(|v| assert!(v.verify())));
|
2018-12-01 12:00:30 -08:00
|
|
|
|
|
|
|
let (_, _, val) = cluster_info
|
|
|
|
.gossip
|
2019-08-13 18:04:14 -07:00
|
|
|
.new_pull_request(timestamp(), &HashMap::new(), ClusterInfo::max_bloom_size())
|
2018-12-01 12:00:30 -08:00
|
|
|
.ok()
|
|
|
|
.unwrap();
|
|
|
|
assert!(val.verify());
|
|
|
|
}
|
2019-01-02 00:46:15 -08:00
|
|
|
|
2019-05-07 13:24:58 -07:00
|
|
|
fn num_layers(nodes: usize, fanout: usize) -> usize {
|
|
|
|
ClusterInfo::describe_data_plane(nodes, fanout).0
|
2019-01-02 00:46:15 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_describe_data_plane() {
|
|
|
|
// no nodes
|
2019-05-07 13:24:58 -07:00
|
|
|
assert_eq!(num_layers(0, 200), 0);
|
2019-01-02 00:46:15 -08:00
|
|
|
|
|
|
|
// 1 node
|
2019-05-07 13:24:58 -07:00
|
|
|
assert_eq!(num_layers(1, 200), 1);
|
2019-01-02 00:46:15 -08:00
|
|
|
|
2019-05-07 13:24:58 -07:00
|
|
|
// 10 nodes with fanout of 2
|
|
|
|
assert_eq!(num_layers(10, 2), 3);
|
2019-01-02 00:46:15 -08:00
|
|
|
|
2019-05-07 13:24:58 -07:00
|
|
|
// fanout + 1 nodes with fanout of 2
|
|
|
|
assert_eq!(num_layers(3, 2), 2);
|
2019-01-02 00:46:15 -08:00
|
|
|
|
|
|
|
// A little more realistic
|
2019-05-07 13:24:58 -07:00
|
|
|
assert_eq!(num_layers(100, 10), 2);
|
2019-01-02 00:46:15 -08:00
|
|
|
|
|
|
|
// A little more realistic with odd numbers
|
2019-05-07 13:24:58 -07:00
|
|
|
assert_eq!(num_layers(103, 13), 2);
|
|
|
|
|
|
|
|
// A little more realistic with just enough for 3 layers
|
|
|
|
assert_eq!(num_layers(111, 10), 3);
|
2019-01-02 00:46:15 -08:00
|
|
|
|
|
|
|
// larger
|
2019-05-07 13:24:58 -07:00
|
|
|
let (layer_cnt, layer_indices) = ClusterInfo::describe_data_plane(10_000, 10);
|
|
|
|
assert_eq!(layer_cnt, 4);
|
|
|
|
// distances between index values should increase by `fanout` for every layer.
|
|
|
|
let mut capacity = 10 * 10;
|
2019-01-02 00:46:15 -08:00
|
|
|
assert_eq!(layer_indices[1], 10);
|
2019-05-07 13:24:58 -07:00
|
|
|
layer_indices[1..].windows(2).for_each(|x| {
|
|
|
|
if x.len() == 2 {
|
|
|
|
assert_eq!(x[1] - x[0], capacity);
|
|
|
|
capacity *= 10;
|
|
|
|
}
|
|
|
|
});
|
2019-01-02 00:46:15 -08:00
|
|
|
|
|
|
|
// massive
|
2019-05-07 13:24:58 -07:00
|
|
|
let (layer_cnt, layer_indices) = ClusterInfo::describe_data_plane(500_000, 200);
|
|
|
|
let mut capacity = 200 * 200;
|
|
|
|
assert_eq!(layer_cnt, 3);
|
|
|
|
// distances between index values should increase by `fanout` for every layer.
|
2019-01-02 00:46:15 -08:00
|
|
|
assert_eq!(layer_indices[1], 200);
|
2019-05-07 13:24:58 -07:00
|
|
|
layer_indices[1..].windows(2).for_each(|x| {
|
|
|
|
if x.len() == 2 {
|
|
|
|
assert_eq!(x[1] - x[0], capacity);
|
|
|
|
capacity *= 200;
|
|
|
|
}
|
|
|
|
});
|
2019-01-02 00:46:15 -08:00
|
|
|
let total_capacity: usize = *layer_indices.last().unwrap();
|
|
|
|
assert!(total_capacity >= 500_000);
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_localize() {
|
|
|
|
// go for gold
|
2019-05-07 13:24:58 -07:00
|
|
|
let (_, layer_indices) = ClusterInfo::describe_data_plane(500_000, 200);
|
2019-01-02 00:46:15 -08:00
|
|
|
let mut me = 0;
|
|
|
|
let mut layer_ix = 0;
|
|
|
|
let locality = ClusterInfo::localize(&layer_indices, 200, me);
|
|
|
|
assert_eq!(locality.layer_ix, layer_ix);
|
|
|
|
assert_eq!(
|
2019-05-07 13:24:58 -07:00
|
|
|
locality.next_layer_bounds,
|
2019-01-02 00:46:15 -08:00
|
|
|
Some((layer_indices[layer_ix + 1], layer_indices[layer_ix + 2]))
|
|
|
|
);
|
|
|
|
me = 201;
|
|
|
|
layer_ix = 1;
|
|
|
|
let locality = ClusterInfo::localize(&layer_indices, 200, me);
|
|
|
|
assert_eq!(
|
|
|
|
locality.layer_ix, layer_ix,
|
|
|
|
"layer_indices[layer_ix] is actually {}",
|
|
|
|
layer_indices[layer_ix]
|
|
|
|
);
|
|
|
|
assert_eq!(
|
2019-05-07 13:24:58 -07:00
|
|
|
locality.next_layer_bounds,
|
2019-01-02 00:46:15 -08:00
|
|
|
Some((layer_indices[layer_ix + 1], layer_indices[layer_ix + 2]))
|
|
|
|
);
|
2019-05-07 13:24:58 -07:00
|
|
|
me = 20_000;
|
|
|
|
layer_ix = 1;
|
2019-01-02 00:46:15 -08:00
|
|
|
let locality = ClusterInfo::localize(&layer_indices, 200, me);
|
|
|
|
assert_eq!(
|
|
|
|
locality.layer_ix, layer_ix,
|
|
|
|
"layer_indices[layer_ix] is actually {}",
|
|
|
|
layer_indices[layer_ix]
|
|
|
|
);
|
|
|
|
assert_eq!(
|
2019-05-07 13:24:58 -07:00
|
|
|
locality.next_layer_bounds,
|
2019-01-02 00:46:15 -08:00
|
|
|
Some((layer_indices[layer_ix + 1], layer_indices[layer_ix + 2]))
|
|
|
|
);
|
|
|
|
|
|
|
|
// test no child layer since last layer should have massive capacity
|
2019-05-07 13:24:58 -07:00
|
|
|
let (_, layer_indices) = ClusterInfo::describe_data_plane(500_000, 200);
|
|
|
|
me = 40_201;
|
2019-01-02 00:46:15 -08:00
|
|
|
layer_ix = 2;
|
|
|
|
let locality = ClusterInfo::localize(&layer_indices, 200, me);
|
|
|
|
assert_eq!(
|
|
|
|
locality.layer_ix, layer_ix,
|
|
|
|
"layer_indices[layer_ix] is actually {}",
|
|
|
|
layer_indices[layer_ix]
|
|
|
|
);
|
2019-05-07 13:24:58 -07:00
|
|
|
assert_eq!(locality.next_layer_bounds, None);
|
2019-01-02 00:46:15 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_localize_child_peer_overlap() {
|
2019-05-07 13:24:58 -07:00
|
|
|
let (_, layer_indices) = ClusterInfo::describe_data_plane(500_000, 200);
|
2019-01-02 00:46:15 -08:00
|
|
|
let last_ix = layer_indices.len() - 1;
|
|
|
|
// sample every 33 pairs to reduce test time
|
|
|
|
for x in (0..*layer_indices.get(last_ix - 2).unwrap()).step_by(33) {
|
|
|
|
let me_locality = ClusterInfo::localize(&layer_indices, 200, x);
|
|
|
|
let buddy_locality = ClusterInfo::localize(&layer_indices, 200, x + 1);
|
2019-05-07 13:24:58 -07:00
|
|
|
assert!(!me_locality.next_layer_peers.is_empty());
|
|
|
|
assert!(!buddy_locality.next_layer_peers.is_empty());
|
2019-01-02 00:46:15 -08:00
|
|
|
me_locality
|
2019-05-07 13:24:58 -07:00
|
|
|
.next_layer_peers
|
2019-01-02 00:46:15 -08:00
|
|
|
.iter()
|
2019-05-07 13:24:58 -07:00
|
|
|
.zip(buddy_locality.next_layer_peers.iter())
|
2019-01-02 00:46:15 -08:00
|
|
|
.for_each(|(x, y)| assert_ne!(x, y));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_network_coverage() {
|
|
|
|
// pretend to be each node in a scaled down network and make sure the set of all the broadcast peers
|
|
|
|
// includes every node in the network.
|
2019-05-07 13:24:58 -07:00
|
|
|
let (_, layer_indices) = ClusterInfo::describe_data_plane(25_000, 10);
|
2019-01-02 00:46:15 -08:00
|
|
|
let mut broadcast_set = HashSet::new();
|
|
|
|
for my_index in 0..25_000 {
|
|
|
|
let my_locality = ClusterInfo::localize(&layer_indices, 10, my_index);
|
|
|
|
broadcast_set.extend(my_locality.neighbor_bounds.0..my_locality.neighbor_bounds.1);
|
2019-05-07 13:24:58 -07:00
|
|
|
broadcast_set.extend(my_locality.next_layer_peers);
|
2019-01-02 00:46:15 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
for i in 0..25_000 {
|
|
|
|
assert!(broadcast_set.contains(&(i as usize)));
|
|
|
|
}
|
|
|
|
assert!(broadcast_set.contains(&(layer_indices.last().unwrap() - 1)));
|
|
|
|
//sanity check for past total capacity.
|
|
|
|
assert!(!broadcast_set.contains(&(layer_indices.last().unwrap())));
|
|
|
|
}
|
2019-01-31 15:51:29 -08:00
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_push_vote() {
|
|
|
|
let keys = Keypair::new();
|
|
|
|
let now = timestamp();
|
2019-03-09 19:28:43 -08:00
|
|
|
let contact_info = ContactInfo::new_localhost(&keys.pubkey(), 0);
|
2019-03-08 17:23:07 -08:00
|
|
|
let mut cluster_info = ClusterInfo::new_with_invalid_keypair(contact_info);
|
2019-01-31 15:51:29 -08:00
|
|
|
|
|
|
|
// make sure empty crds is handled correctly
|
|
|
|
let (votes, max_ts) = cluster_info.get_votes(now);
|
|
|
|
assert_eq!(votes, vec![]);
|
|
|
|
assert_eq!(max_ts, now);
|
|
|
|
|
|
|
|
// add a vote
|
|
|
|
let tx = test_tx();
|
|
|
|
cluster_info.push_vote(tx.clone());
|
|
|
|
|
|
|
|
// -1 to make sure that the clock is strictly lower then when insert occurred
|
|
|
|
let (votes, max_ts) = cluster_info.get_votes(now - 1);
|
|
|
|
assert_eq!(votes, vec![tx]);
|
|
|
|
assert!(max_ts >= now - 1);
|
|
|
|
|
|
|
|
// make sure timestamp filter works
|
|
|
|
let (votes, new_max_ts) = cluster_info.get_votes(max_ts);
|
|
|
|
assert_eq!(votes, vec![]);
|
|
|
|
assert_eq!(max_ts, new_max_ts);
|
|
|
|
}
|
2019-07-30 15:43:17 -07:00
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_add_entrypoint() {
|
|
|
|
let node_keypair = Arc::new(Keypair::new());
|
|
|
|
let mut cluster_info = ClusterInfo::new(
|
|
|
|
ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()),
|
|
|
|
node_keypair,
|
|
|
|
);
|
|
|
|
let entrypoint_pubkey = Pubkey::new_rand();
|
|
|
|
let entrypoint = ContactInfo::new_localhost(&entrypoint_pubkey, timestamp());
|
|
|
|
cluster_info.set_entrypoint(entrypoint.clone());
|
|
|
|
let pulls = cluster_info.new_pull_requests(&HashMap::new());
|
2019-08-13 18:04:14 -07:00
|
|
|
assert_eq!(1, pulls.len() as u64);
|
2019-07-30 15:43:17 -07:00
|
|
|
match pulls.get(0) {
|
|
|
|
Some((addr, msg)) => {
|
|
|
|
assert_eq!(*addr, entrypoint.gossip);
|
|
|
|
match msg {
|
|
|
|
Protocol::PullRequest(_, value) => {
|
|
|
|
assert!(value.verify());
|
|
|
|
assert_eq!(value.pubkey(), cluster_info.id())
|
|
|
|
}
|
|
|
|
_ => panic!("wrong protocol"),
|
2019-03-08 18:08:24 -08:00
|
|
|
}
|
|
|
|
}
|
2019-07-30 15:43:17 -07:00
|
|
|
None => panic!("entrypoint should be a pull destination"),
|
|
|
|
}
|
|
|
|
|
|
|
|
// now add this message back to the table and make sure after the next pull, the entrypoint is unset
|
|
|
|
let entrypoint_crdsvalue = CrdsValue::ContactInfo(entrypoint.clone());
|
|
|
|
let cluster_info = Arc::new(RwLock::new(cluster_info));
|
|
|
|
ClusterInfo::handle_pull_response(
|
|
|
|
&cluster_info,
|
|
|
|
&entrypoint_pubkey,
|
|
|
|
vec![entrypoint_crdsvalue],
|
|
|
|
);
|
|
|
|
let pulls = cluster_info
|
|
|
|
.write()
|
|
|
|
.unwrap()
|
|
|
|
.new_pull_requests(&HashMap::new());
|
2019-08-13 18:04:14 -07:00
|
|
|
assert_eq!(1, pulls.len() as u64);
|
2019-07-30 15:43:17 -07:00
|
|
|
assert_eq!(cluster_info.read().unwrap().entrypoint, Some(entrypoint));
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_split_messages_small() {
|
|
|
|
let value = CrdsValue::ContactInfo(ContactInfo::default());
|
|
|
|
test_split_messages(value);
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_split_messages_large() {
|
|
|
|
let mut btree_slots = BTreeSet::new();
|
|
|
|
for i in 0..128 {
|
|
|
|
btree_slots.insert(i);
|
2019-03-08 18:08:24 -08:00
|
|
|
}
|
2019-07-30 15:43:17 -07:00
|
|
|
let value = CrdsValue::EpochSlots(EpochSlots {
|
|
|
|
from: Pubkey::default(),
|
|
|
|
root: 0,
|
|
|
|
slots: btree_slots,
|
|
|
|
signature: Signature::default(),
|
|
|
|
wallclock: 0,
|
|
|
|
});
|
|
|
|
test_split_messages(value);
|
|
|
|
}
|
|
|
|
|
|
|
|
fn test_split_messages(value: CrdsValue) {
|
|
|
|
const NUM_VALUES: usize = 30;
|
|
|
|
let value_size = value.size();
|
2019-08-13 18:04:14 -07:00
|
|
|
let expected_len = NUM_VALUES / (MAX_PROTOCOL_PAYLOAD_SIZE / value_size).max(1) as usize;
|
2019-07-30 15:43:17 -07:00
|
|
|
let msgs = vec![value; NUM_VALUES];
|
|
|
|
|
|
|
|
let split = ClusterInfo::split_gossip_messages(msgs);
|
|
|
|
assert!(split.len() <= expected_len);
|
|
|
|
}
|
|
|
|
|
2019-08-19 18:14:10 -07:00
|
|
|
#[test]
|
|
|
|
fn test_crds_filter_size() {
|
|
|
|
//sanity test to ensure filter size never exceeds MTU size
|
|
|
|
check_pull_request_size(CrdsFilter::new_rand(1000, 10));
|
|
|
|
check_pull_request_size(CrdsFilter::new_rand(1000, 1000));
|
|
|
|
check_pull_request_size(CrdsFilter::new_rand(100000, 1000));
|
|
|
|
check_pull_request_size(CrdsFilter::new_rand(100000, ClusterInfo::max_bloom_size()));
|
|
|
|
}
|
|
|
|
|
|
|
|
fn check_pull_request_size(filter: CrdsFilter) {
|
|
|
|
let value = CrdsValue::ContactInfo(ContactInfo::default());
|
|
|
|
let protocol = Protocol::PullRequest(filter, value.clone());
|
|
|
|
assert!(serialized_size(&protocol).unwrap() <= PACKET_DATA_SIZE as u64);
|
|
|
|
}
|
2019-03-08 18:08:24 -08:00
|
|
|
}
|