2021-03-09 17:36:05 -08:00
|
|
|
|
use std::{
|
|
|
|
|
mem,
|
|
|
|
|
sync::{Arc, Mutex},
|
|
|
|
|
time::Duration,
|
|
|
|
|
};
|
2019-10-21 15:24:17 -07:00
|
|
|
|
|
2019-10-22 12:48:50 -07:00
|
|
|
|
use chrono::Utc;
|
|
|
|
|
use futures::stream::{FuturesUnordered, StreamExt};
|
2021-03-09 17:36:05 -08:00
|
|
|
|
use tokio::time::{sleep, sleep_until, Sleep};
|
2019-10-21 15:24:17 -07:00
|
|
|
|
use tower::{Service, ServiceExt};
|
|
|
|
|
|
2021-02-17 17:18:32 -08:00
|
|
|
|
use crate::{types::MetaAddr, AddressBook, BoxError, PeerAddrState, Request, Response};
|
2019-10-21 15:24:17 -07:00
|
|
|
|
|
2021-02-17 17:18:32 -08:00
|
|
|
|
/// The `CandidateSet` manages the `PeerSet`'s peer reconnection attempts.
|
2019-10-21 21:25:49 -07:00
|
|
|
|
///
|
2021-02-17 17:18:32 -08:00
|
|
|
|
/// It divides the set of all possible candidate peers into disjoint subsets,
|
|
|
|
|
/// using the `PeerAddrState`:
|
2019-10-21 21:25:49 -07:00
|
|
|
|
///
|
2021-02-17 17:18:32 -08:00
|
|
|
|
/// 1. `Responded` peers, which we previously connected to. If we have not received
|
|
|
|
|
/// any messages from a `Responded` peer within a cutoff time, we assume that it
|
|
|
|
|
/// has disconnected or hung, and attempt reconnection;
|
|
|
|
|
/// 2. `NeverAttempted` peers, which we learned about from other peers or a DNS
|
|
|
|
|
/// seeder, but have never connected to;
|
|
|
|
|
/// 3. `Failed` peers, to whom we attempted to connect but were unable to;
|
|
|
|
|
/// 4. `AttemptPending` peers, which we've recently queued for reconnection.
|
2019-10-21 21:25:49 -07:00
|
|
|
|
///
|
|
|
|
|
/// ```ascii,no_run
|
2021-02-17 17:18:32 -08:00
|
|
|
|
/// ┌──────────────────┐
|
|
|
|
|
/// │ PeerSet │
|
|
|
|
|
/// │GetPeers Responses│
|
|
|
|
|
/// └──────────────────┘
|
2019-10-21 21:25:49 -07:00
|
|
|
|
/// │
|
|
|
|
|
/// │
|
|
|
|
|
/// │
|
|
|
|
|
/// │
|
|
|
|
|
/// ▼
|
2021-02-17 17:18:32 -08:00
|
|
|
|
/// filter by Λ
|
|
|
|
|
/// !contains_addr ╱ ╲
|
|
|
|
|
/// ┌────────────────────────────▶▕ ▏
|
|
|
|
|
/// │ ╲ ╱
|
|
|
|
|
/// │ V
|
|
|
|
|
/// │ │
|
|
|
|
|
/// │ │
|
|
|
|
|
/// │ │
|
|
|
|
|
/// │ │
|
|
|
|
|
/// │ │
|
|
|
|
|
/// │ │
|
|
|
|
|
/// │ │
|
|
|
|
|
/// │ │
|
|
|
|
|
/// ├───────────────────────────────┼───────────────────────────────┐
|
|
|
|
|
/// │ PeerSet AddressBook ▼ │
|
|
|
|
|
/// │ ┌─────────────┐ ┌────────────────┐ ┌─────────────┐ │
|
|
|
|
|
/// │ │ Possibly │ │`NeverAttempted`│ │ `Failed` │ │
|
|
|
|
|
/// │ │Disconnected │ │ Peers │ │ Peers │◀┼┐
|
|
|
|
|
/// │ │ `Responded` │ │ │ │ │ ││
|
|
|
|
|
/// │ │ Peers │ │ │ │ │ ││
|
|
|
|
|
/// │ └─────────────┘ └────────────────┘ └─────────────┘ ││
|
|
|
|
|
/// │ │ │ │ ││
|
|
|
|
|
/// │ #1 oldest_first #2 newest_first #3 oldest_first ││
|
|
|
|
|
/// │ │ │ │ ││
|
|
|
|
|
/// │ ├──────────────────────┴──────────────────────┘ ││
|
|
|
|
|
/// │ │ disjoint `PeerAddrState`s ││
|
|
|
|
|
/// ├────────┼──────────────────────────────────────────────────────┘│
|
|
|
|
|
/// │ ▼ │
|
|
|
|
|
/// │ Λ │
|
|
|
|
|
/// │ ╱ ╲ filter by │
|
|
|
|
|
/// └─────▶▕ ▏!is_potentially_connected │
|
|
|
|
|
/// ╲ ╱ to remove live │
|
|
|
|
|
/// V `Responded` peers │
|
|
|
|
|
/// │ │
|
|
|
|
|
/// │ │
|
|
|
|
|
/// ▼ │
|
|
|
|
|
/// ┌────────────────┐ │
|
|
|
|
|
/// │`AttemptPending`│ │
|
|
|
|
|
/// │ Peers │ │
|
|
|
|
|
/// │ │ │
|
|
|
|
|
/// └────────────────┘ │
|
2019-10-21 21:25:49 -07:00
|
|
|
|
/// │ │
|
|
|
|
|
/// │ │
|
|
|
|
|
/// ▼ │
|
|
|
|
|
/// Λ │
|
|
|
|
|
/// ╱ ╲ │
|
|
|
|
|
/// ▕ ▏─────────────────────────────────────────────────────┘
|
|
|
|
|
/// ╲ ╱ connection failed, update last_seen to now()
|
|
|
|
|
/// V
|
|
|
|
|
/// │
|
|
|
|
|
/// │
|
|
|
|
|
/// ▼
|
|
|
|
|
/// ┌────────────┐
|
|
|
|
|
/// │ send │
|
2019-11-27 11:27:17 -08:00
|
|
|
|
/// │peer::Client│
|
2019-10-21 21:25:49 -07:00
|
|
|
|
/// │to Discover │
|
|
|
|
|
/// └────────────┘
|
2021-02-17 17:18:32 -08:00
|
|
|
|
/// │
|
|
|
|
|
/// │
|
|
|
|
|
/// ▼
|
|
|
|
|
/// ┌───────────────────────────────────────┐
|
|
|
|
|
/// │ every time we receive a peer message: │
|
|
|
|
|
/// │ * update state to `Responded` │
|
|
|
|
|
/// │ * update last_seen to now() │
|
|
|
|
|
/// └───────────────────────────────────────┘
|
|
|
|
|
///
|
2019-10-21 21:25:49 -07:00
|
|
|
|
/// ```
|
2021-02-17 17:18:32 -08:00
|
|
|
|
// TODO:
|
|
|
|
|
// * draw arrow from the "peer message" box into the `Responded` state box
|
|
|
|
|
// * make the "disjoint states" box include `AttemptPending`
|
2019-10-21 15:24:17 -07:00
|
|
|
|
pub(super) struct CandidateSet<S> {
|
|
|
|
|
pub(super) peer_set: Arc<Mutex<AddressBook>>,
|
|
|
|
|
pub(super) peer_service: S,
|
2021-03-09 17:36:05 -08:00
|
|
|
|
next_peer_min_wait: Sleep,
|
2019-10-21 15:24:17 -07:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl<S> CandidateSet<S>
|
|
|
|
|
where
|
2020-09-18 11:20:55 -07:00
|
|
|
|
S: Service<Request, Response = Response, Error = BoxError>,
|
2019-10-21 15:24:17 -07:00
|
|
|
|
S::Future: Send + 'static,
|
|
|
|
|
{
|
2021-03-09 17:36:05 -08:00
|
|
|
|
/// The minimum time between successive calls to `CandidateSet::next()`.
|
|
|
|
|
///
|
|
|
|
|
/// ## Security
|
|
|
|
|
///
|
|
|
|
|
/// Zebra resists distributed denial of service attacks by making sure that new peer connections
|
|
|
|
|
/// are initiated at least `MIN_PEER_CONNECTION_INTERVAL` apart.
|
|
|
|
|
const MIN_PEER_CONNECTION_INTERVAL: Duration = Duration::from_millis(100);
|
|
|
|
|
|
2021-02-17 17:18:32 -08:00
|
|
|
|
/// Uses `peer_set` and `peer_service` to manage a [`CandidateSet`] of peers.
|
2019-10-21 21:25:49 -07:00
|
|
|
|
pub fn new(peer_set: Arc<Mutex<AddressBook>>, peer_service: S) -> CandidateSet<S> {
|
|
|
|
|
CandidateSet {
|
|
|
|
|
peer_set,
|
|
|
|
|
peer_service,
|
2021-03-09 17:36:05 -08:00
|
|
|
|
next_peer_min_wait: sleep(Duration::from_secs(0)),
|
2019-10-21 21:25:49 -07:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2021-02-17 17:18:32 -08:00
|
|
|
|
/// Update the peer set from the network.
|
|
|
|
|
///
|
|
|
|
|
/// - Ask a few live `Responded` peers to send us more peers.
|
|
|
|
|
/// - Process all completed peer responses, adding new peers in the
|
|
|
|
|
/// `NeverAttempted` state.
|
|
|
|
|
///
|
|
|
|
|
/// ## Correctness
|
|
|
|
|
///
|
|
|
|
|
/// The handshaker sets up the peer message receiver so it also sends a
|
|
|
|
|
/// `Responded` peer address update.
|
|
|
|
|
///
|
|
|
|
|
/// `report_failed` puts peers into the `Failed` state.
|
|
|
|
|
///
|
|
|
|
|
/// `next` puts peers into the `AttemptPending` state.
|
2020-09-18 11:20:55 -07:00
|
|
|
|
pub async fn update(&mut self) -> Result<(), BoxError> {
|
2019-10-21 15:24:17 -07:00
|
|
|
|
// Opportunistically crawl the network on every update call to ensure
|
|
|
|
|
// we're actively fetching peers. Continue independently of whether we
|
|
|
|
|
// actually receive any peers, but always ask the network for more.
|
|
|
|
|
// Because requests are load-balanced across existing peers, we can make
|
|
|
|
|
// multiple requests concurrently, which will be randomly assigned to
|
|
|
|
|
// existing peers, but we don't make too many because update may be
|
|
|
|
|
// called while the peer set is already loaded.
|
|
|
|
|
let mut responses = FuturesUnordered::new();
|
2020-12-13 17:00:39 -08:00
|
|
|
|
trace!("sending GetPeers requests");
|
2020-09-21 08:56:03 -07:00
|
|
|
|
// Yes this loops only once (for now), until we add fanout back.
|
2020-09-19 23:54:38 -07:00
|
|
|
|
for _ in 0..1usize {
|
2020-05-26 18:00:58 -07:00
|
|
|
|
self.peer_service.ready_and().await?;
|
2020-02-07 09:25:13 -08:00
|
|
|
|
responses.push(self.peer_service.call(Request::Peers));
|
2019-10-21 15:24:17 -07:00
|
|
|
|
}
|
|
|
|
|
while let Some(rsp) = responses.next().await {
|
2021-02-17 17:18:32 -08:00
|
|
|
|
if let Ok(Response::Peers(rsp_addrs)) = rsp {
|
2020-12-13 17:00:39 -08:00
|
|
|
|
// Filter new addresses to ensure that gossiped addresses are actually new
|
2019-10-21 15:24:17 -07:00
|
|
|
|
let peer_set = &self.peer_set;
|
2021-02-17 17:18:32 -08:00
|
|
|
|
let new_addrs = rsp_addrs
|
|
|
|
|
.iter()
|
|
|
|
|
.filter(|meta| !peer_set.lock().unwrap().contains_addr(&meta.addr))
|
|
|
|
|
.collect::<Vec<_>>();
|
2019-10-21 15:24:17 -07:00
|
|
|
|
trace!(
|
2021-02-17 17:18:32 -08:00
|
|
|
|
?rsp_addrs,
|
|
|
|
|
new_addr_count = ?new_addrs.len(),
|
2019-10-21 15:24:17 -07:00
|
|
|
|
"got response to GetPeers"
|
|
|
|
|
);
|
2021-02-17 17:18:32 -08:00
|
|
|
|
// New addresses are deserialized in the `NeverAttempted` state
|
|
|
|
|
peer_set
|
|
|
|
|
.lock()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.extend(new_addrs.into_iter().cloned());
|
2019-10-21 15:24:17 -07:00
|
|
|
|
} else {
|
|
|
|
|
trace!("got error in GetPeers request");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
|
}
|
|
|
|
|
|
2021-02-17 17:18:32 -08:00
|
|
|
|
/// Returns the next candidate for a connection attempt, if any are available.
|
|
|
|
|
///
|
|
|
|
|
/// Returns peers in this order:
|
|
|
|
|
/// - oldest `Responded` that are not live
|
|
|
|
|
/// - newest `NeverAttempted`
|
|
|
|
|
/// - oldest `Failed`
|
|
|
|
|
///
|
|
|
|
|
/// Skips `AttemptPending` peers and live `Responded` peers.
|
|
|
|
|
///
|
|
|
|
|
/// ## Correctness
|
|
|
|
|
///
|
|
|
|
|
/// `AttemptPending` peers will become `Responded` if they respond, or
|
|
|
|
|
/// become `Failed` if they time out or provide a bad response.
|
|
|
|
|
///
|
|
|
|
|
/// Live `Responded` peers will stay live if they keep responding, or
|
|
|
|
|
/// become a reconnection candidate if they stop responding.
|
2021-03-09 17:36:05 -08:00
|
|
|
|
///
|
|
|
|
|
/// ## Security
|
|
|
|
|
///
|
|
|
|
|
/// Zebra resists distributed denial of service attacks by making sure that
|
|
|
|
|
/// new peer connections are initiated at least
|
|
|
|
|
/// `MIN_PEER_CONNECTION_INTERVAL` apart.
|
|
|
|
|
pub async fn next(&mut self) -> Option<MetaAddr> {
|
|
|
|
|
let current_deadline = self.next_peer_min_wait.deadline();
|
|
|
|
|
let mut sleep = sleep_until(current_deadline + Self::MIN_PEER_CONNECTION_INTERVAL);
|
|
|
|
|
mem::swap(&mut self.next_peer_min_wait, &mut sleep);
|
|
|
|
|
|
|
|
|
|
let reconnect = {
|
|
|
|
|
let mut peer_set_guard = self.peer_set.lock().unwrap();
|
|
|
|
|
// It's okay to early return here because we're returning None
|
|
|
|
|
// instead of yielding the next connection.
|
|
|
|
|
let mut reconnect = peer_set_guard.reconnection_peers().next()?;
|
|
|
|
|
|
|
|
|
|
reconnect.last_seen = Utc::now();
|
|
|
|
|
reconnect.last_connection_state = PeerAddrState::AttemptPending;
|
|
|
|
|
peer_set_guard.update(reconnect);
|
|
|
|
|
reconnect
|
|
|
|
|
};
|
2021-02-17 17:18:32 -08:00
|
|
|
|
|
2021-03-09 17:36:05 -08:00
|
|
|
|
// This is the line that is most relevant to the above ## Security section
|
|
|
|
|
sleep.await;
|
2021-02-17 17:18:32 -08:00
|
|
|
|
|
|
|
|
|
Some(reconnect)
|
2019-10-21 15:24:17 -07:00
|
|
|
|
}
|
|
|
|
|
|
2021-02-17 17:18:32 -08:00
|
|
|
|
/// Mark `addr` as a failed peer.
|
2019-10-21 15:24:17 -07:00
|
|
|
|
pub fn report_failed(&mut self, mut addr: MetaAddr) {
|
|
|
|
|
addr.last_seen = Utc::now();
|
2021-02-17 17:18:32 -08:00
|
|
|
|
addr.last_connection_state = PeerAddrState::Failed;
|
|
|
|
|
self.peer_set.lock().unwrap().update(addr);
|
2019-10-21 15:24:17 -07:00
|
|
|
|
}
|
|
|
|
|
}
|