2018-04-12 09:17:33 -07:00
|
|
|
//! Integration test of the reliable broadcast protocol.
|
|
|
|
|
|
|
|
extern crate hbbft;
|
2018-04-13 10:28:41 -07:00
|
|
|
#[macro_use]
|
|
|
|
extern crate log;
|
|
|
|
extern crate simple_logger;
|
2018-04-12 09:17:33 -07:00
|
|
|
extern crate crossbeam;
|
|
|
|
#[macro_use]
|
|
|
|
extern crate crossbeam_channel;
|
|
|
|
extern crate merkle;
|
|
|
|
|
|
|
|
mod netsim;
|
|
|
|
|
2018-04-20 17:29:54 -07:00
|
|
|
use std::collections::{BTreeMap, HashSet, HashMap, VecDeque};
|
2018-04-16 06:03:32 -07:00
|
|
|
use std::fmt;
|
2018-04-12 09:17:33 -07:00
|
|
|
use std::fmt::Debug;
|
|
|
|
use std::io;
|
2018-04-20 17:29:54 -07:00
|
|
|
use std::net::SocketAddr;
|
2018-04-24 09:31:21 -07:00
|
|
|
use std::sync::{Arc, RwLock};
|
|
|
|
use std::ops::Deref;
|
2018-04-13 10:28:41 -07:00
|
|
|
use crossbeam::{Scope, ScopedJoinHandle};
|
|
|
|
use crossbeam_channel::{bounded, Sender, Receiver};
|
2018-04-12 09:17:33 -07:00
|
|
|
|
|
|
|
use hbbft::proto::*;
|
2018-04-18 01:15:12 -07:00
|
|
|
use hbbft::messaging;
|
2018-04-24 09:31:21 -07:00
|
|
|
use hbbft::messaging::{QMessage, NodeUid, Algorithm, ProposedValue,
|
2018-04-24 03:29:13 -07:00
|
|
|
AlgoMessage, Handler,
|
2018-04-25 02:59:37 -07:00
|
|
|
MessageLoopState, MessageLoop, RemoteMessage, RemoteNode};
|
2018-04-24 09:31:21 -07:00
|
|
|
use hbbft::broadcast::Broadcast;
|
2018-04-13 10:28:41 -07:00
|
|
|
use hbbft::broadcast;
|
2018-04-12 09:17:33 -07:00
|
|
|
|
|
|
|
use netsim::NetSim;
|
|
|
|
|
|
|
|
/// This is a structure to start a consensus node.
|
2018-04-24 03:29:13 -07:00
|
|
|
pub struct TestNode<'a> {
|
2018-04-13 10:28:41 -07:00
|
|
|
/// Node identifier.
|
2018-04-24 09:31:21 -07:00
|
|
|
uid: NodeUid,
|
2018-04-13 10:28:41 -07:00
|
|
|
/// Total number of nodes.
|
|
|
|
num_nodes: usize,
|
2018-04-20 17:29:54 -07:00
|
|
|
/// TX handles indexed with the receiving node address. One handle for each
|
|
|
|
/// other node.
|
2018-04-24 03:29:13 -07:00
|
|
|
txs: HashMap<NodeUid, Sender<Message<ProposedValue>>>,
|
2018-04-20 17:29:54 -07:00
|
|
|
/// RX handle indexed with the transmitting node address. One handle for
|
|
|
|
/// each other node.
|
2018-04-24 03:29:13 -07:00
|
|
|
rxs: HashMap<NodeUid, Receiver<Message<ProposedValue>>>,
|
2018-04-12 09:17:33 -07:00
|
|
|
/// Optionally, a value to be broadcast by this node.
|
2018-04-24 03:29:13 -07:00
|
|
|
value: Option<ProposedValue>,
|
|
|
|
/// Messaging system.
|
2018-04-24 09:31:21 -07:00
|
|
|
message_loop: MessageLoop<'a, Error>,
|
2018-04-12 09:17:33 -07:00
|
|
|
}
|
|
|
|
|
2018-04-24 03:29:13 -07:00
|
|
|
impl<'a> TestNode<'a>
|
2018-04-12 09:17:33 -07:00
|
|
|
{
|
|
|
|
/// Consensus node constructor. It only initialises initial parameters.
|
2018-04-24 09:31:21 -07:00
|
|
|
pub fn new(uid: NodeUid,
|
2018-04-13 10:28:41 -07:00
|
|
|
num_nodes: usize,
|
2018-04-24 03:29:13 -07:00
|
|
|
txs: HashMap<NodeUid, Sender<Message<ProposedValue>>>,
|
|
|
|
rxs: HashMap<NodeUid, Receiver<Message<ProposedValue>>>,
|
2018-04-20 17:29:54 -07:00
|
|
|
value: Option<ProposedValue>) -> Self
|
2018-04-12 09:17:33 -07:00
|
|
|
{
|
|
|
|
TestNode {
|
2018-04-24 09:31:21 -07:00
|
|
|
uid,
|
2018-04-20 17:29:54 -07:00
|
|
|
num_nodes,
|
2018-04-24 03:29:13 -07:00
|
|
|
txs: txs.clone(),
|
2018-04-20 17:29:54 -07:00
|
|
|
rxs,
|
2018-04-24 03:29:13 -07:00
|
|
|
value,
|
2018-04-24 09:31:21 -07:00
|
|
|
message_loop: MessageLoop::new(txs),
|
2018-04-12 09:17:33 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-24 09:31:21 -07:00
|
|
|
pub fn add_handler<H: 'a + Handler<Error>>(&'a self,
|
|
|
|
algo: Algorithm,
|
|
|
|
handler: &'a H)
|
2018-04-13 10:28:41 -07:00
|
|
|
{
|
2018-04-24 09:31:21 -07:00
|
|
|
self.message_loop.insert_algo(algo, handler);
|
2018-04-12 09:17:33 -07:00
|
|
|
}
|
2018-04-18 01:15:12 -07:00
|
|
|
|
2018-04-24 09:31:21 -07:00
|
|
|
pub fn run(&'a self) -> Result<HashSet<ProposedValue>, Error>
|
2018-04-24 03:29:13 -07:00
|
|
|
{
|
2018-04-25 02:59:37 -07:00
|
|
|
let tx = self.message_loop.queue_tx();
|
|
|
|
|
|
|
|
crossbeam::scope(|scope| {
|
|
|
|
// Spawn receive loops for messages from simulated remote
|
|
|
|
// nodes. Each loop receives a message from the simulated remote
|
|
|
|
// node and forwards it to the local message loop having annotated
|
|
|
|
// the message with the sender node UID.
|
|
|
|
for (uid, rx) in &self.rxs {
|
|
|
|
let tx = tx.clone();
|
|
|
|
let self_uid = self.uid;
|
|
|
|
scope.spawn(move || {
|
2018-04-25 06:07:16 -07:00
|
|
|
debug!("Node {} receiver {} starting", self_uid, uid);
|
2018-04-25 02:59:37 -07:00
|
|
|
while let Ok(message) = rx.recv() {
|
|
|
|
// FIXME: error handling
|
|
|
|
tx.send(QMessage::Remote(RemoteMessage {
|
|
|
|
node: RemoteNode::Node(uid.clone()),
|
|
|
|
message
|
|
|
|
})).unwrap();
|
|
|
|
}
|
|
|
|
debug!("Node {} receiver {} terminated", self_uid, uid);
|
|
|
|
});
|
|
|
|
}
|
2018-04-25 06:07:16 -07:00
|
|
|
// Start the local message loop.
|
2018-04-25 02:59:37 -07:00
|
|
|
let _ = self.message_loop.run();
|
|
|
|
});
|
|
|
|
|
2018-04-24 09:31:21 -07:00
|
|
|
Err(Error::NotImplemented)
|
2018-04-24 03:29:13 -07:00
|
|
|
}
|
|
|
|
}
|
2018-04-22 04:40:40 -07:00
|
|
|
|
2018-04-13 10:28:41 -07:00
|
|
|
#[derive(Clone, Debug)]
|
2018-04-14 03:27:17 -07:00
|
|
|
pub enum Error {
|
2018-04-24 09:31:21 -07:00
|
|
|
Messaging(messaging::Error),
|
2018-04-14 03:27:17 -07:00
|
|
|
Broadcast(broadcast::Error),
|
2018-04-12 09:17:33 -07:00
|
|
|
NotImplemented
|
|
|
|
}
|
|
|
|
|
2018-04-24 09:31:21 -07:00
|
|
|
impl From<messaging::Error> for Error {
|
|
|
|
fn from(e: messaging::Error) -> Error { Error::Messaging(e) }
|
|
|
|
}
|
|
|
|
|
2018-04-14 03:27:17 -07:00
|
|
|
impl From<broadcast::Error> for Error {
|
|
|
|
fn from(e: broadcast::Error) -> Error { Error::Broadcast(e) }
|
2018-04-13 10:28:41 -07:00
|
|
|
}
|
|
|
|
|
2018-04-20 17:29:54 -07:00
|
|
|
fn proposed_value(n: usize) -> ProposedValue {
|
|
|
|
let b: u8 = (n & 0xff) as u8;
|
|
|
|
vec![b; 10]
|
2018-04-16 06:03:32 -07:00
|
|
|
}
|
|
|
|
|
2018-04-25 06:07:16 -07:00
|
|
|
fn node_addr(node_index: usize) -> NodeUid {
|
2018-04-22 04:40:40 -07:00
|
|
|
format!("127.0.0.1:{}", node_index).parse().unwrap()
|
|
|
|
}
|
|
|
|
|
2018-04-25 06:07:16 -07:00
|
|
|
/// Creates test nodes but does not run them.
|
2018-04-20 17:29:54 -07:00
|
|
|
fn create_test_nodes(num_nodes: usize,
|
2018-04-24 03:29:13 -07:00
|
|
|
net: &NetSim<Message<Vec<u8>>>) ->
|
2018-04-25 06:07:16 -07:00
|
|
|
HashMap<NodeUid, (TestNode, HashMap<NodeUid, Broadcast>)>
|
2018-04-12 09:17:33 -07:00
|
|
|
{
|
2018-04-25 06:07:16 -07:00
|
|
|
let mut nodes = HashMap::new();
|
2018-04-12 09:17:33 -07:00
|
|
|
for n in 0..num_nodes {
|
2018-04-20 17:29:54 -07:00
|
|
|
let value = proposed_value(n);
|
|
|
|
let mut txs = HashMap::new();
|
|
|
|
let mut rxs = HashMap::new();
|
2018-04-13 10:28:41 -07:00
|
|
|
// Set up comms channels to other nodes.
|
2018-04-12 09:17:33 -07:00
|
|
|
for m in 0..num_nodes {
|
|
|
|
if n == m {
|
|
|
|
// Skip the channel back to the node itself.
|
|
|
|
continue;
|
|
|
|
}
|
2018-04-22 04:40:40 -07:00
|
|
|
let addr = node_addr(m);
|
2018-04-20 17:29:54 -07:00
|
|
|
txs.insert(addr, net.tx(n, m));
|
|
|
|
rxs.insert(addr, net.rx(m, n));
|
2018-04-12 09:17:33 -07:00
|
|
|
}
|
2018-04-24 09:31:21 -07:00
|
|
|
let uid = node_addr(n);
|
2018-04-25 06:07:16 -07:00
|
|
|
|
|
|
|
// Create a broadcast algorithm instance for each node.
|
|
|
|
let mut broadcast_instances = HashMap::new();
|
|
|
|
for k in 0..num_nodes {
|
|
|
|
let them_uid = node_addr(k);
|
|
|
|
broadcast_instances.insert(them_uid, Broadcast::new(them_uid,
|
|
|
|
num_nodes));
|
|
|
|
}
|
|
|
|
|
|
|
|
nodes.insert(uid, (TestNode::new(uid, num_nodes, txs, rxs, Some(value)),
|
|
|
|
broadcast_instances));
|
2018-04-12 09:17:33 -07:00
|
|
|
}
|
|
|
|
nodes
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_4_broadcast_nodes() {
|
2018-04-13 10:28:41 -07:00
|
|
|
simple_logger::init_with_level(log::Level::Debug).unwrap();
|
|
|
|
|
2018-04-12 09:17:33 -07:00
|
|
|
const NUM_NODES: usize = 4;
|
2018-04-24 09:31:21 -07:00
|
|
|
|
2018-04-14 03:27:17 -07:00
|
|
|
let net: NetSim<Message<Vec<u8>>> = NetSim::new(NUM_NODES);
|
2018-04-12 09:17:33 -07:00
|
|
|
let nodes = create_test_nodes(NUM_NODES, &net);
|
2018-04-24 09:31:21 -07:00
|
|
|
let mut join_handles: HashMap<NodeUid, _> = HashMap::new();
|
2018-04-12 09:17:33 -07:00
|
|
|
|
|
|
|
crossbeam::scope(|scope| {
|
2018-04-25 06:07:16 -07:00
|
|
|
// Run the test nodes, each in its own thread.
|
|
|
|
for (uid, (node, broadcast_instances)) in nodes.iter() {
|
|
|
|
join_handles.insert(*uid, scope.spawn(move || {
|
|
|
|
// Register broadcast instance handlers with the message loop.
|
|
|
|
for (instance_uid, instance) in broadcast_instances {
|
|
|
|
node.add_handler(Algorithm::Broadcast(*instance_uid),
|
|
|
|
instance);
|
|
|
|
}
|
2018-04-25 02:59:37 -07:00
|
|
|
debug!("Running {:?}", node.uid);
|
2018-04-24 09:31:21 -07:00
|
|
|
node.run()
|
|
|
|
}));
|
2018-04-12 09:17:33 -07:00
|
|
|
}
|
2018-04-24 09:31:21 -07:00
|
|
|
|
|
|
|
for (uid, join_handle) in join_handles {
|
|
|
|
let result = join_handle.join();
|
|
|
|
println!("Result of {}: {:?}", uid, result);
|
|
|
|
}
|
|
|
|
|
|
|
|
println!("Finished");
|
2018-04-12 09:17:33 -07:00
|
|
|
});
|
|
|
|
}
|