2018-04-12 09:17:33 -07:00
|
|
|
//! Integration test of the reliable broadcast protocol.
|
|
|
|
|
|
|
|
extern crate hbbft;
|
2018-04-13 10:28:41 -07:00
|
|
|
#[macro_use]
|
|
|
|
extern crate log;
|
|
|
|
extern crate simple_logger;
|
2018-04-12 09:17:33 -07:00
|
|
|
extern crate crossbeam;
|
|
|
|
#[macro_use]
|
|
|
|
extern crate crossbeam_channel;
|
|
|
|
extern crate merkle;
|
|
|
|
|
|
|
|
mod netsim;
|
2018-04-13 10:28:41 -07:00
|
|
|
mod node_comms;
|
2018-04-12 09:17:33 -07:00
|
|
|
|
2018-04-13 10:28:41 -07:00
|
|
|
use std::sync::Arc;
|
2018-04-12 09:17:33 -07:00
|
|
|
use std::collections::HashSet;
|
|
|
|
use std::fmt::Debug;
|
|
|
|
use std::io;
|
2018-04-13 10:28:41 -07:00
|
|
|
use crossbeam::{Scope, ScopedJoinHandle};
|
|
|
|
use crossbeam_channel::{bounded, Sender, Receiver};
|
2018-04-12 09:17:33 -07:00
|
|
|
|
|
|
|
use hbbft::proto::*;
|
2018-04-13 10:28:41 -07:00
|
|
|
use hbbft::messaging::{Messaging, SourcedMessage};
|
|
|
|
use hbbft::broadcast;
|
2018-04-12 09:17:33 -07:00
|
|
|
|
|
|
|
use netsim::NetSim;
|
2018-04-13 10:28:41 -07:00
|
|
|
use node_comms::CommsTask;
|
2018-04-12 09:17:33 -07:00
|
|
|
|
|
|
|
/// This is a structure to start a consensus node.
|
|
|
|
pub struct TestNode<'a> {
|
2018-04-13 10:28:41 -07:00
|
|
|
/// Node identifier.
|
|
|
|
node_index: usize,
|
|
|
|
/// Total number of nodes.
|
|
|
|
num_nodes: usize,
|
|
|
|
/// TX handles, one for each other node.
|
2018-04-14 03:27:17 -07:00
|
|
|
txs: Vec<&'a Sender<Message<Vec<u8>>>>,
|
2018-04-13 10:28:41 -07:00
|
|
|
/// RX handle, one for each other node.
|
2018-04-14 03:27:17 -07:00
|
|
|
rxs: Vec<&'a Receiver<Message<Vec<u8>>>>,
|
2018-04-12 09:17:33 -07:00
|
|
|
/// Optionally, a value to be broadcast by this node.
|
|
|
|
value: Option<TestValue>
|
|
|
|
}
|
|
|
|
|
|
|
|
impl<'a> TestNode<'a>
|
|
|
|
{
|
|
|
|
/// Consensus node constructor. It only initialises initial parameters.
|
2018-04-13 10:28:41 -07:00
|
|
|
pub fn new(node_index: usize,
|
|
|
|
num_nodes: usize,
|
2018-04-14 03:27:17 -07:00
|
|
|
txs: Vec<&'a Sender<Message<Vec<u8>>>>,
|
|
|
|
rxs: Vec<&'a Receiver<Message<Vec<u8>>>>,
|
2018-04-12 09:17:33 -07:00
|
|
|
value: Option<TestValue>) -> Self
|
|
|
|
{
|
|
|
|
TestNode {
|
2018-04-13 10:28:41 -07:00
|
|
|
node_index: node_index,
|
|
|
|
num_nodes: num_nodes,
|
2018-04-12 09:17:33 -07:00
|
|
|
txs: txs,
|
|
|
|
rxs: rxs,
|
|
|
|
value: value
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-14 03:27:17 -07:00
|
|
|
pub fn run(&self, messaging: Messaging<Vec<u8>>) ->
|
|
|
|
Result<HashSet<TestValue>, Error>
|
2018-04-13 10:28:41 -07:00
|
|
|
{
|
2018-04-13 10:49:49 -07:00
|
|
|
assert_eq!(self.rxs.len(), self.num_nodes - 1);
|
2018-04-13 10:28:41 -07:00
|
|
|
|
|
|
|
let to_comms_rxs = messaging.to_comms_rxs();
|
|
|
|
let from_comms_tx = messaging.from_comms_tx();
|
|
|
|
let to_algo_rxs = messaging.to_algo_rxs();
|
|
|
|
let from_algo_tx = messaging.from_algo_tx();
|
|
|
|
let ref to_algo_rx0 = to_algo_rxs[0];
|
|
|
|
let value = self.value.to_owned();
|
|
|
|
let num_nodes = self.num_nodes;
|
|
|
|
let mut values = HashSet::new();
|
|
|
|
|
|
|
|
crossbeam::scope(|scope| {
|
|
|
|
let mut handles = Vec::new();
|
|
|
|
|
|
|
|
// Spawn the 0-th instance corresponding to this node. The return
|
|
|
|
// value shall be equal to `value` if computation succeeded or error
|
|
|
|
// otherwise.
|
|
|
|
handles.push(scope.spawn(move || {
|
|
|
|
broadcast::Instance::new(from_algo_tx,
|
|
|
|
to_algo_rx0,
|
|
|
|
value,
|
|
|
|
num_nodes,
|
|
|
|
0)
|
|
|
|
.run()
|
|
|
|
}));
|
|
|
|
|
|
|
|
// Control TX handles to stop all comms threads.
|
|
|
|
let mut comms_stop_txs = Vec::new();
|
|
|
|
|
|
|
|
// Spawn instances 1 through num_nodes-1 together with simulated
|
|
|
|
// remote comms tasks.
|
|
|
|
for i in 1..num_nodes {
|
|
|
|
// Make a channel to be used to stop the comms task.
|
|
|
|
let (comms_stop_tx, comms_stop_rx): (Sender<()>, Receiver<()>) =
|
|
|
|
bounded(1);
|
|
|
|
// Record the TX handle for using it later.
|
|
|
|
comms_stop_txs.push(comms_stop_tx);
|
|
|
|
// Spawn the comms task.
|
|
|
|
scope.spawn(move || {
|
|
|
|
// Termination condition variable.
|
|
|
|
let mut stop = false;
|
|
|
|
|
|
|
|
// Receive messages from the simulated node or locally.
|
|
|
|
while !stop { select_loop! {
|
|
|
|
// Receive from the simulated remote node.
|
|
|
|
recv(self.rxs[i-1], message) => {
|
|
|
|
debug!("Node {}/{} received {:?}",
|
|
|
|
self.node_index, i, message);
|
|
|
|
from_comms_tx.send(
|
|
|
|
SourcedMessage {
|
|
|
|
source: i,
|
|
|
|
message
|
|
|
|
}).unwrap();
|
|
|
|
},
|
|
|
|
// Receive from an algorithm via local
|
|
|
|
// messaging. Forward the message to the simulated
|
|
|
|
// remote node.
|
|
|
|
recv(to_comms_rxs[i-1], message) => {
|
|
|
|
self.txs[i-1].send(message).unwrap();
|
|
|
|
}
|
|
|
|
recv(comms_stop_rx, _) => {
|
2018-04-13 10:49:49 -07:00
|
|
|
debug!("Stopping comms task {}/{}",
|
|
|
|
self.node_index, i);
|
2018-04-13 10:28:41 -07:00
|
|
|
stop = true;
|
|
|
|
}
|
|
|
|
}}
|
|
|
|
});
|
|
|
|
|
|
|
|
let ref to_algo_rx = to_algo_rxs[i];
|
|
|
|
|
|
|
|
// Spawn a broadcast instance associated with the above comms
|
|
|
|
// task.
|
|
|
|
handles.push(scope.spawn(move || {
|
|
|
|
broadcast::Instance::new(from_algo_tx,
|
|
|
|
to_algo_rx,
|
|
|
|
None,
|
|
|
|
num_nodes,
|
|
|
|
i)
|
|
|
|
.run()
|
|
|
|
}));
|
2018-04-12 09:17:33 -07:00
|
|
|
}
|
2018-04-13 10:28:41 -07:00
|
|
|
|
|
|
|
let mut error = None;
|
|
|
|
|
|
|
|
// Collect the values computed by broadcast instances.
|
|
|
|
for h in handles {
|
|
|
|
match h.join() {
|
|
|
|
Ok(v) => {
|
2018-04-14 03:27:17 -07:00
|
|
|
debug!("Received value {:?}", v);
|
2018-04-13 10:28:41 -07:00
|
|
|
values.insert(v);
|
|
|
|
},
|
|
|
|
Err(e) => {
|
|
|
|
error = Some(Error::Broadcast(e));
|
|
|
|
}
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
// Stop the comms tasks.
|
|
|
|
for tx in comms_stop_txs {
|
|
|
|
tx.send(()).unwrap();
|
|
|
|
}
|
|
|
|
|
|
|
|
if error.is_some() {
|
|
|
|
Err(error.unwrap())
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
Ok(values)
|
|
|
|
}
|
|
|
|
})
|
2018-04-12 09:17:33 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-13 10:28:41 -07:00
|
|
|
#[derive(Clone, Debug)]
|
2018-04-14 03:27:17 -07:00
|
|
|
pub enum Error {
|
|
|
|
Broadcast(broadcast::Error),
|
2018-04-12 09:17:33 -07:00
|
|
|
NotImplemented
|
|
|
|
}
|
|
|
|
|
2018-04-14 03:27:17 -07:00
|
|
|
impl From<broadcast::Error> for Error {
|
|
|
|
fn from(e: broadcast::Error) -> Error { Error::Broadcast(e) }
|
2018-04-13 10:28:41 -07:00
|
|
|
}
|
|
|
|
|
2018-04-12 09:17:33 -07:00
|
|
|
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
|
|
|
|
pub struct TestValue {
|
|
|
|
pub value: String
|
|
|
|
}
|
|
|
|
|
|
|
|
/// `TestValue: merkle::Hashable` is derived from `TestValue: AsRef<[u8]>`.
|
|
|
|
impl AsRef<[u8]> for TestValue {
|
|
|
|
fn as_ref(&self) -> &[u8] {
|
|
|
|
self.value.as_ref()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl From<Vec<u8>> for TestValue {
|
|
|
|
fn from(bytes: Vec<u8>) -> TestValue {
|
|
|
|
TestValue {
|
2018-04-14 03:27:17 -07:00
|
|
|
value: String::from_utf8(bytes).expect("Found invalid UTF-8")
|
2018-04-13 10:49:49 -07:00
|
|
|
// conversion from UTF-8 often panics:
|
|
|
|
// String::from_utf8(bytes).expect("Found invalid UTF-8")
|
2018-04-12 09:17:33 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl From<TestValue> for Vec<u8> {
|
|
|
|
fn from(v: TestValue) -> Vec<u8> {
|
|
|
|
match v {
|
|
|
|
TestValue { value } => {
|
|
|
|
value.as_bytes().to_vec()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-13 10:28:41 -07:00
|
|
|
/// Creates a vector of test nodes but does not run them.
|
2018-04-12 09:17:33 -07:00
|
|
|
fn create_test_nodes<'a>(num_nodes: usize,
|
2018-04-14 03:27:17 -07:00
|
|
|
net: &'a NetSim<Message<Vec<u8>>>) ->
|
2018-04-12 09:17:33 -07:00
|
|
|
Vec<TestNode<'a>>
|
|
|
|
{
|
|
|
|
let mut nodes = Vec::new();
|
|
|
|
for n in 0..num_nodes {
|
|
|
|
let value = TestValue {
|
|
|
|
value: format!("-{}-{}-{}-", n, n, n)
|
|
|
|
};
|
|
|
|
let mut txs = Vec::new();
|
|
|
|
let mut rxs = Vec::new();
|
2018-04-13 10:28:41 -07:00
|
|
|
// Set up comms channels to other nodes.
|
2018-04-12 09:17:33 -07:00
|
|
|
for m in 0..num_nodes {
|
|
|
|
if n == m {
|
|
|
|
// Skip the channel back to the node itself.
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
txs.push(net.tx(n, m));
|
|
|
|
rxs.push(net.rx(m, n));
|
|
|
|
}
|
2018-04-13 10:28:41 -07:00
|
|
|
nodes.push(TestNode::new(n, num_nodes, txs, rxs, Some(value)));
|
2018-04-12 09:17:33 -07:00
|
|
|
}
|
|
|
|
nodes
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_4_broadcast_nodes() {
|
2018-04-13 10:28:41 -07:00
|
|
|
simple_logger::init_with_level(log::Level::Debug).unwrap();
|
|
|
|
|
2018-04-12 09:17:33 -07:00
|
|
|
const NUM_NODES: usize = 4;
|
2018-04-14 03:27:17 -07:00
|
|
|
let net: NetSim<Message<Vec<u8>>> = NetSim::new(NUM_NODES);
|
2018-04-12 09:17:33 -07:00
|
|
|
let nodes = create_test_nodes(NUM_NODES, &net);
|
|
|
|
|
|
|
|
crossbeam::scope(|scope| {
|
2018-04-13 10:28:41 -07:00
|
|
|
|
2018-04-12 09:17:33 -07:00
|
|
|
let mut handles = Vec::new();
|
2018-04-13 10:28:41 -07:00
|
|
|
let mut messaging_stop_txs = Vec::new();
|
2018-04-12 09:17:33 -07:00
|
|
|
|
|
|
|
for node in nodes {
|
2018-04-13 10:28:41 -07:00
|
|
|
// Start a local messaging service on the simulated node.
|
2018-04-14 03:27:17 -07:00
|
|
|
let messaging: Messaging<Vec<u8>> =
|
2018-04-13 10:28:41 -07:00
|
|
|
Messaging::new(NUM_NODES);
|
|
|
|
messaging.spawn(scope);
|
|
|
|
// Take the thread control handle.
|
|
|
|
messaging_stop_txs.push(messaging.stop_tx());
|
|
|
|
|
2018-04-12 09:17:33 -07:00
|
|
|
handles.push(scope.spawn(move || {
|
2018-04-13 10:28:41 -07:00
|
|
|
node.run(messaging)
|
2018-04-12 09:17:33 -07:00
|
|
|
}));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Compare the set of values returned by broadcast against the expected
|
|
|
|
// set.
|
|
|
|
for h in handles {
|
2018-04-14 03:27:17 -07:00
|
|
|
match h.join() {
|
|
|
|
Err(Error::NotImplemented) => panic!(),
|
|
|
|
Err(err) => panic!("Error: {:?}", err),
|
|
|
|
Ok(v) => debug!("Finished with values {:?}", v),
|
|
|
|
}
|
2018-04-13 10:28:41 -07:00
|
|
|
}
|
|
|
|
// Stop all messaging tasks.
|
|
|
|
for tx in messaging_stop_txs {
|
|
|
|
tx.send(()).unwrap();
|
2018-04-12 09:17:33 -07:00
|
|
|
}
|
|
|
|
});
|
|
|
|
}
|