diff --git a/tests/network/mod.rs b/tests/network/mod.rs deleted file mode 100644 index f21a245..0000000 --- a/tests/network/mod.rs +++ /dev/null @@ -1,672 +0,0 @@ -use std::collections::{BTreeMap, BTreeSet, VecDeque}; -use std::fmt::{self, Debug}; -use std::mem; -use std::sync::Arc; - -use log::{debug, warn}; -use rand::seq::{IteratorRandom, SliceRandom}; -use rand::{self, Rng}; -use rand_derive::Rand; -use serde_derive::{Deserialize, Serialize}; - -use hbbft::dynamic_honey_badger::Batch; -use hbbft::sender_queue::SenderQueueableOutput; -use hbbft::{ - ConsensusProtocol, Contribution, CpStep, Fault, NetworkInfo, Step, Target, TargetedMessage, -}; - -/// A node identifier. In the tests, nodes are simply numbered. -#[derive(Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Clone, Copy, Serialize, Deserialize, Rand)] -pub struct NodeId(pub usize); - -/// A "node" running an instance of the algorithm `D`. -pub struct TestNode { - /// This node's own ID. - pub id: D::NodeId, - /// The instance of the broadcast algorithm. - algo: D, - /// Incoming messages from other nodes that this node has not yet handled. - pub queue: VecDeque<(D::NodeId, D::Message)>, - /// The values this node has output so far. - outputs: Vec, - /// Outgoing messages to be sent to other nodes. - messages: Vec>, - /// Collected fault logs. - faults: Vec>, -} - -impl TestNode { - /// Returns the list of outputs received by this node. - pub fn outputs(&self) -> &[D::Output] { - &self.outputs - } - - /// Returns whether the algorithm has terminated. - #[allow(unused)] // Not used in all tests. - pub fn terminated(&self) -> bool { - self.algo.terminated() - } - - /// Inputs a value into the instance. - pub fn handle_input(&mut self, input: D::Input, rng: &mut R) { - let step = self.algo.handle_input(input, rng).expect("input"); - self.outputs.extend(step.output); - self.messages.extend(step.messages); - self.faults.extend(step.fault_log.0); - } - - /// Returns the internal algorithm's instance. - #[allow(unused)] // Not used in all tests. - pub fn instance(&self) -> &D { - &self.algo - } - - /// Returns the internal algorithm's mutable instance. - #[allow(unused)] // Not used in all tests. - pub fn instance_mut(&mut self) -> &mut D { - &mut self.algo - } - - /// Creates a new test node with the given broadcast instance. - fn new((algo, step): (D, CpStep)) -> TestNode { - TestNode { - id: algo.our_id().clone(), - algo, - queue: VecDeque::new(), - outputs: step.output.into_iter().collect(), - messages: step.messages, - faults: step.fault_log.0, - } - } - - /// Handles the first message in the node's queue. - fn handle_message(&mut self) { - let mut rng = rand::thread_rng(); - - let (from_id, msg) = self.queue.pop_front().expect("message not found"); - debug!("Handling {:?} -> {:?}: {:?}", from_id, self.id, msg); - let step = self - .algo - .handle_message(&from_id, msg, &mut rng) - .expect("handling message"); - self.outputs.extend(step.output); - self.messages.extend(step.messages); - self.faults.extend(step.fault_log.0); - } - - /// Checks whether the node has messages to process. - fn is_idle(&self) -> bool { - self.queue.is_empty() - } -} - -/// A strategy for picking the next good node to handle a message. -pub enum MessageScheduler { - /// Picks a random node. - Random, - /// Picks the first non-idle node. - First, -} - -impl MessageScheduler { - /// Chooses a node to be the next one to handle a message. - pub fn pick_node( - &self, - nodes: &BTreeMap>, - ) -> D::NodeId { - let mut ids = nodes - .iter() - .filter(|(_, node)| !node.queue.is_empty()) - .map(|(id, _)| id.clone()); - let rand_node = match *self { - MessageScheduler::First => rand::thread_rng().gen_bool(0.1), - MessageScheduler::Random => true, - }; - if rand_node { - ids.choose(&mut rand::thread_rng()) - } else { - ids.next() - } - .expect("no more messages in queue") - } -} - -/// A message combined with a sender. -pub struct MessageWithSender { - /// The sender of the message. - pub sender: ::NodeId, - /// The targeted message (recipient and message body). - pub tm: TargetedMessage<::Message, ::NodeId>, -} - -// The Debug implementation cannot be derived automatically, possibly due to a compiler bug. For -// this reason, it is implemented manually here. -impl fmt::Debug for MessageWithSender { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "MessageWithSender {{ sender: {:?}, tm: {:?} }}", - self.sender, self.tm.target - ) - } -} - -impl MessageWithSender { - /// Creates a new message with a sender. - pub fn new( - sender: D::NodeId, - tm: TargetedMessage, - ) -> MessageWithSender { - MessageWithSender { sender, tm } - } -} - -/// An adversary that can control a set of nodes and pick the next good node to receive a message. -/// -/// See `TestNetwork::step()` for a more detailed description of its capabilities. -pub trait Adversary { - /// Chooses a node to be the next one to handle a message. - /// - /// Starvation is illegal, i.e. in every iteration a node that has pending incoming messages - /// must be chosen. - fn pick_node(&self, nodes: &BTreeMap>) -> D::NodeId; - - /// Called when a node controlled by the adversary receives a message. - fn push_message(&mut self, sender_id: D::NodeId, msg: TargetedMessage); - - /// Produces a list of messages to be sent from the adversary's nodes. - fn step(&mut self) -> Vec>; - - /// Initialize an adversary. This function's primary purpose is to inform the adversary over - /// some aspects of the network, such as which nodes they control. - fn init( - &mut self, - _all_nodes: &BTreeMap>, - _adv_nodes: &BTreeMap>>, - ) { - // default: does nothing - } -} - -/// An adversary whose nodes never send any messages. -pub struct SilentAdversary { - scheduler: MessageScheduler, -} - -impl SilentAdversary { - /// Creates a new silent adversary with the given message scheduler. - pub fn new(scheduler: MessageScheduler) -> SilentAdversary { - SilentAdversary { scheduler } - } -} - -impl Adversary for SilentAdversary { - fn pick_node(&self, nodes: &BTreeMap>) -> D::NodeId { - self.scheduler.pick_node(nodes) - } - - fn push_message(&mut self, _: D::NodeId, _: TargetedMessage) { - // All messages are ignored. - } - - fn step(&mut self) -> Vec> { - vec![] // No messages are sent. - } -} - -/// Return true with a certain `probability` ([0 .. 1.0]). -fn randomly(probability: f32) -> bool { - assert!(probability <= 1.0); - assert!(probability >= 0.0); - - let mut rng = rand::thread_rng(); - rng.gen_range(0.0, 1.0) <= probability -} - -#[test] -fn test_randomly() { - assert!(randomly(1.0)); - assert!(!randomly(0.0)); -} - -/// An adversary that performs naive replay attacks. -/// -/// The adversary will randomly take a message that is sent to one of its nodes and re-send it to -/// a different node. Additionally, it will inject unrelated messages at random. -#[allow(unused)] // not used in all tests -pub struct RandomAdversary { - /// The underlying scheduler used - scheduler: MessageScheduler, - - /// Node ids seen by the adversary. - known_node_ids: Vec, - /// Node ids under control of adversary - known_adversarial_ids: Vec, - - /// Internal queue for messages to be returned on the next `Adversary::step()` call - outgoing: Vec>, - /// Generates random messages to be injected - generator: F, - - /// Probability of a message replay - p_replay: f32, - /// Probability of a message injection - p_inject: f32, -} - -impl RandomAdversary { - /// Creates a new random adversary instance. - #[allow(unused)] - pub fn new(p_replay: f32, p_inject: f32, generator: F) -> RandomAdversary { - assert!( - p_inject < 0.95, - "injections are repeated, p_inject must be smaller than 0.95" - ); - - RandomAdversary { - // The random adversary, true to its name, always schedules randomly. - scheduler: MessageScheduler::Random, - known_node_ids: Vec::new(), - known_adversarial_ids: Vec::new(), - outgoing: Vec::new(), - generator, - p_replay, - p_inject, - } - } -} - -impl TargetedMessage> Adversary - for RandomAdversary -{ - fn init( - &mut self, - all_nodes: &BTreeMap>, - nodes: &BTreeMap>>, - ) { - self.known_adversarial_ids = nodes.keys().cloned().collect(); - self.known_node_ids = all_nodes.keys().cloned().collect(); - } - - fn pick_node(&self, nodes: &BTreeMap>) -> D::NodeId { - // Just let the scheduler pick a node. - self.scheduler.pick_node(nodes) - } - - fn push_message(&mut self, _: D::NodeId, msg: TargetedMessage) { - // If we have not discovered the network topology yet, abort. - if self.known_node_ids.is_empty() { - return; - } - - // only replay a message in some cases - if !randomly(self.p_replay) { - return; - } - - let TargetedMessage { message, target } = msg; - - match target { - Target::All => { - // Ideally, we would want to handle broadcast messages as well; however the - // adversary API is quite cumbersome at the moment in regards to access to the - // network topology. To re-send a broadcast message from one of the attacker - // controlled nodes, we would have to get a list of attacker controlled nodes - // here and use a random one as the origin/sender, this is not done here. - return; - } - Target::Node(our_node_id) => { - // Choose a new target to send the message to. The unwrap never fails, because we - // ensured that `known_node_ids` is non-empty earlier. - let mut rng = rand::thread_rng(); - let new_target_node = self.known_node_ids.iter().choose(&mut rng).unwrap().clone(); - - // TODO: We could randomly broadcast it instead, if we had access to topology - // information. - self.outgoing.push(MessageWithSender::new( - our_node_id, - TargetedMessage { - target: Target::Node(new_target_node), - message, - }, - )); - } - } - } - - fn step(&mut self) -> Vec> { - // Clear messages. - let mut tmp = Vec::new(); - mem::swap(&mut tmp, &mut self.outgoing); - - // Possibly inject more messages: - while randomly(self.p_inject) { - let mut rng = rand::thread_rng(); - - // Pick a random adversarial node and create a message using the generator. - if let Some(sender) = self.known_adversarial_ids[..].choose(&mut rng) { - let tm = (self.generator)(); - - // Add to outgoing queue. - tmp.push(MessageWithSender::new(sender.clone(), tm)); - } - } - - if !tmp.is_empty() { - println!("Injecting random messages: {:?}", tmp); - } - tmp - } -} - -/// A collection of `TestNode`s representing a network. -/// -/// Each `TestNetwork` type is tied to a specific adversary and a consensus protocol. It consists -/// of a set of nodes, some of which are controlled by the adversary and some of which may be -/// observer nodes, as well as a set of threshold-cryptography public keys. -/// -/// In addition to being able to participate correctly in the network using his nodes, the -/// adversary can: -/// -/// 1. Decide which node is the next one to make progress, -/// 2. Send arbitrary messages to any node originating from one of the nodes they control. -/// -/// See the `step` function for details on actual operation of the network. -pub struct TestNetwork, D: ConsensusProtocol> { - pub nodes: BTreeMap>, - pub observer: TestNode, - pub adv_nodes: BTreeMap>>, - adversary: A, -} - -impl, D: ConsensusProtocol> TestNetwork -where - D::Message: Clone, -{ - /// Creates a new network with `good_num` good nodes, and the given `adversary` controlling - /// `adv_num` nodes. - #[allow(unused)] // Not used in all tests. - pub fn new( - good_num: usize, - adv_num: usize, - adversary: G, - new_algo: F, - ) -> TestNetwork - where - F: Fn(Arc>) -> D, - G: Fn(BTreeMap>>) -> A, - { - Self::new_with_step(good_num, adv_num, adversary, |netinfo| { - (new_algo(netinfo), Step::default()) - }) - } - - /// Creates a new network with `good_num` good nodes, and the given `adversary` controlling - /// `adv_num` nodes. - pub fn new_with_step( - good_num: usize, - adv_num: usize, - adversary: G, - new_algo: F, - ) -> TestNetwork - where - F: Fn(Arc>) -> (D, CpStep), - G: Fn(BTreeMap>>) -> A, - { - let mut rng = rand::thread_rng(); - let node_ids = (0..(good_num + adv_num)).map(NodeId); - let mut netinfos = NetworkInfo::generate_map(node_ids, &mut rng) - .expect("Failed to generate `NetworkInfo` map"); - let obs_netinfo = { - let node_ni = netinfos.values().next().unwrap(); - NetworkInfo::new( - NodeId(good_num + adv_num), - None, - node_ni.public_key_set().clone(), - rng.gen(), - node_ni.public_key_map().clone(), - ) - }; - let adv_netinfos = netinfos.split_off(&NodeId(good_num)); - - let new_node = |(id, netinfo): (NodeId, NetworkInfo<_>)| { - (id, TestNode::new(new_algo(Arc::new(netinfo)))) - }; - let new_adv_node = |(id, netinfo): (NodeId, NetworkInfo<_>)| (id, Arc::new(netinfo)); - let adv_nodes: BTreeMap<_, _> = adv_netinfos.into_iter().map(new_adv_node).collect(); - - let observer = TestNode::new(new_algo(Arc::new(obs_netinfo))); - - let mut network = TestNetwork { - nodes: netinfos.into_iter().map(new_node).collect(), - observer, - adversary: adversary(adv_nodes.clone()), - adv_nodes, - }; - - // Inform the adversary about their nodes. - network.adversary.init(&network.nodes, &network.adv_nodes); - - let msgs = network.adversary.step(); - for MessageWithSender { sender, tm } in msgs { - network.dispatch_messages(sender, vec![tm]); - } - let mut initial_msgs: Vec<(D::NodeId, Vec<_>)> = Vec::new(); - for (id, node) in &mut network.nodes { - initial_msgs.push((*id, node.messages.drain(..).collect())); - } - initial_msgs.push(( - network.observer.id, - network.observer.messages.drain(..).collect(), - )); - for (id, msgs) in initial_msgs { - network.dispatch_messages(id, msgs); - } - network - } - - /// Pushes the messages into the queues of the corresponding recipients. - pub fn dispatch_messages(&mut self, sender_id: NodeId, msgs: Q) - where - Q: IntoIterator> + Debug, - { - for msg in msgs { - match msg.target { - Target::All => { - for node in self.nodes.values_mut() { - if node.id != sender_id { - node.queue.push_back((sender_id, msg.message.clone())) - } - } - if self.observer.id != sender_id { - self.observer - .queue - .push_back((sender_id, msg.message.clone())); - } - self.adversary.push_message(sender_id, msg); - } - Target::Node(to_id) => { - if self.adv_nodes.contains_key(&to_id) { - self.adversary.push_message(sender_id, msg); - } else if let Some(node) = self.nodes.get_mut(&to_id) { - node.queue.push_back((sender_id, msg.message)); - } else if self.observer.id == to_id { - self.observer.queue.push_back((sender_id, msg.message)); - } else { - warn!( - "Unknown recipient {:?} for message: {:?}", - to_id, msg.message - ); - } - } - } - } - self.observer_handle_messages(); - self.observer_dispatch_messages(); - } - - /// Handles all messages queued for the observer. - fn observer_handle_messages(&mut self) { - while !self.observer.queue.is_empty() { - self.observer.handle_message(); - let faults: Vec<_> = self.observer.faults.drain(..).collect(); - self.check_faults(faults); - } - } - - /// Dispatches messages from the observer to the queues of the recipients of those messages. - fn observer_dispatch_messages(&mut self) { - self.observer_handle_messages(); - let observer_msgs: Vec<_> = self.observer.messages.drain(..).collect(); - if !observer_msgs.is_empty() { - let observer_id = self.observer.id; - self.dispatch_messages(observer_id, observer_msgs); - } - } - - /// Performs one iteration of the network, consisting of the following steps: - /// - /// 1. Give the adversary a chance to send messages of his choosing through `Adversary::step()`, - /// 2. Let the adversary pick a node that receives its next message through - /// `Adversary::pick_node()`. - /// - /// Returns the node ID of the node that made progress. - pub fn step(&mut self) -> NodeId { - // We let the adversary send out messages to any number of nodes. - let msgs = self.adversary.step(); - for MessageWithSender { sender, tm } in msgs { - self.dispatch_messages(sender, Some(tm)); - } - - // Now one node is chosen to make progress, we let the adversary decide which. - let id = self.adversary.pick_node(&self.nodes); - - // The node handles the incoming message and creates new outgoing ones to be dispatched. - let (msgs, faults): (Vec<_>, Vec<_>) = { - let node = self.nodes.get_mut(&id).unwrap(); - - // Ensure the adversary is playing fair by selecting a node that will result in actual - // progress being made, otherwise `TestNode::handle_message()` will panic on `expect()` - // with a much more cryptic error message. - assert!( - !node.is_idle(), - "adversary illegally selected an idle node in pick_node()" - ); - - node.handle_message(); - ( - node.messages.drain(..).collect(), - node.faults.drain(..).collect(), - ) - }; - self.check_faults(faults); - self.dispatch_messages(id, msgs); - - id - } - - /// Inputs a value in node `id`. - pub fn input(&mut self, id: NodeId, value: D::Input) { - let mut rng = rand::thread_rng(); - - let (msgs, faults): (Vec<_>, Vec<_>) = { - let node = self.nodes.get_mut(&id).expect("input instance"); - node.handle_input(value, &mut rng); - ( - node.messages.drain(..).collect(), - node.faults.drain(..).collect(), - ) - }; - self.check_faults(faults); - self.dispatch_messages(id, msgs); - } - - /// Inputs a value in all nodes. - #[allow(unused)] // Not used in all tests. - pub fn input_all(&mut self, value: D::Input) - where - D::Input: Clone, - { - let ids: Vec = self.nodes.keys().cloned().collect(); - for id in ids { - self.input(id, value.clone()); - } - } - - /// Verifies that no correct node is reported as faulty. - fn check_faults>>(&self, faults: I) { - for fault in faults { - if self.nodes.contains_key(&fault.node_id) { - panic!("Unexpected fault: {:?}", fault); - } - } - } -} - -impl, C, D> TestNetwork -where - D: ConsensusProtocol, NodeId = NodeId>, - C: Contribution + Clone, -{ - /// Verifies that all nodes' outputs agree, given a correct "full" node that output all - /// batches with no gaps. - /// - /// The output of the full node is used to derive in expected output of other nodes in every - /// epoch. After that the check ensures that correct nodes output the same batches in epochs - /// when those nodes were participants (either validators or candidates). - #[allow(unused)] // Not used in all tests. - pub fn verify_batches(&self, full_node: &TestNode) - where - Batch: SenderQueueableOutput, - { - // Participants of epoch 0 are all validators in the test network. - let mut participants: BTreeSet = self - .nodes - .keys() - .cloned() - .chain(self.adv_nodes.keys().cloned()) - .collect(); - let mut expected: BTreeMap> = BTreeMap::new(); - for batch in &full_node.outputs { - for id in &participants { - expected.entry(id.clone()).or_default().push(batch); - } - if let Some(new_participants) = batch.participant_change() { - participants = new_participants; - } - } - for (id, node) in self.nodes.iter().filter(|(&id, _)| id != full_node.id) { - let actual_epochs: BTreeSet<_> = - node.outputs.iter().map(|batch| batch.epoch()).collect(); - let expected_epochs: BTreeSet<_> = - expected[id].iter().map(|batch| batch.epoch()).collect(); - assert_eq!( - expected_epochs, actual_epochs, - "Output epochs of {:?} don't match the expectation.", - id - ); - assert_eq!( - node.outputs.len(), - expected[id].len(), - "Output length of {:?} doesn't match the expectation", - id - ); - assert!( - node.outputs - .iter() - .zip(expected.get(id).expect("node is not expected")) - .all(|(a, b)| a.public_eq(b)), - "Outputs of {:?} don't match the expectation", - id - ); - } - assert!( - self.observer - .outputs - .iter() - .zip(full_node.outputs.iter()) - .all(|(a, b)| a.public_eq(b)), - "Observer outputs don't match the expectation." - ); - } -} diff --git a/tests/queueing_honey_badger.rs b/tests/queueing_honey_badger.rs index 51a314d..adf88a0 100644 --- a/tests/queueing_honey_badger.rs +++ b/tests/queueing_honey_badger.rs @@ -1,56 +1,88 @@ #![deny(unused_must_use)] //! Network tests for Queueing Honey Badger. -mod network; +pub mod net; -use std::collections::{BTreeMap, BTreeSet}; -use std::iter; +use std::collections::BTreeSet; use std::sync::Arc; use log::info; +use proptest::{prelude::ProptestConfig, proptest, proptest_helper}; use rand::{Rng, SeedableRng}; -use rand_xorshift::XorShiftRng; use hbbft::dynamic_honey_badger::{DynamicHoneyBadger, JoinPlan}; use hbbft::queueing_honey_badger::{Change, ChangeState, Input, QueueingHoneyBadger}; use hbbft::sender_queue::{Message, SenderQueue, Step}; use hbbft::{util, NetworkInfo}; -use crate::network::{Adversary, MessageScheduler, NodeId, SilentAdversary, TestNetwork, TestNode}; +use crate::net::adversary::{Adversary, NodeOrderAdversary, ReorderingAdversary}; +use crate::net::proptest::{gen_seed, TestRng, TestRngSeed}; +use crate::net::{NetBuilder, NewNodeInfo, Node, VirtualNet}; +type NodeId = u16; type QHB = SenderQueue>>; +// Send the second half of the transactions to the specified node. +fn input_second_half( + net: &mut VirtualNet, + id: NodeId, + num_txs: usize, + mut rng: &mut TestRng, +) where + A: Adversary, +{ + for tx in (num_txs / 2)..num_txs { + let _ = net.send_input(id, Input::User(tx), &mut rng); + } +} + /// Proposes `num_txs` values and expects nodes to output and order them. -fn test_queueing_honey_badger(mut network: TestNetwork, num_txs: usize) +fn test_queueing_honey_badger(mut net: VirtualNet, num_txs: usize, mut rng: &mut TestRng) where A: Adversary, { - let netinfo = network.observer.instance().algo().netinfo().clone(); + let netinfo = net + .correct_nodes() + .nth(0) + .expect("At least one correct node needs to exist") + .algorithm() + .algo() + .netinfo() + .clone(); + + // Make two copies of all public keys. let pub_keys_add = netinfo.public_key_map().clone(); let mut pub_keys_rm = pub_keys_add.clone(); - pub_keys_rm.remove(&NodeId(0)); - network.input_all(Input::Change(Change::NodeChange(pub_keys_rm.clone()))); - // The second half of the transactions will be input only after a node has been removed and - // readded. + // Get the first correct node id as candidate for removal/re-adding. + let first_correct_node = *net.correct_nodes().nth(0).unwrap().id(); + + // Remove the first correct node, which is to be removed. + pub_keys_rm.remove(&first_correct_node); + + // Broadcast public keys of all nodes except for the node to be removed. + let _ = net.broadcast_input( + &Input::Change(Change::NodeChange(pub_keys_rm.clone())), + &mut rng, + ); + + // Broadcast the first half of the transactions. for tx in 0..(num_txs / 2) { - network.input_all(Input::User(tx)); + let _ = net.broadcast_input(&Input::User(tx), &mut rng); } - let input_second_half = |network: &mut TestNetwork<_, _>, id: NodeId| { - for tx in (num_txs / 2)..num_txs { - network.input(id, Input::User(tx)); - } - }; - - let has_remove = |node: &TestNode| { + // Closure for checking the output of a node for ChangeSet completion containing + // all nodes but the removed node. + let has_remove = |node: &Node| { node.outputs().iter().any(|batch| match batch.change() { ChangeState::Complete(Change::NodeChange(pub_keys)) => pub_keys == &pub_keys_rm, _ => false, }) }; - let has_add = |node: &TestNode| { + // Closure for checking the output of a node for ChangeSet completion containing + // all nodes, including the previously removed node. + let has_add = |node: &Node| { node.outputs().iter().any(|batch| match batch.change() { ChangeState::Complete(Change::NodeChange(pub_keys)) => pub_keys == &pub_keys_add, _ => false, @@ -58,119 +90,135 @@ where }; // Returns `true` if the node has not output all changes or transactions yet. - let node_busy = |node: &TestNode| { - !has_remove(node) || !has_add(node) || !node.instance().algo().queue().is_empty() + let node_busy = |node: &Node| { + !has_remove(node) || !has_add(node) || !node.algorithm().algo().queue().is_empty() }; - let mut awaiting_removal: BTreeSet<_> = network.nodes.iter().map(|(id, _)| *id).collect(); - let mut awaiting_addition: BTreeSet<_> = network - .nodes - .iter() - .map(|(id, _)| *id) - .filter(|id| *id != NodeId(0)) + // All nodes await removal. + let mut awaiting_removal: BTreeSet<_> = net.correct_nodes().map(|node| *node.id()).collect(); + + // All nodes but the removed node await addition. + let mut awaiting_addition: BTreeSet<_> = net + .correct_nodes() + .map(|node| *node.id()) + .filter(|id| *id != first_correct_node) .collect(); - // The set of nodes awaiting the second half of user transactions. + + // All, including the previously removed node, await the second half of transactions. let mut awaiting_second_half: BTreeSet<_> = awaiting_removal.clone(); - // Whether node 0 was rejoined as a validator. - let mut rejoined_node0 = false; - // The removed node 0 which is to be restarted as soon as all remaining validators agree to add - // it back. - let mut saved_node0: Option> = None; + // Whether the first correct node was rejoined as a validator. + let mut rejoined_first_correct = false; + // The removed first correct node which is to be restarted as soon as all remaining + // validators agreed to add it back. + let mut saved_first_correct: Option> = None; // Handle messages in random order until all nodes have output all transactions. - while network.nodes.values().any(node_busy) || !has_add(&network.observer) { - let stepped_id = network.step(); - if awaiting_removal.contains(&stepped_id) && has_remove(&network.nodes[&stepped_id]) { + while net.correct_nodes().any(node_busy) { + let stepped_id = net.crank_expect(&mut rng).0; + if awaiting_removal.contains(&stepped_id) && has_remove(&net.get(stepped_id).unwrap()) { awaiting_removal.remove(&stepped_id); info!( "{:?} has finished waiting for node removal; still waiting: {:?}", stepped_id, awaiting_removal ); + if awaiting_removal.is_empty() { - info!("Removing node 0 from the test network"); - saved_node0 = network.nodes.remove(&NodeId(0)); + info!("Removing first correct node from the test network"); + saved_first_correct = net.remove_node(&first_correct_node); } - // Vote to add node 0 back. - if stepped_id != NodeId(0) { - network.input( + // Vote to add the first correct node back. + if stepped_id != first_correct_node { + let _ = net.send_input( stepped_id, Input::Change(Change::NodeChange(pub_keys_add.clone())), + rng, ); info!( - "Input the vote to add node 0 into {:?} with netinfo {:?}", + "Input the vote to add the first correct node into {:?} with netinfo {:?}", stepped_id, - network.nodes[&stepped_id].instance().algo().netinfo() + net.get(stepped_id).unwrap().algorithm().algo().netinfo() ); } } + if awaiting_removal.is_empty() && awaiting_addition.contains(&stepped_id) { - // If the stepped node started voting to add node 0 back, take a note of that and rejoin - // node 0. - if let Some(join_plan) = network.nodes[&stepped_id] - .outputs() - .iter() - .find_map(|batch| match batch.change() { - ChangeState::InProgress(Change::NodeChange(pub_keys)) - if pub_keys == &pub_keys_add => - { - batch.join_plan() - } - _ => None, - }) + // If the stepped node started voting to add the first correct node back, + // take a note of that and rejoin it. + if let Some(join_plan) = + net.get(stepped_id) + .unwrap() + .outputs() + .iter() + .find_map(|batch| match batch.change() { + ChangeState::InProgress(Change::NodeChange(pub_keys)) + if pub_keys == &pub_keys_add => + { + batch.join_plan() + } + _ => None, + }) { awaiting_addition.remove(&stepped_id); info!( "{:?} has finished waiting for node addition; still waiting: {:?}", stepped_id, awaiting_addition ); - if awaiting_addition.is_empty() && !rejoined_node0 { - let node = saved_node0.take().expect("node 0 wasn't saved"); - let step = restart_node_for_add(&mut network, node, join_plan); - network.dispatch_messages(NodeId(0), step.messages); - rejoined_node0 = true; + if awaiting_addition.is_empty() && !rejoined_first_correct { + let node = saved_first_correct + .take() + .expect("first correct node wasn't saved"); + let step = restart_node_for_add(&mut net, node, join_plan, &mut rng); + net.process_step(first_correct_node, &step) + .expect("processing a step failed"); + rejoined_first_correct = true; } } } - if rejoined_node0 && awaiting_second_half.contains(&stepped_id) { + + if rejoined_first_correct && awaiting_second_half.contains(&stepped_id) { // Input the second half of user transactions into the stepped node. - input_second_half(&mut network, stepped_id); + input_second_half(&mut net, stepped_id, num_txs, &mut rng); awaiting_second_half.remove(&stepped_id); } } - let node1 = network.nodes.get(&NodeId(1)).expect("node 1 is missing"); - network.verify_batches(&node1); + let node_1 = net + .correct_nodes() + .nth(1) + .expect("second correct node is missing"); + net.verify_batches(node_1); } -/// Restarts a stopped and removed node with a given join plan and adds the node back on the test -/// network. -fn restart_node_for_add( - network: &mut TestNetwork, - mut node: TestNode, +/// Restarts specified node on the test network for adding it back as a validator. +fn restart_node_for_add( + net: &mut VirtualNet, + mut node: Node, join_plan: JoinPlan, + mut rng: &mut R, ) -> Step>> where + R: rand::Rng, A: Adversary, { - let our_id = node.id; - info!("Restarting {:?} with {:?}", our_id, join_plan); - let observer = network.observer.id; - let peer_ids: Vec = network - .nodes - .keys() + let our_id = *node.id(); + println!("Restarting node {} with {:?}", node.id(), join_plan); + + // TODO: When an observer node is added to the network, it should also be added to peer_ids. + let peer_ids: Vec<_> = net + .nodes() + .map(|node| node.id()) + .filter(|id| *id != node.id()) .cloned() - .filter(|id| *id != our_id) - .chain(iter::once(observer)) .collect(); - let secret_key = node.instance().algo().netinfo().secret_key().clone(); - let mut rng = XorShiftRng::from_seed(rand::thread_rng().gen::<[u8; 16]>()); + + let secret_key = node.algorithm().algo().netinfo().secret_key().clone(); let (qhb, qhb_step) = QueueingHoneyBadger::builder_joining(our_id, secret_key, join_plan, &mut rng) .and_then(|builder| builder.batch_size(3).build(&mut rng)) .expect("failed to rebuild the node with a join plan"); let (sq, mut sq_step) = SenderQueue::builder(qhb, peer_ids.into_iter()).build(our_id); - *node.instance_mut() = sq; + *node.algorithm_mut() = sq; sq_step.extend(qhb_step.map(|output| output, |fault| fault, Message::from)); - network.nodes.insert(our_id, node); + net.insert_node(node); sq_step } @@ -178,16 +226,12 @@ where #[allow(clippy::needless_pass_by_value)] fn new_queueing_hb( netinfo: Arc>, + seed: TestRngSeed, ) -> (QHB, Step>>) { - let observer = NodeId(netinfo.num_nodes()); + let mut rng: TestRng = TestRng::from_seed(seed); let our_id = *netinfo.our_id(); - let peer_ids = netinfo - .all_ids() - .filter(|&&them| them != our_id) - .cloned() - .chain(iter::once(observer)); + let peer_ids = netinfo.all_ids().filter(|&&them| them != our_id).cloned(); let dhb = DynamicHoneyBadger::builder().build((*netinfo).clone()); - let mut rng = XorShiftRng::from_seed(rand::thread_rng().gen::<[u8; 16]>()); let (qhb, qhb_step) = QueueingHoneyBadger::builder(dhb) .batch_size(3) .build(&mut rng) @@ -198,15 +242,19 @@ fn new_queueing_hb( (sq, step) } -fn test_queueing_honey_badger_different_sizes(new_adversary: F, num_txs: usize) -where +fn test_queueing_honey_badger_different_sizes( + new_adversary: F, + num_txs: usize, + seed: TestRngSeed, +) where A: Adversary, - F: Fn(usize, usize, BTreeMap>>) -> A, + F: Fn() -> A, { // This returns an error in all but the first test. let _ = env_logger::try_init(); - let mut rng = rand::thread_rng(); + let mut rng: TestRng = TestRng::from_seed(seed); + let sizes = vec![3, 5, rng.gen_range(6, 10)]; for size in sizes { // The test is removing one correct node, so we allow fewer faulty ones. @@ -216,21 +264,49 @@ where "Network size: {} good nodes, {} faulty nodes", num_good_nodes, num_adv_nodes ); - let adversary = |adv_nodes| new_adversary(num_good_nodes, num_adv_nodes, adv_nodes); - let network = - TestNetwork::new_with_step(num_good_nodes, num_adv_nodes, adversary, new_queueing_hb); - test_queueing_honey_badger(network, num_txs); + + let (net, _) = NetBuilder::new(0..size as u16) + .num_faulty(num_adv_nodes) + .message_limit(20_000 * size) + .no_time_limit() + .adversary(new_adversary()) + .using_step(move |node_info: NewNodeInfo<_>| { + // Note: The "seed" variable is implicitly copied by the move closure. + // The "Copy" trait is *not* implemented for TestRng, which additionally + // needs to be mutable, while we are in a function which captures immutably. + // To avoid convoluted clone/borrow constructs we pass a TestRngSeed + // rather than a TestRng instance. + new_queueing_hb(Arc::new(node_info.netinfo), seed) + }) + .build(&mut rng) + .expect("Could not construct test network."); + + test_queueing_honey_badger(net, num_txs, &mut rng); } } -#[test] -fn test_queueing_honey_badger_random_delivery_silent() { - let new_adversary = |_: usize, _: usize, _| SilentAdversary::new(MessageScheduler::Random); - test_queueing_honey_badger_different_sizes(new_adversary, 30); +proptest! { + #![proptest_config(ProptestConfig { + cases: 1, .. ProptestConfig::default() + })] + + #[test] + #[allow(clippy::unnecessary_operation)] + fn test_queueing_honey_badger_random_delivery_silent(seed in gen_seed()) { + do_test_queueing_honey_badger_random_delivery_silent(seed) + } + + #[test] + #[allow(clippy::unnecessary_operation)] + fn test_queueing_honey_badger_first_delivery_silent(seed in gen_seed()) { + do_test_queueing_honey_badger_first_delivery_silent(seed) + } } -#[test] -fn test_queueing_honey_badger_first_delivery_silent() { - let new_adversary = |_: usize, _: usize, _| SilentAdversary::new(MessageScheduler::First); - test_queueing_honey_badger_different_sizes(new_adversary, 30); +fn do_test_queueing_honey_badger_random_delivery_silent(seed: TestRngSeed) { + test_queueing_honey_badger_different_sizes(ReorderingAdversary::new, 30, seed); +} + +fn do_test_queueing_honey_badger_first_delivery_silent(seed: TestRngSeed) { + test_queueing_honey_badger_different_sizes(NodeOrderAdversary::new, 30, seed); }