Extract test framework into hbbft_testing crate. (#404)

* Extract test framework into hbbft_testing crate.

Also update Rust and some dependencies.

* Rename DHB tests.

They are only called `net_dynamic_hb` because we used to have two test
frameworks and two DHB test modules.

.travis.yml

@@ -1,6 +1,6 @@
 language: rust
 rust:
-    - 1.33.0
+    - 1.34.0
 cache:
   cargo: true
   timeout: 1200

Cargo.toml

@@ -38,14 +38,13 @@ tiny-keccak = "1.4"
 
 [dev-dependencies]
 colored = "1.7"
-crossbeam = "0.6"
+crossbeam = "0.7.1"
 crossbeam-channel = "0.3"
 docopt = "1.0"
+hbbft_testing = { path = "hbbft_testing" }
 itertools = "0.8.0"
-rand_xorshift = "0.1.1"
-signifix = "0.9"
-proptest = "0.8.7"
-integer-sqrt = "0.1.2"
+signifix = "0.10.0"
+proptest = "0.9.2"
 
 [[example]]
 name = "consensus-node"

ci.sh

@@ -18,3 +18,10 @@ cargo test --features=use-insecure-test-only-mock-crypto --release
 cargo doc
 cargo deadlinks --dir target/doc/hbbft/
 cargo audit
+
+cd hbbft_testing
+cargo clippy --all-targets -- --deny clippy::all
+cargo fmt -- --check
+cargo test --release
+cargo audit
+cd ..

examples/simulation.rs

@@ -1,4 +1,5 @@
 use std::collections::{BTreeMap, VecDeque};
+use std::convert::TryFrom;
 use std::time::{Duration, Instant};
 use std::{cmp, u64};
 
@@ -8,7 +9,7 @@ use itertools::Itertools;
 use rand::{distributions::Standard, rngs::OsRng, seq::SliceRandom, Rng};
 use rand_derive::Rand;
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
-use signifix::{metric, TryFrom};
+use signifix::metric;
 
 use hbbft::dynamic_honey_badger::DynamicHoneyBadger;
 use hbbft::queueing_honey_badger::{Batch, QueueingHoneyBadger};

hbbft_testing/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "hbbft_testing"
+version = "0.1.0"
+authors = [
+    "Vladimir Komendantskiy <komendantsky@gmail.com>",
+    "Andreas Fackler <AndreasFackler@gmx.de>",
+    "Peter van Nostrand <jnz@riseup.net>",
+    "Andrew Gross <andogro@gmail.com>",
+    "Nick Sanders <nsan1129@gmail.com>",
+    "Marc Brinkmann <git@marcbrinkmann.de>",
+    "David Forstenlechner <dforsten@gmail.com>",
+]
+categories = ["algorithms", "asynchronous", "cryptography", "network-programming"]
+keywords = ["consensus", "asynchronous", "threshold"]
+license = "MIT/Apache-2.0"
+readme = "README.md"
+repository = "https://github.com/poanetwork/hbbft"
+description = "Utilities for simulating and testing hbbft-based networks."
+edition = "2018"
+
+[badges]
+travis-ci = { repository = "poanetwork/hbbft" }
+
+[dependencies]
+failure = "0.1.5"
+hbbft = { path = ".." }
+integer-sqrt = "0.1.2"
+proptest = "0.9.2"
+rand = "0.6.5"
+rand_xorshift = "0.1.1"
+threshold_crypto = "0.3.1"

hbbft_testing/src/adversary.rs

@@ -40,8 +40,8 @@ use rand::Rng;
 
 use hbbft::{ConsensusProtocol, CpStep};
 
-use crate::net::util::randomly;
-use crate::net::{CrankError, NetMessage, NetworkMessage, Node, VirtualNet};
+use crate::util::randomly;
+use crate::{CrankError, NetMessage, NetworkMessage, Node, VirtualNet};
 
 /// Immutable network handle.
 ///

hbbft_testing/src/lib.rs

@@ -19,6 +19,9 @@ pub mod err;
 pub mod proptest;
 pub mod util;
 
+#[cfg(test)]
+mod util_tests;
+
 use std::collections::{BTreeMap, BTreeSet, VecDeque};
 use std::io::Write;
 use std::{cmp, env, fmt, fs, io, ops, process, time};
@@ -29,8 +32,6 @@ use hbbft::dynamic_honey_badger::Batch;
 use hbbft::sender_queue::SenderQueueableOutput;
 use hbbft::{self, ConsensusProtocol, Contribution, CpStep, Fault, NetworkInfo, NodeIdT, Step};
 
-use crate::try_some;
-
 pub use self::adversary::Adversary;
 pub use self::err::CrankError;
 
@@ -534,7 +535,7 @@ where
 
         // If the trace setting is not overriden, we use the setting from the environment.
         let trace = self.trace.unwrap_or_else(|| {
-            match env::var("HBBFT_TEST_TRACE").as_ref().map(|s| s.as_str()) {
+            match env::var("HBBFT_TEST_TRACE").as_ref().map(String::as_str) {
                 Ok("true") | Ok("1") => true,
                 _ => false,
             }
@@ -1070,8 +1071,7 @@ where
         }
         for node in self.correct_nodes().filter(|n| n.id() != full_node.id()) {
             let id = node.id();
-            let actual_epochs: BTreeSet<_> =
-                node.outputs.iter().map(|batch| batch.epoch()).collect();
+            let actual_epochs: BTreeSet<_> = node.outputs.iter().map(Batch::epoch).collect();
             let expected_epochs: BTreeSet<_> =
                 expected[id].iter().map(|batch| batch.epoch()).collect();
             assert_eq!(

hbbft_testing/src/util_tests.rs

@@ -1,12 +1,10 @@
-pub mod net;
-
 use proptest::arbitrary::any;
+use proptest::proptest;
 use proptest::strategy::{Strategy, ValueTree};
-use proptest::{proptest, proptest_helper};
 use rand::SeedableRng;
 use rand_xorshift::XorShiftRng;
 
-use crate::net::proptest::{max_sum, NetworkDimension, NetworkDimensionTree};
+use crate::proptest::{max_sum, NetworkDimension, NetworkDimensionTree};
 
 proptest! {
     /// Ensures all generated network dimensions are actually sane.

src/dynamic_honey_badger/dynamic_honey_badger.rs

@@ -294,7 +294,9 @@ where
         let kgs = match self.key_gen_state {
             Some(ref mut kgs) => kgs,
             None => {
-                return Ok(Fault::new(sender_id.clone(), FaultKind::UnexpectedKeyGenMessage).into());
+                return Ok(
+                    Fault::new(sender_id.clone(), FaultKind::UnexpectedKeyGenMessage).into(),
+                );
             }
         };
 

tests/binary_agreement.rs

@@ -13,22 +13,18 @@
 //! - Validity: If any correct node outputs `b`, then at least one correct node received `b` as
 //! input.
 
-pub mod net;
-
 use std::iter::once;
 use std::sync::Arc;
 use std::time;
 
-use proptest::arbitrary::any;
-use proptest::{prelude::ProptestConfig, prop_compose, proptest, proptest_helper};
-use rand::{Rng, SeedableRng};
-
 use hbbft::binary_agreement::BinaryAgreement;
 use hbbft::ConsensusProtocol;
-
-use crate::net::adversary::{Adversary, ReorderingAdversary};
-use crate::net::proptest::{gen_seed, NetworkDimension, TestRng, TestRngSeed};
-use crate::net::{NetBuilder, NewNodeInfo, VirtualNet};
+use hbbft_testing::adversary::{Adversary, ReorderingAdversary};
+use hbbft_testing::proptest::{gen_seed, NetworkDimension, TestRng, TestRngSeed};
+use hbbft_testing::{NetBuilder, NewNodeInfo, VirtualNet};
+use proptest::arbitrary::any;
+use proptest::{prelude::ProptestConfig, prop_compose, proptest};
+use rand::{Rng, SeedableRng};
 
 /// Test configuration for Binary Agreement tests.
 #[derive(Debug)]
@@ -72,36 +68,35 @@ proptest! {
 
 type NodeId = u16;
 
-impl<A> VirtualNet<BinaryAgreement<NodeId, u8>, A>
-where
+fn test_binary_agreement<A, R>(
+    net: &mut VirtualNet<BinaryAgreement<NodeId, u8>, A>,
+    input: Option<bool>,
+    mut rng: R,
+) where
+    R: Rng + 'static,
     A: Adversary<BinaryAgreement<NodeId, u8>>,
 {
-    fn test_binary_agreement<R>(&mut self, input: Option<bool>, mut rng: R)
-    where
-        R: Rng + 'static,
-    {
-        let ids: Vec<NodeId> = self.nodes().map(|n| *n.id()).collect();
-        for id in ids {
-            let _ = self.send_input(id, input.unwrap_or_else(|| rng.gen::<bool>()), &mut rng);
-        }
-
-        // Handle messages in random order until all nodes have output the proposed value.
-        while !self.nodes().all(|node| node.algorithm().terminated()) {
-            let _ = self.crank_expect(&mut rng);
-        }
-        // Verify that all instances output the same value.
-        let mut expected = input;
-        for node in self.nodes() {
-            if let Some(b) = expected {
-                assert!(once(&b).eq(node.outputs()));
-            } else {
-                assert_eq!(1, node.outputs().len());
-                expected = Some(node.outputs()[0]);
-            }
-        }
-        // TODO: As soon as observers are added to the test framework, compare the expected output
-        // against the output of observers.
+    let ids: Vec<NodeId> = net.nodes().map(|n| *n.id()).collect();
+    for id in ids {
+        let _ = net.send_input(id, input.unwrap_or_else(|| rng.gen::<bool>()), &mut rng);
     }
+
+    // Handle messages in random order until all nodes have output the proposed value.
+    while !net.nodes().all(|node| node.algorithm().terminated()) {
+        let _ = net.crank_expect(&mut rng);
+    }
+    // Verify that all instances output the same value.
+    let mut expected = input;
+    for node in net.nodes() {
+        if let Some(b) = expected {
+            assert!(once(&b).eq(node.outputs()));
+        } else {
+            assert_eq!(1, node.outputs().len());
+            expected = Some(node.outputs()[0]);
+        }
+    }
+    // TODO: As soon as observers are added to the test framework, compare the expected output
+    // against the output of observers.
 }
 
 /// Tests Binary Agreement on a given configuration.
@@ -127,7 +122,11 @@ fn binary_agreement(cfg: TestConfig) {
         })
         .build(&mut rng)
         .expect("Could not construct test network.");
-    net.test_binary_agreement(cfg.input, TestRng::from_seed(rng.gen::<TestRngSeed>()));
+    test_binary_agreement(
+        &mut net,
+        cfg.input,
+        TestRng::from_seed(rng.gen::<TestRngSeed>()),
+    );
     println!(
         "Test success: {} good nodes and {} faulty nodes, input: {:?}",
         num_good_nodes, num_faulty_nodes, cfg.input

tests/binary_agreement_mitm.rs

@@ -1,22 +1,19 @@
 #![deny(unused_must_use)]
 //! Tests the BinaryAgreement protocol with a MTIM adversary.
 
-pub mod net;
-
 use std::iter;
 use std::sync::{Arc, Mutex};
 
 use hbbft::binary_agreement::{BinaryAgreement, MessageContent, SbvMessage};
 use hbbft::threshold_sign::ThresholdSign;
 use hbbft::{ConsensusProtocol, CpStep, NetworkInfo};
-use proptest::{proptest, proptest_helper};
+use hbbft_testing::adversary::{NetMutHandle, QueuePosition};
+use hbbft_testing::err::CrankError;
+use hbbft_testing::proptest::{gen_seed, TestRng, TestRngSeed};
+use hbbft_testing::{Adversary, NetBuilder, NetMessage};
+use proptest::proptest;
 use rand::{Rng, SeedableRng};
 
-use crate::net::adversary::{NetMutHandle, QueuePosition};
-use crate::net::err::CrankError;
-use crate::net::proptest::{gen_seed, TestRng, TestRngSeed};
-use crate::net::{Adversary, NetBuilder, NetMessage};
-
 type NodeId = usize;
 type SessionId = u8;
 type Algo = BinaryAgreement<NodeId, SessionId>;

tests/broadcast.rs

@@ -1,21 +1,17 @@
-pub mod net;
-
 use std::collections::BTreeMap;
 use std::iter::once;
 use std::sync::{Arc, Mutex};
 
-use log::info;
-use proptest::{prelude::ProptestConfig, proptest, proptest_helper};
-use rand::{Rng, SeedableRng};
-
 use hbbft::{broadcast::Broadcast, util, ConsensusProtocol, CpStep, NetworkInfo};
-
-use crate::net::adversary::{
+use hbbft_testing::adversary::{
     sort_ascending, swap_random, Adversary, NetMutHandle, NodeOrderAdversary, RandomAdversary,
     ReorderingAdversary,
 };
-use crate::net::proptest::{gen_seed, TestRng, TestRngSeed};
-use crate::net::{CrankError, NetBuilder, NetMessage, NewNodeInfo, VirtualNet};
+use hbbft_testing::proptest::{gen_seed, TestRng, TestRngSeed};
+use hbbft_testing::{CrankError, NetBuilder, NetMessage, NewNodeInfo, VirtualNet};
+use log::info;
+use proptest::{prelude::ProptestConfig, proptest};
+use rand::{Rng, SeedableRng};
 
 type NodeId = u16;
 type NetworkInfoMap = BTreeMap<NodeId, Arc<NetworkInfo<NodeId>>>;

tests/dynamic_honey_badger.rs

@@ -1,18 +1,16 @@
-pub mod net;
-
 use std::collections::{BTreeMap, BTreeSet};
 use std::time;
 
-use hbbft::dynamic_honey_badger::{Change, ChangeState, DynamicHoneyBadger, Input, JoinPlan};
+use hbbft::dynamic_honey_badger::{
+    Batch, Change, ChangeState, DynamicHoneyBadger, Input, JoinPlan,
+};
 use hbbft::sender_queue::{SenderQueue, Step};
-use hbbft::Epoched;
-use proptest::{prelude::ProptestConfig, prop_compose, proptest, proptest_helper};
+use hbbft::{util, Epoched};
+use hbbft_testing::adversary::{Adversary, ReorderingAdversary};
+use hbbft_testing::proptest::{gen_seed, NetworkDimension, TestRng, TestRngSeed};
+use hbbft_testing::{NetBuilder, NewNodeInfo, Node, VirtualNet};
+use proptest::{prelude::ProptestConfig, prop_compose, proptest};
 use rand::{seq::SliceRandom, SeedableRng};
-
-use crate::net::adversary::{Adversary, ReorderingAdversary};
-use crate::net::proptest::{gen_seed, NetworkDimension, TestRng, TestRngSeed};
-use crate::net::{NetBuilder, NewNodeInfo, Node, VirtualNet};
-use hbbft::util;
 use threshold_crypto::PublicKey;
 
 type DHB = SenderQueue<DynamicHoneyBadger<Vec<usize>, usize>>;
@@ -218,7 +216,7 @@ fn do_drop_and_re_add(cfg: TestConfig) {
                 );
             }
         }
-        for change in step.output.iter().map(|output| output.change()) {
+        for change in step.output.iter().map(Batch::change) {
             match change {
                 ChangeState::Complete(Change::NodeChange(ref pub_keys))
                     if *pub_keys == new_pub_keys =>
@@ -389,7 +387,7 @@ fn do_drop_and_re_add(cfg: TestConfig) {
         }
 
         // Check if we are done.
-        if expected_outputs.values().all(|s| s.is_empty())
+        if expected_outputs.values().all(BTreeSet::is_empty)
             && awaiting_apply_old_subset.is_empty()
             && awaiting_apply_new_subset.is_empty()
         {
@@ -443,7 +441,7 @@ where
     // TODO: When an observer node is added to the network, it should also be added to peer_ids.
     let peer_ids: Vec<_> = net
         .nodes()
-        .map(|node| node.id())
+        .map(Node::id)
         .filter(|id| *id != node.id())
         .cloned()
         .collect();

tests/honey_badger.rs

@@ -1,27 +1,23 @@
 #![deny(unused_must_use)]
 //! Network tests for Honey Badger.
 
-pub mod net;
-
 use std::collections::BTreeMap;
 use std::sync::{Arc, Mutex};
 
-use itertools::Itertools;
-use log::info;
-use proptest::{prelude::ProptestConfig, proptest, proptest_helper};
-use rand::{seq::SliceRandom, Rng, SeedableRng};
-
 use hbbft::honey_badger::{Batch, EncryptionSchedule, HoneyBadger, MessageContent};
 use hbbft::sender_queue::{self, SenderQueue, Step};
 use hbbft::transaction_queue::TransactionQueue;
 use hbbft::{threshold_decrypt, util, CpStep, NetworkInfo, Target};
-
-use crate::net::adversary::{
+use hbbft_testing::adversary::{
     sort_by_random_node, Adversary, NetMutHandle, NodeOrderAdversary, RandomAdversary,
     ReorderingAdversary,
 };
-use crate::net::proptest::{gen_seed, TestRng, TestRngSeed};
-use crate::net::{CrankError, NetBuilder, NetMessage, NewNodeInfo, Node, VirtualNet};
+use hbbft_testing::proptest::{gen_seed, TestRng, TestRngSeed};
+use hbbft_testing::{CrankError, NetBuilder, NetMessage, NewNodeInfo, Node, VirtualNet};
+use itertools::Itertools;
+use log::info;
+use proptest::{prelude::ProptestConfig, proptest};
+use rand::{seq::SliceRandom, Rng, SeedableRng};
 
 type NodeId = u16;
 type NetworkInfoMap = BTreeMap<NodeId, Arc<NetworkInfo<NodeId>>>;

tests/net_util.proptest-regressions

@@ -1,7 +0,0 @@
-# Seeds for failure cases proptest has generated in the past. It is
-# automatically read and these particular cases re-run before any
-# novel cases are generated.
-#
-# It is recommended to check this file in to source control so that
-# everyone who runs the test benefits from these saved cases.
-xs 2984819682 1621205622 3953858650 3756562836 # shrinks to seed = [724246048, 1190443809, 4113437293, 1855000933], ops = [Simplify, Complicate]

tests/queueing_honey_badger.rs

@@ -1,23 +1,19 @@
 #![deny(unused_must_use)]
 //! Network tests for Queueing Honey Badger.
 
-pub mod net;
-
 use std::collections::BTreeSet;
 use std::sync::Arc;
 
-use log::info;
-use proptest::{prelude::ProptestConfig, proptest, proptest_helper};
-use rand::{Rng, SeedableRng};
-
 use hbbft::dynamic_honey_badger::{DynamicHoneyBadger, JoinPlan};
 use hbbft::queueing_honey_badger::{Change, ChangeState, Input, QueueingHoneyBadger};
 use hbbft::sender_queue::{Message, SenderQueue, Step};
 use hbbft::{util, NetworkInfo};
-
-use crate::net::adversary::{Adversary, NodeOrderAdversary, ReorderingAdversary};
-use crate::net::proptest::{gen_seed, TestRng, TestRngSeed};
-use crate::net::{NetBuilder, NewNodeInfo, Node, VirtualNet};
+use hbbft_testing::adversary::{Adversary, NodeOrderAdversary, ReorderingAdversary};
+use hbbft_testing::proptest::{gen_seed, TestRng, TestRngSeed};
+use hbbft_testing::{NetBuilder, NewNodeInfo, Node, VirtualNet};
+use log::info;
+use proptest::{prelude::ProptestConfig, proptest};
+use rand::{Rng, SeedableRng};
 
 type NodeId = u16;
 type QHB = SenderQueue<QueueingHoneyBadger<usize, NodeId, Vec<usize>>>;
@@ -205,7 +201,7 @@ where
     // TODO: When an observer node is added to the network, it should also be added to peer_ids.
     let peer_ids: Vec<_> = net
         .nodes()
-        .map(|node| node.id())
+        .map(Node::id)
         .filter(|id| *id != node.id())
         .cloned()
         .collect();

tests/subset.rs

@@ -1,19 +1,16 @@
 #![deny(unused_must_use)]
-pub mod net;
 
 use std::collections::{BTreeMap, BTreeSet};
 use std::iter::once;
 use std::sync::Arc;
 
-use proptest::{prelude::ProptestConfig, proptest, proptest_helper};
-use rand::SeedableRng;
-
 use hbbft::subset::{Subset, SubsetOutput};
 use hbbft::ConsensusProtocol;
-
-use crate::net::adversary::{Adversary, NodeOrderAdversary, ReorderingAdversary};
-use crate::net::proptest::{gen_seed, TestRng, TestRngSeed};
-use crate::net::{NetBuilder, NewNodeInfo, VirtualNet};
+use hbbft_testing::adversary::{Adversary, NodeOrderAdversary, ReorderingAdversary};
+use hbbft_testing::proptest::{gen_seed, TestRng, TestRngSeed};
+use hbbft_testing::{NetBuilder, NewNodeInfo, VirtualNet};
+use proptest::{prelude::ProptestConfig, proptest};
+use rand::SeedableRng;
 
 type NodeId = u16;
 type ProposedValue = Vec<u8>;

tests/threshold_sign.rs

@@ -1,19 +1,15 @@
 #![deny(unused_must_use)]
 //! Non-deterministic tests for the ThresholdSign protocol
 
-pub mod net;
-
 use std::sync::Arc;
 
-use log::info;
-use proptest::{prelude::ProptestConfig, proptest, proptest_helper};
-use rand::{Rng, SeedableRng};
-
 use hbbft::{crypto::Signature, threshold_sign::ThresholdSign, util, ConsensusProtocol};
-
-use crate::net::adversary::{Adversary, NodeOrderAdversary, ReorderingAdversary};
-use crate::net::proptest::{gen_seed, TestRng, TestRngSeed};
-use crate::net::{NetBuilder, NewNodeInfo, VirtualNet};
+use hbbft_testing::adversary::{Adversary, NodeOrderAdversary, ReorderingAdversary};
+use hbbft_testing::proptest::{gen_seed, TestRng, TestRngSeed};
+use hbbft_testing::{NetBuilder, NewNodeInfo, VirtualNet};
+use log::info;
+use proptest::{prelude::ProptestConfig, proptest};
+use rand::{Rng, SeedableRng};
 
 type NodeId = u16;