Network dimension strategy for property based tests. (#225)

* Added missing debug implementations for various test networking parts.

* Initial proptest parts added, including network dimensions.

* Use a test configuration for proptests.

* Tweaks and documentation.

* Improved documentation for `net_dynamic_hb` test.

* Added two missing `;`.

* Allow insane topology creation.

* Renamed `is_sane()` to `is_bft`, `halfway` to `average_higher` and improved comments per suggestions.

* Rename `NetworkTopology` -> `NetworkDimension`.

* Silence newly added clippy warning.

* Smoothed `README.md`.

* Remove workaround for beta/nightly again.

* Caved in to clippy and changed the bft condition.

Cargo.toml

@@ -40,6 +40,7 @@ crossbeam-channel = "0.1"
 docopt = "1.0"
 serde_derive = "1.0.55"
 signifix = "0.9"
+proptest = "0.8.6"
 
 [[example]]
 name = "consensus-node"

tests/README.md

@@ -122,3 +122,71 @@ assert!(net.nodes().all(|node| node.outputs() == first));
 
 println!("End result: {:?}", first);
 ```
+
+### Property based testing
+
+Many higher-level tests allow for a variety of different input parameters like the number of nodes in a network or the amount of faulty ones among them. Other possible parameters include transaction, batch or contribution sizes. To test a variety of randomized combinations of these, the [proptest](https://docs.rs/proptest) crate should be used.
+
+The first step in using `proptest` is parametrizing a test, ensuring that all parameters are passed in and not hardcoded. The resulting function should be wrapped, due to the fact that `rustfmt` will not reformat code inside most macros:
+
+```rust
+proptest! {
+  #[test]
+  fn basic_operations(num_nodes in 3..10u32, num_tx in 40..60u32) {
+      do_basic_operations(num_nodes, num_txs);
+  }
+}
+
+fn do_basic_operations(num_nodes: u32, num_txs: u32) {
+    // ...
+}
+```
+
+Some helper structures and functions are available, e.g. the number of nodes should rarely be specified using a range, but with the `NetworkDimension` strategy instead:
+
+```rust
+use net::NetBuilder;
+use net::proptest::NetworkDimension;
+
+proptest! {
+    #[test]
+    fn basic_operations(dimension in NetworkDimension::range(3, 10), num_txs in 40..60u32) {
+        do_basic_operations(dimension, num_txs)
+    }
+}
+
+fn do_basic_operations(dimension: NetworkDimension, num_txs: u32) {
+    let mut net = NetBuilder::new(0..cfg.dimension.size)
+        .num_faulty(cfg.dimension.faulty)
+        // ...
+}
+```
+
+When specified this way, `dimension` will always be generated with a random valid number of faulty nodes, which is limited by the total amount of nodes. Additionally, `proptest` will automatically try to shrink the solution to a minimum if an error is found. The `NetworkDimension` is reduced in a way that tries to find a minimal combination of size and faulty nodes quicker than independently modified node counts would.
+
+To cut down on the number of parameters passed to each function, a struct containing all parameters for a single test can be added for larger parameter sets:
+
+```rust
+prop_compose! {
+    /// Strategy to generate a test configuration.
+    fn arb_config()
+                 (dimension in NetworkDimension::range(3, 15),
+                  total_txs in 20..60usize,
+                  batch_size in 10..20usize,
+                  contribution_size in 1..10usize)
+                 -> TestConfig {
+        TestConfig{
+            dimension, total_txs, batch_size, contribution_size,
+        }
+    }
+}
+
+proptest!{
+    #[test]
+    fn drop_and_readd(cfg in arb_config()) {
+        do_drop_and_readd(cfg)
+    }
+
+    // ...
+}
+```

tests/net/adversary.rs

@@ -41,6 +41,7 @@ use net::{CrankError, NetMessage, Node, VirtualNet};
 /// Immutable network handle.
 ///
 /// Allows querying public information of the network or getting immutable handles to any node.
+#[derive(Debug)]
 pub struct NetHandle<'a, D: 'a>(&'a VirtualNet<D>)
 where
     D: DistAlgorithm;
@@ -100,6 +101,7 @@ pub enum QueuePosition {
 ///
 /// Allows reordering of messages, injecting new ones into the network queue and getting mutable
 /// handles to nodes.
+#[derive(Debug)]
 pub struct NetMutHandle<'a, D: 'a>(&'a mut VirtualNet<D>)
 where
     D: DistAlgorithm;
@@ -210,6 +212,7 @@ where
 }
 
 /// Immutable node handle.
+#[derive(Debug)]
 pub struct NodeHandle<'a, D: 'a>(&'a Node<D>)
 where
     D: DistAlgorithm;
@@ -253,6 +256,7 @@ where
 }
 
 /// Mutable node handle.
+#[derive(Debug)]
 pub struct NodeMutHandle<'a, D: 'a>(&'a mut Node<D>)
 where
     D: DistAlgorithm;

tests/net/mod.rs

@@ -14,11 +14,12 @@
 
 pub mod adversary;
 pub mod err;
+pub mod proptest;
 #[macro_use]
 pub mod util;
 
 use std::io::Write;
-use std::{cmp, collections, env, fs, io, ops, process};
+use std::{cmp, collections, env, fmt, fs, io, ops, process};
 
 use rand;
 use rand::Rand;
@@ -59,7 +60,6 @@ fn open_trace() -> Result<io::BufWriter<fs::File>, io::Error> {
 }
 
 /// A node in the test network.
-#[derive(Debug)]
 pub struct Node<D: DistAlgorithm> {
     /// Algorithm instance of node.
     algorithm: D,
@@ -69,6 +69,19 @@ pub struct Node<D: DistAlgorithm> {
     outputs: Vec<D::Output>,
 }
 
+impl<D> fmt::Debug for Node<D>
+where
+    D: DistAlgorithm,
+{
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.debug_struct("Node")
+            .field("algorithm", &"yes")
+            .field("is_faulty", &self.is_faulty)
+            .field("outputs", &self.outputs.len())
+            .finish()
+    }
+}
+
 impl<D: DistAlgorithm> Node<D> {
     /// Create a new node.
     #[inline]
@@ -258,6 +271,23 @@ where
     message_limit: Option<usize>,
 }
 
+impl<D, I> fmt::Debug for NetBuilder<D, I>
+where
+    D: DistAlgorithm,
+{
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.debug_struct("NetBuilder")
+            .field("node_ids", &())
+            .field("num_faulty", &self.num_faulty)
+            .field("cons", &self.cons.is_some())
+            .field("adversary", &self.cons.is_some())
+            .field("trace", &self.trace)
+            .field("crank_limit", &self.crank_limit)
+            .field("message_limit", &self.message_limit)
+            .finish()
+    }
+}
+
 impl<D, I> NetBuilder<D, I>
 where
     D: DistAlgorithm,
@@ -428,6 +458,24 @@ where
     message_limit: Option<usize>,
 }
 
+impl<D> fmt::Debug for VirtualNet<D>
+where
+    D: DistAlgorithm,
+{
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.debug_struct("VirtualNet")
+            .field("nodes", &self.nodes.len())
+            .field("messages", &self.messages)
+            .field("adversary", &self.adversary.is_some())
+            .field("trace", &self.trace.is_some())
+            .field("crank_count", &self.crank_count)
+            .field("crank_limit", &self.crank_limit)
+            .field("message_count", &self.message_count)
+            .field("message_limit", &self.message_limit)
+            .finish()
+    }
+}
+
 /// A virtual network
 ///
 /// Virtual networks host a number of nodes that are marked either correct or faulty. Each time a

tests/net/proptest.rs

@@ -0,0 +1,184 @@
+//! Proptest helpers and strategies.
+//!
+//! This module houses strategies to generate (and reduce/expand) various `hbbft` and `net` related
+//! structures.
+
+use proptest::prelude::Rng;
+use proptest::strategy::{Strategy, ValueTree};
+use proptest::test_runner::{Reason, TestRunner};
+
+/// Node network dimension.
+///
+/// A `NetworkDimension` describes the number of correct and faulty nodes in a network. It can also
+/// be checked, "averaged" (using the `average_higher` function) and generated using
+/// `NetworkDimensionTree`.
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+pub struct NetworkDimension {
+    /// Total number of nodes in network.
+    pub size: usize,
+    /// Number of faulty nodes in a network.
+    pub faulty: usize,
+}
+
+impl NetworkDimension {
+    /// Creates a new `NetworkDimension` with the supplied parameters.
+    ///
+    /// Dimensions that do not satisfy BFT conditions (see `is_bft`) can be created using this
+    /// function.
+    pub fn new(size: usize, faulty: usize) -> Self {
+        NetworkDimension { size, faulty }
+    }
+
+    /// Checks whether the network dimension satisfies the `3 * faulty + 1 <= size` condition.
+    pub fn is_bft(&self) -> bool {
+        self.faulty * 3 < self.size
+    }
+
+    /// Creates a new dimension of average complexity.
+    ///
+    /// The new dimension is approximately half way in the interval of `[self, high]` and will
+    /// conform to the constraint checked by `is_bft()`.
+    ///
+    /// # Panics
+    ///
+    /// `high` must be have a higher or equal size and faulty node count.
+    pub fn average_higher(&self, high: NetworkDimension) -> NetworkDimension {
+        assert!(high.size >= self.size);
+        assert!(high.faulty >= self.faulty);
+
+        // We try halving both values, rounding down. If `size` is at the minimum, `faulty` will
+        // shrink afterwards.
+        let mut half = NetworkDimension {
+            size: self.size + (high.size - self.size) / 2,
+            faulty: self.faulty + (high.faulty - self.faulty) / 2,
+        };
+
+        // Reduce the number of faulty nodes, if we are outside our limits.
+        if !half.faulty * 3 <= half.size {
+            half.faulty -= 1;
+        }
+
+        // This assert just checks for bugs.
+        assert!(half.is_bft());
+
+        half
+    }
+
+    /// Creates a proptest strategy to create network dimensions within a certain range.
+    pub fn range(min_size: usize, max_size: usize) -> NetworkDimensionStrategy {
+        NetworkDimensionStrategy { min_size, max_size }
+    }
+}
+
+/// Network dimension tree for proptest generation.
+///
+/// See `proptest::strategy::ValueTree` for a more thorough description.
+#[derive(Copy, Clone, Debug)]
+pub struct NetworkDimensionTree {
+    /// The upper bound for any generated dimension.
+    high: NetworkDimension,
+    /// The currently generated network dimension.
+    current: NetworkDimension,
+    /// The lower bound for any generated dimension value (changes during generation or shrinking).
+    low: NetworkDimension,
+}
+
+impl NetworkDimensionTree {
+    /// Generate a random network dimension tree.
+    ///
+    /// The resulting initial `NetworkDimension` will have a number of nodes within
+    /// [`min_size`, `max_size`] and a valid number of faulty nodes.
+    ///
+    /// # Panics
+    ///
+    /// The minimum `min_size` is 1 and `min_size` must be less than or equal `max_size`.
+    pub fn gen<R: Rng>(mut rng: R, min_size: usize, max_size: usize) -> Self {
+        // A common mistake, add an extra assert for a more helpful error message.
+        assert!(min_size > 0, "minimum network size is 1");
+
+        let total = rng.gen_range(min_size, max_size + 1);
+        let max_faulty = (total - 1) / 3;
+        let faulty = rng.gen_range(0, max_faulty + 1);
+
+        let high = NetworkDimension {
+            size: total,
+            faulty,
+        };
+        assert!(high.is_bft());
+
+        let low = NetworkDimension {
+            size: min_size,
+            faulty: 0,
+        };
+        assert!(low.is_bft());
+
+        NetworkDimensionTree {
+            high,
+            current: high,
+            low,
+        }
+    }
+}
+
+impl ValueTree for NetworkDimensionTree {
+    type Value = NetworkDimension;
+
+    fn current(&self) -> Self::Value {
+        self.current.clone()
+    }
+
+    fn simplify(&mut self) -> bool {
+        // Shrinking is simply done through `average_higher`.
+        let prev = *self;
+
+        self.high = prev.current;
+        self.current = self.low.average_higher(prev.high);
+
+        (prev.high != self.high || prev.current != self.current)
+    }
+
+    fn complicate(&mut self) -> bool {
+        let prev = *self;
+
+        // Minimally increase the faulty-node ratio by adjusting the number of faulty nodes and the
+        // size slightly less. If we are at the maximum number of faulty nodes, we would end up
+        // increasing the network size instead (see branch below though).
+        let mut new_low = self.current;
+        new_low.faulty += 1;
+        new_low.size = (new_low.size + 2).max(new_low.faulty * 3 + 1);
+        assert!(new_low.is_bft());
+
+        // Instead of growing the network, return unchanged if the new network would be larger than
+        // the current high.
+        if new_low.size > self.high.size {
+            return false;
+        }
+
+        self.current = new_low.average_higher(self.high);
+        self.low = new_low;
+
+        (prev.current != self.current || prev.low != self.low)
+    }
+}
+
+/// Network dimension strategy for proptest.
+#[derive(Debug)]
+pub struct NetworkDimensionStrategy {
+    /// Minimum number of nodes for newly generated networks dimensions.
+    pub min_size: usize,
+    /// Maximum number of nodes for newly generated networks dimensions.
+    pub max_size: usize,
+}
+
+impl Strategy for NetworkDimensionStrategy {
+    type Value = NetworkDimension;
+    type Tree = NetworkDimensionTree;
+
+    fn new_tree(&self, runner: &mut TestRunner) -> Result<Self::Tree, Reason> {
+        Ok(NetworkDimensionTree::gen(
+            runner.rng(),
+            self.min_size,
+            self.max_size,
+        ))
+    }
+}

tests/net_dynamic_hb.rs

@@ -1,5 +1,7 @@
 extern crate failure;
 extern crate hbbft;
+#[macro_use]
+extern crate proptest;
 extern crate rand;
 extern crate threshold_crypto;
 
@@ -9,8 +11,9 @@ use std::collections;
 
 use hbbft::dynamic_honey_badger::{Change, ChangeState, DynamicHoneyBadger, Input};
 use hbbft::messaging::DistAlgorithm;
-
+use net::proptest::NetworkDimension;
 use net::NetBuilder;
+use proptest::prelude::ProptestConfig;
 
 /// Choose a node's contribution for an epoch.
 ///
@@ -39,34 +42,52 @@ where
     rand::seq::sample_slice(rng, &queue[0..n], k)
 }
 
-// Note: Still pending: Better batch sizes (configurable).
-#[test]
-fn drop_and_readd() {
-    // Currently fixed settings; to be replace by proptest later on. This wrapper function is
-    // already in place, to avoid with rustfmt not formatting the inside of macros.
-    do_drop_and_readd(3, 10, 20, 10, 3)
+/// Test configuration for dynamic honey badger tests.
+#[derive(Debug)]
+struct TestConfig {
+    /// The desired network dimension.
+    dimension: NetworkDimension,
+    /// Total number of transactions to execute before finishing.
+    total_txs: usize,
+    /// Epoch batch size.
+    batch_size: usize,
+    /// Individual nodes contribution size.
+    contribution_size: usize,
 }
 
-/// Dynamic honey badger: Drop a validator node, demoting it to observer, then re-add it.
-///
-/// * `num_faulty`: The number of faulty nodes.
-/// * `total`: Total number of nodes. Must be >= `3 * num_faulty + 1`.
-/// * `total_txs`: The total number of transactions each node will propose. All nodes will propose
-///                the same transactions, albeit in random order.
-/// * `batch_size`: The number of transaction per epoch, total.
-/// * `contribution_size`: A single nodes contribution to the batch.
-fn do_drop_and_readd(
-    num_faulty: usize,
-    total: usize,
-    total_txs: usize,
-    batch_size: usize,
-    contribution_size: usize,
-) {
+prop_compose! {
+    /// Strategy to generate a test configuration.
+    fn arb_config()
+                 (dimension in NetworkDimension::range(3, 15),
+                  total_txs in 20..60usize,
+                  batch_size in 10..20usize,
+                  contribution_size in 1..10usize)
+                 -> TestConfig {
+        TestConfig{
+            dimension, total_txs, batch_size, contribution_size,
+        }
+    }
+}
+
+/// Proptest wrapper for `do_drop_and_readd`.
+proptest!{
+    #![proptest_config(ProptestConfig {
+        cases: 1, .. ProptestConfig::default()
+    })]
+    #[test]
+    fn drop_and_readd(cfg in arb_config()) {
+        do_drop_and_readd(cfg)
+    }
+}
+
+/// Dynamic honey badger: Drop a validator node, demoting it to observer, then re-add it, all while
+/// running a regular honey badger network.
+fn do_drop_and_readd(cfg: TestConfig) {
     let mut rng = rand::thread_rng();
 
     // First, we create a new test network with Honey Badger instances.
-    let mut net = NetBuilder::new(0..total)
-        .num_faulty(num_faulty)
+    let mut net = NetBuilder::new(0..cfg.dimension.size)
+        .num_faulty(cfg.dimension.faulty)
         .message_limit(200_000)  // Limited to 200k messages for now.
         .using_step(move |node| {
             println!("Constructing new dynamic honey badger node #{}", node.id);
@@ -87,12 +108,12 @@ fn do_drop_and_readd(
     // a number between 0..total_txs, chosen randomly.
     let mut queues: collections::BTreeMap<_, Vec<usize>> = net
         .nodes()
-        .map(|node| (*node.id(), (0..total_txs).collect()))
+        .map(|node| (*node.id(), (0..cfg.total_txs).collect()))
         .collect();
 
     // For each node, select transactions randomly from the queue and propose them.
     for (id, queue) in &mut queues {
-        let proposal = choose_contribution(&mut rng, queue, batch_size, contribution_size);
+        let proposal = choose_contribution(&mut rng, queue, cfg.batch_size, cfg.contribution_size);
         println!("Node {:?} will propose: {:?}", id, proposal);
 
         // The step will have its messages added to the queue automatically, we ignore the output.
@@ -200,7 +221,8 @@ fn do_drop_and_readd(
         // If not done, check if we still want to propose something.
         if has_output {
             // Out of the remaining transactions, select a suitable amount.
-            let proposal = choose_contribution(&mut rng, queue, batch_size, contribution_size);
+            let proposal =
+                choose_contribution(&mut rng, queue, cfg.batch_size, cfg.contribution_size);
 
             let _ = net
                 .send_input(node_id, Input::User(proposal))

tests/net_util.rs

@@ -0,0 +1,129 @@
+extern crate failure;
+extern crate hbbft;
+#[macro_use]
+extern crate proptest;
+extern crate rand;
+extern crate threshold_crypto;
+
+pub mod net;
+
+use proptest::strategy::ValueTree;
+use proptest::test_runner::TestRunner;
+
+use net::proptest::{NetworkDimension, NetworkDimensionTree};
+
+/// Checks the `check_sanity` function with various inputs.
+#[test]
+fn check_sanity_works() {
+    assert!(NetworkDimension::new(3, 0).is_bft());
+    assert!(NetworkDimension::new(4, 0).is_bft());
+    assert!(NetworkDimension::new(5, 0).is_bft());
+    assert!(NetworkDimension::new(6, 0).is_bft());
+    assert!(!NetworkDimension::new(3, 1).is_bft());
+    assert!(NetworkDimension::new(4, 1).is_bft());
+    assert!(NetworkDimension::new(5, 1).is_bft());
+    assert!(NetworkDimension::new(6, 1).is_bft());
+    assert!(NetworkDimension::new(16, 3).is_bft());
+    assert!(NetworkDimension::new(17, 3).is_bft());
+    assert!(NetworkDimension::new(18, 3).is_bft());
+    assert!(NetworkDimension::new(19, 3).is_bft());
+    assert!(NetworkDimension::new(16, 5).is_bft());
+    assert!(NetworkDimension::new(17, 5).is_bft());
+    assert!(NetworkDimension::new(18, 5).is_bft());
+    assert!(NetworkDimension::new(19, 5).is_bft());
+    assert!(!NetworkDimension::new(16, 6).is_bft());
+    assert!(!NetworkDimension::new(17, 6).is_bft());
+    assert!(!NetworkDimension::new(18, 6).is_bft());
+    assert!(NetworkDimension::new(19, 6).is_bft());
+    assert!(!NetworkDimension::new(19, 19).is_bft());
+    assert!(!NetworkDimension::new(19, 21).is_bft());
+
+    // Edge cases:
+    assert!(NetworkDimension::new(1, 0).is_bft());
+    assert!(!NetworkDimension::new(0, 0).is_bft());
+    assert!(!NetworkDimension::new(1, 1).is_bft());
+}
+
+proptest!{
+    /// Ensure that `.average_higher()` produces valid new dimensions.
+    #[test]
+    fn average_higher_is_bft(size in 4..40usize) {
+        let mut faulty: usize = size/3;
+        if faulty > 0 {
+            faulty -= 1;
+        }
+
+        let high = NetworkDimension::new(size, faulty);
+        let low = NetworkDimension::new(size/4, faulty/12);
+
+        println!("high: {:?}, low: {:?}", high, low);
+        assert!(high.is_bft());
+        assert!(low.is_bft());
+
+        let average_higher = low.average_higher(high);
+        println!("average_higher: {:?}", average_higher);
+        assert!(average_higher.is_bft());
+    }
+}
+
+/// Ensure `.average_higher()` works for edge cases.
+#[test]
+fn average_higher_handles_edge_cases() {
+    let high = NetworkDimension::new(1, 0);
+    let low = NetworkDimension::new(1, 0);
+    let average_higher = low.average_higher(high);
+    assert!(average_higher.is_bft());
+
+    let high = NetworkDimension::new(10, 0);
+    let low = NetworkDimension::new(10, 0);
+    let average_higher = low.average_higher(high);
+    assert!(average_higher.is_bft());
+
+    let high = NetworkDimension::new(10, 3);
+    let low = NetworkDimension::new(10, 3);
+    let average_higher = low.average_higher(high);
+    assert!(average_higher.is_bft());
+
+    let high = NetworkDimension::new(11, 3);
+    let low = NetworkDimension::new(10, 3);
+    let average_higher = low.average_higher(high);
+    assert!(average_higher.is_bft());
+}
+
+proptest!{
+    /// Ensures all generated network dimensions are actually sane.
+    #[test]
+    fn generated_network_dimensions_are_sane(nt in NetworkDimension::range(1, 400)) {
+        assert!(nt.is_bft());
+    }
+}
+
+/// Verifies generated network dimensions can be grown and shrunk multiple times.
+#[test]
+fn network_dimensions_shrink_and_grow() {
+    let mut runner = TestRunner::new(Default::default());
+
+    let mut tree = NetworkDimensionTree::gen(runner.rng(), 1, 40);
+    assert!(tree.current().is_bft());
+
+    // We complicate and simplify a few times.
+    for _ in 0..10 {
+        tree.complicate();
+        assert!(tree.current().is_bft());
+    }
+
+    for _ in 0..20 {
+        tree.simplify();
+        assert!(tree.current().is_bft());
+    }
+
+    for _ in 0..10 {
+        tree.complicate();
+        assert!(tree.current().is_bft());
+    }
+
+    for _ in 0..10 {
+        tree.simplify();
+        assert!(tree.current().is_bft());
+    }
+}