Add a test time limit. (#232)

* Added a default time limit to `NetBuilder`.

* Added environment-variable override via `HBBFT_NO_TIME_LIMIT`.

* Check for time limit exceeded when cranking.

* Fix typos and factual errors in API docs.

* Document time limit setting in tests `README.md`.

tests/README.md

@@ -123,6 +123,23 @@ assert!(net.nodes().all(|node| node.outputs() == first));
 println!("End result: {:?}", first);
 ```
 
+### Time-limits
+
+Every `VirtualNet` instance limits execution time to 20 minutes by default, this can be adjusted using the `time_limit` function:
+
+```rust
+use std::time;
+
+let num_nodes = 10;
+let mut net = NetBuilder::new(0..num_nodes)
+    // Change the time limit to five minutes per node total.
+    .time_limit(time::Duration::from_secs(num_nodes * 5 * 60))
+```
+
+If the time limit has been reached, `crank` will return a `TimeLimitHit` error. The time-limit can be disabled completely through `no_time_limit()`.
+
+It's also possible to run tests without a time-limit on a per-run basis by setting the `HBBFT_NO_TIME_LIMIT` environment variable to "true".
+
 ### Property based testing
 
 Many higher-level tests allow for a variety of different input parameters like the number of nodes in a network or the amount of faulty ones among them. Other possible parameters include transaction, batch or contribution sizes. To test a variety of randomized combinations of these, the [proptest](https://docs.rs/proptest) crate should be used.

tests/net/err.rs

@@ -1,6 +1,6 @@
 //! Test network errors
 
-use std::fmt;
+use std::{fmt, time};
 
 use failure;
 use hbbft::messaging::DistAlgorithm;
@@ -27,6 +27,8 @@ where
     CrankLimitExceeded(usize),
     /// The configured maximum number of messages has been reached or exceeded.
     MessageLimitExceeded(usize),
+    /// The execution time limit has been reached or exceeded.
+    TimeLimitHit(time::Duration),
 }
 
 // Note: Deriving [Debug](std::fmt::Debug), [Fail](failure::Fail) and through that,
@@ -60,6 +62,9 @@ where
             CrankError::MessageLimitExceeded(max) => {
                 write!(f, "Maximum number of messages exceeded: {}", max)
             }
+            CrankError::TimeLimitHit(lim) => {
+                write!(f, "Time limit of {} seconds exceeded.", lim.as_secs())
+            }
         }
     }
 }
@@ -82,6 +87,7 @@ where
             CrankError::MessageLimitExceeded(max) => {
                 f.debug_tuple("MessageLimitExceeded").field(max).finish()
             }
+            CrankError::TimeLimitHit(lim) => f.debug_tuple("TimeLimitHit").field(lim).finish(),
         }
     }
 }

tests/net/mod.rs

@@ -19,7 +19,7 @@ pub mod proptest;
 pub mod util;
 
 use std::io::Write;
-use std::{cmp, collections, env, fmt, fs, io, ops, process};
+use std::{cmp, collections, env, fmt, fs, io, ops, process, time};
 
 use rand;
 use rand::Rand;
@@ -30,6 +30,9 @@ use hbbft::messaging::{self, DistAlgorithm, NetworkInfo, Step};
 pub use self::adversary::Adversary;
 pub use self::err::CrankError;
 
+/// The time limit for any network if none was specified.
+const DEFAULT_TIME_LIMIT: Option<time::Duration> = Some(time::Duration::from_secs(60 * 20));
+
 /// Helper macro for tracing.
 ///
 /// If tracing is enabled (that is the `Option` is not `None`), writes out a traced packet.
@@ -269,6 +272,8 @@ where
     crank_limit: Option<usize>,
     /// Optional message limit.
     message_limit: Option<usize>,
+    /// Optional time limit.
+    time_limit: Option<time::Duration>,
 }
 
 impl<D, I> fmt::Debug for NetBuilder<D, I>
@@ -313,6 +318,7 @@ where
             trace: None,
             crank_limit: None,
             message_limit: None,
+            time_limit: DEFAULT_TIME_LIMIT,
         }
     }
 
@@ -330,7 +336,7 @@ where
 
     /// Set a crank limit.
     ///
-    /// Crank limits are useful to limit execution time and reign in adversary. Otherwise, message
+    /// Crank limits are useful to limit execution time and rein in adversary. Otherwise, message
     /// limits are typically more useful. After the limit is hit, any call to `crank` will return a
     /// `CrankError::CrankLimitExceeded`.
     #[inline]
@@ -350,6 +356,15 @@ where
         self
     }
 
+    /// Remove the time limit.
+    ///
+    /// Removes any time limit from the builder.
+    #[inline]
+    pub fn no_time_limit(mut self) -> Self {
+        self.time_limit = None;
+        self
+    }
+
     /// Number of faulty nodes.
     ///
     /// Indicates the number of nodes that should be marked faulty.
@@ -359,6 +374,16 @@ where
         self
     }
 
+    /// Time limit.
+    ///
+    /// Sets the time limit; `crank` will fail if called after this much time as elapsed since
+    /// the network was instantiated.
+    #[inline]
+    pub fn time_limit(mut self, limit: time::Duration) -> Self {
+        self.time_limit = Some(limit);
+        self
+    }
+
     /// Override tracing.
     ///
     /// If set, overrides the environment setting of whether or not tracing should be enabled.
@@ -402,6 +427,20 @@ where
     /// If the total number of nodes is not `> 3 * num_faulty`, construction will panic.
     #[inline]
     pub fn build(self) -> Result<VirtualNet<D>, crypto::error::Error> {
+        // The time limit can be overriden through environment variables:
+        let override_time_limit = env::var("HBBFT_NO_TIME_LIMIT")
+            // We fail early, to avoid tricking the user into thinking that they have set the time
+            // limit when they haven't.
+            .map(|s| s.parse().expect("could not parse `HBBFT_NO_TIME_LIMIT`"))
+            .unwrap_or(false);
+
+        let time_limit = if override_time_limit {
+            eprintln!("WARNING: The time limit for individual tests has been manually disabled through `HBBFT_NO_TIME_LIMIT`.");
+            None
+        } else {
+            self.time_limit
+        };
+
         let cons = self
             .cons
             .as_ref()
@@ -427,6 +466,7 @@ where
 
         net.crank_limit = self.crank_limit;
         net.message_limit = self.message_limit;
+        net.time_limit = time_limit;
 
         Ok(net)
     }
@@ -456,6 +496,10 @@ where
     message_count: usize,
     /// The limit set for the number of messages.
     message_limit: Option<usize>,
+    /// Limits the maximum running time between construction and last call to `crank()`.
+    time_limit: Option<time::Duration>,
+    /// The instant the network was created.
+    start_time: time::Instant,
 }
 
 impl<D> fmt::Debug for VirtualNet<D>
@@ -653,6 +697,8 @@ where
             crank_limit: None,
             message_count,
             message_limit: None,
+            time_limit: None,
+            start_time: time::Instant::now(),
         })
     }
 
@@ -727,6 +773,12 @@ where
             }
         }
 
+        if let Some(limit) = self.time_limit {
+            if time::Instant::now().duration_since(self.start_time) > limit {
+                return Some(Err(CrankError::TimeLimitHit(limit)));
+            }
+        }
+
         // Step 0: We give the Adversary a chance to affect the network.
 
         // We need to swap out the adversary, to avoid ownership/borrowing issues.