added channels to each of the comms threads to send proofs to remote nodes

src/broadcast/mod.rs

@@ -34,6 +34,8 @@ pub struct Instance<'a, T: 'a + Send + Sync> {
     pub tx: &'a channel::Sender<Message<T>>,
     /// The receive side of the multiple producer channel from comms threads.
     pub rx: &'a channel::Receiver<Message<T>>,
+    /// Transmit sides of private channels to comms threads.
+    pub txs_priv: &'a Vec<channel::Sender<Message<T>>>,
     /// Value to be broadcast.
     pub broadcast_value: Option<T>,
 }
@@ -45,11 +47,13 @@ where Vec<u8>: From<T>
 {
     pub fn new(tx: &'a channel::Sender<Message<T>>,
                rx: &'a channel::Receiver<Message<T>>,
+               txs_priv: &'a Vec<channel::Sender<Message<T>>>,
                broadcast_value: Option<T>) -> Self
     {
         Instance {
             tx: tx,
             rx: rx,
+            txs_priv: txs_priv,
             broadcast_value: broadcast_value,
         }
     }

src/commst.rs

@@ -4,6 +4,7 @@
 use std::fmt::Debug;
 use std::sync::{Arc, Mutex};
 use crossbeam;
+#[macro_use]
 use crossbeam_channel as channel;
 
 use proto::Message;
@@ -11,7 +12,7 @@ use task;
 
 /// A communication task connects a remote node to the thread that manages the
 /// consensus algorithm.
-pub struct CommsTask<'a, 'b, T: 'a + Send + Sync +
+pub struct CommsTask<'a, 'b, 'c, T: 'a + 'c + Send + Sync +
                      From<Vec<u8>> + Into<Vec<u8>>>
 where Vec<u8>: From<T>
 {
@@ -19,20 +20,24 @@ where Vec<u8>: From<T>
     tx: &'a channel::Sender<Message<T>>,
     /// The receive side of the multiple consumer channel to comms threads.
     rx: &'a channel::Receiver<Message<T>>,
+    /// The receive side of the private channel to the comms thread.
+    rx_priv: &'c channel::Receiver<Message<T>>,
     /// The socket IO task.
-    task: task::Task<'b>
+    task: task::Task<'b>,
 }
 
-impl<'a, 'b, T: Debug + Send + Sync + From<Vec<u8>> + Into<Vec<u8>>>
-    CommsTask<'a, 'b, T>
+impl<'a, 'b, 'c, T: Debug + Send + Sync + From<Vec<u8>> + Into<Vec<u8>>>
+    CommsTask<'a, 'b, 'c, T>
 where Vec<u8>: From<T>
 {
     pub fn new(tx: &'a channel::Sender<Message<T>>,
                rx: &'a channel::Receiver<Message<T>>,
+               rx_priv: &'c channel::Receiver<Message<T>>,
                stream: &'b ::std::net::TcpStream) -> Self {
         CommsTask {
             tx: tx,
             rx: rx,
+            rx_priv: rx_priv,
             task: task::Task::new(stream)
         }
     }
@@ -43,6 +48,7 @@ where Vec<u8>: From<T>
         // Borrow parts of `self` before entering the thread binding scope.
         let tx = Arc::new(self.tx);
         let rx = Arc::new(self.rx);
+        let rx_priv = Arc::new(self.rx_priv);
         let task = Arc::new(Mutex::new(&mut self.task));
 
         crossbeam::scope(|scope| {
@@ -51,11 +57,25 @@ where Vec<u8>: From<T>
 
             // Local comms receive loop thread.
             scope.spawn(move || {
+                // Unfolded application of `select_loop!`
+                let mut sel = channel::Select::new();
                 loop {
-                    // Receive a message from the manager thread.
-                    let message = rx.recv().unwrap();
-                    // Forward the message to the remote node.
-                    task1.lock().unwrap().send_message(message).unwrap();
+                    // Receive a multicast message from the manager thread.
+                    if let Ok(message) = sel.recv(&rx) {
+                        // Forward the message to the remote node.
+                        task1.lock().unwrap().send_message(message).unwrap();
+                        // Rule: If a selection case fires, the loop must be
+                        // broken.
+                        break;
+                    }
+                    // Receive a private message from the manager thread.
+                    if let Ok(message) = sel.recv(&rx_priv) {
+                        // Forward the message to the remote node.
+                        task1.lock().unwrap().send_message(message).unwrap();
+                        // Rule: If a selection case fires, the loop must be
+                        // broken.
+                        break;
+                    }
                 }
             });
 
@@ -73,6 +93,5 @@ where Vec<u8>: From<T>
                 }
             }
         });
-
     }
 }

src/node.rs

@@ -38,7 +38,7 @@ where Vec<u8>: From<T>
     pub fn run(&self) -> Result<T, ()>
     {
         // Multiple-producer, multiple-consumer channel from comms tasks to
-        // algorithm actor tasks such as Reliable Broadcast.
+        // all algorithm actor tasks such as Reliable Broadcast.
         let (from_comms_tx, from_comms_rx):
         (
             channel::Sender<Message<T>>,
@@ -47,7 +47,7 @@ where Vec<u8>: From<T>
         let (from_comms_tx, from_comms_rx) = (&from_comms_tx, &from_comms_rx);
 
         // Multiple-producer, multiple-consumer channel from algorithm actor
-        // tasks such as Reliable Broadcast to comms tasks.
+        // tasks such as Reliable Broadcast to all comms tasks.
         let (to_comms_tx, to_comms_rx):
         (
             channel::Sender<Message<T>>,
@@ -55,9 +55,22 @@ where Vec<u8>: From<T>
         ) = channel::unbounded();
         let (to_comms_tx, to_comms_rx) = (&to_comms_tx, &to_comms_rx);
 
-        let broadcast_value = self.value.to_owned();
+        let value = &self.value;
         let connections = connection::make(&self.addr, &self.remotes);
 
+        // Single-consumer channels from algorithm actor tasks to comms tasks.
+        let to_comms_1: Vec<(channel::Sender<Message<T>>,
+                             channel::Receiver<Message<T>>)>
+            = (0..connections.len()).map(|_| channel::unbounded()).collect();
+        // All transmit sides of channels to comms tasks.
+        let to_comms_1_txs: Vec<channel::Sender<Message<T>>>
+            = to_comms_1.iter().map(|(tx, _)| tx.to_owned()).collect();
+        let to_comms_1 = &to_comms_1;
+        let to_comms_1_txs = &to_comms_1_txs;
+
+        // FIXME: Compute this index over the vector of connections.
+        const NODE_INDEX: usize = 0;
+
         // All spawned threads will have exited by the end of the scope.
         crossbeam::scope(|scope| {
             // FIXME: Compute [i <- connections | v_i].
@@ -69,12 +82,24 @@ where Vec<u8>: From<T>
                 // - Connect the comms task to the broadcast instance.
                 //
                 // - Broadcast v_i through the broadcast instance?
+                let broadcast_value: Option<T>;
+                if i == NODE_INDEX {
+                    broadcast_value = value.to_owned();
+                }
+                else {
+                    broadcast_value = None;
+                }
+
+                // Receive side of a single-consumer channel from algorithm
+                // actor tasks to the comms task.
+                let ref to_comms_1_rx = to_comms_1[i].1;
 
                 info!("Creating a comms task #{} for {:?}", i,
                       c.stream.peer_addr().unwrap());
                 scope.spawn(move || {
                     commst::CommsTask::new(from_comms_tx,
                                            to_comms_rx,
+                                           to_comms_1_rx,
                                            &c.stream)
                         .run();
                 });
@@ -83,8 +108,8 @@ where Vec<u8>: From<T>
                 scope.spawn(move || {
                     match broadcast::Instance::new(to_comms_tx,
                                                    from_comms_rx,
-                                                   // FIXME
-                                                   None)
+                                                   to_comms_1_txs,
+                                                   broadcast_value)
                         .run()
                     {
                         Ok(_) => debug!("Broadcast instance #{} succeeded", i),