2018-04-02 13:26:40 -07:00
|
|
|
|
//! Reliable broadcast algorithm instance.
|
2018-03-20 09:32:19 -07:00
|
|
|
|
use crossbeam;
|
2018-04-30 08:55:51 -07:00
|
|
|
|
use crossbeam_channel::{Receiver, RecvError, SendError, Sender};
|
|
|
|
|
use merkle::proof::{Lemma, Positioned, Proof};
|
2018-04-11 09:57:30 -07:00
|
|
|
|
use merkle::{Hashable, MerkleTree};
|
2018-04-30 08:55:51 -07:00
|
|
|
|
use proto::*;
|
2018-04-06 09:39:15 -07:00
|
|
|
|
use reed_solomon_erasure as rse;
|
2018-03-27 13:59:38 -07:00
|
|
|
|
use reed_solomon_erasure::ReedSolomon;
|
2018-04-30 08:55:51 -07:00
|
|
|
|
use std::collections::{HashMap, HashSet, VecDeque};
|
2018-05-01 09:32:01 -07:00
|
|
|
|
use std::fmt::{Debug, Display};
|
|
|
|
|
use std::hash::Hash;
|
2018-04-30 08:55:51 -07:00
|
|
|
|
use std::marker::{Send, Sync};
|
|
|
|
|
use std::sync::{Arc, Mutex, RwLock};
|
2018-04-05 05:09:46 -07:00
|
|
|
|
|
2018-04-24 09:31:21 -07:00
|
|
|
|
use messaging;
|
2018-05-01 09:32:01 -07:00
|
|
|
|
use messaging::{AlgoMessage, Algorithm, Handler, LocalMessage, MessageLoopState, ProposedValue,
|
|
|
|
|
QMessage, RemoteMessage, RemoteNode, SourcedMessage, Target, TargetedMessage};
|
|
|
|
|
|
|
|
|
|
/// A `BroadcastMessage` to be sent out, together with a target.
|
|
|
|
|
#[derive(Debug)]
|
|
|
|
|
pub struct TargetedBroadcastMessage<NodeUid> {
|
|
|
|
|
pub target: BroadcastTarget<NodeUid>,
|
|
|
|
|
pub message: BroadcastMessage<ProposedValue>,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl TargetedBroadcastMessage<messaging::NodeUid> {
|
|
|
|
|
pub fn into_remote_message(self) -> RemoteMessage {
|
|
|
|
|
RemoteMessage {
|
|
|
|
|
node: match self.target {
|
|
|
|
|
BroadcastTarget::All => RemoteNode::All,
|
|
|
|
|
BroadcastTarget::Node(node) => RemoteNode::Node(node),
|
|
|
|
|
},
|
|
|
|
|
message: Message::Broadcast(self.message),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// A target node for a `BroadcastMessage`.
|
|
|
|
|
#[derive(Debug)]
|
|
|
|
|
pub enum BroadcastTarget<NodeUid> {
|
|
|
|
|
All,
|
|
|
|
|
Node(NodeUid),
|
|
|
|
|
}
|
2018-03-22 15:47:44 -07:00
|
|
|
|
|
2018-04-25 12:41:46 -07:00
|
|
|
|
struct BroadcastState {
|
2018-04-25 06:07:16 -07:00
|
|
|
|
root_hash: Option<Vec<u8>>,
|
|
|
|
|
leaf_values: Vec<Option<Box<[u8]>>>,
|
|
|
|
|
leaf_values_num: usize,
|
2018-04-25 12:41:46 -07:00
|
|
|
|
echo_num: usize,
|
|
|
|
|
readys: HashMap<Vec<u8>, usize>,
|
|
|
|
|
ready_sent: bool,
|
|
|
|
|
ready_to_decode: bool,
|
|
|
|
|
}
|
|
|
|
|
|
2018-04-27 05:19:39 -07:00
|
|
|
|
/// Reliable Broadcast algorithm instance.
|
2018-05-01 09:32:01 -07:00
|
|
|
|
pub struct Broadcast<NodeUid: Eq + Hash> {
|
2018-04-27 05:19:39 -07:00
|
|
|
|
/// The UID of this node.
|
2018-04-25 12:41:46 -07:00
|
|
|
|
uid: NodeUid,
|
2018-04-27 05:19:39 -07:00
|
|
|
|
/// UIDs of all nodes for iteration purposes.
|
|
|
|
|
all_uids: HashSet<NodeUid>,
|
2018-04-25 12:41:46 -07:00
|
|
|
|
num_nodes: usize,
|
|
|
|
|
num_faulty_nodes: usize,
|
|
|
|
|
data_shard_num: usize,
|
|
|
|
|
coding: ReedSolomon,
|
2018-04-27 05:19:39 -07:00
|
|
|
|
/// All the mutable state is confined to the `state` field. This allows to
|
|
|
|
|
/// mutate state even when the broadcast instance is referred to by an
|
|
|
|
|
/// immutable reference.
|
2018-04-30 08:55:51 -07:00
|
|
|
|
state: RwLock<BroadcastState>,
|
2018-04-24 03:29:13 -07:00
|
|
|
|
}
|
|
|
|
|
|
2018-05-01 09:32:01 -07:00
|
|
|
|
impl Broadcast<messaging::NodeUid> {
|
2018-04-27 05:19:39 -07:00
|
|
|
|
/// The message-driven interface function for calls from the main message
|
|
|
|
|
/// loop.
|
2018-05-01 09:32:01 -07:00
|
|
|
|
pub fn on_message(
|
|
|
|
|
&self,
|
|
|
|
|
m: QMessage,
|
|
|
|
|
tx: &Sender<QMessage>,
|
|
|
|
|
) -> Result<MessageLoopState, Error> {
|
2018-04-30 08:55:51 -07:00
|
|
|
|
match m {
|
|
|
|
|
QMessage::Local(LocalMessage { message, .. }) => match message {
|
2018-05-01 09:32:01 -07:00
|
|
|
|
AlgoMessage::BroadcastInput(value) => self.on_local_message(&mut value.to_owned()),
|
2018-04-25 06:07:16 -07:00
|
|
|
|
|
2018-05-01 09:32:01 -07:00
|
|
|
|
_ => Err(Error::UnexpectedMessage),
|
2018-04-30 08:55:51 -07:00
|
|
|
|
},
|
|
|
|
|
|
|
|
|
|
QMessage::Remote(RemoteMessage {
|
|
|
|
|
node: RemoteNode::Node(uid),
|
|
|
|
|
message,
|
|
|
|
|
}) => {
|
|
|
|
|
if let Message::Broadcast(b) = message {
|
|
|
|
|
self.on_remote_message(uid, &b, tx)
|
|
|
|
|
} else {
|
2018-05-01 09:32:01 -07:00
|
|
|
|
Err(Error::UnexpectedMessage)
|
2018-04-30 08:55:51 -07:00
|
|
|
|
}
|
2018-04-26 06:22:18 -07:00
|
|
|
|
}
|
|
|
|
|
|
2018-05-01 09:32:01 -07:00
|
|
|
|
_ => Err(Error::UnexpectedMessage),
|
2018-04-30 08:55:51 -07:00
|
|
|
|
}
|
|
|
|
|
}
|
2018-04-25 06:07:16 -07:00
|
|
|
|
|
2018-05-01 09:32:01 -07:00
|
|
|
|
fn on_remote_message(
|
|
|
|
|
&self,
|
|
|
|
|
uid: messaging::NodeUid,
|
|
|
|
|
message: &BroadcastMessage<ProposedValue>,
|
|
|
|
|
tx: &Sender<QMessage>,
|
|
|
|
|
) -> Result<MessageLoopState, Error> {
|
|
|
|
|
let (output, messages) = self.handle_broadcast_message(&uid, message)?;
|
|
|
|
|
if let Some(value) = output {
|
|
|
|
|
tx.send(QMessage::Local(LocalMessage {
|
|
|
|
|
dst: Algorithm::CommonSubset,
|
|
|
|
|
message: AlgoMessage::BroadcastOutput(self.uid, value),
|
|
|
|
|
})).map_err(Error::from)?;
|
|
|
|
|
}
|
|
|
|
|
Ok(MessageLoopState::Processing(
|
|
|
|
|
messages
|
|
|
|
|
.into_iter()
|
|
|
|
|
.map(TargetedBroadcastMessage::into_remote_message)
|
|
|
|
|
.collect(),
|
|
|
|
|
))
|
|
|
|
|
}
|
|
|
|
|
|
2018-04-27 05:19:39 -07:00
|
|
|
|
/// Processes the proposed value input by broadcasting it.
|
2018-05-01 09:32:01 -07:00
|
|
|
|
pub fn on_local_message(&self, value: &mut ProposedValue) -> Result<MessageLoopState, Error> {
|
2018-05-01 07:28:31 -07:00
|
|
|
|
let mut state = self.state.write().unwrap();
|
|
|
|
|
// Split the value into chunks/shards, encode them with erasure codes.
|
|
|
|
|
// Assemble a Merkle tree from data and parity shards. Take all proofs
|
|
|
|
|
// from this tree and send them, each to its own node.
|
2018-05-01 09:32:01 -07:00
|
|
|
|
self.send_shards(value.clone())
|
2018-05-01 07:28:31 -07:00
|
|
|
|
.map(|(proof, remote_messages)| {
|
2018-05-01 09:32:01 -07:00
|
|
|
|
// Record the first proof as if it were sent by the node to
|
|
|
|
|
// itself.
|
|
|
|
|
let h = proof.root_hash.clone();
|
|
|
|
|
if proof.validate(h.as_slice()) {
|
|
|
|
|
// Save the leaf value for reconstructing the tree later.
|
|
|
|
|
state.leaf_values[index_of_proof(&proof)] =
|
|
|
|
|
Some(proof.value.clone().into_boxed_slice());
|
|
|
|
|
state.leaf_values_num += 1;
|
|
|
|
|
state.root_hash = Some(h);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
MessageLoopState::Processing(
|
|
|
|
|
remote_messages
|
|
|
|
|
.into_iter()
|
|
|
|
|
.map(TargetedBroadcastMessage::into_remote_message)
|
|
|
|
|
.collect(),
|
|
|
|
|
)
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|
2018-05-01 07:28:31 -07:00
|
|
|
|
|
2018-05-01 09:32:01 -07:00
|
|
|
|
impl<'a, E> Handler<E> for Broadcast<messaging::NodeUid>
|
|
|
|
|
where
|
|
|
|
|
E: From<Error> + From<messaging::Error>,
|
|
|
|
|
{
|
|
|
|
|
fn handle(&self, m: QMessage, tx: Sender<QMessage>) -> Result<MessageLoopState, E> {
|
|
|
|
|
self.on_message(m, &tx).map_err(E::from)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl<NodeUid: Eq + Hash + Debug + Display + Clone> Broadcast<NodeUid> {
|
|
|
|
|
pub fn new(uid: NodeUid, all_uids: HashSet<NodeUid>, num_nodes: usize) -> Result<Self, Error> {
|
|
|
|
|
let num_faulty_nodes = (num_nodes - 1) / 3;
|
|
|
|
|
let parity_shard_num = 2 * num_faulty_nodes;
|
|
|
|
|
let data_shard_num = num_nodes - parity_shard_num;
|
|
|
|
|
let coding = ReedSolomon::new(data_shard_num, parity_shard_num)?;
|
|
|
|
|
|
|
|
|
|
Ok(Broadcast {
|
|
|
|
|
uid,
|
|
|
|
|
all_uids,
|
|
|
|
|
num_nodes,
|
|
|
|
|
num_faulty_nodes,
|
|
|
|
|
data_shard_num,
|
|
|
|
|
coding,
|
|
|
|
|
state: RwLock::new(BroadcastState {
|
|
|
|
|
root_hash: None,
|
|
|
|
|
leaf_values: vec![None; num_nodes],
|
|
|
|
|
leaf_values_num: 0,
|
|
|
|
|
echo_num: 0,
|
|
|
|
|
readys: HashMap::new(),
|
|
|
|
|
ready_sent: false,
|
|
|
|
|
ready_to_decode: false,
|
|
|
|
|
}),
|
2018-05-01 07:28:31 -07:00
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-01 09:32:01 -07:00
|
|
|
|
/// Processes the proposed value input by broadcasting it.
|
|
|
|
|
pub fn propose_value(
|
|
|
|
|
&self,
|
|
|
|
|
value: ProposedValue,
|
|
|
|
|
) -> Result<VecDeque<TargetedBroadcastMessage<NodeUid>>, Error> {
|
2018-04-27 05:19:39 -07:00
|
|
|
|
let mut state = self.state.write().unwrap();
|
|
|
|
|
// Split the value into chunks/shards, encode them with erasure codes.
|
|
|
|
|
// Assemble a Merkle tree from data and parity shards. Take all proofs
|
|
|
|
|
// from this tree and send them, each to its own node.
|
2018-05-01 09:32:01 -07:00
|
|
|
|
self.send_shards(value)
|
|
|
|
|
.map_err(Error::from)
|
|
|
|
|
.map(|(proof, remote_messages)| {
|
|
|
|
|
// Record the first proof as if it were sent by the node to
|
|
|
|
|
// itself.
|
|
|
|
|
let h = proof.root_hash.clone();
|
|
|
|
|
if proof.validate(h.as_slice()) {
|
|
|
|
|
// Save the leaf value for reconstructing the tree later.
|
|
|
|
|
state.leaf_values[index_of_proof(&proof)] =
|
|
|
|
|
Some(proof.value.clone().into_boxed_slice());
|
|
|
|
|
state.leaf_values_num += 1;
|
|
|
|
|
state.root_hash = Some(h);
|
|
|
|
|
}
|
2018-04-27 05:19:39 -07:00
|
|
|
|
|
2018-05-01 09:32:01 -07:00
|
|
|
|
remote_messages
|
|
|
|
|
})
|
2018-04-27 05:19:39 -07:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Breaks the input value into shards of equal length and encodes them --
|
|
|
|
|
/// and some extra parity shards -- with a Reed-Solomon erasure coding
|
|
|
|
|
/// scheme. The returned value contains the shard assigned to this
|
|
|
|
|
/// node. That shard doesn't need to be sent anywhere. It gets recorded in
|
|
|
|
|
/// the broadcast instance.
|
2018-05-01 07:28:31 -07:00
|
|
|
|
fn send_shards(
|
2018-04-30 08:55:51 -07:00
|
|
|
|
&self,
|
2018-05-01 07:28:31 -07:00
|
|
|
|
mut value: ProposedValue,
|
2018-05-01 09:32:01 -07:00
|
|
|
|
) -> Result<
|
|
|
|
|
(
|
|
|
|
|
Proof<ProposedValue>,
|
|
|
|
|
VecDeque<TargetedBroadcastMessage<NodeUid>>,
|
|
|
|
|
),
|
|
|
|
|
Error,
|
|
|
|
|
> {
|
2018-04-27 05:19:39 -07:00
|
|
|
|
let data_shard_num = self.coding.data_shard_count();
|
|
|
|
|
let parity_shard_num = self.coding.parity_shard_count();
|
|
|
|
|
|
2018-04-30 08:55:51 -07:00
|
|
|
|
debug!(
|
|
|
|
|
"Data shards: {}, parity shards: {}",
|
|
|
|
|
self.data_shard_num, parity_shard_num
|
|
|
|
|
);
|
2018-04-27 05:19:39 -07:00
|
|
|
|
// Insert the length of `v` so it can be decoded without the padding.
|
|
|
|
|
let payload_len = value.len() as u8;
|
|
|
|
|
value.insert(0, payload_len); // TODO: Handle messages larger than 255
|
|
|
|
|
// bytes.
|
|
|
|
|
let value_len = value.len();
|
|
|
|
|
// Size of a Merkle tree leaf value, in bytes.
|
|
|
|
|
let shard_len = if value_len % data_shard_num > 0 {
|
|
|
|
|
value_len / data_shard_num + 1
|
2018-04-30 08:55:51 -07:00
|
|
|
|
} else {
|
2018-04-27 05:19:39 -07:00
|
|
|
|
value_len / data_shard_num
|
|
|
|
|
};
|
|
|
|
|
// Pad the last data shard with zeros. Fill the parity shards with
|
|
|
|
|
// zeros.
|
|
|
|
|
value.resize(shard_len * (data_shard_num + parity_shard_num), 0);
|
|
|
|
|
|
|
|
|
|
debug!("value_len {}, shard_len {}", value_len, shard_len);
|
|
|
|
|
|
|
|
|
|
// Divide the vector into chunks/shards.
|
|
|
|
|
let shards_iter = value.chunks_mut(shard_len);
|
|
|
|
|
// Convert the iterator over slices into a vector of slices.
|
|
|
|
|
let mut shards: Vec<&mut [u8]> = Vec::new();
|
|
|
|
|
for s in shards_iter {
|
|
|
|
|
shards.push(s);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
debug!("Shards before encoding: {:?}", shards);
|
|
|
|
|
|
|
|
|
|
// Construct the parity chunks/shards
|
2018-04-30 08:55:51 -07:00
|
|
|
|
self.coding
|
|
|
|
|
.encode(shards.as_mut_slice())
|
|
|
|
|
.map_err(Error::from)?;
|
2018-04-27 05:19:39 -07:00
|
|
|
|
|
|
|
|
|
debug!("Shards: {:?}", shards);
|
|
|
|
|
|
2018-04-30 08:55:51 -07:00
|
|
|
|
let shards_t: Vec<ProposedValue> = shards.into_iter().map(|s| s.to_vec()).collect();
|
2018-04-27 05:19:39 -07:00
|
|
|
|
|
|
|
|
|
// Convert the Merkle tree into a partial binary tree for later
|
|
|
|
|
// deconstruction into compound branches.
|
|
|
|
|
let mtree = MerkleTree::from_vec(&::ring::digest::SHA256, shards_t);
|
|
|
|
|
|
|
|
|
|
// Default result in case of `gen_proof` error.
|
|
|
|
|
let mut result = Err(Error::ProofConstructionFailed);
|
|
|
|
|
let mut outgoing = VecDeque::new();
|
|
|
|
|
|
|
|
|
|
// Send each proof to a node.
|
|
|
|
|
for (leaf_value, uid) in mtree.iter().zip(self.all_uids.clone()) {
|
|
|
|
|
let proof = mtree.gen_proof(leaf_value.to_vec());
|
|
|
|
|
if let Some(proof) = proof {
|
|
|
|
|
if uid == self.uid {
|
|
|
|
|
// The proof is addressed to this node.
|
|
|
|
|
result = Ok(proof);
|
2018-04-30 08:55:51 -07:00
|
|
|
|
} else {
|
2018-04-27 05:19:39 -07:00
|
|
|
|
// Rest of the proofs are sent to remote nodes.
|
2018-05-01 09:32:01 -07:00
|
|
|
|
outgoing.push_back(TargetedBroadcastMessage {
|
|
|
|
|
target: BroadcastTarget::Node(uid),
|
|
|
|
|
message: BroadcastMessage::Value(proof),
|
2018-04-30 08:55:51 -07:00
|
|
|
|
});
|
2018-04-27 05:19:39 -07:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-01 07:28:31 -07:00
|
|
|
|
result.map(|r| (r, outgoing))
|
2018-04-27 05:19:39 -07:00
|
|
|
|
}
|
|
|
|
|
|
2018-05-01 07:01:29 -07:00
|
|
|
|
/// Handler of messages received from remote nodes.
|
2018-05-01 09:32:01 -07:00
|
|
|
|
pub fn handle_broadcast_message(
|
2018-05-01 07:01:29 -07:00
|
|
|
|
&self,
|
2018-05-01 09:32:01 -07:00
|
|
|
|
uid: &NodeUid,
|
2018-05-01 07:01:29 -07:00
|
|
|
|
message: &BroadcastMessage<ProposedValue>,
|
2018-05-01 09:32:01 -07:00
|
|
|
|
) -> Result<
|
|
|
|
|
(
|
|
|
|
|
Option<ProposedValue>,
|
|
|
|
|
VecDeque<TargetedBroadcastMessage<NodeUid>>,
|
|
|
|
|
),
|
|
|
|
|
Error,
|
|
|
|
|
> {
|
2018-04-25 13:26:10 -07:00
|
|
|
|
let mut state = self.state.write().unwrap();
|
2018-05-01 07:01:29 -07:00
|
|
|
|
let no_outgoing = VecDeque::new();
|
2018-04-25 06:07:16 -07:00
|
|
|
|
|
2018-04-25 13:26:10 -07:00
|
|
|
|
// A value received. Record the value and multicast an echo.
|
|
|
|
|
match message {
|
|
|
|
|
BroadcastMessage::Value(p) => {
|
2018-05-01 09:32:01 -07:00
|
|
|
|
if *uid != self.uid {
|
2018-04-25 13:26:10 -07:00
|
|
|
|
// Ignore value messages from unrelated remote nodes.
|
2018-05-01 07:01:29 -07:00
|
|
|
|
Ok((None, no_outgoing))
|
2018-04-30 08:55:51 -07:00
|
|
|
|
} else {
|
2018-04-25 13:26:10 -07:00
|
|
|
|
// Initialise the root hash if not already initialised.
|
2018-04-29 06:27:40 -07:00
|
|
|
|
if state.root_hash.is_none() {
|
2018-04-25 13:26:10 -07:00
|
|
|
|
state.root_hash = Some(p.root_hash.clone());
|
2018-04-30 08:55:51 -07:00
|
|
|
|
debug!(
|
|
|
|
|
"Node {} Value root hash {:?}",
|
|
|
|
|
self.uid,
|
|
|
|
|
HexBytes(&p.root_hash)
|
|
|
|
|
);
|
2018-04-25 13:26:10 -07:00
|
|
|
|
}
|
2018-04-25 06:07:16 -07:00
|
|
|
|
|
2018-04-25 13:26:10 -07:00
|
|
|
|
if let Some(ref h) = state.root_hash.clone() {
|
|
|
|
|
if p.validate(h.as_slice()) {
|
|
|
|
|
// Save the leaf value for reconstructing the tree
|
|
|
|
|
// later.
|
|
|
|
|
state.leaf_values[index_of_proof(&p)] =
|
|
|
|
|
Some(p.value.clone().into_boxed_slice());
|
|
|
|
|
state.leaf_values_num += 1;
|
2018-04-25 12:41:46 -07:00
|
|
|
|
}
|
2018-04-25 13:26:10 -07:00
|
|
|
|
}
|
2018-04-25 12:41:46 -07:00
|
|
|
|
|
2018-04-25 13:26:10 -07:00
|
|
|
|
// Enqueue a broadcast of an echo of this proof.
|
2018-05-01 09:32:01 -07:00
|
|
|
|
let state = VecDeque::from(vec![TargetedBroadcastMessage {
|
|
|
|
|
target: BroadcastTarget::All,
|
|
|
|
|
message: BroadcastMessage::Echo(p.clone()),
|
2018-05-01 07:01:29 -07:00
|
|
|
|
}]);
|
|
|
|
|
Ok((None, state))
|
2018-04-25 13:26:10 -07:00
|
|
|
|
}
|
2018-04-30 08:55:51 -07:00
|
|
|
|
}
|
2018-04-25 12:41:46 -07:00
|
|
|
|
|
2018-04-25 13:26:10 -07:00
|
|
|
|
// An echo received. Verify the proof it contains.
|
|
|
|
|
BroadcastMessage::Echo(p) => {
|
2018-05-01 09:32:01 -07:00
|
|
|
|
if state.root_hash.is_none() && *uid == self.uid {
|
2018-04-29 06:27:40 -07:00
|
|
|
|
state.root_hash = Some(p.root_hash.clone());
|
2018-04-30 08:55:51 -07:00
|
|
|
|
debug!("Node {} Echo root hash {:?}", self.uid, state.root_hash);
|
2018-04-25 13:26:10 -07:00
|
|
|
|
}
|
2018-04-25 12:41:46 -07:00
|
|
|
|
|
2018-04-25 13:26:10 -07:00
|
|
|
|
// Call validate with the root hash as argument.
|
|
|
|
|
if let Some(ref h) = state.root_hash.clone() {
|
|
|
|
|
if p.validate(h.as_slice()) {
|
|
|
|
|
state.echo_num += 1;
|
|
|
|
|
// Save the leaf value for reconstructing the
|
|
|
|
|
// tree later.
|
|
|
|
|
state.leaf_values[index_of_proof(&p)] =
|
|
|
|
|
Some(p.value.clone().into_boxed_slice());
|
|
|
|
|
state.leaf_values_num += 1;
|
|
|
|
|
|
|
|
|
|
// Upon receiving 2f + 1 matching READY(h)
|
|
|
|
|
// messages, wait for N − 2 f ECHO messages,
|
|
|
|
|
// then decode v. Return the decoded v to ACS.
|
2018-04-30 08:55:51 -07:00
|
|
|
|
if state.ready_to_decode
|
|
|
|
|
&& state.leaf_values_num >= self.num_nodes - 2 * self.num_faulty_nodes
|
2018-04-25 13:26:10 -07:00
|
|
|
|
{
|
2018-04-30 08:55:51 -07:00
|
|
|
|
let value = decode_from_shards(
|
|
|
|
|
&mut state.leaf_values,
|
|
|
|
|
&self.coding,
|
|
|
|
|
self.data_shard_num,
|
|
|
|
|
h,
|
|
|
|
|
)?;
|
2018-05-01 07:01:29 -07:00
|
|
|
|
Ok((Some(value), no_outgoing))
|
2018-04-30 08:55:51 -07:00
|
|
|
|
} else if state.leaf_values_num >= self.num_nodes - self.num_faulty_nodes {
|
|
|
|
|
let result: Result<ProposedValue, Error> = decode_from_shards(
|
|
|
|
|
&mut state.leaf_values,
|
|
|
|
|
&self.coding,
|
|
|
|
|
self.data_shard_num,
|
|
|
|
|
h,
|
|
|
|
|
);
|
2018-05-01 09:32:01 -07:00
|
|
|
|
result?;
|
|
|
|
|
// if Ready has not yet been sent, multicast
|
|
|
|
|
// Ready
|
|
|
|
|
if !state.ready_sent {
|
|
|
|
|
state.ready_sent = true;
|
|
|
|
|
|
|
|
|
|
Ok((
|
|
|
|
|
None,
|
|
|
|
|
VecDeque::from(vec![TargetedBroadcastMessage {
|
|
|
|
|
target: BroadcastTarget::All,
|
|
|
|
|
message: BroadcastMessage::Ready(h.to_owned()),
|
|
|
|
|
}]),
|
|
|
|
|
))
|
|
|
|
|
} else {
|
|
|
|
|
Ok((None, no_outgoing))
|
2018-04-30 08:55:51 -07:00
|
|
|
|
}
|
|
|
|
|
} else {
|
2018-05-01 07:01:29 -07:00
|
|
|
|
Ok((None, no_outgoing))
|
2018-04-25 12:41:46 -07:00
|
|
|
|
}
|
2018-04-30 08:55:51 -07:00
|
|
|
|
} else {
|
|
|
|
|
debug!("Broadcast/{} cannot validate Echo {:?}", self.uid, p);
|
2018-05-01 07:01:29 -07:00
|
|
|
|
Ok((None, no_outgoing))
|
2018-04-25 13:26:10 -07:00
|
|
|
|
}
|
2018-04-30 08:55:51 -07:00
|
|
|
|
} else {
|
2018-04-25 13:26:10 -07:00
|
|
|
|
error!("Broadcast/{} root hash not initialised", self.uid);
|
2018-05-01 07:01:29 -07:00
|
|
|
|
Ok((None, no_outgoing))
|
2018-04-25 13:26:10 -07:00
|
|
|
|
}
|
2018-04-30 08:55:51 -07:00
|
|
|
|
}
|
2018-04-25 13:00:22 -07:00
|
|
|
|
|
2018-04-25 13:26:10 -07:00
|
|
|
|
BroadcastMessage::Ready(ref hash) => {
|
|
|
|
|
// Update the number Ready has been received with this hash.
|
|
|
|
|
*state.readys.entry(hash.to_vec()).or_insert(1) += 1;
|
2018-04-25 13:00:22 -07:00
|
|
|
|
|
2018-04-25 13:26:10 -07:00
|
|
|
|
// Check that the root hash matches.
|
|
|
|
|
if let Some(ref h) = state.root_hash.clone() {
|
|
|
|
|
let ready_num = *state.readys.get(h).unwrap_or(&0);
|
|
|
|
|
let mut outgoing = VecDeque::new();
|
2018-04-25 13:00:22 -07:00
|
|
|
|
|
2018-04-25 13:26:10 -07:00
|
|
|
|
// Upon receiving f + 1 matching Ready(h) messages, if Ready
|
|
|
|
|
// has not yet been sent, multicast Ready(h).
|
2018-04-30 08:55:51 -07:00
|
|
|
|
if (ready_num == self.num_faulty_nodes + 1) && !state.ready_sent {
|
2018-04-25 13:26:10 -07:00
|
|
|
|
// Enqueue a broadcast of a ready message.
|
2018-05-01 09:32:01 -07:00
|
|
|
|
outgoing.push_back(TargetedBroadcastMessage {
|
|
|
|
|
target: BroadcastTarget::All,
|
|
|
|
|
message: BroadcastMessage::Ready(h.to_vec()),
|
2018-04-25 13:26:10 -07:00
|
|
|
|
});
|
|
|
|
|
}
|
2018-04-25 13:00:22 -07:00
|
|
|
|
|
2018-05-01 07:01:29 -07:00
|
|
|
|
let mut output = None;
|
|
|
|
|
|
2018-04-25 13:26:10 -07:00
|
|
|
|
// Upon receiving 2f + 1 matching Ready(h) messages, wait
|
|
|
|
|
// for N − 2f Echo messages, then decode v.
|
|
|
|
|
if ready_num > 2 * self.num_faulty_nodes {
|
|
|
|
|
// Wait for N - 2f Echo messages, then decode v.
|
2018-04-30 08:55:51 -07:00
|
|
|
|
if state.echo_num >= self.num_nodes - 2 * self.num_faulty_nodes {
|
2018-04-25 13:26:10 -07:00
|
|
|
|
let value = decode_from_shards(
|
|
|
|
|
&mut state.leaf_values,
|
|
|
|
|
&self.coding,
|
2018-04-30 08:55:51 -07:00
|
|
|
|
self.data_shard_num,
|
|
|
|
|
h,
|
|
|
|
|
)?;
|
2018-04-25 13:00:22 -07:00
|
|
|
|
|
2018-05-01 07:01:29 -07:00
|
|
|
|
output = Some(value);
|
2018-04-30 08:55:51 -07:00
|
|
|
|
} else {
|
2018-04-25 13:26:10 -07:00
|
|
|
|
state.ready_to_decode = true;
|
2018-04-25 13:00:22 -07:00
|
|
|
|
}
|
|
|
|
|
}
|
2018-04-25 06:07:16 -07:00
|
|
|
|
|
2018-05-01 07:01:29 -07:00
|
|
|
|
Ok((output, outgoing))
|
2018-04-30 08:55:51 -07:00
|
|
|
|
} else {
|
2018-05-01 07:01:29 -07:00
|
|
|
|
Ok((None, no_outgoing))
|
2018-04-25 13:26:10 -07:00
|
|
|
|
}
|
|
|
|
|
}
|
2018-04-25 06:07:16 -07:00
|
|
|
|
}
|
2018-04-24 09:31:21 -07:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2018-04-03 15:08:26 -07:00
|
|
|
|
/// Broadcast algorithm instance.
|
2018-03-27 13:59:38 -07:00
|
|
|
|
///
|
2018-04-03 15:08:26 -07:00
|
|
|
|
/// The ACS algorithm requires multiple broadcast instances running
|
2018-04-02 13:26:40 -07:00
|
|
|
|
/// asynchronously, see Figure 4 in the HBBFT paper. Those are N asynchronous
|
2018-04-03 15:08:26 -07:00
|
|
|
|
/// coroutines, each responding to values from one particular remote node. The
|
2018-04-02 13:26:40 -07:00
|
|
|
|
/// paper doesn't make it clear though how other messages - Echo and Ready - are
|
|
|
|
|
/// distributed over the instances. Also it appears that the sender of a message
|
|
|
|
|
/// might become part of the message for this to work.
|
2018-04-05 05:09:46 -07:00
|
|
|
|
pub struct Instance<'a, T: 'a + Clone + Debug + Send + Sync> {
|
|
|
|
|
/// The transmit side of the channel to comms threads.
|
2018-04-24 03:29:13 -07:00
|
|
|
|
tx: &'a Sender<TargetedMessage<ProposedValue>>,
|
2018-04-05 05:09:46 -07:00
|
|
|
|
/// The receive side of the channel from comms threads.
|
2018-04-24 03:29:13 -07:00
|
|
|
|
rx: &'a Receiver<SourcedMessage<ProposedValue>>,
|
2018-03-29 10:19:41 -07:00
|
|
|
|
/// Value to be broadcast.
|
2018-04-03 15:08:26 -07:00
|
|
|
|
broadcast_value: Option<T>,
|
|
|
|
|
/// This instance's index for identification against its comms task.
|
2018-04-05 05:09:46 -07:00
|
|
|
|
node_index: usize,
|
2018-04-04 04:18:57 -07:00
|
|
|
|
/// Number of nodes participating in broadcast.
|
|
|
|
|
num_nodes: usize,
|
|
|
|
|
/// Maximum allowed number of faulty nodes.
|
2018-04-30 08:55:51 -07:00
|
|
|
|
num_faulty_nodes: usize,
|
2018-03-14 17:03:21 -07:00
|
|
|
|
}
|
|
|
|
|
|
2018-04-30 08:55:51 -07:00
|
|
|
|
impl<'a, T: Clone + Debug + Hashable + Send + Sync + Into<Vec<u8>> + From<Vec<u8>>>
|
2018-04-02 13:26:40 -07:00
|
|
|
|
Instance<'a, T>
|
2018-03-27 13:59:38 -07:00
|
|
|
|
{
|
2018-04-30 08:55:51 -07:00
|
|
|
|
pub fn new(
|
|
|
|
|
tx: &'a Sender<TargetedMessage<ProposedValue>>,
|
|
|
|
|
rx: &'a Receiver<SourcedMessage<ProposedValue>>,
|
|
|
|
|
broadcast_value: Option<T>,
|
|
|
|
|
num_nodes: usize,
|
|
|
|
|
node_index: usize,
|
|
|
|
|
) -> Self {
|
2018-04-02 13:26:40 -07:00
|
|
|
|
Instance {
|
2018-04-29 06:27:40 -07:00
|
|
|
|
tx,
|
|
|
|
|
rx,
|
|
|
|
|
broadcast_value,
|
|
|
|
|
node_index,
|
|
|
|
|
num_nodes,
|
2018-04-30 08:55:51 -07:00
|
|
|
|
num_faulty_nodes: (num_nodes - 1) / 3,
|
2018-03-14 17:03:21 -07:00
|
|
|
|
}
|
|
|
|
|
}
|
2018-03-15 16:43:58 -07:00
|
|
|
|
|
2018-03-23 15:54:40 -07:00
|
|
|
|
/// Broadcast stage task returning the computed values in case of success,
|
|
|
|
|
/// and an error in case of failure.
|
2018-04-14 03:27:17 -07:00
|
|
|
|
pub fn run(&mut self) -> Result<T, Error> {
|
2018-03-27 13:59:38 -07:00
|
|
|
|
// Broadcast state machine thread.
|
|
|
|
|
let bvalue = self.broadcast_value.to_owned();
|
2018-04-14 03:27:17 -07:00
|
|
|
|
let result: Result<T, Error>;
|
2018-03-29 10:19:41 -07:00
|
|
|
|
let result_r = Arc::new(Mutex::new(None));
|
|
|
|
|
let result_r_scoped = result_r.clone();
|
2018-03-23 15:54:40 -07:00
|
|
|
|
|
2018-03-20 09:32:19 -07:00
|
|
|
|
crossbeam::scope(|scope| {
|
|
|
|
|
scope.spawn(move || {
|
2018-04-30 08:55:51 -07:00
|
|
|
|
*result_r_scoped.lock().unwrap() = Some(inner_run(
|
|
|
|
|
self.tx,
|
|
|
|
|
self.rx,
|
|
|
|
|
bvalue,
|
|
|
|
|
self.node_index,
|
|
|
|
|
self.num_nodes,
|
|
|
|
|
self.num_faulty_nodes,
|
|
|
|
|
));
|
2018-03-20 09:32:19 -07:00
|
|
|
|
});
|
2018-03-16 11:12:14 -07:00
|
|
|
|
});
|
2018-03-29 10:19:41 -07:00
|
|
|
|
if let Some(ref r) = *result_r.lock().unwrap() {
|
|
|
|
|
result = r.to_owned();
|
2018-04-30 08:55:51 -07:00
|
|
|
|
} else {
|
2018-04-06 09:39:15 -07:00
|
|
|
|
result = Err(Error::Threading);
|
2018-03-29 10:19:41 -07:00
|
|
|
|
}
|
|
|
|
|
result
|
2018-03-14 17:03:21 -07:00
|
|
|
|
}
|
|
|
|
|
}
|
2018-03-22 15:47:44 -07:00
|
|
|
|
|
2018-03-29 09:23:02 -07:00
|
|
|
|
/// Errors returned by the broadcast instance.
|
2018-04-06 09:39:15 -07:00
|
|
|
|
#[derive(Debug, Clone)]
|
2018-04-14 03:27:17 -07:00
|
|
|
|
pub enum Error {
|
2018-03-29 09:23:02 -07:00
|
|
|
|
RootHashMismatch,
|
2018-04-03 15:08:26 -07:00
|
|
|
|
Threading,
|
2018-04-06 09:39:15 -07:00
|
|
|
|
ProofConstructionFailed,
|
|
|
|
|
ReedSolomon(rse::Error),
|
2018-04-25 12:41:46 -07:00
|
|
|
|
Send(SendError<QMessage>),
|
|
|
|
|
SendDeprecated(SendError<TargetedMessage<ProposedValue>>),
|
2018-04-24 09:31:21 -07:00
|
|
|
|
Recv(RecvError),
|
2018-04-25 06:07:16 -07:00
|
|
|
|
UnexpectedMessage,
|
2018-04-30 08:55:51 -07:00
|
|
|
|
NotImplemented,
|
2018-04-06 09:39:15 -07:00
|
|
|
|
}
|
|
|
|
|
|
2018-04-30 08:55:51 -07:00
|
|
|
|
impl From<rse::Error> for Error {
|
|
|
|
|
fn from(err: rse::Error) -> Error {
|
|
|
|
|
Error::ReedSolomon(err)
|
|
|
|
|
}
|
2018-04-06 09:39:15 -07:00
|
|
|
|
}
|
|
|
|
|
|
2018-04-30 08:55:51 -07:00
|
|
|
|
impl From<SendError<QMessage>> for Error {
|
|
|
|
|
fn from(err: SendError<QMessage>) -> Error {
|
|
|
|
|
Error::Send(err)
|
|
|
|
|
}
|
2018-04-25 12:41:46 -07:00
|
|
|
|
}
|
|
|
|
|
|
2018-04-30 08:55:51 -07:00
|
|
|
|
impl From<SendError<TargetedMessage<ProposedValue>>> for Error {
|
|
|
|
|
fn from(err: SendError<TargetedMessage<ProposedValue>>) -> Error {
|
|
|
|
|
Error::SendDeprecated(err)
|
|
|
|
|
}
|
2018-04-06 09:39:15 -07:00
|
|
|
|
}
|
|
|
|
|
|
2018-04-30 08:55:51 -07:00
|
|
|
|
impl From<RecvError> for Error {
|
|
|
|
|
fn from(err: RecvError) -> Error {
|
|
|
|
|
Error::Recv(err)
|
|
|
|
|
}
|
2018-03-29 09:23:02 -07:00
|
|
|
|
}
|
|
|
|
|
|
2018-03-28 06:51:58 -07:00
|
|
|
|
/// Breaks the input value into shards of equal length and encodes them -- and
|
2018-04-03 15:08:26 -07:00
|
|
|
|
/// some extra parity shards -- with a Reed-Solomon erasure coding scheme. The
|
|
|
|
|
/// returned value contains the shard assigned to this node. That shard doesn't
|
|
|
|
|
/// need to be sent anywhere. It is returned to the broadcast instance and gets
|
|
|
|
|
/// recorded immediately.
|
2018-04-30 08:55:51 -07:00
|
|
|
|
fn send_shards<'a, T>(
|
|
|
|
|
value: T,
|
|
|
|
|
tx: &'a Sender<TargetedMessage<ProposedValue>>,
|
|
|
|
|
coding: &ReedSolomon,
|
|
|
|
|
) -> Result<Proof<ProposedValue>, Error>
|
|
|
|
|
where
|
|
|
|
|
T: Clone + Debug + Hashable + Send + Sync + Into<Vec<u8>> + From<Vec<u8>>,
|
2018-03-28 06:51:58 -07:00
|
|
|
|
{
|
2018-04-04 04:18:57 -07:00
|
|
|
|
let data_shard_num = coding.data_shard_count();
|
|
|
|
|
let parity_shard_num = coding.parity_shard_count();
|
|
|
|
|
|
2018-04-30 08:55:51 -07:00
|
|
|
|
debug!(
|
|
|
|
|
"Data shards: {}, parity shards: {}",
|
|
|
|
|
data_shard_num, parity_shard_num
|
|
|
|
|
);
|
2018-04-11 07:44:59 -07:00
|
|
|
|
let mut v: Vec<u8> = T::into(value);
|
2018-04-14 03:27:17 -07:00
|
|
|
|
// Insert the length of `v` so it can be decoded without the padding.
|
|
|
|
|
let payload_len = v.len() as u8;
|
|
|
|
|
v.insert(0, payload_len); // TODO: Handle messages larger than 255 bytes.
|
2018-04-06 11:31:20 -07:00
|
|
|
|
let value_len = v.len();
|
2018-03-28 06:51:58 -07:00
|
|
|
|
// Size of a Merkle tree leaf value, in bytes.
|
2018-04-06 11:31:20 -07:00
|
|
|
|
let shard_len = if value_len % data_shard_num > 0 {
|
|
|
|
|
value_len / data_shard_num + 1
|
2018-04-30 08:55:51 -07:00
|
|
|
|
} else {
|
2018-04-06 11:31:20 -07:00
|
|
|
|
value_len / data_shard_num
|
|
|
|
|
};
|
|
|
|
|
// Pad the last data shard with zeros. Fill the parity shards with zeros.
|
2018-04-14 03:27:17 -07:00
|
|
|
|
v.resize(shard_len * (data_shard_num + parity_shard_num), 0);
|
2018-03-28 06:51:58 -07:00
|
|
|
|
|
2018-04-14 03:27:17 -07:00
|
|
|
|
debug!("value_len {}, shard_len {}", value_len, shard_len);
|
2018-04-06 11:31:20 -07:00
|
|
|
|
|
2018-03-28 06:51:58 -07:00
|
|
|
|
// Divide the vector into chunks/shards.
|
|
|
|
|
let shards_iter = v.chunks_mut(shard_len);
|
|
|
|
|
// Convert the iterator over slices into a vector of slices.
|
|
|
|
|
let mut shards: Vec<&mut [u8]> = Vec::new();
|
|
|
|
|
for s in shards_iter {
|
|
|
|
|
shards.push(s);
|
|
|
|
|
}
|
|
|
|
|
|
2018-04-06 11:31:20 -07:00
|
|
|
|
debug!("Shards before encoding: {:?}", shards);
|
|
|
|
|
|
2018-03-28 06:51:58 -07:00
|
|
|
|
// Construct the parity chunks/shards
|
2018-04-06 09:39:15 -07:00
|
|
|
|
coding.encode(shards.as_mut_slice())?;
|
2018-03-28 06:51:58 -07:00
|
|
|
|
|
2018-04-04 04:18:57 -07:00
|
|
|
|
debug!("Shards: {:?}", shards);
|
|
|
|
|
|
2018-04-30 08:55:51 -07:00
|
|
|
|
let shards_t: Vec<ProposedValue> = shards.into_iter().map(|s| s.to_vec()).collect();
|
2018-03-28 06:51:58 -07:00
|
|
|
|
|
|
|
|
|
// Convert the Merkle tree into a partial binary tree for later
|
|
|
|
|
// deconstruction into compound branches.
|
|
|
|
|
let mtree = MerkleTree::from_vec(&::ring::digest::SHA256, shards_t);
|
|
|
|
|
|
2018-04-04 04:18:57 -07:00
|
|
|
|
// Default result in case of `gen_proof` error.
|
2018-04-06 09:39:15 -07:00
|
|
|
|
let mut result = Err(Error::ProofConstructionFailed);
|
2018-04-03 15:08:26 -07:00
|
|
|
|
|
2018-03-28 06:51:58 -07:00
|
|
|
|
// Send each proof to a node.
|
2018-04-14 03:27:17 -07:00
|
|
|
|
for (i, leaf_value) in mtree.iter().enumerate() {
|
|
|
|
|
let proof = mtree.gen_proof(leaf_value.to_vec());
|
2018-03-28 06:51:58 -07:00
|
|
|
|
if let Some(proof) = proof {
|
2018-04-03 15:08:26 -07:00
|
|
|
|
if i == 0 {
|
|
|
|
|
// The first proof is addressed to this node.
|
|
|
|
|
result = Ok(proof);
|
2018-04-30 08:55:51 -07:00
|
|
|
|
} else {
|
2018-04-03 15:08:26 -07:00
|
|
|
|
// Rest of the proofs are sent to remote nodes.
|
2018-04-30 08:55:51 -07:00
|
|
|
|
tx.send(TargetedMessage {
|
|
|
|
|
target: Target::Node(i),
|
|
|
|
|
message: Message::Broadcast(BroadcastMessage::Value(proof)),
|
|
|
|
|
})?;
|
2018-04-03 15:08:26 -07:00
|
|
|
|
}
|
2018-03-28 06:51:58 -07:00
|
|
|
|
}
|
|
|
|
|
}
|
2018-04-03 15:08:26 -07:00
|
|
|
|
|
|
|
|
|
result
|
2018-03-28 06:51:58 -07:00
|
|
|
|
}
|
|
|
|
|
|
2018-03-23 15:54:40 -07:00
|
|
|
|
/// The main loop of the broadcast task.
|
2018-04-30 08:55:51 -07:00
|
|
|
|
fn inner_run<'a, T>(
|
|
|
|
|
tx: &'a Sender<TargetedMessage<ProposedValue>>,
|
|
|
|
|
rx: &'a Receiver<SourcedMessage<ProposedValue>>,
|
|
|
|
|
broadcast_value: Option<T>,
|
|
|
|
|
node_index: usize,
|
|
|
|
|
num_nodes: usize,
|
|
|
|
|
num_faulty_nodes: usize,
|
|
|
|
|
) -> Result<T, Error>
|
|
|
|
|
where
|
|
|
|
|
T: Clone + Debug + Hashable + Send + Sync + Into<Vec<u8>> + From<Vec<u8>>,
|
2018-03-22 15:47:44 -07:00
|
|
|
|
{
|
2018-03-27 13:59:38 -07:00
|
|
|
|
// Erasure coding scheme: N - 2f value shards and 2f parity shards
|
2018-04-04 04:18:57 -07:00
|
|
|
|
let parity_shard_num = 2 * num_faulty_nodes;
|
|
|
|
|
let data_shard_num = num_nodes - parity_shard_num;
|
2018-04-06 09:39:15 -07:00
|
|
|
|
let coding = ReedSolomon::new(data_shard_num, parity_shard_num)?;
|
2018-04-03 15:08:26 -07:00
|
|
|
|
// currently known leaf values
|
2018-04-04 04:18:57 -07:00
|
|
|
|
let mut leaf_values: Vec<Option<Box<[u8]>>> = vec![None; num_nodes];
|
2018-04-03 15:08:26 -07:00
|
|
|
|
// Write-once root hash of a tree broadcast from the sender associated with
|
|
|
|
|
// this instance.
|
|
|
|
|
let mut root_hash: Option<Vec<u8>> = None;
|
|
|
|
|
// number of non-None leaf values
|
|
|
|
|
let mut leaf_values_num = 0;
|
2018-03-27 13:59:38 -07:00
|
|
|
|
|
|
|
|
|
// Split the value into chunks/shards, encode them with erasure codes.
|
|
|
|
|
// Assemble a Merkle tree from data and parity shards. Take all proofs from
|
|
|
|
|
// this tree and send them, each to its own node.
|
|
|
|
|
if let Some(v) = broadcast_value {
|
2018-04-30 08:55:51 -07:00
|
|
|
|
send_shards(v, tx, &coding).map(|proof| {
|
|
|
|
|
// Record the first proof as if it were sent by the node to
|
|
|
|
|
// itself.
|
|
|
|
|
let h = proof.root_hash.clone();
|
|
|
|
|
if proof.validate(h.as_slice()) {
|
|
|
|
|
// Save the leaf value for reconstructing the tree later.
|
|
|
|
|
leaf_values[index_of_proof(&proof)] = Some(proof.value.clone().into_boxed_slice());
|
|
|
|
|
leaf_values_num += 1;
|
|
|
|
|
root_hash = Some(h);
|
|
|
|
|
}
|
|
|
|
|
})?
|
2018-03-27 13:59:38 -07:00
|
|
|
|
}
|
2018-03-23 15:54:40 -07:00
|
|
|
|
|
2018-03-28 15:38:02 -07:00
|
|
|
|
// return value
|
2018-04-14 03:27:17 -07:00
|
|
|
|
let mut result: Option<Result<T, Error>> = None;
|
2018-03-28 15:38:02 -07:00
|
|
|
|
// Number of times Echo was received with the same root hash.
|
|
|
|
|
let mut echo_num = 0;
|
|
|
|
|
// Number of times Ready was received with the same root hash.
|
2018-04-06 10:42:04 -07:00
|
|
|
|
let mut readys: HashMap<Vec<u8>, usize> = HashMap::new();
|
2018-03-28 15:38:02 -07:00
|
|
|
|
let mut ready_sent = false;
|
2018-03-29 09:23:02 -07:00
|
|
|
|
let mut ready_to_decode = false;
|
2018-03-28 15:38:02 -07:00
|
|
|
|
|
2018-03-22 15:47:44 -07:00
|
|
|
|
// TODO: handle exit conditions
|
2018-04-06 09:39:15 -07:00
|
|
|
|
while result.is_none() {
|
2018-03-22 15:47:44 -07:00
|
|
|
|
// Receive a message from the socket IO task.
|
2018-04-06 09:39:15 -07:00
|
|
|
|
let message = rx.recv()?;
|
2018-04-05 05:09:46 -07:00
|
|
|
|
if let SourcedMessage {
|
|
|
|
|
source: i,
|
2018-04-30 08:55:51 -07:00
|
|
|
|
message: Message::Broadcast(message),
|
|
|
|
|
} = message
|
|
|
|
|
{
|
2018-03-22 15:47:44 -07:00
|
|
|
|
match message {
|
|
|
|
|
// A value received. Record the value and multicast an echo.
|
|
|
|
|
BroadcastMessage::Value(p) => {
|
2018-04-03 15:08:26 -07:00
|
|
|
|
if i != node_index {
|
|
|
|
|
// Ignore value messages from unrelated remote nodes.
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
2018-04-29 06:27:40 -07:00
|
|
|
|
if root_hash.is_none() {
|
2018-03-28 15:38:02 -07:00
|
|
|
|
root_hash = Some(p.root_hash.clone());
|
2018-04-30 08:55:51 -07:00
|
|
|
|
debug!(
|
|
|
|
|
"Node {} Value root hash {:?}",
|
|
|
|
|
node_index,
|
|
|
|
|
HexBytes(&p.root_hash)
|
|
|
|
|
);
|
2018-03-28 15:38:02 -07:00
|
|
|
|
}
|
|
|
|
|
|
2018-04-29 06:27:40 -07:00
|
|
|
|
if let Some(ref h) = root_hash {
|
2018-03-28 15:38:02 -07:00
|
|
|
|
if p.validate(h.as_slice()) {
|
|
|
|
|
// Save the leaf value for reconstructing the tree
|
|
|
|
|
// later.
|
|
|
|
|
leaf_values[index_of_proof(&p)] =
|
2018-04-14 03:27:17 -07:00
|
|
|
|
Some(p.value.clone().into_boxed_slice());
|
2018-04-29 06:27:40 -07:00
|
|
|
|
leaf_values_num += 1;
|
2018-03-28 15:38:02 -07:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// Broadcast an echo of this proof.
|
2018-04-05 05:09:46 -07:00
|
|
|
|
tx.send(TargetedMessage {
|
|
|
|
|
target: Target::All,
|
2018-04-30 08:55:51 -07:00
|
|
|
|
message: Message::Broadcast(BroadcastMessage::Echo(p)),
|
2018-04-06 09:39:15 -07:00
|
|
|
|
})?
|
2018-04-30 08:55:51 -07:00
|
|
|
|
}
|
2018-03-22 15:47:44 -07:00
|
|
|
|
|
|
|
|
|
// An echo received. Verify the proof it contains.
|
|
|
|
|
BroadcastMessage::Echo(p) => {
|
2018-04-29 06:27:40 -07:00
|
|
|
|
if root_hash.is_none() && i == node_index {
|
|
|
|
|
root_hash = Some(p.root_hash.clone());
|
2018-04-30 08:55:51 -07:00
|
|
|
|
debug!("Node {} Echo root hash {:?}", node_index, root_hash);
|
2018-03-28 15:38:02 -07:00
|
|
|
|
}
|
2018-03-22 15:47:44 -07:00
|
|
|
|
|
2018-03-28 15:38:02 -07:00
|
|
|
|
// call validate with the root hash as argument
|
2018-04-29 06:27:40 -07:00
|
|
|
|
if let Some(ref h) = root_hash {
|
2018-03-28 15:38:02 -07:00
|
|
|
|
if p.validate(h.as_slice()) {
|
|
|
|
|
echo_num += 1;
|
|
|
|
|
// Save the leaf value for reconstructing the tree
|
|
|
|
|
// later.
|
|
|
|
|
leaf_values[index_of_proof(&p)] =
|
2018-04-14 03:27:17 -07:00
|
|
|
|
Some(p.value.clone().into_boxed_slice());
|
2018-04-29 06:27:40 -07:00
|
|
|
|
leaf_values_num += 1;
|
2018-03-28 15:38:02 -07:00
|
|
|
|
|
2018-03-29 09:23:02 -07:00
|
|
|
|
// upon receiving 2f + 1 matching READY(h)
|
|
|
|
|
// messages, wait for N − 2 f ECHO messages, then
|
|
|
|
|
// decode v
|
2018-04-30 08:55:51 -07:00
|
|
|
|
if ready_to_decode
|
|
|
|
|
&& leaf_values_num >= num_nodes - 2 * num_faulty_nodes
|
2018-03-29 09:23:02 -07:00
|
|
|
|
{
|
2018-04-30 08:55:51 -07:00
|
|
|
|
result = Some(decode_from_shards(
|
|
|
|
|
&mut leaf_values,
|
|
|
|
|
&coding,
|
|
|
|
|
data_shard_num,
|
|
|
|
|
h,
|
|
|
|
|
));
|
|
|
|
|
} else if leaf_values_num >= num_nodes - num_faulty_nodes {
|
|
|
|
|
result = Some(decode_from_shards(
|
|
|
|
|
&mut leaf_values,
|
|
|
|
|
&coding,
|
|
|
|
|
data_shard_num,
|
|
|
|
|
h,
|
|
|
|
|
));
|
2018-03-29 09:23:02 -07:00
|
|
|
|
// if Ready has not yet been sent, multicast
|
|
|
|
|
// Ready
|
|
|
|
|
if !ready_sent {
|
|
|
|
|
ready_sent = true;
|
2018-04-05 05:09:46 -07:00
|
|
|
|
tx.send(TargetedMessage {
|
|
|
|
|
target: Target::All,
|
2018-04-30 08:55:51 -07:00
|
|
|
|
message: Message::Broadcast(BroadcastMessage::Ready(
|
|
|
|
|
h.to_owned(),
|
|
|
|
|
)),
|
2018-04-06 09:39:15 -07:00
|
|
|
|
})?;
|
2018-03-29 09:23:02 -07:00
|
|
|
|
}
|
2018-03-23 15:54:40 -07:00
|
|
|
|
}
|
2018-03-22 15:47:44 -07:00
|
|
|
|
}
|
|
|
|
|
}
|
2018-04-30 08:55:51 -07:00
|
|
|
|
}
|
2018-03-23 15:54:40 -07:00
|
|
|
|
|
2018-04-06 10:42:04 -07:00
|
|
|
|
BroadcastMessage::Ready(ref hash) => {
|
|
|
|
|
// Update the number Ready has been received with this hash.
|
|
|
|
|
*readys.entry(hash.to_vec()).or_insert(1) += 1;
|
|
|
|
|
|
2018-03-28 15:38:02 -07:00
|
|
|
|
// Check that the root hash matches.
|
2018-04-29 06:27:40 -07:00
|
|
|
|
if let Some(ref h) = root_hash {
|
2018-04-06 10:42:04 -07:00
|
|
|
|
let ready_num: usize = *readys.get(h).unwrap_or(&0);
|
2018-03-23 15:54:40 -07:00
|
|
|
|
|
2018-03-28 15:38:02 -07:00
|
|
|
|
// Upon receiving f + 1 matching Ready(h) messages, if
|
|
|
|
|
// Ready has not yet been sent, multicast Ready(h).
|
2018-04-30 08:55:51 -07:00
|
|
|
|
if (ready_num == num_faulty_nodes + 1) && !ready_sent {
|
2018-04-05 05:09:46 -07:00
|
|
|
|
tx.send(TargetedMessage {
|
|
|
|
|
target: Target::All,
|
2018-04-30 08:55:51 -07:00
|
|
|
|
message: Message::Broadcast(BroadcastMessage::Ready(h.to_vec())),
|
2018-04-06 09:39:15 -07:00
|
|
|
|
})?;
|
2018-03-28 15:38:02 -07:00
|
|
|
|
}
|
2018-03-23 15:54:40 -07:00
|
|
|
|
|
2018-03-28 15:38:02 -07:00
|
|
|
|
// Upon receiving 2f + 1 matching Ready(h) messages,
|
|
|
|
|
// wait for N − 2f Echo messages, then decode v.
|
2018-04-04 04:18:57 -07:00
|
|
|
|
if ready_num > 2 * num_faulty_nodes {
|
2018-03-29 09:23:02 -07:00
|
|
|
|
// Wait for N - 2f Echo messages, then decode v.
|
2018-04-04 04:18:57 -07:00
|
|
|
|
if echo_num >= num_nodes - 2 * num_faulty_nodes {
|
2018-04-30 08:55:51 -07:00
|
|
|
|
result = Some(decode_from_shards(
|
|
|
|
|
&mut leaf_values,
|
|
|
|
|
&coding,
|
|
|
|
|
data_shard_num,
|
|
|
|
|
h,
|
|
|
|
|
));
|
|
|
|
|
} else {
|
2018-03-29 09:23:02 -07:00
|
|
|
|
ready_to_decode = true;
|
|
|
|
|
}
|
2018-03-28 15:38:02 -07:00
|
|
|
|
}
|
2018-03-23 15:54:40 -07:00
|
|
|
|
}
|
|
|
|
|
}
|
2018-03-22 15:47:44 -07:00
|
|
|
|
}
|
2018-04-30 08:55:51 -07:00
|
|
|
|
} else {
|
2018-04-06 08:04:28 -07:00
|
|
|
|
error!("Incorrect message from the socket: {:?}", message);
|
2018-03-22 15:47:44 -07:00
|
|
|
|
}
|
|
|
|
|
}
|
2018-04-06 09:39:15 -07:00
|
|
|
|
// result is not a None, safe to extract value
|
2018-03-29 09:23:02 -07:00
|
|
|
|
result.unwrap()
|
|
|
|
|
}
|
|
|
|
|
|
2018-04-30 08:55:51 -07:00
|
|
|
|
fn decode_from_shards<T>(
|
|
|
|
|
leaf_values: &mut Vec<Option<Box<[u8]>>>,
|
|
|
|
|
coding: &ReedSolomon,
|
|
|
|
|
data_shard_num: usize,
|
|
|
|
|
root_hash: &[u8],
|
|
|
|
|
) -> Result<T, Error>
|
|
|
|
|
where
|
|
|
|
|
T: Clone + Debug + Hashable + Send + Sync + From<Vec<u8>> + Into<Vec<u8>>,
|
2018-03-29 09:23:02 -07:00
|
|
|
|
{
|
|
|
|
|
// Try to interpolate the Merkle tree using the Reed-Solomon erasure coding
|
|
|
|
|
// scheme.
|
2018-04-06 09:39:15 -07:00
|
|
|
|
coding.reconstruct_shards(leaf_values.as_mut_slice())?;
|
2018-03-29 09:23:02 -07:00
|
|
|
|
|
|
|
|
|
// Recompute the Merkle tree root.
|
|
|
|
|
//
|
2018-04-14 03:27:17 -07:00
|
|
|
|
// Collect shards for tree construction.
|
2018-04-24 03:29:13 -07:00
|
|
|
|
let mut shards: Vec<ProposedValue> = Vec::new();
|
2018-03-29 09:23:02 -07:00
|
|
|
|
for l in leaf_values.iter() {
|
|
|
|
|
if let Some(ref v) = *l {
|
2018-04-14 03:27:17 -07:00
|
|
|
|
shards.push(v.to_vec());
|
2018-03-29 09:23:02 -07:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// Construct the Merkle tree.
|
2018-04-14 03:27:17 -07:00
|
|
|
|
let mtree = MerkleTree::from_vec(&::ring::digest::SHA256, shards);
|
2018-03-29 09:23:02 -07:00
|
|
|
|
// If the root hash of the reconstructed tree does not match the one
|
|
|
|
|
// received with proofs then abort.
|
2018-04-29 06:27:40 -07:00
|
|
|
|
if &mtree.root_hash()[..] != root_hash {
|
2018-03-29 09:23:02 -07:00
|
|
|
|
// NOTE: The paper does not define the meaning of *abort*. But it is
|
|
|
|
|
// sensible not to continue trying to reconstruct the tree after this
|
|
|
|
|
// point. This instance must have received incorrect shards.
|
2018-04-06 09:39:15 -07:00
|
|
|
|
Err(Error::RootHashMismatch)
|
2018-04-30 08:55:51 -07:00
|
|
|
|
} else {
|
2018-03-29 09:23:02 -07:00
|
|
|
|
// Reconstruct the value from the data shards.
|
|
|
|
|
Ok(glue_shards(mtree, data_shard_num))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Concatenates the first `n` leaf values of a Merkle tree `m` in one value of
|
|
|
|
|
/// type `T`. This is useful for reconstructing the data value held in the tree
|
|
|
|
|
/// and forgetting the leaves that contain parity information.
|
2018-04-24 03:29:13 -07:00
|
|
|
|
fn glue_shards<T>(m: MerkleTree<ProposedValue>, n: usize) -> T
|
2018-04-30 08:55:51 -07:00
|
|
|
|
where
|
|
|
|
|
T: From<Vec<u8>> + Into<Vec<u8>>,
|
2018-03-29 09:23:02 -07:00
|
|
|
|
{
|
2018-04-29 06:27:40 -07:00
|
|
|
|
let t: Vec<u8> = m.into_iter().take(n).flat_map(|s| s).collect();
|
2018-04-14 03:27:17 -07:00
|
|
|
|
let payload_len = t[0] as usize;
|
|
|
|
|
debug!("Glued data shards {:?}", &t[1..(payload_len + 1)]);
|
2018-04-06 08:04:28 -07:00
|
|
|
|
|
2018-04-14 03:27:17 -07:00
|
|
|
|
Vec::into(t[1..(payload_len + 1)].to_vec())
|
2018-03-23 15:54:40 -07:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// An additional path conversion operation on `Lemma` to allow reconstruction
|
|
|
|
|
/// of erasure-coded `Proof` from `Lemma`s. The output path, when read from left
|
|
|
|
|
/// to right, goes from leaf to root (LSB order).
|
2018-03-28 15:38:02 -07:00
|
|
|
|
fn path_of_lemma(lemma: &Lemma) -> Vec<bool> {
|
2018-03-23 15:54:40 -07:00
|
|
|
|
match lemma.sub_lemma {
|
|
|
|
|
None => {
|
|
|
|
|
match lemma.sibling_hash {
|
|
|
|
|
// lemma terminates with no leaf
|
|
|
|
|
None => vec![],
|
|
|
|
|
// the leaf is on the right
|
|
|
|
|
Some(Positioned::Left(_)) => vec![true],
|
|
|
|
|
// the leaf is on the left
|
|
|
|
|
Some(Positioned::Right(_)) => vec![false],
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
Some(ref l) => {
|
2018-03-28 15:38:02 -07:00
|
|
|
|
let mut p = path_of_lemma(l.as_ref());
|
2018-03-23 15:54:40 -07:00
|
|
|
|
|
|
|
|
|
match lemma.sibling_hash {
|
|
|
|
|
// lemma terminates
|
|
|
|
|
None => (),
|
|
|
|
|
// lemma branches out to the right
|
|
|
|
|
Some(Positioned::Left(_)) => p.push(true),
|
|
|
|
|
// lemma branches out to the left
|
|
|
|
|
Some(Positioned::Right(_)) => p.push(false),
|
|
|
|
|
}
|
|
|
|
|
p
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Further conversion of a binary tree path into an array index.
|
2018-03-28 15:38:02 -07:00
|
|
|
|
fn index_of_path(mut path: Vec<bool>) -> usize {
|
2018-03-23 15:54:40 -07:00
|
|
|
|
let mut idx = 0;
|
|
|
|
|
// Convert to the MSB order.
|
|
|
|
|
path.reverse();
|
|
|
|
|
|
2018-04-29 06:27:40 -07:00
|
|
|
|
for &dir in &path {
|
|
|
|
|
idx <<= 1;
|
|
|
|
|
if dir {
|
|
|
|
|
idx |= 1;
|
2018-03-23 15:54:40 -07:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
idx
|
2018-03-22 15:47:44 -07:00
|
|
|
|
}
|
2018-03-28 15:38:02 -07:00
|
|
|
|
|
|
|
|
|
/// Computes the Merkle tree leaf index of a value in a given proof.
|
2018-04-24 03:29:13 -07:00
|
|
|
|
fn index_of_proof(p: &Proof<ProposedValue>) -> usize {
|
2018-03-28 15:38:02 -07:00
|
|
|
|
index_of_path(path_of_lemma(&p.lemma))
|
|
|
|
|
}
|