blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Erasure refinements, fix generating orders table

This commit is contained in:
Stephen Akridge 2018-05-28 10:25:15 -07:00 committed by Greg Fitzgerald
parent 93b6fceb2f
commit d053f78b74
3 changed files with 153 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
src/crdt.rs
View File

@ -235,29 +235,25 @@ impl Crdt {
transmit_index: &mut u64,
) -> Result<()> {
let me: ReplicatedData = {
// copy to avoid locking during IO
let robj = obj.read().expect("'obj' read lock in pub fn broadcast");
let robj = obj.read().expect("'obj' read lock in crdt::index_blobs");
info!("broadcast table {}", robj.table.len());
robj.table[&robj.me].clone()
};
// enumerate all the blobs, those are the indices
let orders: Vec<_> = blobs
.iter()
.enumerate()
.collect();
let orders: Vec<_> = blobs.iter().enumerate().collect();
info!("orders table {}", orders.len());
let _ : Vec<_> = orders
let _: Vec<_> = orders
.into_iter()
.map(|(i, b)| {
// only leader should be broadcasting
let mut blob = b.write().expect("'b' write lock in pub fn broadcast");
let mut blob = b.write().expect("'blob' write lock in crdt::index_blobs");
blob.set_id(me.id).expect("set_id in pub fn broadcast");
blob.set_index(*transmit_index + i as u64)
.expect("set_index in pub fn broadcast");
//TODO profile this, may need multiple sockets for par_iter
})
.collect();
info!("set blobs index");
Ok(())
}
@ -298,27 +294,29 @@ impl Crdt {
warn!("crdt too small");
return Err(Error::CrdtTooSmall);
}
trace!("nodes table {}", nodes.len());
// enumerate all the blobs, those are the indices
info!("nodes table {}", nodes.len());
// enumerate all the blobs in the window, those are the indices
// transmit them to nodes, starting from a different node
let mut orders = Vec::new();
let window_l = window.write().unwrap();
let orders: Vec<_> = window_l[(*transmit_index as usize)..]
.iter()
.enumerate()
.zip(
nodes
.iter()
.cycle()
.skip((*transmit_index as usize) % nodes.len()),
)
.collect();
trace!("orders table {}", orders.len());
let mut i = (*transmit_index as usize) % window_l.len();
loop {
if window_l[i].is_none() || orders.len() >= window_l.len() {
break;
}
orders.push((window_l[i].clone(), nodes[i % nodes.len()]));
i += 1;
i %= window_l.len();
}
info!("orders table {}", orders.len());
let errs: Vec<_> = orders
.into_iter()
.map(|((_i, b), v)| {
.map(|(b, v)| {
// only leader should be broadcasting
assert!(me.current_leader_id != v.id);
let bl = b.clone().unwrap();
let bl = b.unwrap();
let blob = bl.read().expect("blob read lock in streamer::broadcast");
//TODO profile this, may need multiple sockets for par_iter
trace!("broadcast {} to {}", blob.meta.size, v.replicate_addr);
@ -328,7 +326,7 @@ impl Crdt {
e
})
.collect();
trace!("broadcast results {}", errs.len());
info!("broadcast results {}", errs.len());
for e in errs {
match e {
Err(e) => {

184
src/erasure.rs
View File

@ -150,20 +150,48 @@ pub fn decode_blocks(data: &mut [&mut [u8]], coding: &[&[u8]], erasures: &[i32])
Ok(())
}
// Generate coding blocks in window from consumed to consumed+NUM_DATA
pub fn generate_coding(
re: &BlobRecycler,
window: &mut Vec<Option<SharedBlob>>,
consumed: usize,
) -> Result<()> {
// Allocate some coding blobs and insert into the blobs array
pub fn add_coding_blobs(recycler: &BlobRecycler, blobs: &mut Vec<SharedBlob>, consumed: u64) {
let num_data_segments = blobs.len() / NUM_DATA;
trace!(
"num_data: {} blobs.len(): {}",
num_data_segments,
blobs.len()
);
for i in 0..num_data_segments {
let idx = (i * NUM_CODED) + NUM_DATA - (consumed as usize) % NUM_CODED;
for j in idx..idx + MAX_MISSING {
trace!("putting coding at {}", j);
if j <= blobs.len() {
let new_blob = recycler.allocate();
blobs.insert(j, new_blob);
}
}
}
}
// Generate coding blocks in window starting from consumed
pub fn generate_coding(window: &mut Vec<Option<SharedBlob>>, consumed: usize) -> Result<()> {
let mut data_blobs = Vec::new();
let mut coding_blobs = Vec::new();
let mut data_locks = Vec::new();
let mut data_ptrs: Vec<&[u8]> = Vec::new();
let mut coding_locks = Vec::new();
let mut coding_ptrs: Vec<&mut [u8]> = Vec::new();
for i in consumed..consumed + NUM_DATA {
let block_start = consumed - (consumed % NUM_CODED);
trace!(
"generate start: {} end: {}",
block_start,
block_start + NUM_DATA
);
for i in block_start..block_start + NUM_DATA {
let n = i % window.len();
trace!("window[{}] = {:?}", n, window[n]);
if window[n].is_none() {
trace!("data block is null @ {}", n);
return Ok(());
}
data_blobs.push(
window[n]
.clone()
@ -179,11 +207,14 @@ pub fn generate_coding(
}
// generate coding ptr array
let coding_start = consumed + NUM_DATA;
let coding_end = consumed + NUM_CODED;
let coding_start = block_start + NUM_DATA;
let coding_end = block_start + NUM_CODED;
for i in coding_start..coding_end {
let n = i % window.len();
window[n] = Some(re.allocate());
if window[n].is_none() {
trace!("coding block is null @ {}", n);
return Ok(());
}
coding_blobs.push(
window[n]
.clone()
@ -197,7 +228,7 @@ pub fn generate_coding(
);
}
for (i, l) in coding_locks.iter_mut().enumerate() {
trace!("i: {} data: {}", i, l.data[0]);
trace!("i: {} coding: {}", i, l.data[0]);
coding_ptrs.push(&mut l.data);
}
@ -218,9 +249,16 @@ pub fn recover(
//recover with erasure coding
let mut data_missing = 0;
let mut coded_missing = 0;
let coding_start = consumed + NUM_DATA;
let coding_end = consumed + NUM_CODED;
for i in consumed..coding_end {
let block_start = consumed - (consumed % NUM_CODED);
let coding_start = block_start + NUM_DATA;
let coding_end = block_start + NUM_CODED;
trace!(
"block_start: {} coding_start: {} coding_end: {}",
block_start,
coding_start,
coding_end
);
for i in block_start..coding_end {
let n = i % window.len();
if window[n].is_none() {
if i >= coding_start {
@ -238,7 +276,7 @@ pub fn recover(
let mut data_ptrs: Vec<&mut [u8]> = Vec::new();
let mut coding_ptrs: Vec<&[u8]> = Vec::new();
let mut erasures: Vec<i32> = Vec::new();
for i in consumed..coding_end {
for i in block_start..coding_end {
let j = i % window.len();
let mut b = &mut window[j];
if b.is_some() {
@ -249,7 +287,7 @@ pub fn recover(
*b = Some(n.clone());
//mark the missing memory
blobs.push(n);
erasures.push((i - consumed) as i32);
erasures.push(i as i32);
}
erasures.push(-1);
trace!("erasures: {:?}", erasures);
@ -282,7 +320,8 @@ pub fn recover(
#[cfg(test)]
mod test {
use erasure;
use packet::{BlobRecycler, SharedBlob, PACKET_DATA_SIZE};
use logger;
use packet::{BlobRecycler, SharedBlob};
#[test]
pub fn test_coding() {
@ -350,45 +389,79 @@ mod test {
}
}
#[test]
pub fn test_window_recover() {
let mut window = Vec::new();
let blob_recycler = BlobRecycler::default();
let offset = 4;
for i in 0..(4 * erasure::NUM_CODED + 1) {
fn generate_window(
data_len: usize,
blob_recycler: &BlobRecycler,
offset: usize,
) -> Vec<Option<SharedBlob>> {
let mut window = vec![None; 16];
let mut blobs = Vec::new();
for i in 0..erasure::NUM_DATA + 2 {
let b = blob_recycler.allocate();
let b_ = b.clone();
let data_len = b.read().unwrap().data.len();
let mut w = b.write().unwrap();
w.set_index(i as u64).unwrap();
assert_eq!(i as u64, w.get_index().unwrap());
w.meta.size = PACKET_DATA_SIZE;
w.meta.size = data_len;
for k in 0..data_len {
w.data[k] = (k + i) as u8;
w.data_mut()[k] = (k + i) as u8;
}
window.push(Some(b_));
blobs.push(b_);
}
erasure::add_coding_blobs(blob_recycler, &mut blobs, offset as u64);
for (i, b) in blobs.into_iter().enumerate() {
window[i] = Some(b);
}
window
}
#[test]
pub fn test_window_recover_basic() {
logger::setup();
let data_len = 16;
let blob_recycler = BlobRecycler::default();
// Generate a window
let offset = 4;
let mut window = generate_window(data_len, &blob_recycler, 0);
println!("** after-gen-window:");
print_window(&window);
// Generate the coding blocks
assert!(erasure::generate_coding(&mut window, offset).is_ok());
println!("** after-gen-coding:");
print_window(&window);
// Create a hole in the window
let refwindow = window[offset + 1].clone();
window[offset + 1] = None;
// Recover it from coding
assert!(erasure::recover(&blob_recycler, &mut window, offset).is_ok());
println!("** after-recover:");
print_window(&window);
// Check the result
let window_l = window[offset + 1].clone().unwrap();
let ref_l = refwindow.clone().unwrap();
assert_eq!(
window_l.read().unwrap().data()[..data_len],
ref_l.read().unwrap().data()[..data_len]
);
}
//TODO This needs to be reworked
#[test]
#[ignore]
pub fn test_window_recover() {
logger::setup();
let blob_recycler = BlobRecycler::default();
let offset = 4;
let data_len = 16;
let mut window = generate_window(data_len, &blob_recycler, offset);
println!("** after-gen:");
print_window(&window);
assert!(erasure::generate_coding(&blob_recycler, &mut window, offset).is_ok());
assert!(
erasure::generate_coding(&blob_recycler, &mut window, offset + erasure::NUM_CODED)
.is_ok()
);
assert!(
erasure::generate_coding(
&blob_recycler,
&mut window,
offset + (2 * erasure::NUM_CODED)
).is_ok()
);
assert!(
erasure::generate_coding(
&blob_recycler,
&mut window,
offset + (3 * erasure::NUM_CODED)
).is_ok()
);
assert!(erasure::generate_coding(&mut window, offset).is_ok());
println!("** after-coding:");
print_window(&window);
let refwindow = window[offset + 1].clone();
@ -403,28 +476,13 @@ mod test {
println!("** after-nulling:");
print_window(&window);
assert!(erasure::recover(&blob_recycler, &mut window, offset).is_ok());
assert!(erasure::recover(&blob_recycler, &mut window, offset + erasure::NUM_CODED).is_ok());
assert!(
erasure::recover(
&blob_recycler,
&mut window,
offset + (2 * erasure::NUM_CODED)
).is_err()
);
assert!(
erasure::recover(
&blob_recycler,
&mut window,
offset + (3 * erasure::NUM_CODED)
).is_ok()
);
println!("** after-restore:");
print_window(&window);
let window_l = window[offset + 1].clone().unwrap();
let ref_l = refwindow.clone().unwrap();
assert_eq!(
window_l.read().unwrap().data.to_vec(),
ref_l.read().unwrap().data.to_vec()
window_l.read().unwrap().data()[..data_len],
ref_l.read().unwrap().data()[..data_len]
);
}
}

12
src/streamer.rs
View File

@ -378,6 +378,11 @@ fn broadcast(
}
let mut blobs = dq.into_iter().collect();
// Insert the coding blobs into the blob stream
#[cfg(feature = "erasure")]
erasure::add_coding_blobs(recycler, &mut blobs, *transmit_index);
// Index the blobs
Crdt::index_blobs(crdt, &blobs, transmit_index)?;
// keep the cache of blobs that are broadcast
{
@ -406,15 +411,16 @@ fn broadcast(
}
}
// appends codes to the list of blobs allowing us to reconstruct the stream
// Fill in the coding blob data from the window data blobs
#[cfg(feature = "erasure")]
{
match erasure::generate_coding(recycler, &mut window.write().unwrap(), *transmit_index as usize) {
Err(_e) => { return Err(Error::GenericError) }
match erasure::generate_coding(&mut window.write().unwrap(), *transmit_index as usize) {
Err(_e) => return Err(Error::GenericError),
_ => {}
}
}
// Send blobs out from the window
Crdt::broadcast(crdt, &window, &sock, transmit_index)?;
Ok(())
}