blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

removes erasure::Session, using Reed-Solomon constructs directly 

The extra wrapping and indirection by the Session struct is not used in
any form. The commit removes Session and instead uses Reed-Solomon
constructs directly.
This commit is contained in:
behzad nouri 2022-04-29 14:45:27 -04:00
parent b56c091b37
commit 4b62c93f87
2 changed files with 10 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
ledger/src/erasure.rs
View File

@ -41,10 +41,7 @@
//!
//!
use {
reed_solomon_erasure::{galois_8::Field, ReconstructShard, ReedSolomon},
serde::{Deserialize, Serialize},
};
use serde::{Deserialize, Serialize};
#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
pub struct ErasureConfig {
@ -68,116 +65,3 @@ impl ErasureConfig {
self.num_coding
}
}
type Result<T> = std::result::Result<T, reed_solomon_erasure::Error>;
/// Represents an erasure "session" with a particular configuration and number of data and coding
/// shreds
#[derive(Debug, Clone)]
pub struct Session(ReedSolomon<Field>);
impl Session {
pub fn new(data_count: usize, coding_count: usize) -> Result<Session> {
let rs = ReedSolomon::new(data_count, coding_count)?;
Ok(Session(rs))
}
pub fn new_from_config(config: &ErasureConfig) -> Result<Session> {
let rs = ReedSolomon::new(config.num_data, config.num_coding)?;
Ok(Session(rs))
}
/// Create coding blocks by overwriting `parity`
pub fn encode<T, U>(&self, data: &[T], parity: &mut [U]) -> Result<()>
where
T: AsRef<[u8]>,
U: AsRef<[u8]> + AsMut<[u8]>,
{
self.0.encode_sep(data, parity)
}
/// Recover data + coding blocks into data blocks
pub fn decode_blocks<T>(&self, blocks: &mut [T]) -> Result<()>
where
T: ReconstructShard<Field>,
{
self.0.reconstruct_data(blocks)
}
}
#[cfg(test)]
pub mod test {
use {super::*, log::*, solana_sdk::clock::Slot};
/// Specifies the contents of a 16-data-shred and 4-coding-shred erasure set
/// Exists to be passed to `generate_blockstore_with_coding`
#[derive(Debug, Copy, Clone)]
pub struct ErasureSpec {
/// Which 16-shred erasure set this represents
pub set_index: u64,
pub num_data: usize,
pub num_coding: usize,
}
/// Specifies the contents of a slot
/// Exists to be passed to `generate_blockstore_with_coding`
#[derive(Debug, Clone)]
pub struct SlotSpec {
pub slot: Slot,
pub set_specs: Vec<ErasureSpec>,
}
#[test]
fn test_coding() {
const N_DATA: usize = 4;
const N_CODING: usize = 2;
let session = Session::new(N_DATA, N_CODING).unwrap();
let mut vs: Vec<Vec<u8>> = (0..N_DATA as u8).map(|i| (i..(16 + i)).collect()).collect();
let v_orig: Vec<u8> = vs[0].clone();
let mut coding_blocks: Vec<_> = (0..N_CODING).map(|_| vec![0u8; 16]).collect();
let mut coding_blocks_slices: Vec<_> =
coding_blocks.iter_mut().map(Vec::as_mut_slice).collect();
let v_slices: Vec<_> = vs.iter().map(Vec::as_slice).collect();
session
.encode(v_slices.as_slice(), coding_blocks_slices.as_mut_slice())
.expect("encoding must succeed");
trace!("test_coding: coding blocks:");
for b in &coding_blocks {
trace!("test_coding: {:?}", b);
}
let erasure: usize = 1;
let mut present = vec![true; N_DATA + N_CODING];
present[erasure] = false;
let erased = vs[erasure].clone();
// clear an entry
vs[erasure as usize].copy_from_slice(&[0; 16]);
let mut blocks: Vec<_> = vs
.iter_mut()
.chain(coding_blocks.iter_mut())
.map(Vec::as_mut_slice)
.zip(present)
.collect();
session
.decode_blocks(blocks.as_mut_slice())
.expect("decoding must succeed");
trace!("test_coding: vs:");
for v in &vs {
trace!("test_coding: {:?}", v);
}
assert_eq!(v_orig, vs[0]);
assert_eq!(erased, vs[erasure]);
}
}

14
ledger/src/shredder.rs
View File

@ -1,11 +1,13 @@
use {
crate::{
erasure::Session,
shred::{Error, Shred, MAX_DATA_SHREDS_PER_FEC_BLOCK, SIZE_OF_DATA_SHRED_PAYLOAD},
shred_stats::ProcessShredsStats,
},
rayon::{prelude::*, ThreadPool},
reed_solomon_erasure::Error::{InvalidIndex, TooFewDataShards, TooFewShardsPresent},
reed_solomon_erasure::{
galois_8::Field,
Error::{InvalidIndex, TooFewDataShards, TooFewShardsPresent},
},
solana_entry::entry::Entry,
solana_measure::measure::Measure,
solana_rayon_threadlimit::get_thread_count,
@ -19,6 +21,8 @@ thread_local!(static PAR_THREAD_POOL: RefCell<ThreadPool> = RefCell::new(rayon::
.build()
.unwrap()));
type ReedSolomon = reed_solomon_erasure::ReedSolomon<Field>;
#[derive(Debug)]
pub struct Shredder {
slot: Slot,
@ -235,9 +239,9 @@ impl Shredder {
let data = data.iter().map(Shred::erasure_shard_as_slice);
let data: Vec<_> = data.collect::<Result<_, _>>().unwrap();
let mut parity = vec![vec![0u8; data[0].len()]; num_coding];
Session::new(num_data, num_coding)
ReedSolomon::new(num_data, num_coding)
.unwrap()
.encode(&data, &mut parity[..])
.encode_sep(&data, &mut parity[..])
.unwrap();
let num_data = u16::try_from(num_data).unwrap();
let num_coding = u16::try_from(num_coding).unwrap();
@ -300,7 +304,7 @@ impl Shredder {
mask[index] = true;
}
}
Session::new(num_data_shreds, num_coding_shreds)?.decode_blocks(&mut shards)?;
ReedSolomon::new(num_data_shreds, num_coding_shreds)?.reconstruct_data(&mut shards)?;
let recovered_data = mask
.into_iter()
.zip(shards)