Upgrade to ReedSolomon 4.0 release (#6026)
This commit is contained in:
Pankaj Garg
GitHub
parent
eeda7338cc
commit
dbd337c616
|
|
@ -2427,6 +2427,16 @@ dependencies = [
|
|||
"redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reed-solomon-erasure"
|
||||
version = "4.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cc 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "0.1.80"
|
||||
|
|
@ -3104,6 +3114,7 @@ dependencies = [
|
|||
"rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rayon 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"reed-solomon-erasure 4.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rocksdb 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -3123,7 +3134,6 @@ dependencies = [
|
|||
"solana-metrics 0.20.0",
|
||||
"solana-netutil 0.20.0",
|
||||
"solana-rayon-threadlimit 0.20.0",
|
||||
"solana-reed-solomon-erasure 3.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"solana-runtime 0.20.0",
|
||||
"solana-sdk 0.20.0",
|
||||
"solana-stake-api 0.20.0",
|
||||
|
|
@ -3524,17 +3534,6 @@ dependencies = [
|
|||
"sys-info 0.5.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "solana-reed-solomon-erasure"
|
||||
version = "3.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cc 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rayon 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "solana-replicator"
|
||||
version = "0.20.0"
|
||||
|
|
@ -5470,6 +5469,7 @@ dependencies = [
|
|||
"checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
|
||||
"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
|
||||
"checksum redox_users 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3fe5204c3a17e97dde73f285d49be585df59ed84b50a872baf416e73b62c3828"
|
||||
"checksum reed-solomon-erasure 4.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d8e74ef6375dc3f41597d5865fa60aea3dfc89f7f6d3a29d2935f77c0eada016"
|
||||
"checksum regex 0.1.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4fd4ace6a8cf7860714a2c2280d6c1f7e6a413486c13298bbc86fd3da019402f"
|
||||
"checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd"
|
||||
"checksum regex-syntax 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "f9ec002c35e86791825ed294b50008eea9ddfc8def4420124fbc6b08db834957"
|
||||
|
|
@ -5515,7 +5515,6 @@ dependencies = [
|
|||
"checksum smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)" = "ab606a9c5e214920bb66c458cd7be8ef094f813f20fe77a54cc7dbfff220d4b7"
|
||||
"checksum socket2 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)" = "e8b74de517221a2cb01a53349cf54182acdc31a074727d3079068448c0676d85"
|
||||
"checksum solana-ed25519-dalek 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1c21f9d5aa62959872194dfd086feb4e8efec1c2589d27e6a0339904759e99fc"
|
||||
"checksum solana-reed-solomon-erasure 3.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7948662ae34889f594a1c63ea573db43b7500470325291f43ba8d30c8b9773d8"
|
||||
"checksum solana_libra_bytecode_verifier 0.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7db48f7a723c39b9acfc52e7e0778bb5dbe55d14025fd80e17e8ea48b246415c"
|
||||
"checksum solana_libra_canonical_serialization 0.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d6ef5769b20ca8acd70cc0482fa0cade3c512de9f7e0e35e91b65593b9491205"
|
||||
"checksum solana_libra_compiler 0.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8bccb768b90dd07df05810225db6f09f74e6ba91f2edb0951ce758b2c8c17fb6"
|
||||
|
|
|
|||
|
|
@ -80,9 +80,9 @@ solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "0.20.0" }
|
|||
|
||||
# reed-solomon-erasure's simd_c feature fails to build for x86_64-pc-windows-msvc, use pure-rust
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
reed-solomon-erasure = { package = "solana-reed-solomon-erasure", version = "3.1.1", features = ["pure-rust"] }
|
||||
reed-solomon-erasure = "4.0"
|
||||
[target.'cfg(not(windows))'.dependencies]
|
||||
reed-solomon-erasure = { package = "solana-reed-solomon-erasure", version = "3.1.1" }
|
||||
reed-solomon-erasure = { version = "4.0", features = ["simd-accel"] }
|
||||
|
||||
[dependencies.rocksdb]
|
||||
# Avoid the vendored bzip2 within rocksdb-sys that can cause linker conflicts
|
||||
|
|
|
|||
|
|
@ -41,7 +41,8 @@
|
|||
//!
|
||||
//!
|
||||
|
||||
use reed_solomon_erasure::{ParallelParam, ReedSolomon};
|
||||
use reed_solomon_erasure::galois_8::Field;
|
||||
use reed_solomon_erasure::ReedSolomon;
|
||||
|
||||
//TODO(sakridge) pick these values
|
||||
/// Number of data blobs
|
||||
|
|
@ -86,18 +87,17 @@ type Result<T> = std::result::Result<T, reed_solomon_erasure::Error>;
|
|||
/// Represents an erasure "session" with a particular configuration and number of data and coding
|
||||
/// blobs
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Session(ReedSolomon);
|
||||
pub struct Session(ReedSolomon<Field>);
|
||||
|
||||
impl Session {
|
||||
pub fn new(data_count: usize, coding_count: usize) -> Result<Session> {
|
||||
let rs = ReedSolomon::with_pparam(data_count, coding_count, ParallelParam::Disabled)?;
|
||||
let rs = ReedSolomon::new(data_count, coding_count)?;
|
||||
|
||||
Ok(Session(rs))
|
||||
}
|
||||
|
||||
pub fn new_from_config(config: &ErasureConfig) -> Result<Session> {
|
||||
let rs =
|
||||
ReedSolomon::with_pparam(config.num_data, config.num_coding, ParallelParam::Disabled)?;
|
||||
let rs = ReedSolomon::new(config.num_data, config.num_coding)?;
|
||||
|
||||
Ok(Session(rs))
|
||||
}
|
||||
|
|
@ -114,8 +114,8 @@ impl Session {
|
|||
/// * `data` - array of data blocks to recover into
|
||||
/// * `coding` - array of coding blocks
|
||||
/// * `erasures` - list of indices in data where blocks should be recovered
|
||||
pub fn decode_blocks(&self, blocks: &mut [&mut [u8]], present: &[bool]) -> Result<()> {
|
||||
self.0.reconstruct(blocks, present)?;
|
||||
pub fn decode_blocks(&self, blocks: &mut [(&mut [u8], bool)]) -> Result<()> {
|
||||
self.0.reconstruct(blocks)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -175,7 +175,7 @@ pub mod test {
|
|||
}
|
||||
|
||||
let erasure: usize = 1;
|
||||
let present = &mut [true; N_DATA + N_CODING];
|
||||
let mut present = vec![true; N_DATA + N_CODING];
|
||||
present[erasure] = false;
|
||||
let erased = vs[erasure].clone();
|
||||
|
||||
|
|
@ -186,10 +186,11 @@ pub mod test {
|
|||
.iter_mut()
|
||||
.chain(coding_blocks.iter_mut())
|
||||
.map(Vec::as_mut_slice)
|
||||
.zip(present)
|
||||
.collect();
|
||||
|
||||
session
|
||||
.decode_blocks(blocks.as_mut_slice(), present)
|
||||
.decode_blocks(blocks.as_mut_slice())
|
||||
.expect("decoding must succeed");
|
||||
|
||||
trace!("test_coding: vs:");
|
||||
|
|
|
|||
|
|
@ -593,11 +593,12 @@ impl Shredder {
|
|||
|
||||
let session = Session::new(num_data, num_coding).unwrap();
|
||||
|
||||
let mut blocks: Vec<&mut [u8]> = shred_bufs
|
||||
let mut blocks: Vec<(&mut [u8], bool)> = shred_bufs
|
||||
.iter_mut()
|
||||
.map(|x| x[coding_block_offset..].as_mut())
|
||||
.zip(present.clone())
|
||||
.collect();
|
||||
session.decode_blocks(&mut blocks, &present)?;
|
||||
session.decode_blocks(&mut blocks)?;
|
||||
|
||||
let mut num_drained = 0;
|
||||
present
|
||||
|
|
|
|||
Loading…
Reference in New Issue