moves ErasureConfig to blockstore_meta next to ErasureMeta
This commit is contained in:
parent
4b62c93f87
commit
d7aec2a7e6
|
@ -1,8 +1,5 @@
|
||||||
use {
|
use {
|
||||||
crate::{
|
crate::shred::{Shred, ShredType},
|
||||||
erasure::ErasureConfig,
|
|
||||||
shred::{Shred, ShredType},
|
|
||||||
},
|
|
||||||
serde::{Deserialize, Deserializer, Serialize, Serializer},
|
serde::{Deserialize, Deserializer, Serialize, Serializer},
|
||||||
solana_sdk::{clock::Slot, hash::Hash},
|
solana_sdk::{clock::Slot, hash::Hash},
|
||||||
std::{
|
std::{
|
||||||
|
@ -93,6 +90,12 @@ pub struct ErasureMeta {
|
||||||
config: ErasureConfig,
|
config: ErasureConfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||||
|
pub(crate) struct ErasureConfig {
|
||||||
|
num_data: usize,
|
||||||
|
num_coding: usize,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Deserialize, Serialize)]
|
#[derive(Deserialize, Serialize)]
|
||||||
pub struct DuplicateSlotProof {
|
pub struct DuplicateSlotProof {
|
||||||
#[serde(with = "serde_bytes")]
|
#[serde(with = "serde_bytes")]
|
||||||
|
@ -233,10 +236,10 @@ impl ErasureMeta {
|
||||||
match shred.shred_type() {
|
match shred.shred_type() {
|
||||||
ShredType::Data => None,
|
ShredType::Data => None,
|
||||||
ShredType::Code => {
|
ShredType::Code => {
|
||||||
let config = ErasureConfig::new(
|
let config = ErasureConfig {
|
||||||
usize::from(shred.num_data_shreds().ok()?),
|
num_data: usize::from(shred.num_data_shreds().ok()?),
|
||||||
usize::from(shred.num_coding_shreds().ok()?),
|
num_coding: usize::from(shred.num_coding_shreds().ok()?),
|
||||||
);
|
};
|
||||||
let first_coding_index = u64::from(shred.first_coding_index()?);
|
let first_coding_index = u64::from(shred.first_coding_index()?);
|
||||||
let erasure_meta = ErasureMeta {
|
let erasure_meta = ErasureMeta {
|
||||||
set_index: u64::from(shred.fec_set_index()),
|
set_index: u64::from(shred.fec_set_index()),
|
||||||
|
@ -265,12 +268,12 @@ impl ErasureMeta {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn data_shreds_indices(&self) -> Range<u64> {
|
pub(crate) fn data_shreds_indices(&self) -> Range<u64> {
|
||||||
let num_data = self.config.num_data() as u64;
|
let num_data = self.config.num_data as u64;
|
||||||
self.set_index..self.set_index + num_data
|
self.set_index..self.set_index + num_data
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn coding_shreds_indices(&self) -> Range<u64> {
|
pub(crate) fn coding_shreds_indices(&self) -> Range<u64> {
|
||||||
let num_coding = self.config.num_coding() as u64;
|
let num_coding = self.config.num_coding as u64;
|
||||||
self.first_coding_index..self.first_coding_index + num_coding
|
self.first_coding_index..self.first_coding_index + num_coding
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -281,8 +284,8 @@ impl ErasureMeta {
|
||||||
let num_data = index.data().range(self.data_shreds_indices()).count();
|
let num_data = index.data().range(self.data_shreds_indices()).count();
|
||||||
|
|
||||||
let (data_missing, num_needed) = (
|
let (data_missing, num_needed) = (
|
||||||
self.config.num_data().saturating_sub(num_data),
|
self.config.num_data.saturating_sub(num_data),
|
||||||
self.config.num_data().saturating_sub(num_data + num_coding),
|
self.config.num_data.saturating_sub(num_data + num_coding),
|
||||||
);
|
);
|
||||||
|
|
||||||
if data_missing == 0 {
|
if data_missing == 0 {
|
||||||
|
@ -336,8 +339,10 @@ mod test {
|
||||||
use ErasureMetaStatus::*;
|
use ErasureMetaStatus::*;
|
||||||
|
|
||||||
let set_index = 0;
|
let set_index = 0;
|
||||||
let erasure_config = ErasureConfig::new(8, 16);
|
let erasure_config = ErasureConfig {
|
||||||
|
num_data: 8,
|
||||||
|
num_coding: 16,
|
||||||
|
};
|
||||||
let e_meta = ErasureMeta {
|
let e_meta = ErasureMeta {
|
||||||
set_index,
|
set_index,
|
||||||
first_coding_index: set_index,
|
first_coding_index: set_index,
|
||||||
|
@ -347,10 +352,10 @@ mod test {
|
||||||
let mut rng = thread_rng();
|
let mut rng = thread_rng();
|
||||||
let mut index = Index::new(0);
|
let mut index = Index::new(0);
|
||||||
|
|
||||||
let data_indexes = 0..erasure_config.num_data() as u64;
|
let data_indexes = 0..erasure_config.num_data as u64;
|
||||||
let coding_indexes = 0..erasure_config.num_coding() as u64;
|
let coding_indexes = 0..erasure_config.num_coding as u64;
|
||||||
|
|
||||||
assert_eq!(e_meta.status(&index), StillNeed(erasure_config.num_data()));
|
assert_eq!(e_meta.status(&index), StillNeed(erasure_config.num_data));
|
||||||
|
|
||||||
for ix in data_indexes.clone() {
|
for ix in data_indexes.clone() {
|
||||||
index.data_mut().insert(ix);
|
index.data_mut().insert(ix);
|
||||||
|
@ -365,7 +370,7 @@ mod test {
|
||||||
for &idx in data_indexes
|
for &idx in data_indexes
|
||||||
.clone()
|
.clone()
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.choose_multiple(&mut rng, erasure_config.num_data())
|
.choose_multiple(&mut rng, erasure_config.num_data)
|
||||||
{
|
{
|
||||||
index.data_mut().index.remove(&idx);
|
index.data_mut().index.remove(&idx);
|
||||||
|
|
||||||
|
@ -378,7 +383,7 @@ mod test {
|
||||||
|
|
||||||
for &idx in coding_indexes
|
for &idx in coding_indexes
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.choose_multiple(&mut rng, erasure_config.num_coding())
|
.choose_multiple(&mut rng, erasure_config.num_coding)
|
||||||
{
|
{
|
||||||
index.coding_mut().index.remove(&idx);
|
index.coding_mut().index.remove(&idx);
|
||||||
|
|
||||||
|
|
|
@ -1,67 +0,0 @@
|
||||||
//! # Erasure Coding and Recovery
|
|
||||||
//!
|
|
||||||
//! Shreds are logically grouped into erasure sets or blocks. Each set contains 16 sequential data
|
|
||||||
//! shreds and 4 sequential coding shreds.
|
|
||||||
//!
|
|
||||||
//! Coding shreds in each set starting from `start_idx`:
|
|
||||||
//! For each erasure set:
|
|
||||||
//! generate `NUM_CODING` coding_shreds.
|
|
||||||
//! index the coding shreds from `start_idx` to `start_idx + NUM_CODING - 1`.
|
|
||||||
//!
|
|
||||||
//! model of an erasure set, with top row being data shreds and second being coding
|
|
||||||
//! |<======================= NUM_DATA ==============================>|
|
|
||||||
//! |<==== NUM_CODING ===>|
|
|
||||||
//! +---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+
|
|
||||||
//! | D | | D | | D | | D | | D | | D | | D | | D | | D | | D |
|
|
||||||
//! +---+ +---+ +---+ +---+ +---+ . . . +---+ +---+ +---+ +---+ +---+
|
|
||||||
//! | C | | C | | C | | C | | | | | | | | | | | | |
|
|
||||||
//! +---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+
|
|
||||||
//!
|
|
||||||
//! shred structure for coding shreds
|
|
||||||
//!
|
|
||||||
//! + ------- meta is set and used by transport, meta.size is actual length
|
|
||||||
//! | of data in the byte array shred.data
|
|
||||||
//! |
|
|
||||||
//! | + -- data is stuff shipped over the wire, and has an included
|
|
||||||
//! | | header
|
|
||||||
//! V V
|
|
||||||
//! +----------+------------------------------------------------------------+
|
|
||||||
//! | meta | data |
|
|
||||||
//! |+---+-- |+---+---+---+---+------------------------------------------+|
|
|
||||||
//! || s | . || i | | f | s | ||
|
|
||||||
//! || i | . || n | i | l | i | ||
|
|
||||||
//! || z | . || d | d | a | z | shred.data(), or shred.data_mut() ||
|
|
||||||
//! || e | || e | | g | e | ||
|
|
||||||
//! |+---+-- || x | | s | | ||
|
|
||||||
//! | |+---+---+---+---+------------------------------------------+|
|
|
||||||
//! +----------+------------------------------------------------------------+
|
|
||||||
//! | |<=== coding shred part for "coding" =======>|
|
|
||||||
//! | |
|
|
||||||
//! |<============== data shred part for "coding" ==============>|
|
|
||||||
//!
|
|
||||||
//!
|
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
|
||||||
pub struct ErasureConfig {
|
|
||||||
num_data: usize,
|
|
||||||
num_coding: usize,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ErasureConfig {
|
|
||||||
pub(crate) fn new(num_data: usize, num_coding: usize) -> ErasureConfig {
|
|
||||||
ErasureConfig {
|
|
||||||
num_data,
|
|
||||||
num_coding,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn num_data(self) -> usize {
|
|
||||||
self.num_data
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn num_coding(self) -> usize {
|
|
||||||
self.num_coding
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -15,7 +15,6 @@ pub mod blockstore_db;
|
||||||
pub mod blockstore_meta;
|
pub mod blockstore_meta;
|
||||||
pub mod blockstore_processor;
|
pub mod blockstore_processor;
|
||||||
pub mod builtins;
|
pub mod builtins;
|
||||||
pub mod erasure;
|
|
||||||
pub mod genesis_utils;
|
pub mod genesis_utils;
|
||||||
pub mod leader_schedule;
|
pub mod leader_schedule;
|
||||||
pub mod leader_schedule_cache;
|
pub mod leader_schedule_cache;
|
||||||
|
|
Loading…
Reference in New Issue