save erasure set size, initialize coding blob erasures with that size (#3910)
* save erasure set size, initialize coding blob erasures with that size * fixup * fixup
This commit is contained in:
parent
d83a71d89f
commit
3ab492ccf8
|
@ -462,6 +462,7 @@ impl Blocktree {
|
||||||
.unwrap_or_else(|| ErasureMeta::new(set_index));
|
.unwrap_or_else(|| ErasureMeta::new(set_index));
|
||||||
|
|
||||||
erasure_meta.set_coding_present(index, true);
|
erasure_meta.set_coding_present(index, true);
|
||||||
|
erasure_meta.set_size(bytes.len() - BLOB_HEADER_SIZE);
|
||||||
|
|
||||||
let mut writebatch = self.db.batch()?;
|
let mut writebatch = self.db.batch()?;
|
||||||
|
|
||||||
|
@ -1047,11 +1048,12 @@ impl Blocktree {
|
||||||
let erasure_meta = self.erasure_meta_cf.get((slot, set_index))?.unwrap();
|
let erasure_meta = self.erasure_meta_cf.get((slot, set_index))?.unwrap();
|
||||||
|
|
||||||
let start_idx = erasure_meta.start_index();
|
let start_idx = erasure_meta.start_index();
|
||||||
|
let size = erasure_meta.size();
|
||||||
|
|
||||||
let (data_end_idx, coding_end_idx) = erasure_meta.end_indexes();
|
let (data_end_idx, coding_end_idx) = erasure_meta.end_indexes();
|
||||||
|
|
||||||
let present = &mut [true; ERASURE_SET_SIZE];
|
let present = &mut [true; ERASURE_SET_SIZE];
|
||||||
let mut blobs = Vec::with_capacity(ERASURE_SET_SIZE);
|
let mut blobs = Vec::with_capacity(ERASURE_SET_SIZE);
|
||||||
let mut size = 0;
|
|
||||||
|
|
||||||
for i in start_idx..coding_end_idx {
|
for i in start_idx..coding_end_idx {
|
||||||
if erasure_meta.is_coding_present(i) {
|
if erasure_meta.is_coding_present(i) {
|
||||||
|
@ -1062,10 +1064,6 @@ impl Blocktree {
|
||||||
|
|
||||||
blob_bytes.drain(..BLOB_HEADER_SIZE);
|
blob_bytes.drain(..BLOB_HEADER_SIZE);
|
||||||
|
|
||||||
if size == 0 {
|
|
||||||
size = blob_bytes.len();
|
|
||||||
}
|
|
||||||
|
|
||||||
blobs.push(blob_bytes);
|
blobs.push(blob_bytes);
|
||||||
} else {
|
} else {
|
||||||
let set_relative_idx = (i - start_idx) as usize + NUM_DATA;
|
let set_relative_idx = (i - start_idx) as usize + NUM_DATA;
|
||||||
|
|
|
@ -63,6 +63,8 @@ impl SlotMeta {
|
||||||
pub struct ErasureMeta {
|
pub struct ErasureMeta {
|
||||||
/// Which erasure set in the slot this is
|
/// Which erasure set in the slot this is
|
||||||
pub set_index: u64,
|
pub set_index: u64,
|
||||||
|
/// Size of shards in this erasure set
|
||||||
|
pub size: usize,
|
||||||
/// Bitfield representing presence/absence of data blobs
|
/// Bitfield representing presence/absence of data blobs
|
||||||
pub data: u64,
|
pub data: u64,
|
||||||
/// Bitfield representing presence/absence of coding blobs
|
/// Bitfield representing presence/absence of coding blobs
|
||||||
|
@ -80,6 +82,7 @@ impl ErasureMeta {
|
||||||
pub fn new(set_index: u64) -> ErasureMeta {
|
pub fn new(set_index: u64) -> ErasureMeta {
|
||||||
ErasureMeta {
|
ErasureMeta {
|
||||||
set_index,
|
set_index,
|
||||||
|
size: 0,
|
||||||
data: 0,
|
data: 0,
|
||||||
coding: 0,
|
coding: 0,
|
||||||
}
|
}
|
||||||
|
@ -91,6 +94,7 @@ impl ErasureMeta {
|
||||||
NUM_CODING - self.coding.count_ones() as usize,
|
NUM_CODING - self.coding.count_ones() as usize,
|
||||||
);
|
);
|
||||||
if data_missing > 0 && data_missing + coding_missing <= NUM_CODING {
|
if data_missing > 0 && data_missing + coding_missing <= NUM_CODING {
|
||||||
|
assert!(self.size != 0);
|
||||||
ErasureMetaStatus::CanRecover
|
ErasureMetaStatus::CanRecover
|
||||||
} else if data_missing == 0 {
|
} else if data_missing == 0 {
|
||||||
ErasureMetaStatus::DataFull
|
ErasureMetaStatus::DataFull
|
||||||
|
@ -112,6 +116,14 @@ impl ErasureMeta {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn set_size(&mut self, size: usize) {
|
||||||
|
self.size = size;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn size(&self) -> usize {
|
||||||
|
self.size
|
||||||
|
}
|
||||||
|
|
||||||
pub fn set_coding_present(&mut self, index: u64, present: bool) {
|
pub fn set_coding_present(&mut self, index: u64, present: bool) {
|
||||||
let set_index = Self::set_index_for(index);
|
let set_index = Self::set_index_for(index);
|
||||||
|
|
||||||
|
@ -193,12 +205,7 @@ fn test_meta_indexes() {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_meta_coding_present() {
|
fn test_meta_coding_present() {
|
||||||
let set_index = 0;
|
let mut e_meta = ErasureMeta::default();
|
||||||
let mut e_meta = ErasureMeta {
|
|
||||||
set_index,
|
|
||||||
data: 0,
|
|
||||||
coding: 0,
|
|
||||||
};
|
|
||||||
|
|
||||||
for i in 0..NUM_CODING as u64 {
|
for i in 0..NUM_CODING as u64 {
|
||||||
e_meta.set_coding_present(i, true);
|
e_meta.set_coding_present(i, true);
|
||||||
|
@ -221,12 +228,7 @@ fn test_meta_coding_present() {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_erasure_meta_status() {
|
fn test_erasure_meta_status() {
|
||||||
let set_index = 0;
|
let mut e_meta = ErasureMeta::default();
|
||||||
let mut e_meta = ErasureMeta {
|
|
||||||
set_index,
|
|
||||||
data: 0,
|
|
||||||
coding: 0,
|
|
||||||
};
|
|
||||||
|
|
||||||
assert_eq!(e_meta.status(), ErasureMetaStatus::StillNeed(NUM_DATA));
|
assert_eq!(e_meta.status(), ErasureMetaStatus::StillNeed(NUM_DATA));
|
||||||
|
|
||||||
|
@ -236,6 +238,7 @@ fn test_erasure_meta_status() {
|
||||||
assert_eq!(e_meta.status(), ErasureMetaStatus::DataFull);
|
assert_eq!(e_meta.status(), ErasureMetaStatus::DataFull);
|
||||||
|
|
||||||
e_meta.coding = 0x0e;
|
e_meta.coding = 0x0e;
|
||||||
|
e_meta.size = 1;
|
||||||
assert_eq!(e_meta.status(), ErasureMetaStatus::DataFull);
|
assert_eq!(e_meta.status(), ErasureMetaStatus::DataFull);
|
||||||
|
|
||||||
e_meta.data = 0b0111_1111_1111_1111;
|
e_meta.data = 0b0111_1111_1111_1111;
|
||||||
|
@ -263,12 +266,7 @@ fn test_erasure_meta_status() {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_meta_data_present() {
|
fn test_meta_data_present() {
|
||||||
let set_index = 0;
|
let mut e_meta = ErasureMeta::default();
|
||||||
let mut e_meta = ErasureMeta {
|
|
||||||
set_index,
|
|
||||||
data: 0,
|
|
||||||
coding: 0,
|
|
||||||
};
|
|
||||||
|
|
||||||
for i in 0..NUM_DATA as u64 {
|
for i in 0..NUM_DATA as u64 {
|
||||||
e_meta.set_data_present(i, true);
|
e_meta.set_data_present(i, true);
|
||||||
|
|
Loading…
Reference in New Issue