diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index a02f02cc74..c3c59e7e4b 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -46,7 +46,7 @@ use { borrow::Cow, cell::RefCell, cmp, - collections::{BTreeMap, HashMap, HashSet}, + collections::{hash_map::Entry as HashMapEntry, BTreeMap, HashMap, HashSet}, convert::TryInto, fs, io::{Error as IoError, ErrorKind}, @@ -213,11 +213,19 @@ pub struct BlockstoreInsertionMetrics { pub num_inserted: u64, pub num_repair: u64, pub num_recovered: usize, + num_recovered_blockstore_error: usize, pub num_recovered_inserted: usize, pub num_recovered_failed_sig: usize, pub num_recovered_failed_invalid: usize, pub num_recovered_exists: usize, pub index_meta_time: u64, + num_data_shreds_exists: usize, + num_data_shreds_invalid: usize, + num_data_shreds_blockstore_error: usize, + num_coding_shreds_exists: usize, + num_coding_shreds_invalid: usize, + num_coding_shreds_invalid_erasure_config: usize, + num_coding_shreds_inserted: usize, } impl SlotMetaWorkingSetEntry { @@ -277,6 +285,38 @@ impl BlockstoreInsertionMetrics { self.num_recovered_exists as i64, i64 ), + ( + "num_recovered_blockstore_error", + self.num_recovered_blockstore_error, + i64 + ), + ("num_data_shreds_exists", self.num_data_shreds_exists, i64), + ("num_data_shreds_invalid", self.num_data_shreds_invalid, i64), + ( + "num_data_shreds_blockstore_error", + self.num_data_shreds_blockstore_error, + i64 + ), + ( + "num_coding_shreds_exists", + self.num_coding_shreds_exists, + i64 + ), + ( + "num_coding_shreds_invalid", + self.num_coding_shreds_invalid, + i64 + ), + ( + "num_coding_shreds_invalid_erasure_config", + self.num_coding_shreds_invalid_erasure_config, + i64 + ), + ( + "num_coding_shreds_inserted", + self.num_coding_shreds_inserted, + i64 + ), ); } } @@ -838,7 +878,7 @@ impl Blockstore { } else { ShredSource::Turbine }; - if let Ok(completed_data_sets) = self.check_insert_data_shred( + match self.check_insert_data_shred( shred, &mut erasure_metas, &mut index_working_set, @@ -851,10 +891,18 @@ impl Blockstore { leader_schedule, shred_source, ) { - newly_completed_data_sets.extend(completed_data_sets); - inserted_indices.push(i); - metrics.num_inserted += 1; - } + Err(InsertDataShredError::Exists) => metrics.num_data_shreds_exists += 1, + Err(InsertDataShredError::InvalidShred) => metrics.num_data_shreds_invalid += 1, + Err(InsertDataShredError::BlockstoreError(err)) => { + metrics.num_data_shreds_blockstore_error += 1; + error!("blockstore error: {}", err); + } + Ok(completed_data_sets) => { + newly_completed_data_sets.extend(completed_data_sets); + inserted_indices.push(i); + metrics.num_inserted += 1; + } + }; } else if shred.is_code() { self.check_cache_coding_shred( shred, @@ -865,6 +913,7 @@ impl Blockstore { handle_duplicate, is_trusted, is_repaired, + metrics, ); } else { panic!("There should be no other case"); @@ -914,7 +963,11 @@ impl Blockstore { metrics.num_recovered_failed_invalid += 1; None } - Err(InsertDataShredError::BlockstoreError(_)) => None, + Err(InsertDataShredError::BlockstoreError(err)) => { + metrics.num_recovered_blockstore_error += 1; + error!("blockstore error: {}", err); + None + } Ok(completed_data_sets) => { newly_completed_data_sets.extend(completed_data_sets); metrics.num_recovered_inserted += 1; @@ -1059,6 +1112,7 @@ impl Blockstore { || shred1.coding_header.num_data_shreds != shred2.coding_header.num_data_shreds } + #[allow(clippy::too_many_arguments)] fn check_cache_coding_shred( &self, shred: Shred, @@ -1069,6 +1123,7 @@ impl Blockstore { handle_duplicate: &F, is_trusted: bool, is_repaired: bool, + metrics: &mut BlockstoreInsertionMetrics, ) -> bool where F: Fn(Shred), @@ -1086,11 +1141,13 @@ impl Blockstore { if !is_trusted { if index_meta.coding().is_present(shred_index) { + metrics.num_coding_shreds_exists += 1; handle_duplicate(shred); return false; } if !Blockstore::should_insert_coding_shred(&shred, &self.last_root) { + metrics.num_coding_shreds_invalid += 1; return false; } } @@ -1109,6 +1166,7 @@ impl Blockstore { }); if erasure_config != erasure_meta.config { + metrics.num_coding_shreds_invalid_erasure_config += 1; let conflicting_shred = self.find_conflicting_coding_shred( &shred, slot, @@ -1149,10 +1207,11 @@ impl Blockstore { // committed index_meta.coding_mut().set_present(shred_index, true); - just_received_coding_shreds - .entry((slot, shred_index)) - .or_insert_with(|| shred); - + if let HashMapEntry::Vacant(entry) = just_received_coding_shreds.entry((slot, shred_index)) + { + metrics.num_coding_shreds_inserted += 1; + entry.insert(shred); + } true } @@ -5665,6 +5724,7 @@ pub mod tests { }, false, false, + &mut BlockstoreInsertionMetrics::default(), )); // insert again fails on dupe @@ -5681,6 +5741,7 @@ pub mod tests { }, false, false, + &mut BlockstoreInsertionMetrics::default(), )); assert_eq!(counter.load(Ordering::Relaxed), 1); }