Use the new datapoint macro for RocksDB column family metrics (#25505)
#### Summary of Changes Use the new datapoint macro that supports group-by for RocksDB column family metrics. By using the new macro, we can further remove large chunks of boilerplate code that try to work around the previous datapoint macro that does not support group-by.
This commit is contained in:
parent
56d38e4954
commit
bcff88bf42
|
@ -4,7 +4,7 @@ use {
|
||||||
blockstore_meta,
|
blockstore_meta,
|
||||||
blockstore_metrics::{
|
blockstore_metrics::{
|
||||||
maybe_enable_rocksdb_perf, report_rocksdb_read_perf, report_rocksdb_write_perf,
|
maybe_enable_rocksdb_perf, report_rocksdb_read_perf, report_rocksdb_write_perf,
|
||||||
BlockstoreRocksDbColumnFamilyMetrics, ColumnMetrics, PerfSamplingStatus,
|
BlockstoreRocksDbColumnFamilyMetrics, PerfSamplingStatus,
|
||||||
},
|
},
|
||||||
blockstore_options::{
|
blockstore_options::{
|
||||||
AccessType, BlockstoreOptions, LedgerColumnOptions, ShredStorageType,
|
AccessType, BlockstoreOptions, LedgerColumnOptions, ShredStorageType,
|
||||||
|
@ -953,7 +953,7 @@ pub struct Database {
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct LedgerColumn<C>
|
pub struct LedgerColumn<C>
|
||||||
where
|
where
|
||||||
C: Column + ColumnName + ColumnMetrics,
|
C: Column + ColumnName,
|
||||||
{
|
{
|
||||||
backend: Arc<Rocks>,
|
backend: Arc<Rocks>,
|
||||||
column: PhantomData<C>,
|
column: PhantomData<C>,
|
||||||
|
@ -962,7 +962,7 @@ where
|
||||||
write_perf_status: PerfSamplingStatus,
|
write_perf_status: PerfSamplingStatus,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<C: Column + ColumnName + ColumnMetrics> LedgerColumn<C> {
|
impl<C: Column + ColumnName> LedgerColumn<C> {
|
||||||
pub fn submit_rocksdb_cf_metrics(&self) {
|
pub fn submit_rocksdb_cf_metrics(&self) {
|
||||||
let cf_rocksdb_metrics = BlockstoreRocksDbColumnFamilyMetrics {
|
let cf_rocksdb_metrics = BlockstoreRocksDbColumnFamilyMetrics {
|
||||||
total_sst_files_size: self
|
total_sst_files_size: self
|
||||||
|
@ -1014,7 +1014,7 @@ impl<C: Column + ColumnName + ColumnMetrics> LedgerColumn<C> {
|
||||||
.get_int_property(RocksProperties::BACKGROUND_ERRORS)
|
.get_int_property(RocksProperties::BACKGROUND_ERRORS)
|
||||||
.unwrap_or(BLOCKSTORE_METRICS_ERROR),
|
.unwrap_or(BLOCKSTORE_METRICS_ERROR),
|
||||||
};
|
};
|
||||||
C::report_cf_metrics(cf_rocksdb_metrics, &self.column_options);
|
cf_rocksdb_metrics.report_metrics(C::NAME, &self.column_options);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1076,7 +1076,7 @@ impl Database {
|
||||||
|
|
||||||
pub fn column<C>(&self) -> LedgerColumn<C>
|
pub fn column<C>(&self) -> LedgerColumn<C>
|
||||||
where
|
where
|
||||||
C: Column + ColumnName + ColumnMetrics,
|
C: Column + ColumnName,
|
||||||
{
|
{
|
||||||
LedgerColumn {
|
LedgerColumn {
|
||||||
backend: Arc::clone(&self.backend),
|
backend: Arc::clone(&self.backend),
|
||||||
|
@ -1132,7 +1132,7 @@ impl Database {
|
||||||
|
|
||||||
impl<C> LedgerColumn<C>
|
impl<C> LedgerColumn<C>
|
||||||
where
|
where
|
||||||
C: Column + ColumnName + ColumnMetrics,
|
C: Column + ColumnName,
|
||||||
{
|
{
|
||||||
pub fn get_bytes(&self, key: C::Index) -> Result<Option<Vec<u8>>> {
|
pub fn get_bytes(&self, key: C::Index) -> Result<Option<Vec<u8>>> {
|
||||||
let is_perf_enabled = maybe_enable_rocksdb_perf(
|
let is_perf_enabled = maybe_enable_rocksdb_perf(
|
||||||
|
@ -1241,7 +1241,7 @@ where
|
||||||
|
|
||||||
impl<C> LedgerColumn<C>
|
impl<C> LedgerColumn<C>
|
||||||
where
|
where
|
||||||
C: TypedColumn + ColumnName + ColumnMetrics,
|
C: TypedColumn + ColumnName,
|
||||||
{
|
{
|
||||||
pub fn get(&self, key: C::Index) -> Result<Option<C::Type>> {
|
pub fn get(&self, key: C::Index) -> Result<Option<C::Type>> {
|
||||||
let mut result = Ok(None);
|
let mut result = Ok(None);
|
||||||
|
@ -1303,7 +1303,7 @@ where
|
||||||
|
|
||||||
impl<C> LedgerColumn<C>
|
impl<C> LedgerColumn<C>
|
||||||
where
|
where
|
||||||
C: ProtobufColumn + ColumnName + ColumnMetrics,
|
C: ProtobufColumn + ColumnName,
|
||||||
{
|
{
|
||||||
pub fn get_protobuf_or_bincode<T: DeserializeOwned + Into<C::Type>>(
|
pub fn get_protobuf_or_bincode<T: DeserializeOwned + Into<C::Type>>(
|
||||||
&self,
|
&self,
|
||||||
|
|
|
@ -1,8 +1,5 @@
|
||||||
use {
|
use {
|
||||||
crate::{
|
crate::blockstore_options::LedgerColumnOptions,
|
||||||
blockstore_db::columns,
|
|
||||||
blockstore_options::{LedgerColumnOptions, ShredStorageType},
|
|
||||||
},
|
|
||||||
rocksdb::{
|
rocksdb::{
|
||||||
perf::{set_perf_stats, PerfMetric, PerfStatsLevel},
|
perf::{set_perf_stats, PerfMetric, PerfStatsLevel},
|
||||||
PerfContext,
|
PerfContext,
|
||||||
|
@ -12,63 +9,11 @@ use {
|
||||||
std::{
|
std::{
|
||||||
cell::RefCell,
|
cell::RefCell,
|
||||||
fmt::Debug,
|
fmt::Debug,
|
||||||
sync::{
|
sync::atomic::{AtomicU64, AtomicUsize, Ordering},
|
||||||
atomic::{AtomicU64, AtomicUsize, Ordering},
|
|
||||||
Arc,
|
|
||||||
},
|
|
||||||
time::{Duration, Instant},
|
time::{Duration, Instant},
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
#[macro_export]
|
|
||||||
macro_rules! rocksdb_metric_header {
|
|
||||||
($metric_name:literal, $cf_name:literal, $column_options:expr) => {
|
|
||||||
match $column_options.shred_storage_type {
|
|
||||||
ShredStorageType::RocksLevel =>
|
|
||||||
rocksdb_metric_header!(@compression_type $metric_name, $cf_name, $column_options, "rocks_level"),
|
|
||||||
ShredStorageType::RocksFifo(_) =>
|
|
||||||
rocksdb_metric_header!(@compression_type $metric_name, $cf_name, $column_options, "rocks_fifo"),
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
(@compression_type $metric_name:literal, $cf_name:literal, $column_options:expr, $storage_type:literal) => {
|
|
||||||
match $column_options.compression_type {
|
|
||||||
$crate::blockstore_options::BlockstoreCompressionType::None => rocksdb_metric_header!(@all_fields
|
|
||||||
$metric_name,
|
|
||||||
$cf_name,
|
|
||||||
$storage_type,
|
|
||||||
"None"
|
|
||||||
),
|
|
||||||
$crate::blockstore_options::BlockstoreCompressionType::Snappy => rocksdb_metric_header!(@all_fields
|
|
||||||
$metric_name,
|
|
||||||
$cf_name,
|
|
||||||
$storage_type,
|
|
||||||
"Snappy"
|
|
||||||
),
|
|
||||||
$crate::blockstore_options::BlockstoreCompressionType::Lz4 => rocksdb_metric_header!(@all_fields
|
|
||||||
$metric_name,
|
|
||||||
$cf_name,
|
|
||||||
$storage_type,
|
|
||||||
"Lz4"
|
|
||||||
),
|
|
||||||
$crate::blockstore_options::BlockstoreCompressionType::Zlib => rocksdb_metric_header!(@all_fields
|
|
||||||
$metric_name,
|
|
||||||
$cf_name,
|
|
||||||
$storage_type,
|
|
||||||
"Zlib"
|
|
||||||
),
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
(@all_fields $metric_name:literal, $cf_name:literal, $storage_type:literal, $compression_type:literal) => {
|
|
||||||
concat!($metric_name,
|
|
||||||
",cf_name=", $cf_name,
|
|
||||||
",storage=", $storage_type,
|
|
||||||
",compression=", $compression_type,
|
|
||||||
)
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A metrics struct that exposes RocksDB's column family properties.
|
/// A metrics struct that exposes RocksDB's column family properties.
|
||||||
///
|
///
|
||||||
/// Here we only expose a subset of all the internal properties which are
|
/// Here we only expose a subset of all the internal properties which are
|
||||||
|
@ -170,9 +115,13 @@ impl BlockstoreRocksDbColumnFamilyMetrics {
|
||||||
/// `metric_name_and_cf_tag` with the following format.
|
/// `metric_name_and_cf_tag` with the following format.
|
||||||
///
|
///
|
||||||
/// For example, "blockstore_rocksdb_cfs,cf_name=shred_data".
|
/// For example, "blockstore_rocksdb_cfs,cf_name=shred_data".
|
||||||
pub fn report_metrics(&self, metric_name_and_cf_tag: &'static str) {
|
pub fn report_metrics(&self, cf_name: &'static str, column_options: &LedgerColumnOptions) {
|
||||||
datapoint_info!(
|
datapoint_info!(
|
||||||
metric_name_and_cf_tag,
|
"blockstore_rocksdb_cfs",
|
||||||
|
// tags that support group-by operations
|
||||||
|
"cf_name" => cf_name,
|
||||||
|
"storage" => column_options.get_storage_type_string(),
|
||||||
|
"compression" => column_options.get_compression_type_string(),
|
||||||
// Size related
|
// Size related
|
||||||
(
|
(
|
||||||
"total_sst_files_size",
|
"total_sst_files_size",
|
||||||
|
@ -558,270 +507,3 @@ impl PerfSamplingStatus {
|
||||||
.is_ok()
|
.is_ok()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub trait ColumnMetrics {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::TransactionStatus {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"transaction_status",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::AddressSignatures {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"address_signatures",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::TransactionMemos {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"transaction_memos",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::TransactionStatusIndex {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"transaction_status_index",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::Rewards {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"rewards",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::Blocktime {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"blocktime",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::PerfSamples {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"perf_samples",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::BlockHeight {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"block_height",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::ProgramCosts {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"program_costs",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::ShredCode {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"shred_code",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::ShredData {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"shred_data",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::Index {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"index",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::DeadSlots {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"dead_slots",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::DuplicateSlots {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"duplicate_slots",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::Orphans {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"orphans",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::BankHash {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"bank_hash",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::OptimisticSlots {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"optimistic_slots",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::Root {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"root",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::SlotMeta {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"slot_meta",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ColumnMetrics for columns::ErasureMeta {
|
|
||||||
fn report_cf_metrics(
|
|
||||||
cf_metrics: BlockstoreRocksDbColumnFamilyMetrics,
|
|
||||||
column_options: &Arc<LedgerColumnOptions>,
|
|
||||||
) {
|
|
||||||
cf_metrics.report_metrics(rocksdb_metric_header!(
|
|
||||||
"blockstore_rocksdb_cfs",
|
|
||||||
"erasure_meta",
|
|
||||||
column_options
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
@ -13,7 +13,6 @@ pub mod blockstore;
|
||||||
pub mod ancestor_iterator;
|
pub mod ancestor_iterator;
|
||||||
pub mod blockstore_db;
|
pub mod blockstore_db;
|
||||||
pub mod blockstore_meta;
|
pub mod blockstore_meta;
|
||||||
#[macro_use]
|
|
||||||
pub mod blockstore_metrics;
|
pub mod blockstore_metrics;
|
||||||
pub mod blockstore_options;
|
pub mod blockstore_options;
|
||||||
pub mod blockstore_processor;
|
pub mod blockstore_processor;
|
||||||
|
|
Loading…
Reference in New Issue