blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

(LedgerStore) Include storage type as a tag in RocksDB metric reporting (#23523) 

#### Summary of Changes
This PR further enables group by operation on storage type in blockstore_rocksdb_cfs metrics.
Such group-by allows us to further compare the performance metrics between rocks-level and
rocks-fifo.

To make things extensible, this PR introduces BlockstoreAdvancedOptions and move shred_storage_type. 
All fields in BlockstoreAdvancedOptions will support group-by operation in blockstore_rocksdb_cfs.

Dependency: #23580
This commit is contained in:
Yueh-Hsuan Chiang 2022-03-11 15:17:34 -08:00 committed by GitHub
parent b1da7cff66
commit 1e20bd8f9a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 149 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
core/src/validator.rs
View File

@ -36,7 +36,7 @@ use {
solana_ledger::{
bank_forks_utils,
blockstore::{Blockstore, BlockstoreSignals, CompletedSlotsReceiver, PurgeType},
blockstore_db::{BlockstoreOptions, BlockstoreRecoveryMode, ShredStorageType},
blockstore_db::{BlockstoreAdvancedOptions, BlockstoreOptions, BlockstoreRecoveryMode},
blockstore_processor::{self, TransactionStatusSender},
leader_schedule::FixedSchedule,
leader_schedule_cache::LeaderScheduleCache,
@ -165,7 +165,7 @@ pub struct ValidatorConfig {
pub no_wait_for_vote_to_start_leader: bool,
pub accounts_shrink_ratio: AccountShrinkThreshold,
pub wait_to_vote_slot: Option<Slot>,
pub shred_storage_type: ShredStorageType,
pub blockstore_advanced_options: BlockstoreAdvancedOptions,
}
impl Default for ValidatorConfig {
@ -226,7 +226,7 @@ impl Default for ValidatorConfig {
accounts_shrink_ratio: AccountShrinkThreshold::default(),
accounts_db_config: None,
wait_to_vote_slot: None,
shred_storage_type: ShredStorageType::RocksLevel,
blockstore_advanced_options: BlockstoreAdvancedOptions::default(),
}
}
}
@ -1259,7 +1259,7 @@ fn new_banks_from_ledger(
BlockstoreOptions {
recovery_mode: config.wal_recovery_mode.clone(),
enforce_ulimit_nofile,
shred_storage_type: config.shred_storage_type.clone(),
advanced_options: config.blockstore_advanced_options.clone(),
..BlockstoreOptions::default()
},
)

17
core/tests/ledger_cleanup.rs
View File

@ -9,7 +9,10 @@ mod tests {
solana_core::ledger_cleanup_service::LedgerCleanupService,
solana_ledger::{
blockstore::{make_many_slot_shreds, Blockstore},
blockstore_db::{BlockstoreOptions, BlockstoreRocksFifoOptions, ShredStorageType},
blockstore_db::{
BlockstoreAdvancedOptions, BlockstoreOptions, BlockstoreRocksFifoOptions,
ShredStorageType,
},
get_tmp_ledger_path,
},
solana_measure::measure::Measure,
@ -348,10 +351,14 @@ mod tests {
&ledger_path,
if config.fifo_compaction {
BlockstoreOptions {
shred_storage_type: ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions {
shred_data_cf_size: config.shred_data_cf_size,
..BlockstoreRocksFifoOptions::default()
}),
advanced_options: BlockstoreAdvancedOptions {
shred_storage_type: ShredStorageType::RocksFifo(
BlockstoreRocksFifoOptions {
shred_data_cf_size: config.shred_data_cf_size,
..BlockstoreRocksFifoOptions::default()
},
),
},
..BlockstoreOptions::default()
}
} else {

4
genesis/src/main.rs
View File

@ -15,7 +15,7 @@ use {
solana_genesis::{genesis_accounts::add_genesis_accounts, Base64Account},
solana_ledger::{
blockstore::create_new_ledger,
blockstore_db::{AccessType, ShredStorageType},
blockstore_db::{AccessType, BlockstoreAdvancedOptions},
},
solana_runtime::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
solana_sdk::{
@ -633,7 +633,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
&genesis_config,
max_genesis_archive_unpacked_size,
AccessType::PrimaryOnly,
ShredStorageType::default(),
BlockstoreAdvancedOptions::default(),
)?;
println!("{}", genesis_config);

5
ledger-tool/src/main.rs
View File

@ -24,7 +24,8 @@ use {
bank_forks_utils,
blockstore::{create_new_ledger, Blockstore, PurgeType},
blockstore_db::{
self, AccessType, BlockstoreOptions, BlockstoreRecoveryMode, Database, ShredStorageType,
self, AccessType, BlockstoreAdvancedOptions, BlockstoreOptions, BlockstoreRecoveryMode,
Database,
},
blockstore_processor::ProcessOptions,
shred::Shred,
@ -1721,7 +1722,7 @@ fn main() {
&genesis_config,
solana_runtime::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
AccessType::PrimaryOnly,
ShredStorageType::default(),
BlockstoreAdvancedOptions::default(),
)
.unwrap_or_else(|err| {
eprintln!("Failed to write genesis config: {:?}", err);

121
ledger/src/blockstore.rs
View File

@ -5,8 +5,9 @@ use {
crate::{
ancestor_iterator::AncestorIterator,
blockstore_db::{
columns as cf, AccessType, BlockstoreOptions, Column, ColumnName, Database,
IteratorDirection, IteratorMode, LedgerColumn, Result, ShredStorageType, WriteBatch,
columns as cf, AccessType, BlockstoreAdvancedOptions, BlockstoreOptions, Column,
ColumnName, Database, IteratorDirection, IteratorMode, LedgerColumn, Result,
ShredStorageType, WriteBatch,
},
blockstore_meta::*,
leader_schedule_cache::LeaderScheduleCache,
@ -177,6 +178,7 @@ pub struct Blockstore {
pub lowest_cleanup_slot: Arc<RwLock<Slot>>,
no_compaction: bool,
slots_stats: Arc<Mutex<SlotsStats>>,
advanced_options: BlockstoreAdvancedOptions,
}
struct SlotsStats {
@ -519,8 +521,20 @@ impl BlockstoreRocksDbColumnFamilyMetrics {
}
macro_rules! rocksdb_metric_header {
($metric_name:literal, $cf_name:literal) => {
concat!($metric_name, ",cf_name=", $cf_name)
($metric_name:literal, $cf_name:literal, $advanced_options:expr) => {
match $advanced_options.shred_storage_type {
ShredStorageType::RocksLevel =>
rocksdb_metric_header!(@all_fields $metric_name, $cf_name, "rocks_level"),
ShredStorageType::RocksFifo(_) =>
rocksdb_metric_header!(@all_fields $metric_name, $cf_name, "rocks_fifo"),
}
};
(@all_fields $metric_name:literal, $cf_name:literal, $storage_type:literal) => {
concat!($metric_name,
",cf_name=", $cf_name,
",storage=", $storage_type,
)
};
}
use rocksdb_metric_header;
@ -554,8 +568,10 @@ impl Blockstore {
fn do_open(ledger_path: &Path, options: BlockstoreOptions) -> Result<Blockstore> {
fs::create_dir_all(&ledger_path)?;
let blockstore_path =
ledger_path.join(Self::blockstore_directory(&options.shred_storage_type));
let blockstore_path = ledger_path.join(Self::blockstore_directory(
&options.advanced_options.shred_storage_type,
));
let advanced_options = options.advanced_options.clone();
adjust_ulimit_nofile(options.enforce_ulimit_nofile)?;
@ -648,6 +664,7 @@ impl Blockstore {
lowest_cleanup_slot: Arc::new(RwLock::new(0)),
no_compaction: false,
slots_stats: Arc::new(Mutex::new(SlotsStats::default())),
advanced_options,
};
if initialize_transaction_status_index {
blockstore.initialize_transaction_status_index()?;
@ -944,81 +961,101 @@ impl Blockstore {
/// Collects and reports [`BlockstoreRocksDbColumnFamilyMetrics`] for the
/// all the column families.
pub fn submit_rocksdb_cf_metrics_for_all_cfs(&self) {
let advanced_options = &self.advanced_options;
self.submit_rocksdb_cf_metrics::<cf::SlotMeta>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"slot_meta"
"slot_meta",
advanced_options
));
self.submit_rocksdb_cf_metrics::<cf::DeadSlots>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"dead_slots"
"dead_slots",
advanced_options
));
self.submit_rocksdb_cf_metrics::<cf::DuplicateSlots>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"duplicate_slots"
"duplicate_slots",
advanced_options
));
self.submit_rocksdb_cf_metrics::<cf::ErasureMeta>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"erasure_meta"
"erasure_meta",
advanced_options
));
self.submit_rocksdb_cf_metrics::<cf::Orphans>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"orphans"
"orphans",
advanced_options
));
self.submit_rocksdb_cf_metrics::<cf::BankHash>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"bank_hash"
"bank_hash",
advanced_options
));
self.submit_rocksdb_cf_metrics::<cf::Root>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"root"
"root",
advanced_options
));
self.submit_rocksdb_cf_metrics::<cf::Index>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"index"
"index",
advanced_options
));
self.submit_rocksdb_cf_metrics::<cf::ShredData>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"shred_data"
"shred_data",
advanced_options
));
self.submit_rocksdb_cf_metrics::<cf::ShredCode>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"shred_code"
"shred_code",
advanced_options
));
self.submit_rocksdb_cf_metrics::<cf::TransactionStatus>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"transaction_status"
"transaction_status",
advanced_options
));
self.submit_rocksdb_cf_metrics::<cf::AddressSignatures>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"address_signature"
"address_signature",
advanced_options
));
self.submit_rocksdb_cf_metrics::<cf::TransactionMemos>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"transaction_memos"
"transaction_memos",
advanced_options
));
self.submit_rocksdb_cf_metrics::<cf::TransactionStatusIndex>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"transaction_status_index"
"transaction_status_index",
advanced_options
));
self.submit_rocksdb_cf_metrics::<cf::Rewards>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"rewards"
"rewards",
advanced_options
));
self.submit_rocksdb_cf_metrics::<cf::Blocktime>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"blocktime"
"blocktime",
advanced_options
));
self.submit_rocksdb_cf_metrics::<cf::PerfSamples>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"perf_sample"
"perf_sample",
advanced_options
));
self.submit_rocksdb_cf_metrics::<cf::BlockHeight>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"block_height"
"block_height",
advanced_options
));
self.submit_rocksdb_cf_metrics::<cf::ProgramCosts>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"program_costs"
"program_costs",
advanced_options
));
}
@ -4114,20 +4151,20 @@ pub fn create_new_ledger(
genesis_config: &GenesisConfig,
max_genesis_archive_unpacked_size: u64,
access_type: AccessType,
shred_storage_type: ShredStorageType,
advanced_options: BlockstoreAdvancedOptions,
) -> Result<Hash> {
Blockstore::destroy(ledger_path)?;
genesis_config.write(ledger_path)?;
// Fill slot 0 with ticks that link back to the genesis_config to bootstrap the ledger.
let blockstore_dir = Blockstore::blockstore_directory(&shred_storage_type);
let blockstore_dir = Blockstore::blockstore_directory(&advanced_options.shred_storage_type);
let blockstore = Blockstore::open_with_options(
ledger_path,
BlockstoreOptions {
access_type,
recovery_mode: None,
enforce_ulimit_nofile: false,
shred_storage_type: shred_storage_type.clone(),
advanced_options: advanced_options.clone(),
},
)?;
let ticks_per_slot = genesis_config.ticks_per_slot;
@ -4297,7 +4334,7 @@ macro_rules! create_new_tmp_ledger {
$crate::tmp_ledger_name!(),
$genesis_config,
$crate::blockstore_db::AccessType::PrimaryOnly,
$crate::blockstore_db::ShredStorageType::default(),
$crate::blockstore_db::BlockstoreAdvancedOptions::default(),
)
};
}
@ -4309,7 +4346,7 @@ macro_rules! create_new_tmp_ledger_auto_delete {
$crate::tmp_ledger_name!(),
$genesis_config,
$crate::blockstore_db::AccessType::PrimaryOnly,
$crate::blockstore_db::ShredStorageType::default(),
$crate::blockstore_db::BlockstoreAdvancedOptions::default(),
)
};
}
@ -4321,9 +4358,11 @@ macro_rules! create_new_tmp_ledger_fifo_auto_delete {
$crate::tmp_ledger_name!(),
$genesis_config,
$crate::blockstore_db::AccessType::PrimaryOnly,
$crate::blockstore_db::ShredStorageType::RocksFifo(
$crate::blockstore_db::BlockstoreRocksFifoOptions::default(),
),
$crate::blockstore_db::BlockstoreAdvancedOptions {
shred_storage_type: $crate::blockstore_db::ShredStorageType::RocksFifo(
$crate::blockstore_db::BlockstoreRocksFifoOptions::default(),
),
},
)
};
}
@ -4354,13 +4393,13 @@ pub fn create_new_ledger_from_name(
name: &str,
genesis_config: &GenesisConfig,
access_type: AccessType,
shred_storage_type: ShredStorageType,
advanced_options: BlockstoreAdvancedOptions,
) -> (PathBuf, Hash) {
let (ledger_path, blockhash) = create_new_ledger_from_name_auto_delete(
name,
genesis_config,
access_type,
shred_storage_type,
advanced_options,
);
(ledger_path.into_path(), blockhash)
}
@ -4373,7 +4412,7 @@ pub fn create_new_ledger_from_name_auto_delete(
name: &str,
genesis_config: &GenesisConfig,
access_type: AccessType,
shred_storage_type: ShredStorageType,
advanced_options: BlockstoreAdvancedOptions,
) -> (TempDir, Hash) {
let ledger_path = get_ledger_path_from_name_auto_delete(name);
let blockhash = create_new_ledger(
@ -4381,7 +4420,7 @@ pub fn create_new_ledger_from_name_auto_delete(
genesis_config,
MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
access_type,
shred_storage_type,
advanced_options,
)
.unwrap();
(ledger_path, blockhash)
@ -4690,9 +4729,11 @@ pub mod tests {
let blockstore = Blockstore::open_with_options(
ledger_path.path(),
BlockstoreOptions {
shred_storage_type: ShredStorageType::RocksFifo(
BlockstoreRocksFifoOptions::default(),
),
advanced_options: BlockstoreAdvancedOptions {
shred_storage_type: ShredStorageType::RocksFifo(
BlockstoreRocksFifoOptions::default(),
),
},
..BlockstoreOptions::default()
},
)

29
ledger/src/blockstore_db.rs
View File

@ -411,12 +411,13 @@ impl Rocks {
) -> Vec<ColumnFamilyDescriptor> {
use columns::*;
let access_type = &options.access_type;
let advanced_options = &options.advanced_options;
let (cf_descriptor_shred_data, cf_descriptor_shred_code) =
new_cf_descriptor_pair_shreds::<ShredData, ShredCode>(
&options.shred_storage_type,
access_type,
oldest_slot,
advanced_options,
);
vec![
new_cf_descriptor::<SlotMeta>(access_type, oldest_slot),
@ -996,6 +997,23 @@ impl Default for ShredStorageType {
}
}
/// Advanced options for blockstore.
/// The each advanced option might also be used as a tag that supports
/// group-by operation when reporting Blockstore metrics.
#[derive(Clone)]
pub struct BlockstoreAdvancedOptions {
// Determine how to store both data and coding shreds. Default: RocksLevel.
pub shred_storage_type: ShredStorageType,
}
impl Default for BlockstoreAdvancedOptions {
fn default() -> Self {
Self {
shred_storage_type: ShredStorageType::RocksLevel,
}
}
}
pub struct BlockstoreOptions {
// The access type of blockstore. Default: PrimaryOnly
pub access_type: AccessType,
@ -1003,8 +1021,7 @@ pub struct BlockstoreOptions {
pub recovery_mode: Option<BlockstoreRecoveryMode>,
// Whether to allow unlimited number of open files. Default: true.
pub enforce_ulimit_nofile: bool,
// Determine how to store both data and coding shreds. Default: RocksLevel.
pub shred_storage_type: ShredStorageType,
pub advanced_options: BlockstoreAdvancedOptions,
}
impl Default for BlockstoreOptions {
@ -1014,7 +1031,7 @@ impl Default for BlockstoreOptions {
access_type: AccessType::PrimaryOnly,
recovery_mode: None,
enforce_ulimit_nofile: true,
shred_storage_type: ShredStorageType::RocksLevel,
advanced_options: BlockstoreAdvancedOptions::default(),
}
}
}
@ -1445,11 +1462,11 @@ fn new_cf_descriptor_pair_shreds<
D: 'static + Column + ColumnName, // Column Family for Data Shred
C: 'static + Column + ColumnName, // Column Family for Coding Shred
>(
shred_storage_type: &ShredStorageType,
access_type: &AccessType,
oldest_slot: &OldestSlot,
advanced_options: &BlockstoreAdvancedOptions,
) -> (ColumnFamilyDescriptor, ColumnFamilyDescriptor) {
match shred_storage_type {
match &advanced_options.shred_storage_type {
ShredStorageType::RocksLevel => (
new_cf_descriptor::<D>(access_type, oldest_slot),
new_cf_descriptor::<C>(access_type, oldest_slot),

2
local-cluster/src/validator_configs.rs
View File

@ -62,7 +62,7 @@ pub fn safe_clone_config(config: &ValidatorConfig) -> ValidatorConfig {
accounts_shrink_ratio: config.accounts_shrink_ratio,
accounts_db_config: config.accounts_db_config.clone(),
wait_to_vote_slot: config.wait_to_vote_slot,
shred_storage_type: config.shred_storage_type.clone(),
blockstore_advanced_options: config.blockstore_advanced_options.clone(),
}
}

5
test-validator/src/lib.rs
View File

@ -13,7 +13,8 @@ use {
socketaddr,
},
solana_ledger::{
blockstore::create_new_ledger, blockstore_db::ShredStorageType, create_new_tmp_ledger,
blockstore::create_new_ledger, blockstore_db::BlockstoreAdvancedOptions,
create_new_tmp_ledger,
},
solana_net_utils::PortRange,
solana_rpc::{rpc::JsonRpcConfig, rpc_pubsub_service::PubSubConfig},
@ -582,7 +583,7 @@ impl TestValidator {
.max_genesis_archive_unpacked_size
.unwrap_or(MAX_GENESIS_ARCHIVE_UNPACKED_SIZE),
solana_ledger::blockstore_db::AccessType::PrimaryOnly,
ShredStorageType::default(),
BlockstoreAdvancedOptions::default(),
)
.map_err(|err| {
format!(

38
validator/src/main.rs
View File

@ -33,8 +33,8 @@ use {
contact_info::ContactInfo,
},
solana_ledger::blockstore_db::{
BlockstoreRecoveryMode, BlockstoreRocksFifoOptions, ShredStorageType,
DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
BlockstoreAdvancedOptions, BlockstoreRecoveryMode, BlockstoreRocksFifoOptions,
ShredStorageType, DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
},
solana_perf::recycler::enable_recycler_warming,
solana_poh::poh_service,
@ -2559,22 +2559,24 @@ pub fn main() {
validator_config.max_ledger_shreds = Some(limit_ledger_size);
}
validator_config.shred_storage_type = match matches.value_of("rocksdb_shred_compaction") {
None => ShredStorageType::default(),
Some(shred_compaction_string) => match shred_compaction_string {
"level" => ShredStorageType::RocksLevel,
"fifo" => {
let shred_storage_size =
value_t_or_exit!(matches, "rocksdb_fifo_shred_storage_size", u64);
ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions {
shred_data_cf_size: shred_storage_size / 2,
shred_code_cf_size: shred_storage_size / 2,
})
}
_ => panic!(
"Unrecognized rocksdb-shred-compaction: {}",
shred_compaction_string
),
validator_config.blockstore_advanced_options = BlockstoreAdvancedOptions {
shred_storage_type: match matches.value_of("rocksdb_shred_compaction") {
None => ShredStorageType::default(),
Some(shred_compaction_string) => match shred_compaction_string {
"level" => ShredStorageType::RocksLevel,
"fifo" => {
let shred_storage_size =
value_t_or_exit!(matches, "rocksdb_fifo_shred_storage_size", u64);
ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions {
shred_data_cf_size: shred_storage_size / 2,
shred_code_cf_size: shred_storage_size / 2,
})
}
_ => panic!(
"Unrecognized rocksdb-shred-compaction: {}",
shred_compaction_string
),
},
},
};