blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

(LedgerStore) Rename BlockstoreAdvancedOptions to LedgerColumnOptions (#23764) 

This PR renames BlockstoreAdvancedOptions to LedgerColumnOptions, as we will
pass-down this struct to LedgerColumn to allow it to perform metric reporting.
This commit is contained in:
Yueh-Hsuan Chiang 2022-03-18 11:13:35 -07:00 committed by GitHub
parent 56428be629
commit f999eef452
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 64 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
core/src/validator.rs
View File

@ -38,7 +38,7 @@ use {
blockstore::{
Blockstore, BlockstoreError, BlockstoreSignals, CompletedSlotsReceiver, PurgeType,
},
blockstore_db::{BlockstoreAdvancedOptions, BlockstoreOptions, BlockstoreRecoveryMode},
blockstore_db::{BlockstoreOptions, BlockstoreRecoveryMode, LedgerColumnOptions},
blockstore_processor::{self, TransactionStatusSender},
leader_schedule::FixedSchedule,
leader_schedule_cache::LeaderScheduleCache,
@ -168,7 +168,7 @@ pub struct ValidatorConfig {
pub no_wait_for_vote_to_start_leader: bool,
pub accounts_shrink_ratio: AccountShrinkThreshold,
pub wait_to_vote_slot: Option<Slot>,
pub blockstore_advanced_options: BlockstoreAdvancedOptions,
pub ledger_column_options: LedgerColumnOptions,
}
impl Default for ValidatorConfig {
@ -230,7 +230,7 @@ impl Default for ValidatorConfig {
accounts_shrink_ratio: AccountShrinkThreshold::default(),
accounts_db_config: None,
wait_to_vote_slot: None,
blockstore_advanced_options: BlockstoreAdvancedOptions::default(),
ledger_column_options: LedgerColumnOptions::default(),
}
}
}
@ -1297,7 +1297,7 @@ fn load_blockstore(
ledger_path,
BlockstoreOptions {
recovery_mode: config.wal_recovery_mode.clone(),
advanced_options: config.blockstore_advanced_options.clone(),
column_options: config.ledger_column_options.clone(),
..BlockstoreOptions::default()
},
)

4
core/tests/ledger_cleanup.rs
View File

@ -10,7 +10,7 @@ mod tests {
solana_ledger::{
blockstore::{make_many_slot_shreds, Blockstore},
blockstore_db::{
BlockstoreAdvancedOptions, BlockstoreOptions, BlockstoreRocksFifoOptions,
BlockstoreOptions, BlockstoreRocksFifoOptions, LedgerColumnOptions,
ShredStorageType,
},
get_tmp_ledger_path,
@ -351,7 +351,7 @@ mod tests {
&ledger_path,
if config.fifo_compaction {
BlockstoreOptions {
advanced_options: BlockstoreAdvancedOptions {
column_options: LedgerColumnOptions {
shred_storage_type: ShredStorageType::RocksFifo(
BlockstoreRocksFifoOptions {
shred_data_cf_size: config.shred_data_cf_size,

4
genesis/src/main.rs
View File

@ -13,7 +13,7 @@ use {
},
solana_entry::poh::compute_hashes_per_tick,
solana_genesis::{genesis_accounts::add_genesis_accounts, Base64Account},
solana_ledger::{blockstore::create_new_ledger, blockstore_db::BlockstoreAdvancedOptions},
solana_ledger::{blockstore::create_new_ledger, blockstore_db::LedgerColumnOptions},
solana_runtime::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
solana_sdk::{
account::{Account, AccountSharedData, ReadableAccount, WritableAccount},
@ -629,7 +629,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
&ledger_path,
&genesis_config,
max_genesis_archive_unpacked_size,
BlockstoreAdvancedOptions::default(),
LedgerColumnOptions::default(),
)?;
println!("{}", genesis_config);

6
ledger-tool/src/main.rs
View File

@ -25,8 +25,8 @@ use {
bank_forks_utils,
blockstore::{create_new_ledger, Blockstore, PurgeType},
blockstore_db::{
self, AccessType, BlockstoreAdvancedOptions, BlockstoreOptions, BlockstoreRecoveryMode,
Database,
self, AccessType, BlockstoreOptions, BlockstoreRecoveryMode, Database,
LedgerColumnOptions,
},
blockstore_processor::{BlockstoreProcessorError, ProcessOptions},
shred::Shred,
@ -1719,7 +1719,7 @@ fn main() {
&output_directory,
&genesis_config,
solana_runtime::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
BlockstoreAdvancedOptions::default(),
LedgerColumnOptions::default(),
)
.unwrap_or_else(|err| {
eprintln!("Failed to write genesis config: {:?}", err);

78
ledger/src/blockstore.rs
View File

@ -5,8 +5,8 @@ use {
crate::{
ancestor_iterator::AncestorIterator,
blockstore_db::{
columns as cf, AccessType, BlockstoreAdvancedOptions, BlockstoreOptions, Column,
ColumnName, Database, IteratorDirection, IteratorMode, LedgerColumn, Result,
columns as cf, AccessType, BlockstoreOptions, Column, ColumnName, Database,
IteratorDirection, IteratorMode, LedgerColumn, LedgerColumnOptions, Result,
ShredStorageType, WriteBatch,
},
blockstore_meta::*,
@ -178,7 +178,7 @@ pub struct Blockstore {
pub lowest_cleanup_slot: RwLock<Slot>,
no_compaction: bool,
slots_stats: Mutex<SlotsStats>,
advanced_options: BlockstoreAdvancedOptions,
column_options: LedgerColumnOptions,
}
struct SlotsStats {
@ -521,8 +521,8 @@ impl BlockstoreRocksDbColumnFamilyMetrics {
}
macro_rules! rocksdb_metric_header {
($metric_name:literal, $cf_name:literal, $advanced_options:expr) => {
match $advanced_options.shred_storage_type {
($metric_name:literal, $cf_name:literal, $column_options:expr) => {
match $column_options.shred_storage_type {
ShredStorageType::RocksLevel =>
rocksdb_metric_header!(@all_fields $metric_name, $cf_name, "rocks_level"),
ShredStorageType::RocksFifo(_) =>
@ -569,9 +569,9 @@ impl Blockstore {
fn do_open(ledger_path: &Path, options: BlockstoreOptions) -> Result<Blockstore> {
fs::create_dir_all(&ledger_path)?;
let blockstore_path = ledger_path.join(Self::blockstore_directory(
&options.advanced_options.shred_storage_type,
&options.column_options.shred_storage_type,
));
let advanced_options = options.advanced_options.clone();
let column_options = options.column_options.clone();
adjust_ulimit_nofile(options.enforce_ulimit_nofile)?;
@ -664,7 +664,7 @@ impl Blockstore {
lowest_cleanup_slot: RwLock::<Slot>::default(),
no_compaction: false,
slots_stats: Mutex::<SlotsStats>::default(),
advanced_options,
column_options,
};
if initialize_transaction_status_index {
blockstore.initialize_transaction_status_index()?;
@ -961,101 +961,101 @@ impl Blockstore {
/// Collects and reports [`BlockstoreRocksDbColumnFamilyMetrics`] for the
/// all the column families.
pub fn submit_rocksdb_cf_metrics_for_all_cfs(&self) {
let advanced_options = &self.advanced_options;
let column_options = &self.column_options;
self.submit_rocksdb_cf_metrics::<cf::SlotMeta>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"slot_meta",
advanced_options
column_options
));
self.submit_rocksdb_cf_metrics::<cf::DeadSlots>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"dead_slots",
advanced_options
column_options
));
self.submit_rocksdb_cf_metrics::<cf::DuplicateSlots>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"duplicate_slots",
advanced_options
column_options
));
self.submit_rocksdb_cf_metrics::<cf::ErasureMeta>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"erasure_meta",
advanced_options
column_options
));
self.submit_rocksdb_cf_metrics::<cf::Orphans>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"orphans",
advanced_options
column_options
));
self.submit_rocksdb_cf_metrics::<cf::BankHash>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"bank_hash",
advanced_options
column_options
));
self.submit_rocksdb_cf_metrics::<cf::Root>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"root",
advanced_options
column_options
));
self.submit_rocksdb_cf_metrics::<cf::Index>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"index",
advanced_options
column_options
));
self.submit_rocksdb_cf_metrics::<cf::ShredData>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"shred_data",
advanced_options
column_options
));
self.submit_rocksdb_cf_metrics::<cf::ShredCode>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"shred_code",
advanced_options
column_options
));
self.submit_rocksdb_cf_metrics::<cf::TransactionStatus>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"transaction_status",
advanced_options
column_options
));
self.submit_rocksdb_cf_metrics::<cf::AddressSignatures>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"address_signature",
advanced_options
column_options
));
self.submit_rocksdb_cf_metrics::<cf::TransactionMemos>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"transaction_memos",
advanced_options
column_options
));
self.submit_rocksdb_cf_metrics::<cf::TransactionStatusIndex>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"transaction_status_index",
advanced_options
column_options
));
self.submit_rocksdb_cf_metrics::<cf::Rewards>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"rewards",
advanced_options
column_options
));
self.submit_rocksdb_cf_metrics::<cf::Blocktime>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"blocktime",
advanced_options
column_options
));
self.submit_rocksdb_cf_metrics::<cf::PerfSamples>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"perf_sample",
advanced_options
column_options
));
self.submit_rocksdb_cf_metrics::<cf::BlockHeight>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"block_height",
advanced_options
column_options
));
self.submit_rocksdb_cf_metrics::<cf::ProgramCosts>(rocksdb_metric_header!(
"blockstore_rocksdb_cfs",
"program_costs",
advanced_options
column_options
));
}
@ -4150,20 +4150,20 @@ pub fn create_new_ledger(
ledger_path: &Path,
genesis_config: &GenesisConfig,
max_genesis_archive_unpacked_size: u64,
advanced_options: BlockstoreAdvancedOptions,
column_options: LedgerColumnOptions,
) -> Result<Hash> {
Blockstore::destroy(ledger_path)?;
genesis_config.write(ledger_path)?;
// Fill slot 0 with ticks that link back to the genesis_config to bootstrap the ledger.
let blockstore_dir = Blockstore::blockstore_directory(&advanced_options.shred_storage_type);
let blockstore_dir = Blockstore::blockstore_directory(&column_options.shred_storage_type);
let blockstore = Blockstore::open_with_options(
ledger_path,
BlockstoreOptions {
access_type: AccessType::PrimaryOnly,
recovery_mode: None,
enforce_ulimit_nofile: false,
advanced_options: advanced_options.clone(),
column_options: column_options.clone(),
},
)?;
let ticks_per_slot = genesis_config.ticks_per_slot;
@ -4332,7 +4332,7 @@ macro_rules! create_new_tmp_ledger {
$crate::blockstore::create_new_ledger_from_name(
$crate::tmp_ledger_name!(),
$genesis_config,
$crate::blockstore_db::BlockstoreAdvancedOptions::default(),
$crate::blockstore_db::LedgerColumnOptions::default(),
)
};
}
@ -4343,7 +4343,7 @@ macro_rules! create_new_tmp_ledger_auto_delete {
$crate::blockstore::create_new_ledger_from_name_auto_delete(
$crate::tmp_ledger_name!(),
$genesis_config,
$crate::blockstore_db::BlockstoreAdvancedOptions::default(),
$crate::blockstore_db::LedgerColumnOptions::default(),
)
};
}
@ -4354,7 +4354,7 @@ macro_rules! create_new_tmp_ledger_fifo_auto_delete {
$crate::blockstore::create_new_ledger_from_name_auto_delete(
$crate::tmp_ledger_name!(),
$genesis_config,
$crate::blockstore_db::BlockstoreAdvancedOptions {
$crate::blockstore_db::LedgerColumnOptions {
shred_storage_type: $crate::blockstore_db::ShredStorageType::RocksFifo(
$crate::blockstore_db::BlockstoreRocksFifoOptions::default(),
),
@ -4388,10 +4388,10 @@ pub fn verify_shred_slots(slot: Slot, parent_slot: Slot, last_root: Slot) -> boo
pub fn create_new_ledger_from_name(
name: &str,
genesis_config: &GenesisConfig,
advanced_options: BlockstoreAdvancedOptions,
column_options: LedgerColumnOptions,
) -> (PathBuf, Hash) {
let (ledger_path, blockhash) =
create_new_ledger_from_name_auto_delete(name, genesis_config, advanced_options);
create_new_ledger_from_name_auto_delete(name, genesis_config, column_options);
(ledger_path.into_path(), blockhash)
}
@ -4402,14 +4402,14 @@ pub fn create_new_ledger_from_name(
pub fn create_new_ledger_from_name_auto_delete(
name: &str,
genesis_config: &GenesisConfig,
advanced_options: BlockstoreAdvancedOptions,
column_options: LedgerColumnOptions,
) -> (TempDir, Hash) {
let ledger_path = get_ledger_path_from_name_auto_delete(name);
let blockhash = create_new_ledger(
ledger_path.path(),
genesis_config,
MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
advanced_options,
column_options,
)
.unwrap();
(ledger_path, blockhash)
@ -4718,7 +4718,7 @@ pub mod tests {
let blockstore = Blockstore::open_with_options(
ledger_path.path(),
BlockstoreOptions {
advanced_options: BlockstoreAdvancedOptions {
column_options: LedgerColumnOptions {
shred_storage_type: ShredStorageType::RocksFifo(
BlockstoreRocksFifoOptions::default(),
),

16
ledger/src/blockstore_db.rs
View File

@ -991,16 +991,16 @@ impl Default for ShredStorageType {
}
}
/// Advanced options for blockstore.
/// The each advanced option might also be used as a tag that supports
/// group-by operation when reporting Blockstore metrics.
/// Options for LedgerColumn.
/// Each field might also be used as a tag that supports group-by operation when
/// reporting metrics.
#[derive(Clone)]
pub struct BlockstoreAdvancedOptions {
pub struct LedgerColumnOptions {
// Determine how to store both data and coding shreds. Default: RocksLevel.
pub shred_storage_type: ShredStorageType,
}
impl Default for BlockstoreAdvancedOptions {
impl Default for LedgerColumnOptions {
fn default() -> Self {
Self {
shred_storage_type: ShredStorageType::RocksLevel,
@ -1015,7 +1015,7 @@ pub struct BlockstoreOptions {
pub recovery_mode: Option<BlockstoreRecoveryMode>,
// Whether to allow unlimited number of open files. Default: true.
pub enforce_ulimit_nofile: bool,
pub advanced_options: BlockstoreAdvancedOptions,
pub column_options: LedgerColumnOptions,
}
impl Default for BlockstoreOptions {
@ -1025,7 +1025,7 @@ impl Default for BlockstoreOptions {
access_type: AccessType::PrimaryOnly,
recovery_mode: None,
enforce_ulimit_nofile: true,
advanced_options: BlockstoreAdvancedOptions::default(),
column_options: LedgerColumnOptions::default(),
}
}
}
@ -1459,7 +1459,7 @@ fn new_cf_descriptor_pair_shreds<
options: &BlockstoreOptions,
oldest_slot: &OldestSlot,
) -> (ColumnFamilyDescriptor, ColumnFamilyDescriptor) {
match &options.advanced_options.shred_storage_type {
match &options.column_options.shred_storage_type {
ShredStorageType::RocksLevel => (
new_cf_descriptor::<D>(options, oldest_slot),
new_cf_descriptor::<C>(options, oldest_slot),

2
local-cluster/src/validator_configs.rs
View File

@ -63,7 +63,7 @@ pub fn safe_clone_config(config: &ValidatorConfig) -> ValidatorConfig {
accounts_shrink_ratio: config.accounts_shrink_ratio,
accounts_db_config: config.accounts_db_config.clone(),
wait_to_vote_slot: config.wait_to_vote_slot,
blockstore_advanced_options: config.blockstore_advanced_options.clone(),
ledger_column_options: config.ledger_column_options.clone(),
}
}

5
test-validator/src/lib.rs
View File

@ -13,8 +13,7 @@ use {
socketaddr,
},
solana_ledger::{
blockstore::create_new_ledger, blockstore_db::BlockstoreAdvancedOptions,
create_new_tmp_ledger,
blockstore::create_new_ledger, blockstore_db::LedgerColumnOptions, create_new_tmp_ledger,
},
solana_net_utils::PortRange,
solana_rpc::{rpc::JsonRpcConfig, rpc_pubsub_service::PubSubConfig},
@ -582,7 +581,7 @@ impl TestValidator {
config
.max_genesis_archive_unpacked_size
.unwrap_or(MAX_GENESIS_ARCHIVE_UNPACKED_SIZE),
BlockstoreAdvancedOptions::default(),
LedgerColumnOptions::default(),
)
.map_err(|err| {
format!(

6
validator/src/main.rs
View File

@ -33,8 +33,8 @@ use {
contact_info::ContactInfo,
},
solana_ledger::blockstore_db::{
BlockstoreAdvancedOptions, BlockstoreRecoveryMode, BlockstoreRocksFifoOptions,
ShredStorageType, DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
BlockstoreRecoveryMode, BlockstoreRocksFifoOptions, LedgerColumnOptions, ShredStorageType,
DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
},
solana_perf::recycler::enable_recycler_warming,
solana_poh::poh_service,
@ -2567,7 +2567,7 @@ pub fn main() {
validator_config.max_ledger_shreds = Some(limit_ledger_size);
}
validator_config.blockstore_advanced_options = BlockstoreAdvancedOptions {
validator_config.ledger_column_options = LedgerColumnOptions {
shred_storage_type: match matches.value_of("rocksdb_shred_compaction") {
None => ShredStorageType::default(),
Some(shred_compaction_string) => match shred_compaction_string {