(Refactor) Move blocktore options related stuff to blockstore_options.rs (#25509)
#### Problem blockstore_db.rs has a mutual dependency between blockstore_metrics.rs. #### Summary of Changes This PR removes the mutual dependency by moving the option-related stuff out from blockstore_db.rs to its new home --- blockstore_options.rs. By doing this, we address the mutual dependency and also make the code cleaner.
This commit is contained in:
parent
da28badcaa
commit
5b67960c76
|
@ -346,7 +346,7 @@ pub mod tests {
|
|||
solana_gossip::cluster_info::{ClusterInfo, Node},
|
||||
solana_ledger::{
|
||||
blockstore::BlockstoreSignals,
|
||||
blockstore_db::BlockstoreOptions,
|
||||
blockstore_options::BlockstoreOptions,
|
||||
create_new_tmp_ledger,
|
||||
genesis_utils::{create_genesis_config, GenesisConfigInfo},
|
||||
},
|
||||
|
|
|
@ -41,7 +41,7 @@ use {
|
|||
blockstore::{
|
||||
Blockstore, BlockstoreError, BlockstoreSignals, CompletedSlotsReceiver, PurgeType,
|
||||
},
|
||||
blockstore_db::{BlockstoreOptions, BlockstoreRecoveryMode, LedgerColumnOptions},
|
||||
blockstore_options::{BlockstoreOptions, BlockstoreRecoveryMode, LedgerColumnOptions},
|
||||
blockstore_processor::{self, TransactionStatusSender},
|
||||
leader_schedule::FixedSchedule,
|
||||
leader_schedule_cache::LeaderScheduleCache,
|
||||
|
|
|
@ -9,7 +9,7 @@ mod tests {
|
|||
solana_core::ledger_cleanup_service::LedgerCleanupService,
|
||||
solana_ledger::{
|
||||
blockstore::{make_many_slot_shreds, Blockstore},
|
||||
blockstore_db::{
|
||||
blockstore_options::{
|
||||
BlockstoreOptions, BlockstoreRocksFifoOptions, LedgerColumnOptions,
|
||||
ShredStorageType,
|
||||
},
|
||||
|
|
|
@ -13,7 +13,7 @@ use {
|
|||
},
|
||||
solana_entry::poh::compute_hashes_per_tick,
|
||||
solana_genesis::{genesis_accounts::add_genesis_accounts, Base64Account},
|
||||
solana_ledger::{blockstore::create_new_ledger, blockstore_db::LedgerColumnOptions},
|
||||
solana_ledger::{blockstore::create_new_ledger, blockstore_options::LedgerColumnOptions},
|
||||
solana_runtime::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
|
||||
solana_sdk::{
|
||||
account::{Account, AccountSharedData, ReadableAccount, WritableAccount},
|
||||
|
|
|
@ -16,7 +16,7 @@ use {
|
|||
},
|
||||
solana_ledger::{
|
||||
bigtable_upload::ConfirmedBlockUploadConfig, blockstore::Blockstore,
|
||||
blockstore_db::AccessType,
|
||||
blockstore_options::AccessType,
|
||||
},
|
||||
solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature},
|
||||
solana_storage_bigtable::CredentialType,
|
||||
|
|
|
@ -24,9 +24,9 @@ use {
|
|||
ancestor_iterator::AncestorIterator,
|
||||
bank_forks_utils,
|
||||
blockstore::{create_new_ledger, Blockstore, PurgeType},
|
||||
blockstore_db::{
|
||||
self, AccessType, BlockstoreOptions, BlockstoreRecoveryMode, Database,
|
||||
LedgerColumnOptions,
|
||||
blockstore_db::{self, Database},
|
||||
blockstore_options::{
|
||||
AccessType, BlockstoreOptions, BlockstoreRecoveryMode, LedgerColumnOptions,
|
||||
},
|
||||
blockstore_processor::{BlockstoreProcessorError, ProcessOptions},
|
||||
shred::Shred,
|
||||
|
|
|
@ -5,10 +5,13 @@ use {
|
|||
crate::{
|
||||
ancestor_iterator::AncestorIterator,
|
||||
blockstore_db::{
|
||||
columns as cf, AccessType, BlockstoreOptions, Column, Database, IteratorDirection,
|
||||
IteratorMode, LedgerColumn, LedgerColumnOptions, Result, ShredStorageType, WriteBatch,
|
||||
columns as cf, Column, Database, IteratorDirection, IteratorMode, LedgerColumn, Result,
|
||||
WriteBatch,
|
||||
},
|
||||
blockstore_meta::*,
|
||||
blockstore_options::{
|
||||
AccessType, BlockstoreOptions, LedgerColumnOptions, ShredStorageType,
|
||||
},
|
||||
leader_schedule_cache::LeaderScheduleCache,
|
||||
next_slots_iterator::NextSlotsIterator,
|
||||
shred::{self, max_ticks_per_n_shreds, ErasureSetId, Shred, ShredId, ShredType, Shredder},
|
||||
|
@ -3999,7 +4002,7 @@ macro_rules! create_new_tmp_ledger {
|
|||
$crate::blockstore::create_new_ledger_from_name(
|
||||
$crate::tmp_ledger_name!(),
|
||||
$genesis_config,
|
||||
$crate::blockstore_db::LedgerColumnOptions::default(),
|
||||
$crate::blockstore_options::LedgerColumnOptions::default(),
|
||||
)
|
||||
};
|
||||
}
|
||||
|
@ -4010,7 +4013,7 @@ macro_rules! create_new_tmp_ledger_auto_delete {
|
|||
$crate::blockstore::create_new_ledger_from_name_auto_delete(
|
||||
$crate::tmp_ledger_name!(),
|
||||
$genesis_config,
|
||||
$crate::blockstore_db::LedgerColumnOptions::default(),
|
||||
$crate::blockstore_options::LedgerColumnOptions::default(),
|
||||
)
|
||||
};
|
||||
}
|
||||
|
@ -4021,11 +4024,11 @@ macro_rules! create_new_tmp_ledger_fifo_auto_delete {
|
|||
$crate::blockstore::create_new_ledger_from_name_auto_delete(
|
||||
$crate::tmp_ledger_name!(),
|
||||
$genesis_config,
|
||||
$crate::blockstore_db::LedgerColumnOptions {
|
||||
shred_storage_type: $crate::blockstore_db::ShredStorageType::RocksFifo(
|
||||
$crate::blockstore_db::BlockstoreRocksFifoOptions::default(),
|
||||
$crate::blockstore_options::LedgerColumnOptions {
|
||||
shred_storage_type: $crate::blockstore_options::ShredStorageType::RocksFifo(
|
||||
$crate::blockstore_options::BlockstoreRocksFifoOptions::default(),
|
||||
),
|
||||
..$crate::blockstore_db::LedgerColumnOptions::default()
|
||||
..$crate::blockstore_options::LedgerColumnOptions::default()
|
||||
},
|
||||
)
|
||||
};
|
||||
|
@ -4313,7 +4316,7 @@ pub mod tests {
|
|||
use {
|
||||
super::*,
|
||||
crate::{
|
||||
blockstore_db::BlockstoreRocksFifoOptions,
|
||||
blockstore_options::BlockstoreRocksFifoOptions,
|
||||
genesis_utils::{create_genesis_config, GenesisConfigInfo},
|
||||
leader_schedule::{FixedSchedule, LeaderSchedule},
|
||||
shred::{max_ticks_per_n_shreds, ShredFlags},
|
||||
|
|
|
@ -6,6 +6,9 @@ use {
|
|||
maybe_enable_rocksdb_perf, report_rocksdb_read_perf, report_rocksdb_write_perf,
|
||||
BlockstoreRocksDbColumnFamilyMetrics, ColumnMetrics, PerfSamplingStatus,
|
||||
},
|
||||
blockstore_options::{
|
||||
AccessType, BlockstoreOptions, LedgerColumnOptions, ShredStorageType,
|
||||
},
|
||||
rocksdb_metric_header,
|
||||
},
|
||||
bincode::{deserialize, serialize},
|
||||
|
@ -17,9 +20,8 @@ use {
|
|||
compaction_filter::CompactionFilter,
|
||||
compaction_filter_factory::{CompactionFilterContext, CompactionFilterFactory},
|
||||
properties as RocksProperties, ColumnFamily, ColumnFamilyDescriptor, CompactionDecision,
|
||||
DBCompactionStyle, DBCompressionType as RocksCompressionType, DBIterator, DBRawIterator,
|
||||
DBRecoveryMode, FifoCompactOptions, IteratorMode as RocksIteratorMode, Options,
|
||||
WriteBatch as RWriteBatch, DB,
|
||||
DBCompactionStyle, DBIterator, DBRawIterator, FifoCompactOptions,
|
||||
IteratorMode as RocksIteratorMode, Options, WriteBatch as RWriteBatch, DB,
|
||||
},
|
||||
serde::{de::DeserializeOwned, Serialize},
|
||||
solana_runtime::hardened_unpack::UnpackError,
|
||||
|
@ -52,16 +54,6 @@ pub const DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES: u64 = 250 * 1024 * 1024 *
|
|||
|
||||
const MAX_WRITE_BUFFER_SIZE: u64 = 256 * 1024 * 1024; // 256MB
|
||||
const FIFO_WRITE_BUFFER_SIZE: u64 = 2 * MAX_WRITE_BUFFER_SIZE;
|
||||
// Maximum size of cf::DataShred. Used when `shred_storage_type`
|
||||
// is set to ShredStorageType::RocksFifo. The default value is set
|
||||
// to 125GB, assuming 500GB total storage for ledger and 25% is
|
||||
// used by data shreds.
|
||||
const DEFAULT_FIFO_COMPACTION_DATA_CF_SIZE: u64 = 125 * 1024 * 1024 * 1024;
|
||||
// Maximum size of cf::CodeShred. Used when `shred_storage_type`
|
||||
// is set to ShredStorageType::RocksFifo. The default value is set
|
||||
// to 100GB, assuming 500GB total storage for ledger and 20% is
|
||||
// used by coding shreds.
|
||||
const DEFAULT_FIFO_COMPACTION_CODING_CF_SIZE: u64 = 100 * 1024 * 1024 * 1024;
|
||||
|
||||
// Column family for metadata about a leader slot
|
||||
const META_CF: &str = "meta";
|
||||
|
@ -236,55 +228,6 @@ pub mod columns {
|
|||
// - Account for column in `analyze_storage()` in ledger-tool/src/main.rs
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum AccessType {
|
||||
/// Primary (read/write) access; only one process can have Primary access.
|
||||
Primary,
|
||||
/// Primary (read/write) access with RocksDB automatic compaction disabled.
|
||||
PrimaryForMaintenance,
|
||||
/// Secondary (read) access; multiple processes can have Secondary access.
|
||||
/// Additionally, Secondary access can be obtained while another process
|
||||
/// already has Primary access.
|
||||
Secondary,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum BlockstoreRecoveryMode {
|
||||
TolerateCorruptedTailRecords,
|
||||
AbsoluteConsistency,
|
||||
PointInTime,
|
||||
SkipAnyCorruptedRecord,
|
||||
}
|
||||
|
||||
impl From<&str> for BlockstoreRecoveryMode {
|
||||
fn from(string: &str) -> Self {
|
||||
match string {
|
||||
"tolerate_corrupted_tail_records" => {
|
||||
BlockstoreRecoveryMode::TolerateCorruptedTailRecords
|
||||
}
|
||||
"absolute_consistency" => BlockstoreRecoveryMode::AbsoluteConsistency,
|
||||
"point_in_time" => BlockstoreRecoveryMode::PointInTime,
|
||||
"skip_any_corrupted_record" => BlockstoreRecoveryMode::SkipAnyCorruptedRecord,
|
||||
bad_mode => panic!("Invalid recovery mode: {}", bad_mode),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<BlockstoreRecoveryMode> for DBRecoveryMode {
|
||||
fn from(brm: BlockstoreRecoveryMode) -> Self {
|
||||
match brm {
|
||||
BlockstoreRecoveryMode::TolerateCorruptedTailRecords => {
|
||||
DBRecoveryMode::TolerateCorruptedTailRecords
|
||||
}
|
||||
BlockstoreRecoveryMode::AbsoluteConsistency => DBRecoveryMode::AbsoluteConsistency,
|
||||
BlockstoreRecoveryMode::PointInTime => DBRecoveryMode::PointInTime,
|
||||
BlockstoreRecoveryMode::SkipAnyCorruptedRecord => {
|
||||
DBRecoveryMode::SkipAnyCorruptedRecord
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Clone, Debug)]
|
||||
struct OldestSlot(Arc<AtomicU64>);
|
||||
|
||||
|
@ -1083,130 +1026,6 @@ pub struct WriteBatch<'a> {
|
|||
map: HashMap<&'static str, &'a ColumnFamily>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum ShredStorageType {
|
||||
// Stores shreds under RocksDB's default compaction (level).
|
||||
RocksLevel,
|
||||
// (Experimental) Stores shreds under RocksDB's FIFO compaction which
|
||||
// allows ledger store to reclaim storage more efficiently with
|
||||
// lower I/O overhead.
|
||||
RocksFifo(BlockstoreRocksFifoOptions),
|
||||
}
|
||||
|
||||
impl Default for ShredStorageType {
|
||||
fn default() -> Self {
|
||||
Self::RocksLevel
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum BlockstoreCompressionType {
|
||||
None,
|
||||
Snappy,
|
||||
Lz4,
|
||||
Zlib,
|
||||
}
|
||||
|
||||
impl Default for BlockstoreCompressionType {
|
||||
fn default() -> Self {
|
||||
Self::None
|
||||
}
|
||||
}
|
||||
|
||||
impl BlockstoreCompressionType {
|
||||
fn to_rocksdb_compression_type(&self) -> RocksCompressionType {
|
||||
match self {
|
||||
Self::None => RocksCompressionType::None,
|
||||
Self::Snappy => RocksCompressionType::Snappy,
|
||||
Self::Lz4 => RocksCompressionType::Lz4,
|
||||
Self::Zlib => RocksCompressionType::Zlib,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Options for LedgerColumn.
|
||||
/// Each field might also be used as a tag that supports group-by operation when
|
||||
/// reporting metrics.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct LedgerColumnOptions {
|
||||
// Determine how to store both data and coding shreds. Default: RocksLevel.
|
||||
pub shred_storage_type: ShredStorageType,
|
||||
|
||||
// Determine the way to compress column families which are eligible for
|
||||
// compression.
|
||||
pub compression_type: BlockstoreCompressionType,
|
||||
|
||||
// Control how often RocksDB read/write performance samples are collected.
|
||||
// If the value is greater than 0, then RocksDB read/write perf sample
|
||||
// will be collected once for every `rocks_perf_sample_interval` ops.
|
||||
pub rocks_perf_sample_interval: usize,
|
||||
}
|
||||
|
||||
impl Default for LedgerColumnOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
shred_storage_type: ShredStorageType::RocksLevel,
|
||||
compression_type: BlockstoreCompressionType::default(),
|
||||
rocks_perf_sample_interval: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct BlockstoreOptions {
|
||||
// The access type of blockstore. Default: Primary
|
||||
pub access_type: AccessType,
|
||||
// Whether to open a blockstore under a recovery mode. Default: None.
|
||||
pub recovery_mode: Option<BlockstoreRecoveryMode>,
|
||||
// Whether to allow unlimited number of open files. Default: true.
|
||||
pub enforce_ulimit_nofile: bool,
|
||||
pub column_options: LedgerColumnOptions,
|
||||
}
|
||||
|
||||
impl Default for BlockstoreOptions {
|
||||
/// The default options are the values used by [`Blockstore::open`].
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
access_type: AccessType::Primary,
|
||||
recovery_mode: None,
|
||||
enforce_ulimit_nofile: true,
|
||||
column_options: LedgerColumnOptions::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BlockstoreRocksFifoOptions {
|
||||
// The maximum storage size for storing data shreds in column family
|
||||
// [`cf::DataShred`]. Typically, data shreds contribute around 25% of the
|
||||
// ledger store storage size if the RPC service is enabled, or 50% if RPC
|
||||
// service is not enabled.
|
||||
//
|
||||
// Note that this number must be greater than FIFO_WRITE_BUFFER_SIZE
|
||||
// otherwise we won't be able to write any file. If not, the blockstore
|
||||
// will panic.
|
||||
pub shred_data_cf_size: u64,
|
||||
// The maximum storage size for storing coding shreds in column family
|
||||
// [`cf::CodeShred`]. Typically, coding shreds contribute around 20% of the
|
||||
// ledger store storage size if the RPC service is enabled, or 40% if RPC
|
||||
// service is not enabled.
|
||||
//
|
||||
// Note that this number must be greater than FIFO_WRITE_BUFFER_SIZE
|
||||
// otherwise we won't be able to write any file. If not, the blockstore
|
||||
// will panic.
|
||||
pub shred_code_cf_size: u64,
|
||||
}
|
||||
|
||||
impl Default for BlockstoreRocksFifoOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
// Maximum size of cf::ShredData.
|
||||
shred_data_cf_size: DEFAULT_FIFO_COMPACTION_DATA_CF_SIZE,
|
||||
// Maximum size of cf::ShredCode.
|
||||
shred_code_cf_size: DEFAULT_FIFO_COMPACTION_CODING_CF_SIZE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Database {
|
||||
pub fn open(path: &Path, options: BlockstoreOptions) -> Result<Self> {
|
||||
let column_options = Arc::new(options.column_options.clone());
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
use {
|
||||
crate::blockstore_db::{
|
||||
columns, BlockstoreCompressionType, LedgerColumnOptions, ShredStorageType,
|
||||
crate::{
|
||||
blockstore_db::columns,
|
||||
blockstore_options::{LedgerColumnOptions, ShredStorageType},
|
||||
},
|
||||
rocksdb::{
|
||||
perf::{set_perf_stats, PerfMetric, PerfStatsLevel},
|
||||
|
@ -32,25 +33,25 @@ macro_rules! rocksdb_metric_header {
|
|||
|
||||
(@compression_type $metric_name:literal, $cf_name:literal, $column_options:expr, $storage_type:literal) => {
|
||||
match $column_options.compression_type {
|
||||
BlockstoreCompressionType::None => rocksdb_metric_header!(@all_fields
|
||||
$crate::blockstore_options::BlockstoreCompressionType::None => rocksdb_metric_header!(@all_fields
|
||||
$metric_name,
|
||||
$cf_name,
|
||||
$storage_type,
|
||||
"None"
|
||||
),
|
||||
BlockstoreCompressionType::Snappy => rocksdb_metric_header!(@all_fields
|
||||
$crate::blockstore_options::BlockstoreCompressionType::Snappy => rocksdb_metric_header!(@all_fields
|
||||
$metric_name,
|
||||
$cf_name,
|
||||
$storage_type,
|
||||
"Snappy"
|
||||
),
|
||||
BlockstoreCompressionType::Lz4 => rocksdb_metric_header!(@all_fields
|
||||
$crate::blockstore_options::BlockstoreCompressionType::Lz4 => rocksdb_metric_header!(@all_fields
|
||||
$metric_name,
|
||||
$cf_name,
|
||||
$storage_type,
|
||||
"Lz4"
|
||||
),
|
||||
BlockstoreCompressionType::Zlib => rocksdb_metric_header!(@all_fields
|
||||
$crate::blockstore_options::BlockstoreCompressionType::Zlib => rocksdb_metric_header!(@all_fields
|
||||
$metric_name,
|
||||
$cf_name,
|
||||
$storage_type,
|
||||
|
|
|
@ -0,0 +1,203 @@
|
|||
use rocksdb::{DBCompressionType as RocksCompressionType, DBRecoveryMode};
|
||||
|
||||
pub struct BlockstoreOptions {
|
||||
// The access type of blockstore. Default: Primary
|
||||
pub access_type: AccessType,
|
||||
// Whether to open a blockstore under a recovery mode. Default: None.
|
||||
pub recovery_mode: Option<BlockstoreRecoveryMode>,
|
||||
// Whether to allow unlimited number of open files. Default: true.
|
||||
pub enforce_ulimit_nofile: bool,
|
||||
pub column_options: LedgerColumnOptions,
|
||||
}
|
||||
|
||||
impl Default for BlockstoreOptions {
|
||||
/// The default options are the values used by [`Blockstore::open`].
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
access_type: AccessType::Primary,
|
||||
recovery_mode: None,
|
||||
enforce_ulimit_nofile: true,
|
||||
column_options: LedgerColumnOptions::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum AccessType {
|
||||
/// Primary (read/write) access; only one process can have Primary access.
|
||||
Primary,
|
||||
/// Primary (read/write) access with RocksDB automatic compaction disabled.
|
||||
PrimaryForMaintenance,
|
||||
/// Secondary (read) access; multiple processes can have Secondary access.
|
||||
/// Additionally, Secondary access can be obtained while another process
|
||||
/// already has Primary access.
|
||||
Secondary,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum BlockstoreRecoveryMode {
|
||||
TolerateCorruptedTailRecords,
|
||||
AbsoluteConsistency,
|
||||
PointInTime,
|
||||
SkipAnyCorruptedRecord,
|
||||
}
|
||||
|
||||
impl From<&str> for BlockstoreRecoveryMode {
|
||||
fn from(string: &str) -> Self {
|
||||
match string {
|
||||
"tolerate_corrupted_tail_records" => {
|
||||
BlockstoreRecoveryMode::TolerateCorruptedTailRecords
|
||||
}
|
||||
"absolute_consistency" => BlockstoreRecoveryMode::AbsoluteConsistency,
|
||||
"point_in_time" => BlockstoreRecoveryMode::PointInTime,
|
||||
"skip_any_corrupted_record" => BlockstoreRecoveryMode::SkipAnyCorruptedRecord,
|
||||
bad_mode => panic!("Invalid recovery mode: {}", bad_mode),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<BlockstoreRecoveryMode> for DBRecoveryMode {
|
||||
fn from(brm: BlockstoreRecoveryMode) -> Self {
|
||||
match brm {
|
||||
BlockstoreRecoveryMode::TolerateCorruptedTailRecords => {
|
||||
DBRecoveryMode::TolerateCorruptedTailRecords
|
||||
}
|
||||
BlockstoreRecoveryMode::AbsoluteConsistency => DBRecoveryMode::AbsoluteConsistency,
|
||||
BlockstoreRecoveryMode::PointInTime => DBRecoveryMode::PointInTime,
|
||||
BlockstoreRecoveryMode::SkipAnyCorruptedRecord => {
|
||||
DBRecoveryMode::SkipAnyCorruptedRecord
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Options for LedgerColumn.
|
||||
/// Each field might also be used as a tag that supports group-by operation when
|
||||
/// reporting metrics.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct LedgerColumnOptions {
|
||||
// Determine how to store both data and coding shreds. Default: RocksLevel.
|
||||
pub shred_storage_type: ShredStorageType,
|
||||
|
||||
// Determine the way to compress column families which are eligible for
|
||||
// compression.
|
||||
pub compression_type: BlockstoreCompressionType,
|
||||
|
||||
// Control how often RocksDB read/write performance samples are collected.
|
||||
// If the value is greater than 0, then RocksDB read/write perf sample
|
||||
// will be collected once for every `rocks_perf_sample_interval` ops.
|
||||
pub rocks_perf_sample_interval: usize,
|
||||
}
|
||||
|
||||
impl Default for LedgerColumnOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
shred_storage_type: ShredStorageType::RocksLevel,
|
||||
compression_type: BlockstoreCompressionType::default(),
|
||||
rocks_perf_sample_interval: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl LedgerColumnOptions {
|
||||
pub fn get_storage_type_string(&self) -> &'static str {
|
||||
match self.shred_storage_type {
|
||||
ShredStorageType::RocksLevel => "rocks_level",
|
||||
ShredStorageType::RocksFifo(_) => "rocks_fifo",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_compression_type_string(&self) -> &'static str {
|
||||
match self.compression_type {
|
||||
BlockstoreCompressionType::None => "None",
|
||||
BlockstoreCompressionType::Snappy => "Snappy",
|
||||
BlockstoreCompressionType::Lz4 => "Lz4",
|
||||
BlockstoreCompressionType::Zlib => "Zlib",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum ShredStorageType {
|
||||
// Stores shreds under RocksDB's default compaction (level).
|
||||
RocksLevel,
|
||||
// (Experimental) Stores shreds under RocksDB's FIFO compaction which
|
||||
// allows ledger store to reclaim storage more efficiently with
|
||||
// lower I/O overhead.
|
||||
RocksFifo(BlockstoreRocksFifoOptions),
|
||||
}
|
||||
|
||||
impl Default for ShredStorageType {
|
||||
fn default() -> Self {
|
||||
Self::RocksLevel
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BlockstoreRocksFifoOptions {
|
||||
// The maximum storage size for storing data shreds in column family
|
||||
// [`cf::DataShred`]. Typically, data shreds contribute around 25% of the
|
||||
// ledger store storage size if the RPC service is enabled, or 50% if RPC
|
||||
// service is not enabled.
|
||||
//
|
||||
// Note that this number must be greater than FIFO_WRITE_BUFFER_SIZE
|
||||
// otherwise we won't be able to write any file. If not, the blockstore
|
||||
// will panic.
|
||||
pub shred_data_cf_size: u64,
|
||||
// The maximum storage size for storing coding shreds in column family
|
||||
// [`cf::CodeShred`]. Typically, coding shreds contribute around 20% of the
|
||||
// ledger store storage size if the RPC service is enabled, or 40% if RPC
|
||||
// service is not enabled.
|
||||
//
|
||||
// Note that this number must be greater than FIFO_WRITE_BUFFER_SIZE
|
||||
// otherwise we won't be able to write any file. If not, the blockstore
|
||||
// will panic.
|
||||
pub shred_code_cf_size: u64,
|
||||
}
|
||||
|
||||
// Maximum size of cf::DataShred. Used when `shred_storage_type`
|
||||
// is set to ShredStorageType::RocksFifo. The default value is set
|
||||
// to 125GB, assuming 500GB total storage for ledger and 25% is
|
||||
// used by data shreds.
|
||||
const DEFAULT_FIFO_COMPACTION_DATA_CF_SIZE: u64 = 125 * 1024 * 1024 * 1024;
|
||||
// Maximum size of cf::CodeShred. Used when `shred_storage_type`
|
||||
// is set to ShredStorageType::RocksFifo. The default value is set
|
||||
// to 100GB, assuming 500GB total storage for ledger and 20% is
|
||||
// used by coding shreds.
|
||||
const DEFAULT_FIFO_COMPACTION_CODING_CF_SIZE: u64 = 100 * 1024 * 1024 * 1024;
|
||||
|
||||
impl Default for BlockstoreRocksFifoOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
// Maximum size of cf::ShredData.
|
||||
shred_data_cf_size: DEFAULT_FIFO_COMPACTION_DATA_CF_SIZE,
|
||||
// Maximum size of cf::ShredCode.
|
||||
shred_code_cf_size: DEFAULT_FIFO_COMPACTION_CODING_CF_SIZE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum BlockstoreCompressionType {
|
||||
None,
|
||||
Snappy,
|
||||
Lz4,
|
||||
Zlib,
|
||||
}
|
||||
|
||||
impl Default for BlockstoreCompressionType {
|
||||
fn default() -> Self {
|
||||
Self::None
|
||||
}
|
||||
}
|
||||
|
||||
impl BlockstoreCompressionType {
|
||||
pub(crate) fn to_rocksdb_compression_type(&self) -> RocksCompressionType {
|
||||
match self {
|
||||
Self::None => RocksCompressionType::None,
|
||||
Self::Snappy => RocksCompressionType::Snappy,
|
||||
Self::Lz4 => RocksCompressionType::Lz4,
|
||||
Self::Zlib => RocksCompressionType::Zlib,
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1597,7 +1597,7 @@ pub mod tests {
|
|||
use {
|
||||
super::*,
|
||||
crate::{
|
||||
blockstore_db::{AccessType, BlockstoreOptions},
|
||||
blockstore_options::{AccessType, BlockstoreOptions},
|
||||
genesis_utils::{
|
||||
create_genesis_config, create_genesis_config_with_leader, GenesisConfigInfo,
|
||||
},
|
||||
|
|
|
@ -15,6 +15,7 @@ pub mod blockstore_db;
|
|||
pub mod blockstore_meta;
|
||||
#[macro_use]
|
||||
pub mod blockstore_metrics;
|
||||
pub mod blockstore_options;
|
||||
pub mod blockstore_processor;
|
||||
pub mod builtins;
|
||||
pub mod genesis_utils;
|
||||
|
|
|
@ -12,7 +12,7 @@ use {
|
|||
solana_ledger::{
|
||||
ancestor_iterator::AncestorIterator,
|
||||
blockstore::{Blockstore, PurgeType},
|
||||
blockstore_db::{AccessType, BlockstoreOptions},
|
||||
blockstore_options::{AccessType, BlockstoreOptions},
|
||||
leader_schedule::{FixedSchedule, LeaderSchedule},
|
||||
},
|
||||
solana_local_cluster::{
|
||||
|
|
|
@ -13,7 +13,8 @@ use {
|
|||
socketaddr,
|
||||
},
|
||||
solana_ledger::{
|
||||
blockstore::create_new_ledger, blockstore_db::LedgerColumnOptions, create_new_tmp_ledger,
|
||||
blockstore::create_new_ledger, blockstore_options::LedgerColumnOptions,
|
||||
create_new_tmp_ledger,
|
||||
},
|
||||
solana_net_utils::PortRange,
|
||||
solana_program_runtime::compute_budget::ComputeBudget,
|
||||
|
|
|
@ -30,9 +30,12 @@ use {
|
|||
validator::{is_snapshot_config_valid, Validator, ValidatorConfig, ValidatorStartProgress},
|
||||
},
|
||||
solana_gossip::{cluster_info::Node, contact_info::ContactInfo},
|
||||
solana_ledger::blockstore_db::{
|
||||
BlockstoreCompressionType, BlockstoreRecoveryMode, BlockstoreRocksFifoOptions,
|
||||
LedgerColumnOptions, ShredStorageType, DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
|
||||
solana_ledger::{
|
||||
blockstore_db::DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
|
||||
blockstore_options::{
|
||||
BlockstoreCompressionType, BlockstoreRecoveryMode, BlockstoreRocksFifoOptions,
|
||||
LedgerColumnOptions, ShredStorageType,
|
||||
},
|
||||
},
|
||||
solana_net_utils::VALIDATOR_PORT_RANGE,
|
||||
solana_perf::recycler::enable_recycler_warming,
|
||||
|
|
Loading…
Reference in New Issue