blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

(LedgerStore) Use different path for different blockstore storage type. (#23236) 

#### Summary of Changes
To avoid mixing the use of different shred storage types, each shred storage type
will have its blockstore in a different directory.

This PR still keeps the RocksFifo setting hidden.  The default ShredStorageType and
blockstore directory are still RocksLevel and `rocksdb`.

Will follow-up with PRs on making FIFO option public in ledger-tool and validator.

#### Test Plan
* Added a new test to verify the existence of `rocksdb-fifo` directory when FIFO compaction is used.
* Updated existing test to verify the current setting still store ledger under `rocksdb` directory.
* Manually ran ledger_cleanup_test with both level and fifo compaction and verified the resulting ledger.
* Ran a validator with this PR.
This commit is contained in:
Yueh-Hsuan Chiang 2022-03-02 18:30:22 -08:00 committed by GitHub
parent 39387e8446
commit 634f4eb37d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 161 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
genesis/src/main.rs
View File

@ -13,7 +13,10 @@ use {
},
solana_entry::poh::compute_hashes_per_tick,
solana_genesis::{genesis_accounts::add_genesis_accounts, Base64Account},
solana_ledger::{blockstore::create_new_ledger, blockstore_db::AccessType},
solana_ledger::{
blockstore::create_new_ledger,
blockstore_db::{AccessType, ShredStorageType},
},
solana_runtime::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
solana_sdk::{
account::{Account, AccountSharedData, ReadableAccount, WritableAccount},
@ -630,6 +633,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
&genesis_config,
max_genesis_archive_unpacked_size,
AccessType::PrimaryOnly,
ShredStorageType::default(),
)?;
println!("{}", genesis_config);

5
ledger-tool/src/main.rs
View File

@ -23,7 +23,9 @@ use {
ancestor_iterator::AncestorIterator,
bank_forks_utils,
blockstore::{create_new_ledger, Blockstore, PurgeType},
blockstore_db::{self, AccessType, BlockstoreOptions, BlockstoreRecoveryMode, Database},
blockstore_db::{
self, AccessType, BlockstoreOptions, BlockstoreRecoveryMode, Database, ShredStorageType,
},
blockstore_processor::ProcessOptions,
shred::Shred,
},
@ -1715,6 +1717,7 @@ fn main() {
&genesis_config,
solana_runtime::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
AccessType::PrimaryOnly,
ShredStorageType::default(),
)
.unwrap_or_else(|err| {
eprintln!("Failed to write genesis config: {:?}", err);

114
ledger/src/blockstore.rs
View File

@ -6,7 +6,7 @@ use {
ancestor_iterator::AncestorIterator,
blockstore_db::{
columns as cf, AccessType, BlockstoreOptions, Column, Database, IteratorDirection,
IteratorMode, LedgerColumn, Result, WriteBatch,
IteratorMode, LedgerColumn, Result, ShredStorageType, WriteBatch,
},
blockstore_meta::*,
leader_schedule_cache::LeaderScheduleCache,
@ -72,7 +72,8 @@ pub use {
pub mod blockstore_purge;
pub const BLOCKSTORE_DIRECTORY: &str = "rocksdb";
pub const BLOCKSTORE_DIRECTORY_ROCKS_LEVEL: &str = "rocksdb";
pub const BLOCKSTORE_DIRECTORY_ROCKS_FIFO: &str = "rocksdb_fifo";
thread_local!(static PAR_THREAD_POOL: RefCell<ThreadPool> = RefCell::new(rayon::ThreadPoolBuilder::new()
.num_threads(get_thread_count())
@ -354,10 +355,19 @@ impl Blockstore {
self.db
}
/// The path to the ledger store
pub fn ledger_path(&self) -> &PathBuf {
&self.ledger_path
}
/// The directory under `ledger_path` to the underlying blockstore.
pub fn blockstore_directory(shred_storage_type: &ShredStorageType) -> &str {
match shred_storage_type {
ShredStorageType::RocksLevel => BLOCKSTORE_DIRECTORY_ROCKS_LEVEL,
ShredStorageType::RocksFifo(_) => BLOCKSTORE_DIRECTORY_ROCKS_FIFO,
}
}
/// Opens a Ledger in directory, provides "infinite" window of shreds
pub fn open(ledger_path: &Path) -> Result<Blockstore> {
Self::do_open(ledger_path, BlockstoreOptions::default())
@ -369,7 +379,8 @@ impl Blockstore {
fn do_open(ledger_path: &Path, options: BlockstoreOptions) -> Result<Blockstore> {
fs::create_dir_all(&ledger_path)?;
let blockstore_path = ledger_path.join(BLOCKSTORE_DIRECTORY);
let blockstore_path =
ledger_path.join(Self::blockstore_directory(&options.shred_storage_type));
adjust_ulimit_nofile(options.enforce_ulimit_nofile)?;
@ -547,11 +558,15 @@ impl Blockstore {
}
/// Deletes the blockstore at the specified path.
///
/// Note that if the `ledger_path` has multiple rocksdb instances, this
/// function will destroy all.
pub fn destroy(ledger_path: &Path) -> Result<()> {
// Database::destroy() fails if the path doesn't exist
// Database::destroy() fails if the root directory doesn't exist
fs::create_dir_all(ledger_path)?;
let blockstore_path = ledger_path.join(BLOCKSTORE_DIRECTORY);
Database::destroy(&blockstore_path)
Database::destroy(&Path::new(ledger_path).join(BLOCKSTORE_DIRECTORY_ROCKS_LEVEL)).and(
Database::destroy(&Path::new(ledger_path).join(BLOCKSTORE_DIRECTORY_ROCKS_FIFO)),
)
}
/// Returns the SlotMeta of the specified slot.
@ -3783,18 +3798,20 @@ pub fn create_new_ledger(
genesis_config: &GenesisConfig,
max_genesis_archive_unpacked_size: u64,
access_type: AccessType,
shred_storage_type: ShredStorageType,
) -> Result<Hash> {
Blockstore::destroy(ledger_path)?;
genesis_config.write(ledger_path)?;
// Fill slot 0 with ticks that link back to the genesis_config to bootstrap the ledger.
let blockstore_dir = Blockstore::blockstore_directory(&shred_storage_type);
let blockstore = Blockstore::open_with_options(
ledger_path,
BlockstoreOptions {
access_type,
recovery_mode: None,
enforce_ulimit_nofile: false,
..BlockstoreOptions::default()
shred_storage_type: shred_storage_type.clone(),
},
)?;
let ticks_per_slot = genesis_config.ticks_per_slot;
@ -3827,7 +3844,7 @@ pub fn create_new_ledger(
"-C",
ledger_path.to_str().unwrap(),
DEFAULT_GENESIS_FILE,
"rocksdb",
blockstore_dir,
];
let output = std::process::Command::new("tar")
.args(&args)
@ -3884,11 +3901,11 @@ pub fn create_new_ledger(
)
});
fs::rename(
&ledger_path.join("rocksdb"),
ledger_path.join("rocksdb.failed"),
&ledger_path.join(blockstore_dir),
ledger_path.join(format!("{}.failed", blockstore_dir)),
)
.unwrap_or_else(|e| {
error_messages += &format!("/failed to stash problematic rocksdb: {}", e)
error_messages += &format!("/failed to stash problematic {}: {}", blockstore_dir, e)
});
return Err(BlockstoreError::Io(IoError::new(
@ -3964,6 +3981,7 @@ macro_rules! create_new_tmp_ledger {
$crate::tmp_ledger_name!(),
$genesis_config,
$crate::blockstore_db::AccessType::PrimaryOnly,
$crate::blockstore_db::ShredStorageType::default(),
)
};
}
@ -3975,6 +3993,21 @@ macro_rules! create_new_tmp_ledger_auto_delete {
$crate::tmp_ledger_name!(),
$genesis_config,
$crate::blockstore_db::AccessType::PrimaryOnly,
$crate::blockstore_db::ShredStorageType::default(),
)
};
}
#[macro_export]
macro_rules! create_new_tmp_ledger_fifo_auto_delete {
($genesis_config:expr) => {
$crate::blockstore::create_new_ledger_from_name_auto_delete(
$crate::tmp_ledger_name!(),
$genesis_config,
$crate::blockstore_db::AccessType::PrimaryOnly,
$crate::blockstore_db::ShredStorageType::RocksFifo(
$crate::blockstore_db::BlockstoreRocksFifoOptions::default(),
),
)
};
}
@ -4005,9 +4038,14 @@ pub fn create_new_ledger_from_name(
name: &str,
genesis_config: &GenesisConfig,
access_type: AccessType,
shred_storage_type: ShredStorageType,
) -> (PathBuf, Hash) {
let (ledger_path, blockhash) =
create_new_ledger_from_name_auto_delete(name, genesis_config, access_type);
let (ledger_path, blockhash) = create_new_ledger_from_name_auto_delete(
name,
genesis_config,
access_type,
shred_storage_type,
);
(ledger_path.into_path(), blockhash)
}
@ -4019,6 +4057,7 @@ pub fn create_new_ledger_from_name_auto_delete(
name: &str,
genesis_config: &GenesisConfig,
access_type: AccessType,
shred_storage_type: ShredStorageType,
) -> (TempDir, Hash) {
let ledger_path = get_ledger_path_from_name_auto_delete(name);
let blockhash = create_new_ledger(
@ -4026,6 +4065,7 @@ pub fn create_new_ledger_from_name_auto_delete(
genesis_config,
MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
access_type,
shred_storage_type,
)
.unwrap();
(ledger_path, blockhash)
@ -4168,6 +4208,7 @@ pub mod tests {
use {
super::*,
crate::{
blockstore_db::BlockstoreRocksFifoOptions,
genesis_utils::{create_genesis_config, GenesisConfigInfo},
leader_schedule::{FixedSchedule, LeaderSchedule},
shred::{max_ticks_per_n_shreds, DataShredHeader},
@ -4224,6 +4265,53 @@ pub mod tests {
let entries = blockstore.get_slot_entries(0, 0).unwrap();
assert_eq!(ticks, entries);
assert!(Path::new(ledger_path.path())
.join(Blockstore::blockstore_directory(
&ShredStorageType::RocksLevel,
))
.exists());
}
#[test]
fn test_create_new_ledger_with_options_fifo() {
solana_logger::setup();
let mint_total = 1_000_000_000_000;
let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(mint_total);
let (ledger_path, _blockhash) = create_new_tmp_ledger_fifo_auto_delete!(&genesis_config);
let blockstore = Blockstore::open_with_options(
ledger_path.path(),
BlockstoreOptions {
shred_storage_type: ShredStorageType::RocksFifo(
BlockstoreRocksFifoOptions::default(),
),
..BlockstoreOptions::default()
},
)
.unwrap();
let ticks = create_ticks(genesis_config.ticks_per_slot, 0, genesis_config.hash());
let entries = blockstore.get_slot_entries(0, 0).unwrap();
assert_eq!(ticks, entries);
assert!(Path::new(ledger_path.path())
.join(Blockstore::blockstore_directory(
&ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions::default())
))
.exists());
}
#[test]
fn test_rocksdb_directory() {
assert_eq!(
Blockstore::blockstore_directory(&ShredStorageType::RocksLevel),
BLOCKSTORE_DIRECTORY_ROCKS_LEVEL
);
assert_eq!(
Blockstore::blockstore_directory(&ShredStorageType::RocksFifo(
BlockstoreRocksFifoOptions::default()
)),
BLOCKSTORE_DIRECTORY_ROCKS_FIFO
);
}
#[test]

8
ledger/src/blockstore_db.rs
View File

@ -971,6 +971,7 @@ pub struct WriteBatch<'a> {
map: HashMap<&'static str, &'a ColumnFamily>,
}
#[derive(Clone)]
pub enum ShredStorageType {
// Stores shreds under RocksDB's default compaction (level).
RocksLevel,
@ -980,6 +981,12 @@ pub enum ShredStorageType {
RocksFifo(BlockstoreRocksFifoOptions),
}
impl Default for ShredStorageType {
fn default() -> Self {
Self::RocksLevel
}
}
pub struct BlockstoreOptions {
// The access type of blockstore. Default: PrimaryOnly
pub access_type: AccessType,
@ -1003,6 +1010,7 @@ impl Default for BlockstoreOptions {
}
}
#[derive(Clone)]
pub struct BlockstoreRocksFifoOptions {
// The maximum storage size for storing data shreds in column family
// [`cf::DataShred`]. Typically, data shreds contribute around 25% of the

39
runtime/src/hardened_unpack.rs
View File

@ -467,6 +467,9 @@ fn is_valid_genesis_archive_entry(parts: &[&str], kind: tar::EntryType) -> bool
(["rocksdb"], Directory) => true,
(["rocksdb", _], GNUSparse) => true,
(["rocksdb", _], Regular) => true,
(["rocksdb_fifo"], Directory) => true,
(["rocksdb_fifo", _], GNUSparse) => true,
(["rocksdb_fifo", _], Regular) => true,
_ => false,
}
}
@ -600,6 +603,18 @@ mod tests {
&["rocksdb", "foo"],
tar::EntryType::GNUSparse,
));
assert!(is_valid_genesis_archive_entry(
&["rocksdb_fifo"],
tar::EntryType::Directory
));
assert!(is_valid_genesis_archive_entry(
&["rocksdb_fifo", "foo"],
tar::EntryType::Regular
));
assert!(is_valid_genesis_archive_entry(
&["rocksdb_fifo", "foo"],
tar::EntryType::GNUSparse,
));
assert!(!is_valid_genesis_archive_entry(
&["aaaa"],
@ -633,6 +648,30 @@ mod tests {
&["rocksdb", "foo", "bar"],
tar::EntryType::GNUSparse
));
assert!(!is_valid_genesis_archive_entry(
&["rocksdb_fifo"],
tar::EntryType::Regular
));
assert!(!is_valid_genesis_archive_entry(
&["rocksdb_fifo"],
tar::EntryType::GNUSparse,
));
assert!(!is_valid_genesis_archive_entry(
&["rocksdb_fifo", "foo"],
tar::EntryType::Directory,
));
assert!(!is_valid_genesis_archive_entry(
&["rocksdb_fifo", "foo", "bar"],
tar::EntryType::Directory,
));
assert!(!is_valid_genesis_archive_entry(
&["rocksdb_fifo", "foo", "bar"],
tar::EntryType::Regular
));
assert!(!is_valid_genesis_archive_entry(
&["rocksdb_fifo", "foo", "bar"],
tar::EntryType::GNUSparse
));
}
fn with_finalize_and_unpack<C>(archive: tar::Builder<Vec<u8>>, checker: C) -> Result<()>

5
test-validator/src/lib.rs
View File

@ -12,7 +12,9 @@ use {
gossip_service::discover_cluster,
socketaddr,
},
solana_ledger::{blockstore::create_new_ledger, create_new_tmp_ledger},
solana_ledger::{
blockstore::create_new_ledger, blockstore_db::ShredStorageType, create_new_tmp_ledger,
},
solana_net_utils::PortRange,
solana_rpc::{rpc::JsonRpcConfig, rpc_pubsub_service::PubSubConfig},
solana_runtime::{
@ -580,6 +582,7 @@ impl TestValidator {
.max_genesis_archive_unpacked_size
.unwrap_or(MAX_GENESIS_ARCHIVE_UNPACKED_SIZE),
solana_ledger::blockstore_db::AccessType::PrimaryOnly,
ShredStorageType::default(),
)
.map_err(|err| {
format!(