blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

ledger path reform: use Path/PathBuf instead of strings (#5344)

This commit is contained in:
Michael Vines 2019-07-30 15:53:41 -07:00 committed by GitHub
parent 294d9288d2
commit c78db6a94b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 76 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
core/benches/blocktree.rs
View File

@ -11,12 +11,13 @@ use rand::{thread_rng, Rng};
use solana::blocktree::{get_tmp_ledger_path, Blocktree};
use solana::entry::{make_large_test_entries, make_tiny_test_entries, EntrySlice};
use solana::packet::{Blob, BLOB_HEADER_SIZE};
use std::path::Path;
use test::Bencher;
// Given some blobs and a ledger at ledger_path, benchmark writing the blobs to the ledger
fn bench_write_blobs(bench: &mut Bencher, blobs: &mut Vec<Blob>, ledger_path: &str) {
fn bench_write_blobs(bench: &mut Bencher, blobs: &mut Vec<Blob>, ledger_path: &Path) {
let blocktree =
Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
Blocktree::open(ledger_path).expect("Expected to be able to open database ledger");
let num_blobs = blobs.len();
@ -36,7 +37,7 @@ fn bench_write_blobs(bench: &mut Bencher, blobs: &mut Vec<Blob>, ledger_path: &s
}
});
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
Blocktree::destroy(ledger_path).expect("Expected successful database destruction");
}
// Insert some blobs into the ledger in preparation for read benchmarks

37
core/src/blocktree.rs
View File

@ -27,6 +27,7 @@ use std::cell::RefCell;
use std::cmp;
use std::fs;
use std::io;
use std::path::{Path, PathBuf};
use std::rc::Rc;
use std::sync::mpsc::{sync_channel, Receiver, SyncSender, TrySendError};
use std::sync::{Arc, RwLock};
@ -113,14 +114,12 @@ pub const INDEX_CF: &str = "index";
impl Blocktree {
/// Opens a Ledger in directory, provides "infinite" window of blobs
pub fn open(ledger_path: &str) -> Result<Blocktree> {
use std::path::Path;
pub fn open(ledger_path: &Path) -> Result<Blocktree> {
fs::create_dir_all(&ledger_path)?;
let ledger_path = Path::new(&ledger_path).join(BLOCKTREE_DIRECTORY);
let blocktree_path = ledger_path.join(BLOCKTREE_DIRECTORY);
// Open the database
let db = Database::open(&ledger_path)?;
let db = Database::open(&blocktree_path)?;
let batch_processor = unsafe { Arc::new(RwLock::new(db.batch_processor())) };
@ -162,7 +161,7 @@ impl Blocktree {
}
pub fn open_with_signal(
ledger_path: &str,
ledger_path: &Path,
) -> Result<(Self, Receiver<bool>, CompletedSlotsReceiver)> {
let mut blocktree = Self::open(ledger_path)?;
let (signal_sender, signal_receiver) = sync_channel(1);
@ -174,11 +173,11 @@ impl Blocktree {
Ok((blocktree, signal_receiver, completed_slots_receiver))
}
pub fn destroy(ledger_path: &str) -> Result<()> {
// Database::destroy() fails is the path doesn't exist
pub fn destroy(ledger_path: &Path) -> Result<()> {
// Database::destroy() fails if the path doesn't exist
fs::create_dir_all(ledger_path)?;
let path = std::path::Path::new(ledger_path).join(BLOCKTREE_DIRECTORY);
Database::destroy(&path)
let blocktree_path = ledger_path.join(BLOCKTREE_DIRECTORY);
Database::destroy(&blocktree_path)
}
pub fn meta(&self, slot: u64) -> Result<Option<SlotMeta>> {
@ -1958,7 +1957,7 @@ fn slot_has_updates(slot_meta: &SlotMeta, slot_meta_backup: &Option<SlotMeta>) -
// Creates a new ledger with slot 0 full of ticks (and only ticks).
//
// Returns the blockhash that can be used to append entries with.
pub fn create_new_ledger(ledger_path: &str, genesis_block: &GenesisBlock) -> Result<Hash> {
pub fn create_new_ledger(ledger_path: &Path, genesis_block: &GenesisBlock) -> Result<Hash> {
let ticks_per_slot = genesis_block.ticks_per_slot;
Blocktree::destroy(ledger_path)?;
genesis_block.write(&ledger_path)?;
@ -1971,7 +1970,7 @@ pub fn create_new_ledger(ledger_path: &str, genesis_block: &GenesisBlock) -> Res
Ok(entries.last().unwrap().hash)
}
pub fn genesis<'a, I>(ledger_path: &str, keypair: &Keypair, entries: I) -> Result<()>
pub fn genesis<'a, I>(ledger_path: &Path, keypair: &Keypair, entries: I) -> Result<()>
where
I: IntoIterator<Item = &'a Entry>,
{
@ -2008,12 +2007,18 @@ macro_rules! get_tmp_ledger_path {
};
}
pub fn get_tmp_ledger_path(name: &str) -> String {
pub fn get_tmp_ledger_path(name: &str) -> PathBuf {
use std::env;
let out_dir = env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string());
let keypair = Keypair::new();
let path = format!("{}/ledger/{}-{}", out_dir, name, keypair.pubkey());
let path = [
out_dir,
"ledger".to_string(),
format!("{}-{}", name, keypair.pubkey()),
]
.iter()
.collect();
// whack any possible collision
let _ignored = fs::remove_dir_all(&path);
@ -2032,7 +2037,7 @@ macro_rules! create_new_tmp_ledger {
//
// Note: like `create_new_ledger` the returned ledger will have slot 0 full of ticks (and only
// ticks)
pub fn create_new_tmp_ledger(name: &str, genesis_block: &GenesisBlock) -> (String, Hash) {
pub fn create_new_tmp_ledger(name: &str, genesis_block: &GenesisBlock) -> (PathBuf, Hash) {
let ledger_path = get_tmp_ledger_path(name);
let blockhash = create_new_ledger(&ledger_path, genesis_block).unwrap();
(ledger_path, blockhash)
@ -2045,7 +2050,7 @@ macro_rules! tmp_copy_blocktree {
};
}
pub fn tmp_copy_blocktree(from: &str, name: &str) -> String {
pub fn tmp_copy_blocktree(from: &Path, name: &str) -> PathBuf {
let path = get_tmp_ledger_path(name);
let blocktree = Blocktree::open(from).unwrap();

3
core/src/broadcast_stage.rs
View File

@ -241,6 +241,7 @@ mod test {
use solana_sdk::hash::Hash;
use solana_sdk::pubkey::Pubkey;
use solana_sdk::signature::{Keypair, KeypairUtil};
use std::path::Path;
use std::sync::atomic::AtomicBool;
use std::sync::mpsc::channel;
use std::sync::{Arc, RwLock};
@ -255,7 +256,7 @@ mod test {
fn setup_dummy_broadcast_service(
leader_pubkey: &Pubkey,
ledger_path: &str,
ledger_path: &Path,
entry_receiver: Receiver<WorkingBankEntries>,
) -> MockBroadcastStage {
// Make the database ledger

3
core/src/cluster_tests.rs
View File

@ -22,6 +22,7 @@ use solana_sdk::timing::{
NUM_CONSECUTIVE_LEADER_SLOTS,
};
use solana_sdk::transport::TransportError;
use std::path::Path;
use std::thread::sleep;
use std::time::Duration;
@ -94,7 +95,7 @@ pub fn fullnode_exit(entry_point_info: &ContactInfo, nodes: usize) {
}
}
pub fn verify_ledger_ticks(ledger_path: &str, ticks_per_slot: usize) {
pub fn verify_ledger_ticks(ledger_path: &Path, ticks_per_slot: usize) {
let ledger = Blocktree::open(ledger_path).unwrap();
let zeroth_slot = ledger.get_slot_entries(0, 0, None).unwrap();
let last_id = zeroth_slot.last().unwrap().hash;

3
core/src/erasure.rs
View File

@ -334,6 +334,7 @@ pub mod test {
use solana_sdk::signature::Signable;
use solana_sdk::signature::{Keypair, KeypairUtil};
use std::borrow::Borrow;
use std::path::Path;
/// Specifies the contents of a 16-data-blob and 4-coding-blob erasure set
/// Exists to be passed to `generate_blocktree_with_coding`
@ -748,7 +749,7 @@ pub mod test {
/// Genarates a ledger according to the given specs.
/// Blocktree should have correct SlotMeta and ErasureMeta and so on but will not have done any
/// possible recovery.
pub fn generate_blocktree_with_coding(ledger_path: &str, specs: &[SlotSpec]) -> Blocktree {
pub fn generate_blocktree_with_coding(ledger_path: &Path, specs: &[SlotSpec]) -> Blocktree {
let blocktree = Blocktree::open(ledger_path).unwrap();
let model = generate_ledger_model(specs);

9
core/src/local_cluster.rs
View File

@ -27,6 +27,7 @@ use solana_vote_api::vote_state::VoteState;
use std::collections::HashMap;
use std::fs::remove_dir_all;
use std::io::{Error, ErrorKind, Result};
use std::path::PathBuf;
use std::sync::Arc;
use solana_librapay_api::librapay_transaction;
@ -36,17 +37,17 @@ pub struct ValidatorInfo {
pub keypair: Arc<Keypair>,
pub voting_keypair: Arc<Keypair>,
pub storage_keypair: Arc<Keypair>,
pub ledger_path: String,
pub ledger_path: PathBuf,
pub contact_info: ContactInfo,
}
pub struct ReplicatorInfo {
pub replicator_storage_pubkey: Pubkey,
pub ledger_path: String,
pub ledger_path: PathBuf,
}
impl ReplicatorInfo {
fn new(storage_pubkey: Pubkey, ledger_path: String) -> Self {
fn new(storage_pubkey: Pubkey, ledger_path: PathBuf) -> Self {
Self {
replicator_storage_pubkey: storage_pubkey,
ledger_path,
@ -395,7 +396,7 @@ impl LocalCluster {
.chain(self.replicator_infos.values().map(|info| &info.ledger_path))
{
remove_dir_all(&ledger_path)
.unwrap_or_else(|_| panic!("Unable to remove {}", ledger_path));
.unwrap_or_else(|_| panic!("Unable to remove {:?}", ledger_path));
}
}

9
core/src/replicator.rs
View File

@ -62,7 +62,7 @@ pub struct Replicator {
struct ReplicatorMeta {
slot: u64,
slots_per_segment: u64,
ledger_path: String,
ledger_path: PathBuf,
signature: Signature,
ledger_data_file_encrypted: PathBuf,
sampling_offsets: Vec<u64>,
@ -200,7 +200,7 @@ impl Replicator {
/// * `keypair` - Keypair for this replicator
#[allow(clippy::new_ret_no_self)]
pub fn new(
ledger_path: &str,
ledger_path: &Path,
node: Node,
cluster_entrypoint: ContactInfo,
keypair: Arc<Keypair>,
@ -263,7 +263,7 @@ impl Replicator {
let exit = exit.clone();
let node_info = node.info.clone();
let mut meta = ReplicatorMeta {
ledger_path: ledger_path.to_string(),
ledger_path: ledger_path.to_path_buf(),
..ReplicatorMeta::default()
};
spawn(move || {
@ -516,8 +516,7 @@ impl Replicator {
}
fn encrypt_ledger(meta: &mut ReplicatorMeta, blocktree: &Arc<Blocktree>) -> Result<()> {
let ledger_path = Path::new(&meta.ledger_path);
meta.ledger_data_file_encrypted = ledger_path.join(ENCRYPTED_FILENAME);
meta.ledger_data_file_encrypted = meta.ledger_path.join(ENCRYPTED_FILENAME);
{
let mut ivec = [0u8; 64];

7
core/src/validator.rs
View File

@ -26,6 +26,7 @@ use solana_sdk::pubkey::Pubkey;
use solana_sdk::signature::{Keypair, KeypairUtil};
use solana_sdk::timing::{timestamp, DEFAULT_SLOTS_PER_TURN};
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::Receiver;
use std::sync::{Arc, Mutex, RwLock};
@ -79,7 +80,7 @@ impl Validator {
pub fn new(
mut node: Node,
keypair: &Arc<Keypair>,
ledger_path: &str,
ledger_path: &Path,
vote_account: &Pubkey,
voting_keypair: &Arc<Keypair>,
storage_keypair: &Arc<Keypair>,
@ -340,7 +341,7 @@ fn get_bank_forks(
}
pub fn new_banks_from_blocktree(
blocktree_path: &str,
blocktree_path: &Path,
account_paths: Option<String>,
snapshot_path: Option<String>,
verify_ledger: bool,
@ -401,7 +402,7 @@ impl Service for Validator {
}
}
pub fn new_validator_for_tests() -> (Validator, ContactInfo, Keypair, String) {
pub fn new_validator_for_tests() -> (Validator, ContactInfo, Keypair, PathBuf) {
use crate::blocktree::create_new_tmp_ledger;
use crate::genesis_utils::{create_genesis_block_with_leader, GenesisBlockInfo};

2
core/tests/replicator.rs
View File

@ -93,7 +93,7 @@ fn test_replicator_startup_leader_hang() {
solana_logger::setup();
info!("starting replicator test");
let leader_ledger_path = "replicator_test_leader_ledger";
let leader_ledger_path = std::path::PathBuf::from("replicator_test_leader_ledger");
let (genesis_block, _mint_keypair) = create_genesis_block(10_000);
let (replicator_ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_block);

5
genesis/src/main.rs
View File

@ -34,6 +34,7 @@ use std::collections::HashMap;
use std::error;
use std::fs::File;
use std::io;
use std::path::PathBuf;
use std::str::FromStr;
use std::time::{Duration, Instant};
@ -248,7 +249,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
let bootstrap_storage_keypair_file =
matches.value_of("bootstrap_storage_keypair_file").unwrap();
let mint_keypair_file = matches.value_of("mint_keypair_file").unwrap();
let ledger_path = matches.value_of("ledger_path").unwrap();
let ledger_path = PathBuf::from(matches.value_of("ledger_path").unwrap());
let lamports = value_t_or_exit!(matches, "lamports", u64);
let bootstrap_leader_lamports = value_t_or_exit!(matches, "bootstrap_leader_lamports", u64);
let bootstrap_leader_stake_lamports =
@ -359,7 +360,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
builder = solana_storage_api::rewards_pools::genesis(builder);
builder = solana_stake_api::rewards_pools::genesis(builder);
create_new_ledger(ledger_path, &builder.build())?;
create_new_ledger(&ledger_path, &builder.build())?;
Ok(())
}

11
ledger-tool/src/main.rs
View File

@ -5,6 +5,7 @@ use solana_sdk::genesis_block::GenesisBlock;
use std::collections::BTreeMap;
use std::fs::File;
use std::io::{stdout, Write};
use std::path::PathBuf;
use std::process::exit;
use std::str::FromStr;
@ -119,20 +120,20 @@ fn main() {
))
.get_matches();
let ledger_path = matches.value_of("ledger").unwrap();
let ledger_path = PathBuf::from(matches.value_of("ledger").unwrap());
let genesis_block = GenesisBlock::load(ledger_path).unwrap_or_else(|err| {
let genesis_block = GenesisBlock::load(&ledger_path).unwrap_or_else(|err| {
eprintln!(
"Failed to open ledger genesis_block at {}: {}",
"Failed to open ledger genesis_block at {:?}: {}",
ledger_path, err
);
exit(1);
});
let blocktree = match Blocktree::open(ledger_path) {
let blocktree = match Blocktree::open(&ledger_path) {
Ok(blocktree) => blocktree,
Err(err) => {
eprintln!("Failed to open ledger at {}: {}", ledger_path, err);
eprintln!("Failed to open ledger at {:?}: {}", ledger_path, err);
exit(1);
}
};

2
ledger-tool/tests/basic.rs
View File

@ -38,6 +38,8 @@ fn nominal() {
let (ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_block);
let ticks = ticks_per_slot as usize;
let ledger_path = ledger_path.to_str().unwrap();
// Basic validation
let output = run_ledger_tool(&["-l", &ledger_path, "verify"]);
assert!(output.status.success());

5
replicator/src/main.rs
View File

@ -4,6 +4,7 @@ use solana::contact_info::ContactInfo;
use solana::replicator::Replicator;
use solana_sdk::signature::{read_keypair, Keypair, KeypairUtil};
use std::net::SocketAddr;
use std::path::PathBuf;
use std::process::exit;
use std::sync::Arc;
@ -50,7 +51,7 @@ fn main() {
)
.get_matches();
let ledger_path = matches.value_of("ledger").unwrap();
let ledger_path = PathBuf::from(matches.value_of("ledger").unwrap());
let keypair = if let Some(identity) = matches.value_of("identity") {
read_keypair(identity).unwrap_or_else(|err| {
@ -93,7 +94,7 @@ fn main() {
let entrypoint_info = ContactInfo::new_gossip_entry_point(&entrypoint_addr);
let replicator = Replicator::new(
ledger_path,
&ledger_path,
node,
entrypoint_info,
Arc::new(keypair),

20
sdk/src/genesis_block.rs
View File

@ -166,7 +166,7 @@ impl GenesisBlock {
hash(&serialized.into_bytes())
}
pub fn load(ledger_path: &str) -> Result<Self, std::io::Error> {
pub fn load(ledger_path: &Path) -> Result<Self, std::io::Error> {
let file = OpenOptions::new()
.read(true)
.open(&Path::new(ledger_path).join("genesis.bin"))
@ -179,14 +179,13 @@ impl GenesisBlock {
Ok(genesis_block)
}
pub fn write(&self, ledger_path: &str) -> Result<(), std::io::Error> {
pub fn write(&self, ledger_path: &Path) -> Result<(), std::io::Error> {
let serialized = serialize(&self)
.map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, format!("{:?}", err)))?;
let dir = Path::new(ledger_path);
std::fs::create_dir_all(&dir)?;
std::fs::create_dir_all(&ledger_path)?;
let mut file = File::create(&dir.join("genesis.bin"))?;
let mut file = File::create(&ledger_path.join("genesis.bin"))?;
file.write_all(&serialized)
}
}
@ -195,12 +194,19 @@ impl GenesisBlock {
mod tests {
use super::*;
use crate::signature::{Keypair, KeypairUtil};
use std::path::PathBuf;
fn make_tmp_path(name: &str) -> String {
fn make_tmp_path(name: &str) -> PathBuf {
let out_dir = std::env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string());
let keypair = Keypair::new();
let path = format!("{}/tmp/{}-{}", out_dir, name, keypair.pubkey());
let path = [
out_dir,
"tmp".to_string(),
format!("{}-{}", name, keypair.pubkey()),
]
.iter()
.collect();
// whack any possible collision
let _ignored = std::fs::remove_dir_all(&path);

7
validator/src/main.rs
View File

@ -11,6 +11,7 @@ use solana_netutil::parse_port_range;
use solana_sdk::signature::{read_keypair, Keypair, KeypairUtil};
use std::fs::File;
use std::net::SocketAddr;
use std::path::PathBuf;
use std::process::exit;
use std::sync::Arc;
@ -75,7 +76,7 @@ fn main() {
.help("Create this file, if it doesn't already exist, once node initialization is complete"),
)
.arg(
Arg::with_name("ledger")
Arg::with_name("ledger_path")
.short("l")
.long("ledger")
.value_name("DIR")
@ -208,7 +209,7 @@ fn main() {
pubkey.parse().expect("failed to parse vote_account")
});
let ledger_path = matches.value_of("ledger").unwrap();
let ledger_path = PathBuf::from(matches.value_of("ledger_path").unwrap());
validator_config.sigverify_disabled = matches.is_present("no_sigverify");
@ -283,7 +284,7 @@ fn main() {
let validator = Validator::new(
node,
&keypair,
ledger_path,
&ledger_path,
&vote_account,
&Arc::new(voting_keypair),
&Arc::new(storage_keypair),