From 4d52f47f87d9b6e8ceefb8da7fe9e908226d363f Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Mon, 4 Nov 2019 19:10:06 -0700 Subject: [PATCH] Move get_bank_forks() into ledger/ so its available for use by ledger-tool/ (#6720) --- core/src/validator.rs | 85 ++++++---------------------------- core/tests/bank_forks.rs | 2 +- ledger-tool/src/main.rs | 32 +++++++------ ledger/src/bank_forks.rs | 4 +- ledger/src/bank_forks_utils.rs | 62 +++++++++++++++++++++++++ ledger/src/lib.rs | 1 + 6 files changed, 98 insertions(+), 88 deletions(-) create mode 100644 ledger/src/bank_forks_utils.rs diff --git a/core/src/validator.rs b/core/src/validator.rs index 82ee4e81c..4ef6edbf3 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -20,10 +20,10 @@ use crate::{ }; use solana_ledger::{ bank_forks::{BankForks, SnapshotConfig}, + bank_forks_utils, blocktree::{Blocktree, CompletedSlotsReceiver}, blocktree_processor::{self, BankForksInfo}, leader_schedule_cache::LeaderScheduleCache, - snapshot_utils, }; use solana_metrics::datapoint_info; use solana_sdk::{ @@ -37,7 +37,6 @@ use solana_sdk::{ }; use std::{ - fs, net::{IpAddr, Ipv4Addr, SocketAddr}, path::{Path, PathBuf}, process, @@ -409,67 +408,6 @@ impl Validator { } } -fn get_bank_forks( - genesis_block: &GenesisBlock, - blocktree: &Blocktree, - account_paths: Option, - snapshot_config: Option<&SnapshotConfig>, - verify_ledger: bool, - dev_halt_at_slot: Option, -) -> (BankForks, Vec, LeaderScheduleCache) { - let process_options = blocktree_processor::ProcessOptions { - verify_ledger, - dev_halt_at_slot, - ..blocktree_processor::ProcessOptions::default() - }; - - if let Some(snapshot_config) = snapshot_config.as_ref() { - info!( - "Initializing snapshot path: {:?}", - snapshot_config.snapshot_path - ); - let _ = fs::remove_dir_all(&snapshot_config.snapshot_path); - fs::create_dir_all(&snapshot_config.snapshot_path) - .expect("Couldn't create snapshot directory"); - - let tar = - snapshot_utils::get_snapshot_tar_path(&snapshot_config.snapshot_package_output_path); - if tar.exists() { - info!("Loading snapshot package: {:?}", tar); - // Fail hard here if snapshot fails to load, don't silently continue - let deserialized_bank = snapshot_utils::bank_from_archive( - account_paths - .clone() - .expect("Account paths not present when booting from snapshot"), - &snapshot_config.snapshot_path, - &tar, - ) - .expect("Load from snapshot failed"); - - return blocktree_processor::process_blocktree_from_root( - genesis_block, - blocktree, - Arc::new(deserialized_bank), - &process_options, - ) - .expect("processing blocktree after loading snapshot failed"); - } else { - info!("Snapshot package does not exist: {:?}", tar); - } - } else { - info!("Snapshots disabled"); - } - - info!("Processing ledger from genesis"); - blocktree_processor::process_blocktree( - &genesis_block, - &blocktree, - account_paths, - process_options, - ) - .expect("process_blocktree failed") -} - #[cfg(not(unix))] fn adjust_ulimit_nofile() {} @@ -549,18 +487,25 @@ pub fn new_banks_from_blocktree( let (blocktree, ledger_signal_receiver, completed_slots_receiver) = Blocktree::open_with_signal(blocktree_path).expect("Failed to open ledger database"); - let (mut bank_forks, bank_forks_info, leader_schedule_cache) = get_bank_forks( + let process_options = blocktree_processor::ProcessOptions { + verify_ledger, + dev_halt_at_slot, + ..blocktree_processor::ProcessOptions::default() + }; + + let (mut bank_forks, bank_forks_info, leader_schedule_cache) = bank_forks_utils::load( &genesis_block, &blocktree, account_paths, snapshot_config.as_ref(), - verify_ledger, - dev_halt_at_slot, - ); + process_options, + ) + .unwrap_or_else(|err| { + error!("Failed to load ledger: {:?}", err); + std::process::exit(1); + }); - if let Some(snapshot_config) = snapshot_config { - bank_forks.set_snapshot_config(snapshot_config); - } + bank_forks.set_snapshot_config(snapshot_config); ( genesis_blockhash, diff --git a/core/tests/bank_forks.rs b/core/tests/bank_forks.rs index a50486493..2c3344a7e 100644 --- a/core/tests/bank_forks.rs +++ b/core/tests/bank_forks.rs @@ -63,7 +63,7 @@ mod tests { snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()), snapshot_path: PathBuf::from(snapshot_dir.path()), }; - bank_forks.set_snapshot_config(snapshot_config.clone()); + bank_forks.set_snapshot_config(Some(snapshot_config.clone())); SnapshotTestConfig { accounts_dir, snapshot_dir, diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index d343fdd8e..e446a53c1 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -1,15 +1,17 @@ use clap::{crate_description, crate_name, crate_version, value_t_or_exit, App, Arg, SubCommand}; -use solana_ledger::blocktree::Blocktree; -use solana_ledger::blocktree_processor::{process_blocktree, ProcessOptions}; -use solana_ledger::rooted_slot_iterator::RootedSlotIterator; -use solana_sdk::clock::Slot; -use solana_sdk::genesis_block::GenesisBlock; -use std::collections::BTreeMap; -use std::fs::File; -use std::io::{stdout, Write}; -use std::path::PathBuf; -use std::process::exit; -use std::str::FromStr; +use solana_ledger::{ + bank_forks_utils, blocktree::Blocktree, blocktree_processor, + rooted_slot_iterator::RootedSlotIterator, +}; +use solana_sdk::{clock::Slot, genesis_block::GenesisBlock}; +use std::{ + collections::BTreeMap, + fs::File, + io::{stdout, Write}, + path::PathBuf, + process::exit, + str::FromStr, +}; #[derive(PartialEq)] enum LedgerOutputMethod { @@ -169,12 +171,12 @@ fn main() { } ("verify", _) => { println!("Verifying ledger..."); - let options = ProcessOptions { + let process_options = blocktree_processor::ProcessOptions { verify_ledger: true, - ..ProcessOptions::default() + ..blocktree_processor::ProcessOptions::default() }; - match process_blocktree(&genesis_block, &blocktree, None, options) { - Ok((_bank_forks, bank_forks_info, _)) => { + match bank_forks_utils::load(&genesis_block, &blocktree, None, None, process_options) { + Ok((_bank_forks, bank_forks_info, _leader_schedule_cache)) => { println!("{:?}", bank_forks_info); } Err(err) => { diff --git a/ledger/src/bank_forks.rs b/ledger/src/bank_forks.rs index 7c3f94792..a4c13bd74 100644 --- a/ledger/src/bank_forks.rs +++ b/ledger/src/bank_forks.rs @@ -292,8 +292,8 @@ impl BankForks { .retain(|slot, _| slot == &root || descendants[&root].contains(slot)); } - pub fn set_snapshot_config(&mut self, snapshot_config: SnapshotConfig) { - self.snapshot_config = Some(snapshot_config); + pub fn set_snapshot_config(&mut self, snapshot_config: Option) { + self.snapshot_config = snapshot_config; } pub fn snapshot_config(&self) -> &Option { diff --git a/ledger/src/bank_forks_utils.rs b/ledger/src/bank_forks_utils.rs new file mode 100644 index 000000000..00659ae07 --- /dev/null +++ b/ledger/src/bank_forks_utils.rs @@ -0,0 +1,62 @@ +use crate::{ + bank_forks::{BankForks, SnapshotConfig}, + blocktree::Blocktree, + blocktree_processor::{self, BankForksInfo, BlocktreeProcessorError, ProcessOptions}, + leader_schedule_cache::LeaderScheduleCache, + snapshot_utils, +}; +use log::*; +use solana_sdk::genesis_block::GenesisBlock; +use std::{fs, sync::Arc}; + +pub fn load( + genesis_block: &GenesisBlock, + blocktree: &Blocktree, + account_paths: Option, + snapshot_config: Option<&SnapshotConfig>, + process_options: ProcessOptions, +) -> Result<(BankForks, Vec, LeaderScheduleCache), BlocktreeProcessorError> { + if let Some(snapshot_config) = snapshot_config.as_ref() { + info!( + "Initializing snapshot path: {:?}", + snapshot_config.snapshot_path + ); + let _ = fs::remove_dir_all(&snapshot_config.snapshot_path); + fs::create_dir_all(&snapshot_config.snapshot_path) + .expect("Couldn't create snapshot directory"); + + let tar = + snapshot_utils::get_snapshot_tar_path(&snapshot_config.snapshot_package_output_path); + if tar.exists() { + info!("Loading snapshot package: {:?}", tar); + // Fail hard here if snapshot fails to load, don't silently continue + let deserialized_bank = snapshot_utils::bank_from_archive( + account_paths + .clone() + .expect("Account paths not present when booting from snapshot"), + &snapshot_config.snapshot_path, + &tar, + ) + .expect("Load from snapshot failed"); + + return blocktree_processor::process_blocktree_from_root( + genesis_block, + blocktree, + Arc::new(deserialized_bank), + &process_options, + ); + } else { + info!("Snapshot package does not exist: {:?}", tar); + } + } else { + info!("Snapshots disabled"); + } + + info!("Processing ledger from genesis"); + blocktree_processor::process_blocktree( + &genesis_block, + &blocktree, + account_paths, + process_options, + ) +} diff --git a/ledger/src/lib.rs b/ledger/src/lib.rs index c64a98dec..29b9a3074 100644 --- a/ledger/src/lib.rs +++ b/ledger/src/lib.rs @@ -1,4 +1,5 @@ pub mod bank_forks; +pub mod bank_forks_utils; pub mod block_error; #[macro_use] pub mod blocktree;