ledger-tool can now load a ledger snapshot (#6729)

ci/testnet-manager.sh

@@ -216,7 +216,7 @@ maybe_deploy_software() {
   (
     echo "--- net.sh restart"
     set -x
-    time net/net.sh restart --skip-setup -t "$CHANNEL_OR_TAG" --skip-ledger-verify "$arg"
+    time net/net.sh restart --skip-setup -t "$CHANNEL_OR_TAG" --skip-poh-verify "$arg"
   ) || ok=false
   if ! $ok; then
     net/net.sh logs

core/src/validator.rs

@@ -118,7 +118,7 @@ impl Validator {
         voting_keypair: &Arc<Keypair>,
         storage_keypair: &Arc<Keypair>,
         entrypoint_info_option: Option<&ContactInfo>,
-        verify_ledger: bool,
+        poh_verify: bool,
         config: &ValidatorConfig,
     ) -> Self {
         let id = keypair.pubkey();
@@ -171,7 +171,7 @@ impl Validator {
             ledger_path,
             config.account_paths.clone(),
             config.snapshot_config.clone(),
-            verify_ledger,
+            poh_verify,
             config.dev_halt_at_slot,
         );
 
@@ -452,7 +452,7 @@ pub fn new_banks_from_blocktree(
     blocktree_path: &Path,
     account_paths: Option<String>,
     snapshot_config: Option<SnapshotConfig>,
-    verify_ledger: bool,
+    poh_verify: bool,
     dev_halt_at_slot: Option<Slot>,
 ) -> (
     Hash,
@@ -488,7 +488,7 @@ pub fn new_banks_from_blocktree(
         Blocktree::open_with_signal(blocktree_path).expect("Failed to open ledger database");
 
     let process_options = blocktree_processor::ProcessOptions {
-        verify_ledger,
+        poh_verify,
         dev_halt_at_slot,
         ..blocktree_processor::ProcessOptions::default()
     };

ledger-tool/src/main.rs

@@ -1,6 +1,8 @@
-use clap::{crate_description, crate_name, crate_version, value_t_or_exit, App, Arg, SubCommand};
+use clap::{
+    crate_description, crate_name, crate_version, value_t, value_t_or_exit, App, Arg, SubCommand,
+};
 use solana_ledger::{
-    bank_forks_utils, blocktree::Blocktree, blocktree_processor,
+    bank_forks::SnapshotConfig, bank_forks_utils, blocktree::Blocktree, blocktree_processor,
     rooted_slot_iterator::RootedSlotIterator,
 };
 use solana_sdk::{clock::Slot, genesis_block::GenesisBlock};
@@ -71,7 +73,7 @@ fn output_ledger(blocktree: Blocktree, starting_slot: Slot, method: LedgerOutput
 
 fn main() {
     const DEFAULT_ROOT_COUNT: &str = "1";
-    solana_logger::setup();
+    solana_logger::setup_with_filter("solana=info");
 
     let starting_slot_arg = Arg::with_name("starting_slot")
         .long("starting-slot")
@@ -103,8 +105,37 @@ fn main() {
         ))
         .subcommand(SubCommand::with_name("bounds").about("Print lowest and highest non-empty slots. Note: This ignores gaps in slots"))
         .subcommand(SubCommand::with_name("json").about("Print the ledger in JSON format").arg(&starting_slot_arg))
-        .subcommand(SubCommand::with_name("verify").about("Verify the ledger's PoH"))
+        .subcommand(
-        .subcommand(SubCommand::with_name("prune").about("Prune the ledger at the block height").arg(
+            SubCommand::with_name("verify")
+            .about("Verify the ledger's PoH")
+            .arg(
+                Arg::with_name("no_snapshot")
+                    .long("no-snapshot")
+                    .takes_value(false)
+                    .help("Do not start from a local snapshot if present"),
+            )
+            .arg(
+                Arg::with_name("account_paths")
+                    .long("accounts")
+                    .value_name("PATHS")
+                    .takes_value(true)
+                    .help("Comma separated persistent accounts location"),
+            )
+        .arg(
+            Arg::with_name("halt_at_slot")
+                .long("halt-at-slot")
+                .value_name("SLOT")
+                .takes_value(true)
+                .help("Halt processing at the given slot"),
+        )
+        .arg(
+            clap::Arg::with_name("skip_poh_verify")
+                .long("skip-poh-verify")
+                .takes_value(false)
+                .help("Skip ledger PoH verification"),
+        )
+
+        ).subcommand(SubCommand::with_name("prune").about("Prune the ledger at the block height").arg(
             Arg::with_name("slot_list")
                 .long("slot-list")
                 .value_name("FILENAME")
@@ -169,15 +200,42 @@ fn main() {
             let starting_slot = value_t_or_exit!(args_matches, "starting_slot", Slot);
             output_ledger(blocktree, starting_slot, LedgerOutputMethod::Json);
         }
-        ("verify", _) => {
+        ("verify", Some(arg_matches)) => {
             println!("Verifying ledger...");
+
+            let dev_halt_at_slot = value_t!(arg_matches, "halt_at_slot", Slot).ok();
+            let poh_verify = !arg_matches.is_present("skip_poh_verify");
+
+            let snapshot_config = if arg_matches.is_present("no_snapshot") {
+                None
+            } else {
+                Some(SnapshotConfig {
+                    snapshot_interval_slots: 0, // Value doesn't matter
+                    snapshot_package_output_path: ledger_path.clone(),
+                    snapshot_path: ledger_path.clone().join("snapshot"),
+                })
+            };
+            let account_paths = if let Some(account_paths) = matches.value_of("account_paths") {
+                Some(account_paths.to_string())
+            } else {
+                Some(ledger_path.join("accounts").to_str().unwrap().to_string())
+            };
+
             let process_options = blocktree_processor::ProcessOptions {
-                verify_ledger: true,
+                poh_verify,
+                dev_halt_at_slot,
                 ..blocktree_processor::ProcessOptions::default()
             };
-            match bank_forks_utils::load(&genesis_block, &blocktree, None, None, process_options) {
+
-                Ok((_bank_forks, bank_forks_info, _leader_schedule_cache)) => {
+            match bank_forks_utils::load(
-                    println!("{:?}", bank_forks_info);
+                &genesis_block,
+                &blocktree,
+                account_paths,
+                snapshot_config.as_ref(),
+                process_options,
+            ) {
+                Ok((_bank_forks, _bank_forks_info, _leader_schedule_cache)) => {
+                    println!("Ok");
                 }
                 Err(err) => {
                     eprintln!("Ledger verification failed: {:?}", err);

ledger/src/blocktree_processor.rs

@@ -1,29 +1,32 @@
-use crate::bank_forks::BankForks;
+use crate::{
-use crate::block_error::BlockError;
+    bank_forks::BankForks,
-use crate::blocktree::Blocktree;
+    block_error::BlockError,
-use crate::blocktree_meta::SlotMeta;
+    blocktree::Blocktree,
-use crate::entry::{create_ticks, Entry, EntrySlice};
+    blocktree_meta::SlotMeta,
-use crate::leader_schedule_cache::LeaderScheduleCache;
+    entry::{create_ticks, Entry, EntrySlice},
+    leader_schedule_cache::LeaderScheduleCache,
+};
+use itertools::Itertools;
 use log::*;
-use rand::seq::SliceRandom;
+use rand::{seq::SliceRandom, thread_rng};
-use rand::thread_rng;
+use rayon::{prelude::*, ThreadPool};
-use rayon::prelude::*;
-use rayon::ThreadPool;
 use solana_metrics::{datapoint, datapoint_error, inc_new_counter_debug};
-use solana_runtime::bank::Bank;
-use solana_runtime::transaction_batch::TransactionBatch;
-use solana_sdk::clock::{Slot, MAX_RECENT_BLOCKHASHES};
-use solana_sdk::genesis_block::GenesisBlock;
-use solana_sdk::hash::Hash;
-use solana_sdk::signature::{Keypair, KeypairUtil};
-use solana_sdk::timing::duration_as_ms;
-use solana_sdk::transaction::Result;
-use std::result;
-use std::sync::Arc;
-use std::time::{Duration, Instant};
-
 use solana_rayon_threadlimit::get_thread_count;
-use std::cell::RefCell;
+use solana_runtime::{bank::Bank, transaction_batch::TransactionBatch};
+use solana_sdk::{
+    clock::{Slot, MAX_RECENT_BLOCKHASHES},
+    genesis_block::GenesisBlock,
+    hash::Hash,
+    signature::{Keypair, KeypairUtil},
+    timing::duration_as_ms,
+    transaction::Result,
+};
+use std::{
+    cell::RefCell,
+    result,
+    sync::Arc,
+    time::{Duration, Instant},
+};
 
 thread_local!(static PAR_THREAD_POOL: RefCell<ThreadPool> = RefCell::new(rayon::ThreadPoolBuilder::new()
                     .num_threads(get_thread_count())
@@ -193,7 +196,7 @@ pub type ProcessCallback = Arc<dyn Fn(&Bank) -> () + Sync + Send>;
 
 #[derive(Default)]
 pub struct ProcessOptions {
-    pub verify_ledger: bool,
+    pub poh_verify: bool,
     pub full_leader_cache: bool,
     pub dev_halt_at_slot: Option<Slot>,
     pub entry_callback: Option<ProcessCallback>,
@@ -279,9 +282,14 @@ pub fn process_blocktree_from_root(
     };
 
     info!(
-        "processing ledger...complete in {}ms, forks={}...",
+        "ledger processed in {}ms. {} fork{} at {}",
         duration_as_ms(&now.elapsed()),
         bank_forks_info.len(),
+        if bank_forks_info.len() > 1 { "s" } else { "" },
+        bank_forks_info
+            .iter()
+            .map(|bfi| bfi.bank_slot.to_string())
+            .join(", ")
     );
 
     Ok((bank_forks, bank_forks_info, leader_schedule_cache))
@@ -295,7 +303,7 @@ fn verify_and_process_slot_entries(
 ) -> result::Result<Hash, BlocktreeProcessorError> {
     assert!(!entries.is_empty());
 
-    if opts.verify_ledger {
+    if opts.poh_verify {
         let next_bank_tick_height = bank.tick_height() + entries.tick_count();
         let max_bank_tick_height = bank.max_tick_height();
         if next_bank_tick_height != max_bank_tick_height {
@@ -569,7 +577,7 @@ pub mod tests {
             .expect("Expected to write shredded entries to blocktree");
 
         let opts = ProcessOptions {
-            verify_ledger: true,
+            poh_verify: true,
             ..ProcessOptions::default()
         };
         assert_eq!(
@@ -609,7 +617,7 @@ pub mod tests {
             .expect("Expected to write shredded entries to blocktree");
 
         let opts = ProcessOptions {
-            verify_ledger: true,
+            poh_verify: true,
             ..ProcessOptions::default()
         };
         assert_eq!(
@@ -660,7 +668,7 @@ pub mod tests {
             .expect("Expected to write shredded entries to blocktree");
 
         let opts = ProcessOptions {
-            verify_ledger: true,
+            poh_verify: true,
             ..ProcessOptions::default()
         };
         assert_eq!(
@@ -726,7 +734,7 @@ pub mod tests {
         fill_blocktree_slot_with_ticks(&blocktree, ticks_per_slot, 2, 1, blockhash);
 
         let opts = ProcessOptions {
-            verify_ledger: true,
+            poh_verify: true,
             ..ProcessOptions::default()
         };
         let (mut _bank_forks, bank_forks_info, _) =
@@ -788,7 +796,7 @@ pub mod tests {
         blocktree.set_roots(&[0, 1, 4]).unwrap();
 
         let opts = ProcessOptions {
-            verify_ledger: true,
+            poh_verify: true,
             ..ProcessOptions::default()
         };
         let (bank_forks, bank_forks_info, _) =
@@ -862,7 +870,7 @@ pub mod tests {
         blocktree.set_roots(&[0, 1]).unwrap();
 
         let opts = ProcessOptions {
-            verify_ledger: true,
+            poh_verify: true,
             ..ProcessOptions::default()
         };
         let (bank_forks, bank_forks_info, _) =
@@ -942,7 +950,7 @@ pub mod tests {
 
         // Check that we can properly restart the ledger / leader scheduler doesn't fail
         let opts = ProcessOptions {
-            verify_ledger: true,
+            poh_verify: true,
             ..ProcessOptions::default()
         };
         let (bank_forks, bank_forks_info, _) =
@@ -1089,7 +1097,7 @@ pub mod tests {
             )
             .unwrap();
         let opts = ProcessOptions {
-            verify_ledger: true,
+            poh_verify: true,
             ..ProcessOptions::default()
         };
         let (bank_forks, bank_forks_info, _) =
@@ -1118,7 +1126,7 @@ pub mod tests {
 
         let blocktree = Blocktree::open(&ledger_path).unwrap();
         let opts = ProcessOptions {
-            verify_ledger: true,
+            poh_verify: true,
             ..ProcessOptions::default()
         };
         let (bank_forks, bank_forks_info, _) =
@@ -1857,7 +1865,7 @@ pub mod tests {
         // Set up bank1
         let bank0 = Arc::new(Bank::new(&genesis_block));
         let opts = ProcessOptions {
-            verify_ledger: true,
+            poh_verify: true,
             ..ProcessOptions::default()
         };
         process_bank_0(&bank0, &blocktree, &opts).unwrap();

multinode-demo/bootstrap-leader.sh

@@ -34,7 +34,7 @@ while [[ -n $1 ]]; do
     elif [[ $1 = --limit-ledger-size ]]; then
       args+=("$1")
       shift
-    elif [[ $1 = --skip-ledger-verify ]]; then
+    elif [[ $1 = --skip-poh-verify ]]; then
       args+=("$1")
       shift
     else

multinode-demo/validator.sh

@@ -98,7 +98,7 @@ while [[ -n $1 ]]; do
     elif [[ $1 = --limit-ledger-size ]]; then
       args+=("$1")
       shift
-    elif [[ $1 = --skip-ledger-verify ]]; then
+    elif [[ $1 = --skip-poh-verify ]]; then
       args+=("$1")
       shift
     elif [[ $1 = --no-snapshot-fetch ]]; then
@@ -140,7 +140,7 @@ while [[ -n $1 ]]; do
     elif [[ $1 = --limit-ledger-size ]]; then
       args+=("$1")
       shift
-    elif [[ $1 = --skip-ledger-verify ]]; then
+    elif [[ $1 = --skip-poh-verify ]]; then
       args+=("$1")
       shift
     elif [[ $1 = -h ]]; then

net/net.sh

@@ -72,7 +72,7 @@ Operate a configured testnet
                                         in genesis block for external nodes
    --no-snapshot-fetch
                                       - If set, disables booting validators from a snapshot
-   --skip-ledger-verify
+   --skip-poh-verify
                                       - If set, validators will skip verifying
                                         the ledger they already have saved to disk at
                                         boot (results in a much faster boot)
@@ -176,7 +176,7 @@ while [[ -n $1 ]]; do
     elif [[ $1 = --limit-ledger-size ]]; then
       maybeLimitLedgerSize="$1"
       shift 1
-    elif [[ $1 = --skip-ledger-verify ]]; then
+    elif [[ $1 = --skip-poh-verify ]]; then
       maybeSkipLedgerVerify="$1"
       shift 1
     elif [[ $1 = --skip-setup ]]; then

validator/src/main.rs

@@ -392,8 +392,8 @@ pub fn main() {
                 .help("drop older slots in the ledger"),
         )
         .arg(
-            clap::Arg::with_name("skip_ledger_verify")
+            clap::Arg::with_name("skip_poh_verify")
-                .long("skip-ledger-verify")
+                .long("skip-poh-verify")
                 .takes_value(false)
                 .help("Skip ledger verification at node bootup"),
         )
@@ -539,7 +539,7 @@ pub fn main() {
     }
 
     let init_complete_file = matches.value_of("init_complete_file");
-    let verify_ledger = !matches.is_present("skip_ledger_verify");
+    let skip_poh_verify = matches.is_present("skip_poh_verify");
     validator_config.blockstream_unix_socket = matches
         .value_of("blockstream_unix_socket")
         .map(PathBuf::from);
@@ -640,7 +640,7 @@ pub fn main() {
         &Arc::new(voting_keypair),
         &Arc::new(storage_keypair),
         cluster_entrypoint.as_ref(),
-        verify_ledger,
+        !skip_poh_verify,
         &validator_config,
     );