Restore ledger-tool print and json commands (#5048)
* Restore ledger-tool print and json commands * Remove obsolete read_ledger()
This commit is contained in:
Michael Vines
GitHub
parent
22315d88e7
commit
ebcdc06dc3
|
|
@ -793,70 +793,6 @@ impl Blocktree {
|
|||
iter.map(|(_, blob_data)| Blob::new(&blob_data))
|
||||
}
|
||||
|
||||
/// Return an iterator for all the entries in the given file.
|
||||
pub fn read_ledger(&self) -> Result<impl Iterator<Item = Entry>> {
|
||||
use crate::entry::EntrySlice;
|
||||
use std::collections::VecDeque;
|
||||
|
||||
struct EntryIterator {
|
||||
db_iterator: Cursor<cf::Data>,
|
||||
|
||||
// TODO: remove me when replay_stage is iterating by block (Blocktree)
|
||||
// this verification is duplicating that of replay_stage, which
|
||||
// can do this in parallel
|
||||
blockhash: Option<Hash>,
|
||||
// https://github.com/rust-rocksdb/rust-rocksdb/issues/234
|
||||
// rocksdb issue: the _blocktree member must be lower in the struct to prevent a crash
|
||||
// when the db_iterator member above is dropped.
|
||||
// _blocktree is unused, but dropping _blocktree results in a broken db_iterator
|
||||
// you have to hold the database open in order to iterate over it, and in order
|
||||
// for db_iterator to be able to run Drop
|
||||
// _blocktree: Blocktree,
|
||||
entries: VecDeque<Entry>,
|
||||
}
|
||||
|
||||
impl Iterator for EntryIterator {
|
||||
type Item = Entry;
|
||||
|
||||
fn next(&mut self) -> Option<Entry> {
|
||||
if !self.entries.is_empty() {
|
||||
return Some(self.entries.pop_front().unwrap());
|
||||
}
|
||||
|
||||
if self.db_iterator.valid() {
|
||||
if let Some(value) = self.db_iterator.value_bytes() {
|
||||
if let Ok(next_entries) =
|
||||
deserialize::<Vec<Entry>>(&value[BLOB_HEADER_SIZE..])
|
||||
{
|
||||
if let Some(blockhash) = self.blockhash {
|
||||
if !next_entries.verify(&blockhash) {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
self.db_iterator.next();
|
||||
if next_entries.is_empty() {
|
||||
return None;
|
||||
}
|
||||
self.entries = VecDeque::from(next_entries);
|
||||
let entry = self.entries.pop_front().unwrap();
|
||||
self.blockhash = Some(entry.hash);
|
||||
return Some(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
let mut db_iterator = self.db.cursor::<cf::Data>()?;
|
||||
|
||||
db_iterator.seek_to_first();
|
||||
Ok(EntryIterator {
|
||||
entries: VecDeque::new(),
|
||||
db_iterator,
|
||||
blockhash: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_slot_entries_with_blob_count(
|
||||
&self,
|
||||
slot: u64,
|
||||
|
|
@ -1937,9 +1873,7 @@ pub fn tmp_copy_blocktree(from: &str, name: &str) -> String {
|
|||
#[cfg(test)]
|
||||
pub mod tests {
|
||||
use super::*;
|
||||
use crate::entry::{
|
||||
create_ticks, make_tiny_test_entries, make_tiny_test_entries_from_hash, Entry, EntrySlice,
|
||||
};
|
||||
use crate::entry::{create_ticks, make_tiny_test_entries, Entry, EntrySlice};
|
||||
use crate::erasure::{CodingGenerator, ErasureConfig};
|
||||
use crate::packet;
|
||||
use rand::seq::SliceRandom;
|
||||
|
|
@ -2467,59 +2401,6 @@ pub mod tests {
|
|||
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
|
||||
}
|
||||
|
||||
#[test]
|
||||
pub fn test_genesis_and_entry_iterator() {
|
||||
let entries = make_tiny_test_entries_from_hash(&Hash::default(), 10);
|
||||
|
||||
let ledger_path = get_tmp_ledger_path("test_genesis_and_entry_iterator");
|
||||
{
|
||||
genesis(&ledger_path, &Keypair::new(), &entries).unwrap();
|
||||
|
||||
let ledger = Blocktree::open(&ledger_path).expect("open failed");
|
||||
|
||||
let read_entries: Vec<Entry> =
|
||||
ledger.read_ledger().expect("read_ledger failed").collect();
|
||||
assert!(read_entries.verify(&Hash::default()));
|
||||
assert_eq!(entries, read_entries);
|
||||
}
|
||||
|
||||
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
||||
}
|
||||
#[test]
|
||||
pub fn test_entry_iterator_up_to_consumed() {
|
||||
let entries = make_tiny_test_entries_from_hash(&Hash::default(), 3);
|
||||
let ledger_path = get_tmp_ledger_path("test_genesis_and_entry_iterator");
|
||||
{
|
||||
// put entries except last 2 into ledger
|
||||
genesis(&ledger_path, &Keypair::new(), &entries[..entries.len() - 2]).unwrap();
|
||||
|
||||
let ledger = Blocktree::open(&ledger_path).expect("open failed");
|
||||
|
||||
// now write the last entry, ledger has a hole in it one before the end
|
||||
// +-+-+-+-+-+-+-+ +-+
|
||||
// | | | | | | | | | |
|
||||
// +-+-+-+-+-+-+-+ +-+
|
||||
ledger
|
||||
.write_entries(
|
||||
0u64,
|
||||
0,
|
||||
(entries.len() - 1) as u64,
|
||||
16,
|
||||
&entries[entries.len() - 1..],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let read_entries: Vec<Entry> =
|
||||
ledger.read_ledger().expect("read_ledger failed").collect();
|
||||
assert!(read_entries.verify(&Hash::default()));
|
||||
|
||||
// enumeration should stop at the hole
|
||||
assert_eq!(entries[..entries.len() - 2].to_vec(), read_entries);
|
||||
}
|
||||
|
||||
Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
|
||||
}
|
||||
|
||||
#[test]
|
||||
pub fn test_new_blobs_signal() {
|
||||
// Initialize ledger
|
||||
|
|
|
|||
|
|
@ -1,13 +1,66 @@
|
|||
use clap::{crate_description, crate_name, crate_version, App, Arg, SubCommand};
|
||||
use clap::{crate_description, crate_name, crate_version, value_t, App, Arg, SubCommand};
|
||||
use solana::blocktree::Blocktree;
|
||||
use solana::blocktree_processor::process_blocktree;
|
||||
use solana_sdk::genesis_block::GenesisBlock;
|
||||
use std::io::{stdout, Write};
|
||||
use std::process::exit;
|
||||
|
||||
#[derive(PartialEq)]
|
||||
enum LedgerOutputMethod {
|
||||
Print,
|
||||
Json,
|
||||
}
|
||||
fn output_ledger(blocktree: Blocktree, starting_slot: u64, method: LedgerOutputMethod) {
|
||||
let rooted_slot_iterator = blocktree
|
||||
.rooted_slot_iterator(starting_slot)
|
||||
.unwrap_or_else(|err| {
|
||||
eprintln!(
|
||||
"Failed to load entries starting from slot {}: {:?}",
|
||||
starting_slot, err
|
||||
);
|
||||
exit(1);
|
||||
});
|
||||
|
||||
if method == LedgerOutputMethod::Json {
|
||||
stdout().write_all(b"{\"ledger\":[\n").expect("open array");
|
||||
}
|
||||
|
||||
for (slot, slot_meta) in rooted_slot_iterator {
|
||||
match method {
|
||||
LedgerOutputMethod::Print => println!("Slot {}", slot),
|
||||
LedgerOutputMethod::Json => {
|
||||
serde_json::to_writer(stdout(), &slot_meta).expect("serialize slot_meta");
|
||||
stdout().write_all(b",\n").expect("newline");
|
||||
}
|
||||
}
|
||||
|
||||
let entries = blocktree
|
||||
.get_slot_entries(slot, 0, None)
|
||||
.unwrap_or_else(|err| {
|
||||
eprintln!("Failed to load entries for slot {}: {:?}", slot, err);
|
||||
exit(1);
|
||||
});
|
||||
|
||||
for entry in entries {
|
||||
match method {
|
||||
LedgerOutputMethod::Print => println!("{:?}", entry),
|
||||
LedgerOutputMethod::Json => {
|
||||
serde_json::to_writer(stdout(), &entry).expect("serialize entry");
|
||||
stdout().write_all(b",\n").expect("newline");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if method == LedgerOutputMethod::Json {
|
||||
stdout().write_all(b"\n]}\n").expect("close array");
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
solana_logger::setup();
|
||||
let matches = App::new(crate_name!()).about(crate_description!())
|
||||
let matches = App::new(crate_name!())
|
||||
.about(crate_description!())
|
||||
.version(crate_version!())
|
||||
.arg(
|
||||
Arg::with_name("ledger")
|
||||
|
|
@ -19,26 +72,12 @@ fn main() {
|
|||
.help("Use directory for ledger location"),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("head")
|
||||
.short("n")
|
||||
.long("head")
|
||||
Arg::with_name("starting_slot")
|
||||
.long("starting-slot")
|
||||
.value_name("NUM")
|
||||
.takes_value(true)
|
||||
.help("Limit to at most the first NUM entries in ledger\n (only applies to print and json commands)"),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("min-hashes")
|
||||
.short("h")
|
||||
.long("min-hashes")
|
||||
.value_name("NUM")
|
||||
.takes_value(true)
|
||||
.help("Skip entries with fewer than NUM hashes\n (only applies to print and json commands)"),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("continue")
|
||||
.short("c")
|
||||
.long("continue")
|
||||
.help("Continue verify even if verification fails"),
|
||||
.default_value("0")
|
||||
.help("Start at this slot (only applies to print and json commands)"),
|
||||
)
|
||||
.subcommand(SubCommand::with_name("print").about("Print the ledger"))
|
||||
.subcommand(SubCommand::with_name("json").about("Print the ledger in JSON format"))
|
||||
|
|
@ -63,63 +102,27 @@ fn main() {
|
|||
}
|
||||
};
|
||||
|
||||
let entries = match blocktree.read_ledger() {
|
||||
Ok(entries) => entries,
|
||||
Err(err) => {
|
||||
eprintln!("Failed to read ledger at {}: {}", ledger_path, err);
|
||||
exit(1);
|
||||
}
|
||||
};
|
||||
|
||||
let head = match matches.value_of("head") {
|
||||
Some(head) => head.parse().expect("please pass a number for --head"),
|
||||
None => <usize>::max_value(),
|
||||
};
|
||||
|
||||
let min_hashes = match matches.value_of("min-hashes") {
|
||||
Some(hashes) => hashes
|
||||
.parse()
|
||||
.expect("please pass a number for --min-hashes"),
|
||||
None => 0,
|
||||
} as u64;
|
||||
let starting_slot = value_t!(matches, "starting_slot", u64).unwrap_or_else(|e| e.exit());
|
||||
|
||||
match matches.subcommand() {
|
||||
("print", _) => {
|
||||
for (i, entry) in entries.enumerate() {
|
||||
if i >= head {
|
||||
break;
|
||||
}
|
||||
|
||||
if entry.num_hashes < min_hashes {
|
||||
continue;
|
||||
}
|
||||
println!("{:?}", entry);
|
||||
}
|
||||
output_ledger(blocktree, starting_slot, LedgerOutputMethod::Print);
|
||||
}
|
||||
("json", _) => {
|
||||
stdout().write_all(b"{\"ledger\":[\n").expect("open array");
|
||||
for (i, entry) in entries.enumerate() {
|
||||
if i >= head {
|
||||
break;
|
||||
}
|
||||
|
||||
if entry.num_hashes < min_hashes {
|
||||
continue;
|
||||
}
|
||||
serde_json::to_writer(stdout(), &entry).expect("serialize");
|
||||
stdout().write_all(b",\n").expect("newline");
|
||||
}
|
||||
stdout().write_all(b"\n]}\n").expect("close array");
|
||||
output_ledger(blocktree, starting_slot, LedgerOutputMethod::Json);
|
||||
}
|
||||
("verify", _) => match process_blocktree(&genesis_block, &blocktree, None) {
|
||||
Ok((_bank_forks, bank_forks_info, _)) => {
|
||||
println!("{:?}", bank_forks_info);
|
||||
("verify", _) => {
|
||||
println!("Verifying ledger...");
|
||||
match process_blocktree(&genesis_block, &blocktree, None) {
|
||||
Ok((_bank_forks, bank_forks_info, _)) => {
|
||||
println!("{:?}", bank_forks_info);
|
||||
}
|
||||
Err(err) => {
|
||||
eprintln!("Ledger verification failed: {:?}", err);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
eprintln!("Ledger verification failed: {:?}", err);
|
||||
exit(1);
|
||||
}
|
||||
},
|
||||
}
|
||||
("", _) => {
|
||||
eprintln!("{}", matches.usage());
|
||||
exit(1);
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ extern crate solana;
|
|||
use assert_cmd::prelude::*;
|
||||
use solana::blocktree::create_new_tmp_ledger;
|
||||
use solana::genesis_utils::create_genesis_block;
|
||||
use std::cmp;
|
||||
use std::process::Command;
|
||||
use std::process::Output;
|
||||
|
||||
|
|
@ -46,22 +45,5 @@ fn nominal() {
|
|||
// Print everything
|
||||
let output = run_ledger_tool(&["-l", &ledger_path, "print"]);
|
||||
assert!(output.status.success());
|
||||
assert_eq!(count_newlines(&output.stdout), ticks);
|
||||
|
||||
// Only print the first N items
|
||||
let count = cmp::min(genesis_block.ticks_per_slot, 5);
|
||||
let output = run_ledger_tool(&["-l", &ledger_path, "-n", &count.to_string(), "print"]);
|
||||
println!("{:?}", output);
|
||||
assert!(output.status.success());
|
||||
assert_eq!(count_newlines(&output.stdout), count as usize);
|
||||
|
||||
// Skip entries with no hashes
|
||||
let output = run_ledger_tool(&["-l", &ledger_path, "-h", "1", "print"]);
|
||||
assert!(output.status.success());
|
||||
assert_eq!(count_newlines(&output.stdout), ticks);
|
||||
|
||||
// Skip entries with fewer than 2 hashes (skip everything)
|
||||
let output = run_ledger_tool(&["-l", &ledger_path, "-h", "2", "print"]);
|
||||
assert!(output.status.success());
|
||||
assert_eq!(count_newlines(&output.stdout), 0);
|
||||
assert_eq!(count_newlines(&output.stdout), ticks + 1);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue