blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Facility to generate a blocktree prune list using ledger tool (#5041) 

automerge
This commit is contained in:
Pankaj Garg 2019-07-12 16:58:13 -07:00 committed by Grimes
parent d2b6c2e0ce
commit 1c966aac25
8 changed files with 174 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.lock generated
View File

@ -2605,7 +2605,10 @@ version = "0.17.0"
dependencies = [
"assert_cmd 0.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_yaml 0.8.9 (registry+https://github.com/rust-lang/crates.io-index)",
"solana 0.17.0",
"solana-logger 0.17.0",
"solana-runtime 0.17.0",

15
core/src/blocktree_processor.rs
View File

@ -142,6 +142,7 @@ pub fn process_blocktree(
genesis_block: &GenesisBlock,
blocktree: &Blocktree,
account_paths: Option<String>,
verify_ledger: bool,
) -> result::Result<(BankForks, Vec<BankForksInfo>, LeaderScheduleCache), BlocktreeProcessorError> {
let now = Instant::now();
info!("processing ledger...");
@ -205,7 +206,7 @@ pub fn process_blocktree(
}
if !entries.is_empty() {
if !entries.verify(&last_entry_hash) {
if verify_ledger && !entries.verify(&last_entry_hash) {
warn!(
"Ledger proof of history failed at slot: {}, entry: {}",
slot, entry_height
@ -374,7 +375,7 @@ pub mod tests {
fill_blocktree_slot_with_ticks(&blocktree, ticks_per_slot, 2, 1, blockhash);
let (mut _bank_forks, bank_forks_info, _) =
process_blocktree(&genesis_block, &blocktree, None).unwrap();
process_blocktree(&genesis_block, &blocktree, None, true).unwrap();
assert_eq!(bank_forks_info.len(), 1);
assert_eq!(
@ -433,7 +434,7 @@ pub mod tests {
blocktree.set_roots(&[4, 1, 0]).unwrap();
let (bank_forks, bank_forks_info, _) =
process_blocktree(&genesis_block, &blocktree, None).unwrap();
process_blocktree(&genesis_block, &blocktree, None, true).unwrap();
assert_eq!(bank_forks_info.len(), 1); // One fork, other one is ignored b/c not a descendant of the root
@ -507,7 +508,7 @@ pub mod tests {
blocktree.set_roots(&[0, 1]).unwrap();
let (bank_forks, bank_forks_info, _) =
process_blocktree(&genesis_block, &blocktree, None).unwrap();
process_blocktree(&genesis_block, &blocktree, None, true).unwrap();
assert_eq!(bank_forks_info.len(), 2); // There are two forks
assert_eq!(
@ -588,7 +589,7 @@ pub mod tests {
// Check that we can properly restart the ledger / leader scheduler doesn't fail
let (bank_forks, bank_forks_info, _) =
process_blocktree(&genesis_block, &blocktree, None).unwrap();
process_blocktree(&genesis_block, &blocktree, None, true).unwrap();
assert_eq!(bank_forks_info.len(), 1); // There is one fork
assert_eq!(
@ -724,7 +725,7 @@ pub mod tests {
.unwrap();
let entry_height = genesis_block.ticks_per_slot + entries.len() as u64;
let (bank_forks, bank_forks_info, _) =
process_blocktree(&genesis_block, &blocktree, None).unwrap();
process_blocktree(&genesis_block, &blocktree, None, true).unwrap();
assert_eq!(bank_forks_info.len(), 1);
assert_eq!(bank_forks.root(), 0);
@ -755,7 +756,7 @@ pub mod tests {
let blocktree = Blocktree::open(&ledger_path).unwrap();
let (bank_forks, bank_forks_info, _) =
process_blocktree(&genesis_block, &blocktree, None).unwrap();
process_blocktree(&genesis_block, &blocktree, None, true).unwrap();
assert_eq!(bank_forks_info.len(), 1);
assert_eq!(

3
core/src/local_cluster.rs
View File

@ -177,6 +177,7 @@ impl LocalCluster {
&leader_voting_keypair,
&leader_storage_keypair,
None,
true,
&config.validator_configs[0],
);
@ -308,6 +309,7 @@ impl LocalCluster {
&voting_keypair,
&storage_keypair,
Some(&self.entry_point_info),
true,
&validator_config,
);
@ -561,6 +563,7 @@ impl Cluster for LocalCluster {
&fullnode_info.voting_keypair,
&fullnode_info.storage_keypair,
None,
true,
config,
);

25
core/src/validator.rs
View File

@ -82,6 +82,7 @@ impl Validator {
voting_keypair: &Arc<Keypair>,
storage_keypair: &Arc<Keypair>,
entrypoint_info_option: Option<&ContactInfo>,
verify_ledger: bool,
config: &ValidatorConfig,
) -> Self {
warn!("CUDA is {}abled", if cfg!(cuda) { "en" } else { "dis" });
@ -102,6 +103,7 @@ impl Validator {
ledger_path,
config.account_paths.clone(),
config.snapshot_path.clone(),
verify_ledger,
);
let leader_schedule_cache = Arc::new(leader_schedule_cache);
@ -302,6 +304,7 @@ fn get_bank_forks(
blocktree: &Blocktree,
account_paths: Option<String>,
snapshot_path: Option<String>,
verify_ledger: bool,
) -> (BankForks, Vec<BankForksInfo>, LeaderScheduleCache) {
if snapshot_path.is_some() {
let bank_forks =
@ -319,8 +322,13 @@ fn get_bank_forks(
}
}
let (mut bank_forks, bank_forks_info, leader_schedule_cache) =
blocktree_processor::process_blocktree(&genesis_block, &blocktree, account_paths)
.expect("process_blocktree failed");
blocktree_processor::process_blocktree(
&genesis_block,
&blocktree,
account_paths,
verify_ledger,
)
.expect("process_blocktree failed");
if snapshot_path.is_some() {
bank_forks.set_snapshot_config(snapshot_path);
let _ = bank_forks.add_snapshot(0, 0);
@ -332,6 +340,7 @@ pub fn new_banks_from_blocktree(
blocktree_path: &str,
account_paths: Option<String>,
snapshot_path: Option<String>,
verify_ledger: bool,
) -> (
BankForks,
Vec<BankForksInfo>,
@ -348,8 +357,13 @@ pub fn new_banks_from_blocktree(
Blocktree::open_with_signal(blocktree_path)
.expect("Expected to successfully open database ledger");
let (bank_forks, bank_forks_info, leader_schedule_cache) =
get_bank_forks(&genesis_block, &blocktree, account_paths, snapshot_path);
let (bank_forks, bank_forks_info, leader_schedule_cache) = get_bank_forks(
&genesis_block,
&blocktree,
account_paths,
snapshot_path,
verify_ledger,
);
(
bank_forks,
@ -413,6 +427,7 @@ pub fn new_validator_for_tests() -> (Validator, ContactInfo, Keypair, String) {
&voting_keypair,
&storage_keypair,
None,
true,
&ValidatorConfig::default(),
);
discover_cluster(&contact_info.gossip, 1).expect("Node startup failed");
@ -448,6 +463,7 @@ mod tests {
&voting_keypair,
&storage_keypair,
Some(&leader_node.info),
true,
&ValidatorConfig::default(),
);
validator.close().unwrap();
@ -479,6 +495,7 @@ mod tests {
&voting_keypair,
&storage_keypair,
Some(&leader_node.info),
true,
&ValidatorConfig::default(),
)
})

2
core/tests/tvu.rs
View File

@ -97,7 +97,7 @@ fn test_replay() {
completed_slots_receiver,
leader_schedule_cache,
_,
) = validator::new_banks_from_blocktree(&blocktree_path, None, None);
) = validator::new_banks_from_blocktree(&blocktree_path, None, None, true);
let working_bank = bank_forks.working_bank();
assert_eq!(
working_bank.get_balance(&mint_keypair.pubkey()),

3
ledger-tool/Cargo.toml
View File

@ -10,7 +10,10 @@ homepage = "https://solana.com/"
[dependencies]
clap = "2.33.0"
serde = "1.0.94"
serde_derive = "1.0.94"
serde_json = "1.0.40"
serde_yaml = "0.8.9"
solana = { path = "../core", version = "0.17.0" }
solana-logger = { path = "../logger", version = "0.17.0" }
solana-runtime = { path = "../runtime", version = "0.17.0" }

127
ledger-tool/src/main.rs
View File

@ -2,8 +2,11 @@ use clap::{crate_description, crate_name, crate_version, value_t, App, Arg, SubC
use solana::blocktree::Blocktree;
use solana::blocktree_processor::process_blocktree;
use solana_sdk::genesis_block::GenesisBlock;
use std::collections::BTreeMap;
use std::fs::File;
use std::io::{stdout, Write};
use std::process::exit;
use std::str::FromStr;
#[derive(PartialEq)]
enum LedgerOutputMethod {
@ -58,6 +61,7 @@ fn output_ledger(blocktree: Blocktree, starting_slot: u64, method: LedgerOutputM
}
fn main() {
const DEFAULT_ROOT_COUNT: &str = "1";
solana_logger::setup();
let matches = App::new(crate_name!())
.about(crate_description!())
@ -82,6 +86,36 @@ fn main() {
.subcommand(SubCommand::with_name("print").about("Print the ledger"))
.subcommand(SubCommand::with_name("json").about("Print the ledger in JSON format"))
.subcommand(SubCommand::with_name("verify").about("Verify the ledger's PoH"))
.subcommand(SubCommand::with_name("prune").about("Prune the ledger at the block height").arg(
Arg::with_name("slot_list")
.long("slot-list")
.value_name("FILENAME")
.takes_value(true)
.help("The location of the YAML file with a list of rollback slot heights and hashes"),
))
.subcommand(SubCommand::with_name("list-roots").about("Output upto last <num-roots> root hashes and their heights starting at the given block height").arg(
Arg::with_name("max_height")
.long("max-height")
.value_name("NUM")
.takes_value(true)
.required(true)
.help("Maximum block height"),
).arg(
Arg::with_name("slot_list")
.long("slot-list")
.value_name("FILENAME")
.required(false)
.takes_value(true)
.help("The location of the output YAML file. A list of rollback slot heights and hashes will be written to the file."),
).arg(
Arg::with_name("num_roots")
.long("num-roots")
.value_name("NUM")
.takes_value(true)
.default_value(DEFAULT_ROOT_COUNT)
.required(false)
.help("Number of roots in the output"),
))
.get_matches();
let ledger_path = matches.value_of("ledger").unwrap();
@ -113,7 +147,7 @@ fn main() {
}
("verify", _) => {
println!("Verifying ledger...");
match process_blocktree(&genesis_block, &blocktree, None) {
match process_blocktree(&genesis_block, &blocktree, None, true) {
Ok((_bank_forks, bank_forks_info, _)) => {
println!("{:?}", bank_forks_info);
}
@ -123,6 +157,97 @@ fn main() {
}
}
}
("prune", Some(args_matches)) => {
if let Some(prune_file_path) = args_matches.value_of("slot_list") {
let prune_file = File::open(prune_file_path.to_string()).unwrap();
let slot_hashes: BTreeMap<u64, String> =
serde_yaml::from_reader(prune_file).unwrap();
let iter = blocktree
.rooted_slot_iterator(0)
.expect("Failed to get rooted slot");
let potential_hashes: Vec<_> = iter
.filter_map(|(slot, meta)| {
let blockhash = blocktree
.get_slot_entries(slot, meta.last_index, Some(1))
.unwrap()
.first()
.unwrap()
.hash
.to_string();
slot_hashes.get(&slot).and_then(|hash| {
if *hash == blockhash {
Some((slot, blockhash))
} else {
None
}
})
})
.collect();
let (target_slot, target_hash) = potential_hashes
.last()
.expect("Failed to find a valid slot");
println!("Prune at slot {:?} hash {:?}", target_slot, target_hash);
// ToDo: Do the actual pruning of the database
}
}
("list-roots", Some(args_matches)) => {
let max_height = if let Some(height) = args_matches.value_of("max_height") {
usize::from_str(height).expect("Maximum height must be a number")
} else {
panic!("Maximum height must be provided");
};
let num_roots = if let Some(roots) = args_matches.value_of("num_roots") {
usize::from_str(roots).expect("Number of roots must be a number")
} else {
usize::from_str(DEFAULT_ROOT_COUNT).unwrap()
};
let iter = blocktree
.rooted_slot_iterator(0)
.expect("Failed to get rooted slot");
let slot_hash: Vec<_> = iter
.filter_map(|(slot, meta)| {
if slot <= max_height as u64 {
let blockhash = blocktree
.get_slot_entries(slot, meta.last_index, Some(1))
.unwrap()
.first()
.unwrap()
.hash;
Some((slot, blockhash))
} else {
None
}
})
.collect();
let mut output_file: Box<Write> = if let Some(path) = args_matches.value_of("slot_list")
{
match File::create(path) {
Ok(file) => Box::new(file),
_ => Box::new(stdout()),
}
} else {
Box::new(stdout())
};
slot_hash
.into_iter()
.rev()
.enumerate()
.for_each(|(i, (slot, hash))| {
if i < num_roots {
output_file
.write_all(format!("{:?}: {:?}\n", slot, hash).as_bytes())
.expect("failed to write");
}
});
}
("", _) => {
eprintln!("{}", matches.usage());
exit(1);

9
validator/src/main.rs
View File

@ -162,6 +162,12 @@ fn main() {
.value_name("PATHS")
.takes_value(true)
.help("Snapshot path"),
)
.arg(
clap::Arg::with_name("skip_ledger_verify")
.long("skip-ledger-verify")
.takes_value(false)
.help("Skip ledger verification at node bootup"),
)
.get_matches();
@ -267,6 +273,8 @@ fn main() {
node.info.rpc_pubsub = SocketAddr::new(gossip_addr.ip(), port_number + 1);
};
let verify_ledger = !matches.is_present("skip_ledger_verify");
let validator = Validator::new(
node,
&keypair,
@ -275,6 +283,7 @@ fn main() {
&Arc::new(voting_keypair),
&Arc::new(storage_keypair),
cluster_entrypoint.as_ref(),
verify_ledger,
&validator_config,
);