More guard rails for restart with ledger procedure (#10853)
* Add expected_bank_hash required for supermajority * Print snapshot hash in ledger-tool create-snapshot.
This commit is contained in:
parent
3a3b7a399a
commit
1ffad2d051
|
@ -64,6 +64,7 @@ use std::{
|
||||||
pub struct ValidatorConfig {
|
pub struct ValidatorConfig {
|
||||||
pub dev_halt_at_slot: Option<Slot>,
|
pub dev_halt_at_slot: Option<Slot>,
|
||||||
pub expected_genesis_hash: Option<Hash>,
|
pub expected_genesis_hash: Option<Hash>,
|
||||||
|
pub expected_bank_hash: Option<Hash>,
|
||||||
pub expected_shred_version: Option<u16>,
|
pub expected_shred_version: Option<u16>,
|
||||||
pub voting_disabled: bool,
|
pub voting_disabled: bool,
|
||||||
pub account_paths: Vec<PathBuf>,
|
pub account_paths: Vec<PathBuf>,
|
||||||
|
@ -90,6 +91,7 @@ impl Default for ValidatorConfig {
|
||||||
Self {
|
Self {
|
||||||
dev_halt_at_slot: None,
|
dev_halt_at_slot: None,
|
||||||
expected_genesis_hash: None,
|
expected_genesis_hash: None,
|
||||||
|
expected_bank_hash: None,
|
||||||
expected_shred_version: None,
|
expected_shred_version: None,
|
||||||
voting_disabled: false,
|
voting_disabled: false,
|
||||||
max_ledger_shreds: None,
|
max_ledger_shreds: None,
|
||||||
|
@ -227,8 +229,8 @@ impl Validator {
|
||||||
if let Some(expected_shred_version) = config.expected_shred_version {
|
if let Some(expected_shred_version) = config.expected_shred_version {
|
||||||
if expected_shred_version != node.info.shred_version {
|
if expected_shred_version != node.info.shred_version {
|
||||||
error!(
|
error!(
|
||||||
"shred version mismatch: expected {}",
|
"shred version mismatch: expected {} found: {}",
|
||||||
expected_shred_version
|
expected_shred_version, node.info.shred_version,
|
||||||
);
|
);
|
||||||
process::exit(1);
|
process::exit(1);
|
||||||
}
|
}
|
||||||
|
@ -392,7 +394,9 @@ impl Validator {
|
||||||
(None, None)
|
(None, None)
|
||||||
};
|
};
|
||||||
|
|
||||||
wait_for_supermajority(config, &bank, &cluster_info, rpc_override_health_check);
|
if wait_for_supermajority(config, &bank, &cluster_info, rpc_override_health_check) {
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
let poh_service = PohService::new(poh_recorder.clone(), &poh_config, &exit);
|
let poh_service = PohService::new(poh_recorder.clone(), &poh_config, &exit);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
@ -698,14 +702,35 @@ fn backup_and_clear_blockstore(ledger_path: &Path, start_slot: Slot, shred_versi
|
||||||
drop(blockstore);
|
drop(blockstore);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Return true on error, indicating the validator should exit.
|
||||||
fn wait_for_supermajority(
|
fn wait_for_supermajority(
|
||||||
config: &ValidatorConfig,
|
config: &ValidatorConfig,
|
||||||
bank: &Bank,
|
bank: &Bank,
|
||||||
cluster_info: &ClusterInfo,
|
cluster_info: &ClusterInfo,
|
||||||
rpc_override_health_check: Arc<AtomicBool>,
|
rpc_override_health_check: Arc<AtomicBool>,
|
||||||
) {
|
) -> bool {
|
||||||
if config.wait_for_supermajority != Some(bank.slot()) {
|
if let Some(wait_for_supermajority) = config.wait_for_supermajority {
|
||||||
return;
|
match wait_for_supermajority.cmp(&bank.slot()) {
|
||||||
|
std::cmp::Ordering::Less => return false,
|
||||||
|
std::cmp::Ordering::Greater => {
|
||||||
|
error!("Ledger does not have enough data to wait for supermajority, please enable snapshot fetch. Has {} needs {}", bank.slot(), wait_for_supermajority);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(expected_bank_hash) = config.expected_bank_hash {
|
||||||
|
if bank.hash() != expected_bank_hash {
|
||||||
|
error!(
|
||||||
|
"Bank hash({}) does not match expected value: {}",
|
||||||
|
bank.hash(),
|
||||||
|
expected_bank_hash
|
||||||
|
);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
info!(
|
info!(
|
||||||
|
@ -725,6 +750,7 @@ fn wait_for_supermajority(
|
||||||
sleep(Duration::new(1, 0));
|
sleep(Duration::new(1, 0));
|
||||||
}
|
}
|
||||||
rpc_override_health_check.store(false, Ordering::Relaxed);
|
rpc_override_health_check.store(false, Ordering::Relaxed);
|
||||||
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct TestValidator {
|
pub struct TestValidator {
|
||||||
|
@ -1058,4 +1084,56 @@ mod tests {
|
||||||
remove_dir_all(path).unwrap();
|
remove_dir_all(path).unwrap();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_wait_for_supermajority() {
|
||||||
|
solana_logger::setup();
|
||||||
|
use solana_sdk::genesis_config::create_genesis_config;
|
||||||
|
use solana_sdk::hash::hash;
|
||||||
|
let node_keypair = Arc::new(Keypair::new());
|
||||||
|
let cluster_info = ClusterInfo::new(
|
||||||
|
ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()),
|
||||||
|
node_keypair,
|
||||||
|
);
|
||||||
|
|
||||||
|
let (genesis_config, _mint_keypair) = create_genesis_config(1);
|
||||||
|
let bank = Arc::new(Bank::new(&genesis_config));
|
||||||
|
let mut config = ValidatorConfig::default();
|
||||||
|
let rpc_override_health_check = Arc::new(AtomicBool::new(false));
|
||||||
|
assert!(!wait_for_supermajority(
|
||||||
|
&config,
|
||||||
|
&bank,
|
||||||
|
&cluster_info,
|
||||||
|
rpc_override_health_check.clone()
|
||||||
|
));
|
||||||
|
|
||||||
|
// bank=0, wait=1, should fail
|
||||||
|
config.wait_for_supermajority = Some(1);
|
||||||
|
assert!(wait_for_supermajority(
|
||||||
|
&config,
|
||||||
|
&bank,
|
||||||
|
&cluster_info,
|
||||||
|
rpc_override_health_check.clone()
|
||||||
|
));
|
||||||
|
|
||||||
|
// bank=1, wait=0, should pass, bank is past the wait slot
|
||||||
|
let bank = Bank::new_from_parent(&bank, &Pubkey::default(), 1);
|
||||||
|
config.wait_for_supermajority = Some(0);
|
||||||
|
assert!(!wait_for_supermajority(
|
||||||
|
&config,
|
||||||
|
&bank,
|
||||||
|
&cluster_info,
|
||||||
|
rpc_override_health_check.clone()
|
||||||
|
));
|
||||||
|
|
||||||
|
// bank=1, wait=1, equal, but bad hash provided
|
||||||
|
config.wait_for_supermajority = Some(1);
|
||||||
|
config.expected_bank_hash = Some(hash(&[1]));
|
||||||
|
assert!(wait_for_supermajority(
|
||||||
|
&config,
|
||||||
|
&bank,
|
||||||
|
&cluster_info,
|
||||||
|
rpc_override_health_check
|
||||||
|
));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1363,8 +1363,9 @@ fn main() {
|
||||||
.and_then(|package| {
|
.and_then(|package| {
|
||||||
snapshot_utils::archive_snapshot_package(&package).map(|ok| {
|
snapshot_utils::archive_snapshot_package(&package).map(|ok| {
|
||||||
println!(
|
println!(
|
||||||
"Successfully created snapshot for slot {}: {:?}",
|
"Successfully created snapshot for slot {}, hash {}: {:?}",
|
||||||
bank.slot(),
|
bank.slot(),
|
||||||
|
bank.hash(),
|
||||||
package.tar_output_file
|
package.tar_output_file
|
||||||
);
|
);
|
||||||
println!(
|
println!(
|
||||||
|
|
|
@ -735,6 +735,14 @@ pub fn main() {
|
||||||
.validator(hash_validator)
|
.validator(hash_validator)
|
||||||
.help("Require the genesis have this hash"),
|
.help("Require the genesis have this hash"),
|
||||||
)
|
)
|
||||||
|
.arg(
|
||||||
|
Arg::with_name("expected_bank_hash")
|
||||||
|
.long("expected-bank-hash")
|
||||||
|
.value_name("HASH")
|
||||||
|
.takes_value(true)
|
||||||
|
.validator(hash_validator)
|
||||||
|
.help("When wait-for-supermajority <x>, require the bank at <x> to have this hash"),
|
||||||
|
)
|
||||||
.arg(
|
.arg(
|
||||||
Arg::with_name("expected_shred_version")
|
Arg::with_name("expected_shred_version")
|
||||||
.long("expected-shred-version")
|
.long("expected-shred-version")
|
||||||
|
@ -755,6 +763,7 @@ pub fn main() {
|
||||||
.arg(
|
.arg(
|
||||||
Arg::with_name("wait_for_supermajority")
|
Arg::with_name("wait_for_supermajority")
|
||||||
.long("wait-for-supermajority")
|
.long("wait-for-supermajority")
|
||||||
|
.requires("expected_bank_hash")
|
||||||
.value_name("SLOT")
|
.value_name("SLOT")
|
||||||
.validator(is_slot)
|
.validator(is_slot)
|
||||||
.help("After processing the ledger and the next slot is SLOT, wait until a supermajority of stake is visible on gossip before starting PoH"),
|
.help("After processing the ledger and the next slot is SLOT, wait until a supermajority of stake is visible on gossip before starting PoH"),
|
||||||
|
@ -891,6 +900,9 @@ pub fn main() {
|
||||||
expected_genesis_hash: matches
|
expected_genesis_hash: matches
|
||||||
.value_of("expected_genesis_hash")
|
.value_of("expected_genesis_hash")
|
||||||
.map(|s| Hash::from_str(&s).unwrap()),
|
.map(|s| Hash::from_str(&s).unwrap()),
|
||||||
|
expected_bank_hash: matches
|
||||||
|
.value_of("expected_bank_hash")
|
||||||
|
.map(|s| Hash::from_str(&s).unwrap()),
|
||||||
expected_shred_version: value_t!(matches, "expected_shred_version", u16).ok(),
|
expected_shred_version: value_t!(matches, "expected_shred_version", u16).ok(),
|
||||||
new_hard_forks: hardforks_of(&matches, "hard_forks"),
|
new_hard_forks: hardforks_of(&matches, "hard_forks"),
|
||||||
rpc_config: JsonRpcConfig {
|
rpc_config: JsonRpcConfig {
|
||||||
|
|
Loading…
Reference in New Issue