2020-06-17 20:54:52 -07:00
|
|
|
/// Cluster independent integration tests
|
2019-03-01 10:36:52 -08:00
|
|
|
///
|
|
|
|
/// All tests must start from an entry point and a funding keypair and
|
|
|
|
/// discover the rest of the network.
|
2021-06-04 08:23:06 -07:00
|
|
|
use log::*;
|
2021-12-03 09:00:31 -08:00
|
|
|
use {
|
|
|
|
rand::{thread_rng, Rng},
|
|
|
|
rayon::prelude::*,
|
2022-06-08 04:57:12 -07:00
|
|
|
solana_client::{connection_cache::ConnectionCache, thin_client::ThinClient},
|
2021-12-03 09:00:31 -08:00
|
|
|
solana_core::consensus::VOTE_THRESHOLD_DEPTH,
|
|
|
|
solana_entry::entry::{Entry, EntrySlice},
|
|
|
|
solana_gossip::{
|
2022-03-31 12:47:00 -07:00
|
|
|
cluster_info,
|
2021-12-03 09:00:31 -08:00
|
|
|
contact_info::ContactInfo,
|
|
|
|
crds_value::{self, CrdsData, CrdsValue},
|
|
|
|
gossip_error::GossipError,
|
|
|
|
gossip_service::discover_cluster,
|
|
|
|
},
|
|
|
|
solana_ledger::blockstore::Blockstore,
|
|
|
|
solana_sdk::{
|
|
|
|
client::SyncClient,
|
|
|
|
clock::{self, Slot, NUM_CONSECUTIVE_LEADER_SLOTS},
|
|
|
|
commitment_config::CommitmentConfig,
|
|
|
|
epoch_schedule::MINIMUM_SLOTS_PER_EPOCH,
|
|
|
|
exit::Exit,
|
|
|
|
hash::Hash,
|
|
|
|
poh_config::PohConfig,
|
|
|
|
pubkey::Pubkey,
|
|
|
|
signature::{Keypair, Signature, Signer},
|
|
|
|
system_transaction,
|
|
|
|
timing::{duration_as_ms, timestamp},
|
|
|
|
transport::TransportError,
|
|
|
|
},
|
|
|
|
solana_streamer::socket::SocketAddrSpace,
|
|
|
|
solana_vote_program::vote_transaction,
|
|
|
|
std::{
|
|
|
|
collections::{HashMap, HashSet},
|
2022-08-10 23:17:27 -07:00
|
|
|
net::{IpAddr, Ipv4Addr, SocketAddr},
|
2021-12-03 09:00:31 -08:00
|
|
|
path::Path,
|
|
|
|
sync::{Arc, RwLock},
|
|
|
|
thread::sleep,
|
|
|
|
time::{Duration, Instant},
|
|
|
|
},
|
2019-08-21 23:59:11 -07:00
|
|
|
};
|
2019-03-01 10:36:52 -08:00
|
|
|
|
2022-08-12 06:20:04 -07:00
|
|
|
pub fn get_client_facing_addr(contact_info: &ContactInfo) -> (SocketAddr, SocketAddr) {
|
2022-08-10 23:17:27 -07:00
|
|
|
let (rpc, mut tpu) = contact_info.client_facing_addr();
|
|
|
|
// QUIC certificate authentication requires the IP Address to match. ContactInfo might have
|
|
|
|
// 0.0.0.0 as the IP instead of 127.0.0.1.
|
|
|
|
tpu.set_ip(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)));
|
|
|
|
(rpc, tpu)
|
|
|
|
}
|
|
|
|
|
2019-03-01 10:36:52 -08:00
|
|
|
/// Spend and verify from every node in the network
|
2021-03-25 12:27:07 -07:00
|
|
|
pub fn spend_and_verify_all_nodes<S: ::std::hash::BuildHasher + Sync + Send>(
|
2019-03-01 10:36:52 -08:00
|
|
|
entry_point_info: &ContactInfo,
|
|
|
|
funding_keypair: &Keypair,
|
|
|
|
nodes: usize,
|
2019-06-19 00:13:19 -07:00
|
|
|
ignore_nodes: HashSet<Pubkey, S>,
|
2021-07-23 08:25:03 -07:00
|
|
|
socket_addr_space: SocketAddrSpace,
|
2022-06-08 04:57:12 -07:00
|
|
|
connection_cache: &Arc<ConnectionCache>,
|
2019-03-01 10:36:52 -08:00
|
|
|
) {
|
2021-07-23 08:25:03 -07:00
|
|
|
let cluster_nodes =
|
|
|
|
discover_cluster(&entry_point_info.gossip, nodes, socket_addr_space).unwrap();
|
2019-03-01 10:36:52 -08:00
|
|
|
assert!(cluster_nodes.len() >= nodes);
|
2021-03-25 12:27:07 -07:00
|
|
|
let ignore_nodes = Arc::new(ignore_nodes);
|
|
|
|
cluster_nodes.par_iter().for_each(|ingress_node| {
|
2019-06-19 00:13:19 -07:00
|
|
|
if ignore_nodes.contains(&ingress_node.id) {
|
2021-03-25 12:27:07 -07:00
|
|
|
return;
|
2019-06-19 00:13:19 -07:00
|
|
|
}
|
2019-03-01 10:36:52 -08:00
|
|
|
let random_keypair = Keypair::new();
|
2022-08-10 23:17:27 -07:00
|
|
|
let (rpc, tpu) = get_client_facing_addr(ingress_node);
|
2022-06-08 04:57:12 -07:00
|
|
|
let client = ThinClient::new(rpc, tpu, connection_cache.clone());
|
2019-03-01 10:36:52 -08:00
|
|
|
let bal = client
|
2021-01-26 11:23:07 -08:00
|
|
|
.poll_get_balance_with_commitment(
|
|
|
|
&funding_keypair.pubkey(),
|
|
|
|
CommitmentConfig::processed(),
|
|
|
|
)
|
2019-03-01 10:36:52 -08:00
|
|
|
.expect("balance in source");
|
|
|
|
assert!(bal > 0);
|
2021-08-13 09:08:20 -07:00
|
|
|
let (blockhash, _) = client
|
|
|
|
.get_latest_blockhash_with_commitment(CommitmentConfig::confirmed())
|
2019-11-06 13:15:00 -08:00
|
|
|
.unwrap();
|
2019-05-20 10:03:19 -07:00
|
|
|
let mut transaction =
|
2021-06-18 06:34:46 -07:00
|
|
|
system_transaction::transfer(funding_keypair, &random_keypair.pubkey(), 1, blockhash);
|
2019-03-21 07:43:21 -07:00
|
|
|
let confs = VOTE_THRESHOLD_DEPTH + 1;
|
2019-03-01 10:36:52 -08:00
|
|
|
let sig = client
|
2021-06-18 06:34:46 -07:00
|
|
|
.retry_transfer_until_confirmed(funding_keypair, &mut transaction, 10, confs)
|
2019-03-01 10:36:52 -08:00
|
|
|
.unwrap();
|
|
|
|
for validator in &cluster_nodes {
|
2019-06-19 00:13:19 -07:00
|
|
|
if ignore_nodes.contains(&validator.id) {
|
|
|
|
continue;
|
|
|
|
}
|
2022-08-10 23:17:27 -07:00
|
|
|
let (rpc, tpu) = get_client_facing_addr(validator);
|
2022-06-08 04:57:12 -07:00
|
|
|
let client = ThinClient::new(rpc, tpu, connection_cache.clone());
|
2019-03-21 07:43:21 -07:00
|
|
|
client.poll_for_signature_confirmation(&sig, confs).unwrap();
|
2019-03-01 10:36:52 -08:00
|
|
|
}
|
2021-03-25 12:27:07 -07:00
|
|
|
});
|
2019-03-01 10:36:52 -08:00
|
|
|
}
|
2019-03-03 22:01:09 -08:00
|
|
|
|
2019-08-21 23:59:11 -07:00
|
|
|
pub fn verify_balances<S: ::std::hash::BuildHasher>(
|
|
|
|
expected_balances: HashMap<Pubkey, u64, S>,
|
|
|
|
node: &ContactInfo,
|
2022-06-08 04:57:12 -07:00
|
|
|
connection_cache: Arc<ConnectionCache>,
|
2019-08-21 23:59:11 -07:00
|
|
|
) {
|
2022-08-10 23:17:27 -07:00
|
|
|
let (rpc, tpu) = get_client_facing_addr(node);
|
2022-06-08 04:57:12 -07:00
|
|
|
let client = ThinClient::new(rpc, tpu, connection_cache);
|
2019-08-21 23:59:11 -07:00
|
|
|
for (pk, b) in expected_balances {
|
2019-11-06 13:15:00 -08:00
|
|
|
let bal = client
|
2021-01-26 11:23:07 -08:00
|
|
|
.poll_get_balance_with_commitment(&pk, CommitmentConfig::processed())
|
2019-11-06 13:15:00 -08:00
|
|
|
.expect("balance in source");
|
2019-08-21 23:59:11 -07:00
|
|
|
assert_eq!(bal, b);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn send_many_transactions(
|
|
|
|
node: &ContactInfo,
|
|
|
|
funding_keypair: &Keypair,
|
2022-06-08 04:57:12 -07:00
|
|
|
connection_cache: &Arc<ConnectionCache>,
|
2019-08-21 23:59:11 -07:00
|
|
|
max_tokens_per_transfer: u64,
|
|
|
|
num_txs: u64,
|
|
|
|
) -> HashMap<Pubkey, u64> {
|
2022-08-10 23:17:27 -07:00
|
|
|
let (rpc, tpu) = get_client_facing_addr(node);
|
2022-06-08 04:57:12 -07:00
|
|
|
let client = ThinClient::new(rpc, tpu, connection_cache.clone());
|
2019-08-21 23:59:11 -07:00
|
|
|
let mut expected_balances = HashMap::new();
|
2019-03-09 16:25:38 -08:00
|
|
|
for _ in 0..num_txs {
|
|
|
|
let random_keypair = Keypair::new();
|
|
|
|
let bal = client
|
2021-01-26 11:23:07 -08:00
|
|
|
.poll_get_balance_with_commitment(
|
|
|
|
&funding_keypair.pubkey(),
|
|
|
|
CommitmentConfig::processed(),
|
|
|
|
)
|
2019-03-09 16:25:38 -08:00
|
|
|
.expect("balance in source");
|
|
|
|
assert!(bal > 0);
|
2021-08-13 09:08:20 -07:00
|
|
|
let (blockhash, _) = client
|
|
|
|
.get_latest_blockhash_with_commitment(CommitmentConfig::processed())
|
2019-11-06 13:15:00 -08:00
|
|
|
.unwrap();
|
2019-08-21 23:59:11 -07:00
|
|
|
let transfer_amount = thread_rng().gen_range(1, max_tokens_per_transfer);
|
|
|
|
|
|
|
|
let mut transaction = system_transaction::transfer(
|
2021-06-18 06:34:46 -07:00
|
|
|
funding_keypair,
|
2019-08-21 23:59:11 -07:00
|
|
|
&random_keypair.pubkey(),
|
|
|
|
transfer_amount,
|
|
|
|
blockhash,
|
|
|
|
);
|
|
|
|
|
2019-03-09 16:25:38 -08:00
|
|
|
client
|
2021-06-18 06:34:46 -07:00
|
|
|
.retry_transfer(funding_keypair, &mut transaction, 5)
|
2019-03-09 16:25:38 -08:00
|
|
|
.unwrap();
|
2019-08-21 23:59:11 -07:00
|
|
|
|
|
|
|
expected_balances.insert(random_keypair.pubkey(), transfer_amount);
|
2019-03-09 16:25:38 -08:00
|
|
|
}
|
2019-08-21 23:59:11 -07:00
|
|
|
|
|
|
|
expected_balances
|
2019-03-09 16:25:38 -08:00
|
|
|
}
|
|
|
|
|
2019-07-30 15:53:41 -07:00
|
|
|
pub fn verify_ledger_ticks(ledger_path: &Path, ticks_per_slot: usize) {
|
2020-01-13 13:13:52 -08:00
|
|
|
let ledger = Blockstore::open(ledger_path).unwrap();
|
2020-04-09 13:09:59 -07:00
|
|
|
let zeroth_slot = ledger.get_slot_entries(0, 0).unwrap();
|
2019-03-07 15:49:07 -08:00
|
|
|
let last_id = zeroth_slot.last().unwrap().hash;
|
|
|
|
let next_slots = ledger.get_slots_since(&[0]).unwrap().remove(&0).unwrap();
|
|
|
|
let mut pending_slots: Vec<_> = next_slots
|
|
|
|
.into_iter()
|
|
|
|
.map(|slot| (slot, 0, last_id))
|
|
|
|
.collect();
|
|
|
|
while !pending_slots.is_empty() {
|
|
|
|
let (slot, parent_slot, last_id) = pending_slots.pop().unwrap();
|
|
|
|
let next_slots = ledger
|
|
|
|
.get_slots_since(&[slot])
|
|
|
|
.unwrap()
|
|
|
|
.remove(&slot)
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
// If you're not the last slot, you should have a full set of ticks
|
|
|
|
let should_verify_ticks = if !next_slots.is_empty() {
|
|
|
|
Some((slot - parent_slot) as usize * ticks_per_slot)
|
|
|
|
} else {
|
|
|
|
None
|
|
|
|
};
|
|
|
|
|
|
|
|
let last_id = verify_slot_ticks(&ledger, slot, &last_id, should_verify_ticks);
|
|
|
|
pending_slots.extend(
|
|
|
|
next_slots
|
|
|
|
.into_iter()
|
|
|
|
.map(|child_slot| (child_slot, slot, last_id)),
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-03-23 19:19:55 -07:00
|
|
|
pub fn sleep_n_epochs(
|
|
|
|
num_epochs: f64,
|
2019-05-18 14:01:36 -07:00
|
|
|
config: &PohConfig,
|
2019-03-23 19:19:55 -07:00
|
|
|
ticks_per_slot: u64,
|
|
|
|
slots_per_epoch: u64,
|
|
|
|
) {
|
2019-05-18 14:01:36 -07:00
|
|
|
let num_ticks_per_second = (1000 / duration_as_ms(&config.target_tick_duration)) as f64;
|
2019-03-23 19:19:55 -07:00
|
|
|
let num_ticks_to_sleep = num_epochs * ticks_per_slot as f64 * slots_per_epoch as f64;
|
2019-05-18 14:01:36 -07:00
|
|
|
let secs = ((num_ticks_to_sleep + num_ticks_per_second - 1.0) / num_ticks_per_second) as u64;
|
|
|
|
warn!("sleep_n_epochs: {} seconds", secs);
|
|
|
|
sleep(Duration::from_secs(secs));
|
2019-03-23 19:19:55 -07:00
|
|
|
}
|
|
|
|
|
2019-03-05 17:56:51 -08:00
|
|
|
pub fn kill_entry_and_spend_and_verify_rest(
|
|
|
|
entry_point_info: &ContactInfo,
|
2021-06-03 20:06:13 -07:00
|
|
|
entry_point_validator_exit: &Arc<RwLock<Exit>>,
|
2019-03-05 17:56:51 -08:00
|
|
|
funding_keypair: &Keypair,
|
2022-06-08 04:57:12 -07:00
|
|
|
connection_cache: &Arc<ConnectionCache>,
|
2019-03-05 17:56:51 -08:00
|
|
|
nodes: usize,
|
2019-04-29 15:26:52 -07:00
|
|
|
slot_millis: u64,
|
2021-07-23 08:25:03 -07:00
|
|
|
socket_addr_space: SocketAddrSpace,
|
2019-03-05 17:56:51 -08:00
|
|
|
) {
|
2021-02-26 21:42:09 -08:00
|
|
|
info!("kill_entry_and_spend_and_verify_rest...");
|
2021-07-23 08:25:03 -07:00
|
|
|
let cluster_nodes =
|
|
|
|
discover_cluster(&entry_point_info.gossip, nodes, socket_addr_space).unwrap();
|
2019-03-05 17:56:51 -08:00
|
|
|
assert!(cluster_nodes.len() >= nodes);
|
2022-08-10 23:17:27 -07:00
|
|
|
let (rpc, tpu) = get_client_facing_addr(entry_point_info);
|
2022-06-08 04:57:12 -07:00
|
|
|
let client = ThinClient::new(rpc, tpu, connection_cache.clone());
|
2022-04-06 08:03:38 -07:00
|
|
|
|
2019-11-15 08:36:33 -08:00
|
|
|
// sleep long enough to make sure we are in epoch 3
|
|
|
|
let first_two_epoch_slots = MINIMUM_SLOTS_PER_EPOCH * (3 + 1);
|
2019-05-23 14:50:23 -07:00
|
|
|
|
|
|
|
for ingress_node in &cluster_nodes {
|
|
|
|
client
|
2021-01-26 11:23:07 -08:00
|
|
|
.poll_get_balance_with_commitment(&ingress_node.id, CommitmentConfig::processed())
|
2019-05-23 14:50:23 -07:00
|
|
|
.unwrap_or_else(|err| panic!("Node {} has no balance: {}", ingress_node.id, err));
|
|
|
|
}
|
|
|
|
|
2019-03-21 07:43:21 -07:00
|
|
|
info!("sleeping for 2 leader fortnights");
|
|
|
|
sleep(Duration::from_millis(
|
2019-04-29 15:26:52 -07:00
|
|
|
slot_millis * first_two_epoch_slots as u64,
|
2019-03-21 07:43:21 -07:00
|
|
|
));
|
2019-04-29 15:26:52 -07:00
|
|
|
info!("done sleeping for first 2 warmup epochs");
|
2019-05-23 14:50:23 -07:00
|
|
|
info!("killing entry point: {}", entry_point_info.id);
|
2021-02-26 21:42:09 -08:00
|
|
|
entry_point_validator_exit.write().unwrap().exit();
|
2019-04-29 15:26:52 -07:00
|
|
|
info!("sleeping for some time");
|
2019-03-21 07:43:21 -07:00
|
|
|
sleep(Duration::from_millis(
|
2019-04-29 15:26:52 -07:00
|
|
|
slot_millis * NUM_CONSECUTIVE_LEADER_SLOTS,
|
2019-03-21 07:43:21 -07:00
|
|
|
));
|
|
|
|
info!("done sleeping for 2 fortnights");
|
2019-03-05 17:56:51 -08:00
|
|
|
for ingress_node in &cluster_nodes {
|
|
|
|
if ingress_node.id == entry_point_info.id {
|
2019-08-16 15:46:19 -07:00
|
|
|
info!("ingress_node.id == entry_point_info.id, continuing...");
|
2019-03-05 17:56:51 -08:00
|
|
|
continue;
|
|
|
|
}
|
2019-03-12 13:48:02 -07:00
|
|
|
|
2022-08-10 23:17:27 -07:00
|
|
|
let (rpc, tpu) = get_client_facing_addr(ingress_node);
|
2022-06-08 04:57:12 -07:00
|
|
|
let client = ThinClient::new(rpc, tpu, connection_cache.clone());
|
2019-05-23 14:50:23 -07:00
|
|
|
let balance = client
|
2021-01-26 11:23:07 -08:00
|
|
|
.poll_get_balance_with_commitment(
|
|
|
|
&funding_keypair.pubkey(),
|
|
|
|
CommitmentConfig::processed(),
|
|
|
|
)
|
2019-03-05 17:56:51 -08:00
|
|
|
.expect("balance in source");
|
2019-05-23 14:50:23 -07:00
|
|
|
assert_ne!(balance, 0);
|
2019-03-12 13:48:02 -07:00
|
|
|
|
|
|
|
let mut result = Ok(());
|
|
|
|
let mut retries = 0;
|
|
|
|
loop {
|
|
|
|
retries += 1;
|
|
|
|
if retries > 5 {
|
|
|
|
result.unwrap();
|
|
|
|
}
|
|
|
|
|
|
|
|
let random_keypair = Keypair::new();
|
2021-08-13 09:08:20 -07:00
|
|
|
let (blockhash, _) = client
|
|
|
|
.get_latest_blockhash_with_commitment(CommitmentConfig::processed())
|
2019-11-06 13:15:00 -08:00
|
|
|
.unwrap();
|
2019-04-03 08:45:57 -07:00
|
|
|
let mut transaction = system_transaction::transfer(
|
2021-06-18 06:34:46 -07:00
|
|
|
funding_keypair,
|
2019-03-12 13:48:02 -07:00
|
|
|
&random_keypair.pubkey(),
|
|
|
|
1,
|
2019-05-13 12:49:37 -07:00
|
|
|
blockhash,
|
2019-03-12 13:48:02 -07:00
|
|
|
);
|
|
|
|
|
2019-03-21 07:43:21 -07:00
|
|
|
let confs = VOTE_THRESHOLD_DEPTH + 1;
|
2019-03-12 13:48:02 -07:00
|
|
|
let sig = {
|
2019-03-21 07:43:21 -07:00
|
|
|
let sig = client.retry_transfer_until_confirmed(
|
2021-06-18 06:34:46 -07:00
|
|
|
funding_keypair,
|
2019-03-21 07:43:21 -07:00
|
|
|
&mut transaction,
|
|
|
|
5,
|
|
|
|
confs,
|
|
|
|
);
|
|
|
|
match sig {
|
2019-03-12 13:48:02 -07:00
|
|
|
Err(e) => {
|
2020-03-12 23:20:49 -07:00
|
|
|
result = Err(e);
|
2019-03-12 13:48:02 -07:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
Ok(sig) => sig,
|
|
|
|
}
|
|
|
|
};
|
2019-08-16 15:46:19 -07:00
|
|
|
info!("poll_all_nodes_for_signature()");
|
2022-06-08 04:57:12 -07:00
|
|
|
match poll_all_nodes_for_signature(
|
|
|
|
entry_point_info,
|
|
|
|
&cluster_nodes,
|
|
|
|
connection_cache,
|
|
|
|
&sig,
|
|
|
|
confs,
|
|
|
|
) {
|
2019-03-12 13:48:02 -07:00
|
|
|
Err(e) => {
|
2019-08-16 15:46:19 -07:00
|
|
|
info!("poll_all_nodes_for_signature() failed {:?}", e);
|
2019-03-12 13:48:02 -07:00
|
|
|
result = Err(e);
|
|
|
|
}
|
|
|
|
Ok(()) => {
|
2019-08-16 15:46:19 -07:00
|
|
|
info!("poll_all_nodes_for_signature() succeeded, done.");
|
2019-03-12 13:48:02 -07:00
|
|
|
break;
|
|
|
|
}
|
2019-03-05 17:56:51 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2019-03-07 15:49:07 -08:00
|
|
|
|
2022-06-08 04:57:12 -07:00
|
|
|
pub fn check_for_new_roots(
|
|
|
|
num_new_roots: usize,
|
|
|
|
contact_infos: &[ContactInfo],
|
|
|
|
connection_cache: &Arc<ConnectionCache>,
|
|
|
|
test_name: &str,
|
|
|
|
) {
|
2020-05-05 14:07:21 -07:00
|
|
|
let mut roots = vec![HashSet::new(); contact_infos.len()];
|
|
|
|
let mut done = false;
|
|
|
|
let mut last_print = Instant::now();
|
2021-03-25 12:27:07 -07:00
|
|
|
let loop_start = Instant::now();
|
2021-10-15 11:40:03 -07:00
|
|
|
let loop_timeout = Duration::from_secs(180);
|
2021-06-09 15:01:48 -07:00
|
|
|
let mut num_roots_map = HashMap::new();
|
2020-05-05 14:07:21 -07:00
|
|
|
while !done {
|
2021-03-25 12:27:07 -07:00
|
|
|
assert!(loop_start.elapsed() < loop_timeout);
|
2021-06-09 15:01:48 -07:00
|
|
|
|
2020-05-05 14:07:21 -07:00
|
|
|
for (i, ingress_node) in contact_infos.iter().enumerate() {
|
2022-08-10 23:17:27 -07:00
|
|
|
let (rpc, tpu) = get_client_facing_addr(ingress_node);
|
2022-06-08 04:57:12 -07:00
|
|
|
let client = ThinClient::new(rpc, tpu, connection_cache.clone());
|
2021-06-09 15:01:48 -07:00
|
|
|
let root_slot = client
|
|
|
|
.get_slot_with_commitment(CommitmentConfig::finalized())
|
|
|
|
.unwrap_or(0);
|
|
|
|
roots[i].insert(root_slot);
|
|
|
|
num_roots_map.insert(ingress_node.id, roots[i].len());
|
|
|
|
let num_roots = roots.iter().map(|r| r.len()).min().unwrap();
|
|
|
|
done = num_roots >= num_new_roots;
|
2021-03-25 12:27:07 -07:00
|
|
|
if done || last_print.elapsed().as_secs() > 3 {
|
2021-05-04 00:51:42 -07:00
|
|
|
info!(
|
2021-06-09 15:01:48 -07:00
|
|
|
"{} waiting for {} new roots.. observed: {:?}",
|
|
|
|
test_name, num_new_roots, num_roots_map
|
2021-05-04 00:51:42 -07:00
|
|
|
);
|
2020-05-05 14:07:21 -07:00
|
|
|
last_print = Instant::now();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
sleep(Duration::from_millis(clock::DEFAULT_MS_PER_SLOT / 2));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-07-21 23:04:24 -07:00
|
|
|
pub fn check_no_new_roots(
|
|
|
|
num_slots_to_wait: usize,
|
|
|
|
contact_infos: &[ContactInfo],
|
2022-06-08 04:57:12 -07:00
|
|
|
connection_cache: &Arc<ConnectionCache>,
|
2020-07-21 23:04:24 -07:00
|
|
|
test_name: &str,
|
|
|
|
) {
|
|
|
|
assert!(!contact_infos.is_empty());
|
|
|
|
let mut roots = vec![0; contact_infos.len()];
|
|
|
|
let max_slot = contact_infos
|
|
|
|
.iter()
|
|
|
|
.enumerate()
|
|
|
|
.map(|(i, ingress_node)| {
|
2022-08-10 23:17:27 -07:00
|
|
|
let (rpc, tpu) = get_client_facing_addr(ingress_node);
|
2022-06-08 04:57:12 -07:00
|
|
|
let client = ThinClient::new(rpc, tpu, connection_cache.clone());
|
2020-07-21 23:04:24 -07:00
|
|
|
let initial_root = client
|
|
|
|
.get_slot()
|
|
|
|
.unwrap_or_else(|_| panic!("get_slot for {} failed", ingress_node.id));
|
|
|
|
roots[i] = initial_root;
|
|
|
|
client
|
2021-01-26 11:23:07 -08:00
|
|
|
.get_slot_with_commitment(CommitmentConfig::processed())
|
2020-07-21 23:04:24 -07:00
|
|
|
.unwrap_or_else(|_| panic!("get_slot for {} failed", ingress_node.id))
|
|
|
|
})
|
|
|
|
.max()
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
let end_slot = max_slot + num_slots_to_wait as u64;
|
|
|
|
let mut current_slot;
|
|
|
|
let mut last_print = Instant::now();
|
2020-07-25 21:30:08 -07:00
|
|
|
let mut reached_end_slot = false;
|
2020-07-21 23:04:24 -07:00
|
|
|
loop {
|
2020-07-25 21:30:08 -07:00
|
|
|
for contact_info in contact_infos {
|
2022-08-10 23:17:27 -07:00
|
|
|
let (rpc, tpu) = get_client_facing_addr(contact_info);
|
2022-06-08 04:57:12 -07:00
|
|
|
let client = ThinClient::new(rpc, tpu, connection_cache.clone());
|
2020-07-25 21:30:08 -07:00
|
|
|
current_slot = client
|
2021-01-26 11:23:07 -08:00
|
|
|
.get_slot_with_commitment(CommitmentConfig::processed())
|
2020-07-25 21:30:08 -07:00
|
|
|
.unwrap_or_else(|_| panic!("get_slot for {} failed", contact_infos[0].id));
|
|
|
|
if current_slot > end_slot {
|
|
|
|
reached_end_slot = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if last_print.elapsed().as_secs() > 3 {
|
|
|
|
info!(
|
|
|
|
"{} current slot: {} on validator: {}, waiting for any validator with slot: {}",
|
|
|
|
test_name, current_slot, contact_info.id, end_slot
|
|
|
|
);
|
|
|
|
last_print = Instant::now();
|
|
|
|
}
|
2020-07-21 23:04:24 -07:00
|
|
|
}
|
2020-07-25 21:30:08 -07:00
|
|
|
if reached_end_slot {
|
|
|
|
break;
|
2020-07-21 23:04:24 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i, ingress_node) in contact_infos.iter().enumerate() {
|
2022-08-10 23:17:27 -07:00
|
|
|
let (rpc, tpu) = get_client_facing_addr(ingress_node);
|
2022-06-08 04:57:12 -07:00
|
|
|
let client = ThinClient::new(rpc, tpu, connection_cache.clone());
|
2020-07-21 23:04:24 -07:00
|
|
|
assert_eq!(
|
|
|
|
client
|
|
|
|
.get_slot()
|
|
|
|
.unwrap_or_else(|_| panic!("get_slot for {} failed", ingress_node.id)),
|
|
|
|
roots[i]
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-03-12 13:48:02 -07:00
|
|
|
fn poll_all_nodes_for_signature(
|
|
|
|
entry_point_info: &ContactInfo,
|
|
|
|
cluster_nodes: &[ContactInfo],
|
2022-06-08 04:57:12 -07:00
|
|
|
connection_cache: &Arc<ConnectionCache>,
|
2019-03-12 13:48:02 -07:00
|
|
|
sig: &Signature,
|
2019-03-21 07:43:21 -07:00
|
|
|
confs: usize,
|
2019-04-25 12:46:40 -07:00
|
|
|
) -> Result<(), TransportError> {
|
2019-03-12 13:48:02 -07:00
|
|
|
for validator in cluster_nodes {
|
|
|
|
if validator.id == entry_point_info.id {
|
|
|
|
continue;
|
|
|
|
}
|
2022-08-10 23:17:27 -07:00
|
|
|
let (rpc, tpu) = get_client_facing_addr(validator);
|
2022-06-08 04:57:12 -07:00
|
|
|
let client = ThinClient::new(rpc, tpu, connection_cache.clone());
|
2021-06-18 06:34:46 -07:00
|
|
|
client.poll_for_signature_confirmation(sig, confs)?;
|
2019-03-12 13:48:02 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2020-01-13 13:13:52 -08:00
|
|
|
fn get_and_verify_slot_entries(
|
|
|
|
blockstore: &Blockstore,
|
|
|
|
slot: Slot,
|
|
|
|
last_entry: &Hash,
|
|
|
|
) -> Vec<Entry> {
|
2020-04-09 13:09:59 -07:00
|
|
|
let entries = blockstore.get_slot_entries(slot, 0).unwrap();
|
2021-05-19 07:31:47 -07:00
|
|
|
assert!(entries.verify(last_entry));
|
2019-03-07 15:49:07 -08:00
|
|
|
entries
|
|
|
|
}
|
|
|
|
|
|
|
|
fn verify_slot_ticks(
|
2020-01-13 13:13:52 -08:00
|
|
|
blockstore: &Blockstore,
|
2019-11-02 00:38:30 -07:00
|
|
|
slot: Slot,
|
2019-03-07 15:49:07 -08:00
|
|
|
last_entry: &Hash,
|
|
|
|
expected_num_ticks: Option<usize>,
|
|
|
|
) -> Hash {
|
2020-01-13 13:13:52 -08:00
|
|
|
let entries = get_and_verify_slot_entries(blockstore, slot, last_entry);
|
2019-03-07 15:49:07 -08:00
|
|
|
let num_ticks: usize = entries.iter().map(|entry| entry.is_tick() as usize).sum();
|
|
|
|
if let Some(expected_num_ticks) = expected_num_ticks {
|
|
|
|
assert_eq!(num_ticks, expected_num_ticks);
|
|
|
|
}
|
|
|
|
entries.last().unwrap().hash
|
|
|
|
}
|
2021-07-10 22:22:07 -07:00
|
|
|
|
|
|
|
pub fn submit_vote_to_cluster_gossip(
|
|
|
|
node_keypair: &Keypair,
|
|
|
|
vote_keypair: &Keypair,
|
|
|
|
vote_slot: Slot,
|
|
|
|
vote_hash: Hash,
|
|
|
|
blockhash: Hash,
|
|
|
|
gossip_addr: SocketAddr,
|
2021-07-23 08:25:03 -07:00
|
|
|
socket_addr_space: &SocketAddrSpace,
|
2021-07-10 22:22:07 -07:00
|
|
|
) -> Result<(), GossipError> {
|
|
|
|
let vote_tx = vote_transaction::new_vote_transaction(
|
|
|
|
vec![vote_slot],
|
|
|
|
vote_hash,
|
|
|
|
blockhash,
|
|
|
|
node_keypair,
|
|
|
|
vote_keypair,
|
|
|
|
vote_keypair,
|
|
|
|
None,
|
|
|
|
);
|
|
|
|
|
|
|
|
cluster_info::push_messages_to_peer(
|
|
|
|
vec![CrdsValue::new_signed(
|
|
|
|
CrdsData::Vote(
|
|
|
|
0,
|
2021-12-29 11:31:26 -08:00
|
|
|
crds_value::Vote::new(node_keypair.pubkey(), vote_tx, timestamp()).unwrap(),
|
2021-07-10 22:22:07 -07:00
|
|
|
),
|
|
|
|
node_keypair,
|
|
|
|
)],
|
|
|
|
node_keypair.pubkey(),
|
|
|
|
gossip_addr,
|
2021-07-23 08:25:03 -07:00
|
|
|
socket_addr_space,
|
2021-07-10 22:22:07 -07:00
|
|
|
)
|
|
|
|
}
|