2019-12-06 22:16:28 -08:00
|
|
|
use assert_matches::assert_matches;
|
2019-05-24 19:20:09 -07:00
|
|
|
use log::*;
|
2019-07-02 17:35:03 -07:00
|
|
|
use serial_test_derive::serial;
|
2020-01-27 16:49:25 -08:00
|
|
|
use solana_client::rpc_client::RpcClient;
|
2019-10-23 11:50:10 -07:00
|
|
|
use solana_client::thin_client::create_client;
|
2019-08-21 10:23:33 -07:00
|
|
|
use solana_core::{
|
2020-01-27 16:49:25 -08:00
|
|
|
broadcast_stage::BroadcastStageType, consensus::VOTE_THRESHOLD_DEPTH,
|
|
|
|
gossip_service::discover_cluster, validator::ValidatorConfig,
|
2019-08-16 00:00:38 -07:00
|
|
|
};
|
2019-12-03 16:31:59 -08:00
|
|
|
use solana_ledger::{
|
2020-01-13 13:13:52 -08:00
|
|
|
bank_forks::SnapshotConfig, blockstore::Blockstore, leader_schedule::FixedSchedule,
|
2019-12-03 16:31:59 -08:00
|
|
|
leader_schedule::LeaderSchedule, snapshot_utils,
|
|
|
|
};
|
2019-11-12 19:30:35 -08:00
|
|
|
use solana_local_cluster::{
|
|
|
|
cluster::Cluster,
|
|
|
|
cluster_tests,
|
|
|
|
local_cluster::{ClusterConfig, LocalCluster},
|
|
|
|
};
|
2019-10-08 22:34:26 -07:00
|
|
|
use solana_sdk::{
|
|
|
|
client::SyncClient,
|
|
|
|
clock,
|
2019-11-06 13:15:00 -08:00
|
|
|
commitment_config::CommitmentConfig,
|
2019-08-21 23:59:11 -07:00
|
|
|
epoch_schedule::{EpochSchedule, MINIMUM_SLOTS_PER_EPOCH},
|
2019-11-08 20:56:57 -08:00
|
|
|
genesis_config::OperatingMode,
|
2019-10-08 22:34:26 -07:00
|
|
|
poh_config::PohConfig,
|
2019-12-03 16:31:59 -08:00
|
|
|
signature::{Keypair, KeypairUtil},
|
2019-08-21 23:59:11 -07:00
|
|
|
};
|
2020-01-27 16:49:25 -08:00
|
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
2019-08-21 23:59:11 -07:00
|
|
|
use std::{
|
|
|
|
collections::{HashMap, HashSet},
|
2019-12-03 16:31:59 -08:00
|
|
|
fs, iter,
|
2019-11-06 13:15:00 -08:00
|
|
|
path::{Path, PathBuf},
|
2019-12-03 16:31:59 -08:00
|
|
|
sync::Arc,
|
2019-08-21 23:59:11 -07:00
|
|
|
thread::sleep,
|
2019-12-03 16:31:59 -08:00
|
|
|
time::{Duration, Instant},
|
2019-08-21 23:59:11 -07:00
|
|
|
};
|
|
|
|
use tempfile::TempDir;
|
2019-03-01 10:36:52 -08:00
|
|
|
|
2019-07-20 13:13:55 -07:00
|
|
|
#[test]
|
2019-08-21 15:27:42 -07:00
|
|
|
#[serial]
|
2019-07-20 13:13:55 -07:00
|
|
|
fn test_ledger_cleanup_service() {
|
|
|
|
solana_logger::setup();
|
2019-07-30 13:48:46 -07:00
|
|
|
error!("test_ledger_cleanup_service");
|
2019-07-20 13:13:55 -07:00
|
|
|
let num_nodes = 3;
|
|
|
|
let mut validator_config = ValidatorConfig::default();
|
|
|
|
validator_config.max_ledger_slots = Some(100);
|
|
|
|
let config = ClusterConfig {
|
|
|
|
cluster_lamports: 10_000,
|
|
|
|
poh_config: PohConfig::new_sleep(Duration::from_millis(50)),
|
|
|
|
node_stakes: vec![100; num_nodes],
|
|
|
|
validator_configs: vec![validator_config.clone(); num_nodes],
|
|
|
|
..ClusterConfig::default()
|
|
|
|
};
|
|
|
|
let mut cluster = LocalCluster::new(&config);
|
|
|
|
// 200ms/per * 100 = 20 seconds, so sleep a little longer than that.
|
|
|
|
sleep(Duration::from_secs(60));
|
|
|
|
|
|
|
|
cluster_tests::spend_and_verify_all_nodes(
|
|
|
|
&cluster.entry_point_info,
|
|
|
|
&cluster.funding_keypair,
|
|
|
|
num_nodes,
|
|
|
|
HashSet::new(),
|
|
|
|
);
|
|
|
|
cluster.close_preserve_ledgers();
|
|
|
|
//check everyone's ledgers and make sure only ~100 slots are stored
|
2019-12-03 16:31:59 -08:00
|
|
|
for (_, info) in &cluster.validators {
|
2019-07-20 13:13:55 -07:00
|
|
|
let mut slots = 0;
|
2020-01-13 13:13:52 -08:00
|
|
|
let blockstore = Blockstore::open(&info.info.ledger_path).unwrap();
|
|
|
|
blockstore
|
2019-07-20 13:13:55 -07:00
|
|
|
.slot_meta_iterator(0)
|
|
|
|
.unwrap()
|
|
|
|
.for_each(|_| slots += 1);
|
2020-01-13 13:13:52 -08:00
|
|
|
// with 3 nodes upto 3 slots can be in progress and not complete so max slots in blockstore should be upto 103
|
2019-07-20 13:13:55 -07:00
|
|
|
assert!(slots <= 103, "got {}", slots);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-03-01 10:36:52 -08:00
|
|
|
#[test]
|
2019-07-02 17:35:03 -07:00
|
|
|
#[serial]
|
2019-03-05 17:56:51 -08:00
|
|
|
fn test_spend_and_verify_all_nodes_1() {
|
2019-03-01 10:36:52 -08:00
|
|
|
solana_logger::setup();
|
2019-07-30 13:48:46 -07:00
|
|
|
error!("test_spend_and_verify_all_nodes_1");
|
2019-03-01 10:36:52 -08:00
|
|
|
let num_nodes = 1;
|
2019-04-15 15:27:45 -07:00
|
|
|
let local = LocalCluster::new_with_equal_stakes(num_nodes, 10_000, 100);
|
2019-03-01 10:36:52 -08:00
|
|
|
cluster_tests::spend_and_verify_all_nodes(
|
|
|
|
&local.entry_point_info,
|
|
|
|
&local.funding_keypair,
|
|
|
|
num_nodes,
|
2019-06-19 00:13:19 -07:00
|
|
|
HashSet::new(),
|
2019-03-01 10:36:52 -08:00
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
2019-08-21 15:27:42 -07:00
|
|
|
#[serial]
|
2019-03-05 17:56:51 -08:00
|
|
|
fn test_spend_and_verify_all_nodes_2() {
|
2019-03-01 10:36:52 -08:00
|
|
|
solana_logger::setup();
|
2019-07-30 13:48:46 -07:00
|
|
|
error!("test_spend_and_verify_all_nodes_2");
|
2019-03-01 10:36:52 -08:00
|
|
|
let num_nodes = 2;
|
2019-04-15 15:27:45 -07:00
|
|
|
let local = LocalCluster::new_with_equal_stakes(num_nodes, 10_000, 100);
|
2019-03-01 10:36:52 -08:00
|
|
|
cluster_tests::spend_and_verify_all_nodes(
|
|
|
|
&local.entry_point_info,
|
|
|
|
&local.funding_keypair,
|
|
|
|
num_nodes,
|
2019-06-19 00:13:19 -07:00
|
|
|
HashSet::new(),
|
2019-03-01 10:36:52 -08:00
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
2019-08-21 15:27:42 -07:00
|
|
|
#[serial]
|
2019-03-05 17:56:51 -08:00
|
|
|
fn test_spend_and_verify_all_nodes_3() {
|
2019-03-01 10:36:52 -08:00
|
|
|
solana_logger::setup();
|
2019-07-30 13:48:46 -07:00
|
|
|
error!("test_spend_and_verify_all_nodes_3");
|
2019-03-01 10:36:52 -08:00
|
|
|
let num_nodes = 3;
|
2019-04-15 15:27:45 -07:00
|
|
|
let local = LocalCluster::new_with_equal_stakes(num_nodes, 10_000, 100);
|
2019-03-01 10:36:52 -08:00
|
|
|
cluster_tests::spend_and_verify_all_nodes(
|
|
|
|
&local.entry_point_info,
|
|
|
|
&local.funding_keypair,
|
|
|
|
num_nodes,
|
2019-06-19 00:13:19 -07:00
|
|
|
HashSet::new(),
|
2019-03-01 10:36:52 -08:00
|
|
|
);
|
|
|
|
}
|
2019-03-03 22:01:09 -08:00
|
|
|
|
2019-04-19 15:47:03 -07:00
|
|
|
#[test]
|
2019-09-12 11:39:39 -07:00
|
|
|
#[allow(unused_attributes)]
|
2019-04-19 15:47:03 -07:00
|
|
|
#[ignore]
|
|
|
|
fn test_spend_and_verify_all_nodes_env_num_nodes() {
|
|
|
|
solana_logger::setup();
|
|
|
|
let num_nodes: usize = std::env::var("NUM_NODES")
|
|
|
|
.expect("please set environment variable NUM_NODES")
|
|
|
|
.parse()
|
|
|
|
.expect("could not parse NUM_NODES as a number");
|
|
|
|
let local = LocalCluster::new_with_equal_stakes(num_nodes, 10_000, 100);
|
|
|
|
cluster_tests::spend_and_verify_all_nodes(
|
|
|
|
&local.entry_point_info,
|
|
|
|
&local.funding_keypair,
|
|
|
|
num_nodes,
|
2019-06-19 00:13:19 -07:00
|
|
|
HashSet::new(),
|
2019-04-19 15:47:03 -07:00
|
|
|
);
|
|
|
|
}
|
|
|
|
|
2019-07-02 17:35:03 -07:00
|
|
|
#[allow(unused_attributes)]
|
2019-03-03 22:01:09 -08:00
|
|
|
#[test]
|
|
|
|
#[should_panic]
|
2019-10-11 12:30:52 -07:00
|
|
|
fn test_validator_exit_default_config_should_panic() {
|
2019-03-03 22:01:09 -08:00
|
|
|
solana_logger::setup();
|
2019-10-11 12:30:52 -07:00
|
|
|
error!("test_validator_exit_default_config_should_panic");
|
2019-03-03 22:01:09 -08:00
|
|
|
let num_nodes = 2;
|
2019-04-15 15:27:45 -07:00
|
|
|
let local = LocalCluster::new_with_equal_stakes(num_nodes, 10_000, 100);
|
2019-10-11 12:30:52 -07:00
|
|
|
cluster_tests::validator_exit(&local.entry_point_info, num_nodes);
|
2019-03-03 22:01:09 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
2019-07-02 17:35:03 -07:00
|
|
|
#[serial]
|
2019-10-11 12:30:52 -07:00
|
|
|
fn test_validator_exit_2() {
|
2019-03-03 22:01:09 -08:00
|
|
|
solana_logger::setup();
|
2019-10-11 12:30:52 -07:00
|
|
|
error!("test_validator_exit_2");
|
2019-03-03 22:01:09 -08:00
|
|
|
let num_nodes = 2;
|
2019-05-23 22:05:16 -07:00
|
|
|
let mut validator_config = ValidatorConfig::default();
|
2019-10-11 12:30:52 -07:00
|
|
|
validator_config.rpc_config.enable_validator_exit = true;
|
2020-01-07 19:35:09 -08:00
|
|
|
validator_config.wait_for_supermajority = true;
|
|
|
|
|
2019-04-15 15:27:45 -07:00
|
|
|
let config = ClusterConfig {
|
|
|
|
cluster_lamports: 10_000,
|
2019-06-19 00:13:19 -07:00
|
|
|
node_stakes: vec![100; num_nodes],
|
|
|
|
validator_configs: vec![validator_config.clone(); num_nodes],
|
2019-04-15 15:27:45 -07:00
|
|
|
..ClusterConfig::default()
|
|
|
|
};
|
|
|
|
let local = LocalCluster::new(&config);
|
2019-10-11 12:30:52 -07:00
|
|
|
cluster_tests::validator_exit(&local.entry_point_info, num_nodes);
|
2019-03-03 22:01:09 -08:00
|
|
|
}
|
2019-03-05 17:56:51 -08:00
|
|
|
|
2019-03-21 07:43:21 -07:00
|
|
|
// Cluster needs a supermajority to remain, so the minimum size for this test is 4
|
2019-03-05 17:56:51 -08:00
|
|
|
#[test]
|
2019-07-02 17:35:03 -07:00
|
|
|
#[serial]
|
2019-03-21 07:43:21 -07:00
|
|
|
fn test_leader_failure_4() {
|
|
|
|
solana_logger::setup();
|
2019-07-30 13:48:46 -07:00
|
|
|
error!("test_leader_failure_4");
|
2019-03-21 07:43:21 -07:00
|
|
|
let num_nodes = 4;
|
2019-05-23 22:05:16 -07:00
|
|
|
let mut validator_config = ValidatorConfig::default();
|
2019-10-11 12:30:52 -07:00
|
|
|
validator_config.rpc_config.enable_validator_exit = true;
|
2019-04-15 15:27:45 -07:00
|
|
|
let config = ClusterConfig {
|
|
|
|
cluster_lamports: 10_000,
|
|
|
|
node_stakes: vec![100; 4],
|
2019-06-19 00:13:19 -07:00
|
|
|
validator_configs: vec![validator_config.clone(); num_nodes],
|
2019-04-15 15:27:45 -07:00
|
|
|
..ClusterConfig::default()
|
|
|
|
};
|
|
|
|
let local = LocalCluster::new(&config);
|
2019-03-05 17:56:51 -08:00
|
|
|
cluster_tests::kill_entry_and_spend_and_verify_rest(
|
|
|
|
&local.entry_point_info,
|
|
|
|
&local.funding_keypair,
|
|
|
|
num_nodes,
|
2019-05-18 14:01:36 -07:00
|
|
|
config.ticks_per_slot * config.poh_config.target_tick_duration.as_millis() as u64,
|
2019-03-05 17:56:51 -08:00
|
|
|
);
|
|
|
|
}
|
2019-11-15 08:36:33 -08:00
|
|
|
|
2019-12-03 16:31:59 -08:00
|
|
|
/// This function runs a network, initiates a partition based on a
|
|
|
|
/// configuration, resolve the partition, then checks that the network
|
|
|
|
/// continues to achieve consensus
|
|
|
|
/// # Arguments
|
|
|
|
/// * `partitions` - A slice of partition configurations, where each partition
|
|
|
|
/// configuration is a slice of (usize, bool), representing a node's stake and
|
|
|
|
/// whether or not it should be killed during the partition
|
|
|
|
/// * `leader_schedule` - An option that specifies whether the cluster should
|
|
|
|
/// run with a fixed, predetermined leader schedule
|
|
|
|
fn run_cluster_partition(
|
|
|
|
partitions: &[&[(usize, bool)]],
|
|
|
|
leader_schedule: Option<(LeaderSchedule, Vec<Arc<Keypair>>)>,
|
|
|
|
) {
|
2019-11-15 08:36:33 -08:00
|
|
|
solana_logger::setup();
|
|
|
|
info!("PARTITION_TEST!");
|
2019-12-03 16:31:59 -08:00
|
|
|
let num_nodes = partitions.len();
|
|
|
|
let node_stakes: Vec<_> = partitions
|
|
|
|
.iter()
|
|
|
|
.flat_map(|p| p.iter().map(|(stake_weight, _)| 100 * *stake_weight as u64))
|
|
|
|
.collect();
|
|
|
|
assert_eq!(node_stakes.len(), num_nodes);
|
|
|
|
let cluster_lamports = node_stakes.iter().sum::<u64>() * 2;
|
|
|
|
let partition_start_epoch = 2;
|
|
|
|
let mut validator_config = ValidatorConfig::default();
|
|
|
|
|
|
|
|
// Returns:
|
|
|
|
// 1) The keys for the validiators
|
|
|
|
// 2) The amount of time it would take to iterate through one full iteration of the given
|
|
|
|
// leader schedule
|
|
|
|
let (validator_keys, leader_schedule_time): (Vec<_>, u64) = {
|
|
|
|
if let Some((leader_schedule, validator_keys)) = leader_schedule {
|
|
|
|
assert_eq!(validator_keys.len(), num_nodes);
|
|
|
|
let num_slots_per_rotation = leader_schedule.num_slots() as u64;
|
|
|
|
let fixed_schedule = FixedSchedule {
|
|
|
|
start_epoch: partition_start_epoch,
|
|
|
|
leader_schedule: Arc::new(leader_schedule),
|
|
|
|
};
|
|
|
|
validator_config.fixed_leader_schedule = Some(fixed_schedule);
|
|
|
|
(
|
|
|
|
validator_keys,
|
|
|
|
num_slots_per_rotation * clock::DEFAULT_MS_PER_SLOT,
|
|
|
|
)
|
|
|
|
} else {
|
|
|
|
(
|
|
|
|
iter::repeat_with(|| Arc::new(Keypair::new()))
|
|
|
|
.take(partitions.len())
|
|
|
|
.collect(),
|
|
|
|
10_000,
|
|
|
|
)
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
let validator_pubkeys: Vec<_> = validator_keys.iter().map(|v| v.pubkey()).collect();
|
2020-01-27 16:49:25 -08:00
|
|
|
let config = ClusterConfig {
|
2019-12-03 16:31:59 -08:00
|
|
|
cluster_lamports,
|
|
|
|
node_stakes,
|
2019-11-15 08:36:33 -08:00
|
|
|
validator_configs: vec![validator_config.clone(); num_nodes],
|
2019-12-03 16:31:59 -08:00
|
|
|
validator_keys: Some(validator_keys),
|
2019-11-15 08:36:33 -08:00
|
|
|
..ClusterConfig::default()
|
|
|
|
};
|
2019-12-03 16:31:59 -08:00
|
|
|
|
2020-01-27 16:49:25 -08:00
|
|
|
let enable_partition = Some(Arc::new(AtomicBool::new(true)));
|
2019-11-15 08:36:33 -08:00
|
|
|
info!(
|
2020-01-27 16:49:25 -08:00
|
|
|
"PARTITION_TEST starting cluster with {:?} partitions slots_per_epoch: {}",
|
|
|
|
partitions, config.slots_per_epoch,
|
2019-11-15 08:36:33 -08:00
|
|
|
);
|
2019-12-03 16:31:59 -08:00
|
|
|
let mut cluster = LocalCluster::new(&config);
|
|
|
|
|
2020-01-27 16:49:25 -08:00
|
|
|
let (cluster_nodes, _) = discover_cluster(&cluster.entry_point_info.gossip, num_nodes).unwrap();
|
|
|
|
|
|
|
|
info!("PARTITION_TEST sleeping until partition starting condition",);
|
|
|
|
loop {
|
|
|
|
let mut reached_epoch = true;
|
|
|
|
for node in &cluster_nodes {
|
|
|
|
let node_client = RpcClient::new_socket(node.rpc);
|
|
|
|
if let Ok(epoch_info) = node_client.get_epoch_info() {
|
|
|
|
info!("slots_per_epoch: {:?}", epoch_info);
|
|
|
|
if epoch_info.slots_in_epoch <= (1 << VOTE_THRESHOLD_DEPTH) {
|
|
|
|
reached_epoch = false;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
reached_epoch = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if reached_epoch {
|
|
|
|
info!("PARTITION_TEST start partition");
|
|
|
|
enable_partition
|
|
|
|
.clone()
|
|
|
|
.unwrap()
|
|
|
|
.store(false, Ordering::Relaxed);
|
|
|
|
break;
|
|
|
|
} else {
|
|
|
|
sleep(Duration::from_millis(100));
|
|
|
|
}
|
2019-11-15 08:36:33 -08:00
|
|
|
}
|
2020-01-27 16:49:25 -08:00
|
|
|
sleep(Duration::from_millis(leader_schedule_time));
|
|
|
|
|
|
|
|
info!("PARTITION_TEST remove partition");
|
|
|
|
enable_partition.unwrap().store(true, Ordering::Relaxed);
|
|
|
|
|
|
|
|
let mut dead_nodes = HashSet::new();
|
2019-12-03 16:31:59 -08:00
|
|
|
let mut alive_node_contact_infos = vec![];
|
|
|
|
let should_exits: Vec<_> = partitions
|
|
|
|
.iter()
|
|
|
|
.flat_map(|p| p.iter().map(|(_, should_exit)| should_exit))
|
|
|
|
.collect();
|
|
|
|
assert_eq!(should_exits.len(), validator_pubkeys.len());
|
2020-01-27 16:49:25 -08:00
|
|
|
let timeout = 10;
|
2019-11-15 08:36:33 -08:00
|
|
|
if timeout > 0 {
|
2020-01-27 16:49:25 -08:00
|
|
|
// Give partitions time to propagate their blocks from during the partition
|
2019-12-03 16:31:59 -08:00
|
|
|
// after the partition resolves
|
|
|
|
let propagation_time = leader_schedule_time;
|
2020-01-27 16:49:25 -08:00
|
|
|
info!("PARTITION_TEST resolving partition. sleeping {}ms", timeout);
|
|
|
|
sleep(Duration::from_millis(10_000));
|
|
|
|
info!(
|
|
|
|
"PARTITION_TEST waiting for blocks to propagate after partition {}ms",
|
|
|
|
propagation_time
|
|
|
|
);
|
2019-12-03 16:31:59 -08:00
|
|
|
sleep(Duration::from_millis(propagation_time));
|
|
|
|
info!("PARTITION_TEST resuming normal operation");
|
|
|
|
for (pubkey, should_exit) in validator_pubkeys.iter().zip(should_exits) {
|
|
|
|
if *should_exit {
|
|
|
|
info!("Killing validator with id: {}", pubkey);
|
|
|
|
cluster.exit_node(pubkey);
|
|
|
|
dead_nodes.insert(*pubkey);
|
|
|
|
} else {
|
|
|
|
alive_node_contact_infos.push(
|
|
|
|
cluster
|
|
|
|
.validators
|
|
|
|
.get(pubkey)
|
|
|
|
.unwrap()
|
|
|
|
.info
|
|
|
|
.contact_info
|
|
|
|
.clone(),
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
2019-11-15 08:36:33 -08:00
|
|
|
}
|
2019-12-03 16:31:59 -08:00
|
|
|
|
|
|
|
assert!(alive_node_contact_infos.len() > 0);
|
2019-11-20 11:25:18 -08:00
|
|
|
info!("PARTITION_TEST discovering nodes");
|
2019-12-03 16:31:59 -08:00
|
|
|
let (cluster_nodes, _) = discover_cluster(
|
|
|
|
&alive_node_contact_infos[0].gossip,
|
|
|
|
alive_node_contact_infos.len(),
|
|
|
|
)
|
|
|
|
.unwrap();
|
2019-11-20 11:25:18 -08:00
|
|
|
info!("PARTITION_TEST discovered {} nodes", cluster_nodes.len());
|
|
|
|
info!("PARTITION_TEST looking for new roots on all nodes");
|
2019-12-03 16:31:59 -08:00
|
|
|
let mut roots = vec![HashSet::new(); alive_node_contact_infos.len()];
|
2019-11-20 11:25:18 -08:00
|
|
|
let mut done = false;
|
2020-01-27 16:49:25 -08:00
|
|
|
let mut last_print = Instant::now();
|
2019-11-20 11:25:18 -08:00
|
|
|
while !done {
|
2019-12-03 16:31:59 -08:00
|
|
|
for (i, ingress_node) in alive_node_contact_infos.iter().enumerate() {
|
2019-11-20 11:25:18 -08:00
|
|
|
let client = create_client(
|
|
|
|
ingress_node.client_facing_addr(),
|
|
|
|
solana_core::cluster_info::VALIDATOR_PORT_RANGE,
|
|
|
|
);
|
|
|
|
let slot = client.get_slot().unwrap_or(0);
|
|
|
|
roots[i].insert(slot);
|
|
|
|
let min_node = roots.iter().map(|r| r.len()).min().unwrap_or(0);
|
2020-01-27 16:49:25 -08:00
|
|
|
if last_print.elapsed().as_secs() > 3 {
|
|
|
|
info!("PARTITION_TEST min observed roots {}/16", min_node);
|
|
|
|
last_print = Instant::now();
|
|
|
|
}
|
2019-11-20 11:25:18 -08:00
|
|
|
done = min_node >= 16;
|
|
|
|
}
|
|
|
|
sleep(Duration::from_millis(clock::DEFAULT_MS_PER_SLOT / 2));
|
|
|
|
}
|
2020-01-27 16:49:25 -08:00
|
|
|
info!("PARTITION_TEST done waiting for roots");
|
2019-11-15 08:36:33 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[allow(unused_attributes)]
|
|
|
|
#[ignore]
|
|
|
|
#[test]
|
|
|
|
#[serial]
|
2019-12-03 16:31:59 -08:00
|
|
|
fn test_cluster_partition_1_2() {
|
|
|
|
run_cluster_partition(&[&[(1, false)], &[(1, false), (1, false)]], None)
|
2019-11-15 08:36:33 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[allow(unused_attributes)]
|
|
|
|
#[ignore]
|
|
|
|
#[test]
|
|
|
|
#[serial]
|
2019-12-03 16:31:59 -08:00
|
|
|
fn test_cluster_partition_1_1() {
|
|
|
|
run_cluster_partition(&[&[(1, false)], &[(1, false)]], None)
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
#[serial]
|
|
|
|
fn test_cluster_partition_1_1_1() {
|
|
|
|
run_cluster_partition(&[&[(1, false)], &[(1, false)], &[(1, false)]], None)
|
2019-11-15 08:36:33 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
#[serial]
|
2019-12-03 16:31:59 -08:00
|
|
|
fn test_kill_partition() {
|
|
|
|
// This test:
|
|
|
|
// 1) Spins up three partitions
|
|
|
|
// 2) Forces more slots in the leader schedule for the first partition so
|
|
|
|
// that this partition will be the heaviiest
|
|
|
|
// 3) Schedules the other validators for sufficient slots in the schedule
|
|
|
|
// so that they will still be locked out of voting for the major partitoin
|
|
|
|
// when the partition resolves
|
|
|
|
// 4) Kills the major partition. Validators are locked out, but should be
|
|
|
|
// able to reset to the major partition
|
|
|
|
// 5) Check for recovery
|
|
|
|
let mut leader_schedule = vec![];
|
|
|
|
let num_slots_per_validator = 8;
|
|
|
|
let partitions: [&[(usize, bool)]; 3] = [&[(9, true)], &[(10, false)], &[(10, false)]];
|
|
|
|
let validator_keys: Vec<_> = iter::repeat_with(|| Arc::new(Keypair::new()))
|
|
|
|
.take(partitions.len())
|
|
|
|
.collect();
|
|
|
|
for (i, k) in validator_keys.iter().enumerate() {
|
|
|
|
let num_slots = {
|
|
|
|
if i == 0 {
|
|
|
|
// Set up the leader to have 50% of the slots
|
|
|
|
num_slots_per_validator * (partitions.len() - 1)
|
|
|
|
} else {
|
|
|
|
num_slots_per_validator
|
|
|
|
}
|
|
|
|
};
|
|
|
|
for _ in 0..num_slots {
|
|
|
|
leader_schedule.push(k.pubkey())
|
|
|
|
}
|
|
|
|
}
|
2020-01-27 16:49:25 -08:00
|
|
|
info!("leader_schedule: {}", leader_schedule.len());
|
2019-12-03 16:31:59 -08:00
|
|
|
|
|
|
|
run_cluster_partition(
|
|
|
|
&partitions,
|
|
|
|
Some((
|
|
|
|
LeaderSchedule::new_from_schedule(leader_schedule),
|
|
|
|
validator_keys,
|
|
|
|
)),
|
|
|
|
)
|
2019-11-15 08:36:33 -08:00
|
|
|
}
|
|
|
|
|
2019-03-07 15:49:07 -08:00
|
|
|
#[test]
|
2019-07-02 17:35:03 -07:00
|
|
|
#[serial]
|
2019-03-07 15:49:07 -08:00
|
|
|
fn test_two_unbalanced_stakes() {
|
2019-04-15 17:15:50 -07:00
|
|
|
solana_logger::setup();
|
2019-07-30 13:48:46 -07:00
|
|
|
error!("test_two_unbalanced_stakes");
|
2019-05-23 22:05:16 -07:00
|
|
|
let mut validator_config = ValidatorConfig::default();
|
2019-03-07 15:49:07 -08:00
|
|
|
let num_ticks_per_second = 100;
|
2019-05-18 14:01:36 -07:00
|
|
|
let num_ticks_per_slot = 10;
|
2019-06-14 11:38:37 -07:00
|
|
|
let num_slots_per_epoch = MINIMUM_SLOTS_PER_EPOCH as u64;
|
2019-05-18 14:01:36 -07:00
|
|
|
|
2019-10-11 12:30:52 -07:00
|
|
|
validator_config.rpc_config.enable_validator_exit = true;
|
2019-04-15 15:27:45 -07:00
|
|
|
let mut cluster = LocalCluster::new(&ClusterConfig {
|
|
|
|
node_stakes: vec![999_990, 3],
|
|
|
|
cluster_lamports: 1_000_000,
|
2019-06-19 00:13:19 -07:00
|
|
|
validator_configs: vec![validator_config.clone(); 2],
|
2019-04-15 15:27:45 -07:00
|
|
|
ticks_per_slot: num_ticks_per_slot,
|
|
|
|
slots_per_epoch: num_slots_per_epoch,
|
2020-01-03 09:30:36 -08:00
|
|
|
stakers_slot_offset: num_slots_per_epoch,
|
2019-05-18 14:01:36 -07:00
|
|
|
poh_config: PohConfig::new_sleep(Duration::from_millis(1000 / num_ticks_per_second)),
|
2019-04-15 15:27:45 -07:00
|
|
|
..ClusterConfig::default()
|
|
|
|
});
|
2019-04-29 15:26:52 -07:00
|
|
|
|
2019-03-23 19:19:55 -07:00
|
|
|
cluster_tests::sleep_n_epochs(
|
|
|
|
10.0,
|
2019-11-08 20:56:57 -08:00
|
|
|
&cluster.genesis_config.poh_config,
|
2019-03-23 19:19:55 -07:00
|
|
|
num_ticks_per_slot,
|
|
|
|
num_slots_per_epoch,
|
|
|
|
);
|
2019-03-07 15:49:07 -08:00
|
|
|
cluster.close_preserve_ledgers();
|
2019-05-23 23:20:04 -07:00
|
|
|
let leader_pubkey = cluster.entry_point_info.id;
|
2019-12-03 16:31:59 -08:00
|
|
|
let leader_ledger = cluster.validators[&leader_pubkey].info.ledger_path.clone();
|
2019-03-19 06:36:45 -07:00
|
|
|
cluster_tests::verify_ledger_ticks(&leader_ledger, num_ticks_per_slot as usize);
|
2019-03-07 15:49:07 -08:00
|
|
|
}
|
2019-03-09 16:25:38 -08:00
|
|
|
|
|
|
|
#[test]
|
2019-09-12 11:39:39 -07:00
|
|
|
#[serial]
|
2019-03-09 16:25:38 -08:00
|
|
|
fn test_forwarding() {
|
|
|
|
// Set up a cluster where one node is never the leader, so all txs sent to this node
|
2019-03-10 17:33:01 -07:00
|
|
|
// will be have to be forwarded in order to be confirmed
|
2019-04-15 15:27:45 -07:00
|
|
|
let config = ClusterConfig {
|
|
|
|
node_stakes: vec![999_990, 3],
|
|
|
|
cluster_lamports: 2_000_000,
|
2019-09-12 11:39:39 -07:00
|
|
|
validator_configs: vec![ValidatorConfig::default(); 2],
|
2019-04-15 15:27:45 -07:00
|
|
|
..ClusterConfig::default()
|
|
|
|
};
|
|
|
|
let cluster = LocalCluster::new(&config);
|
2019-03-09 16:25:38 -08:00
|
|
|
|
2019-05-16 07:14:58 -07:00
|
|
|
let (cluster_nodes, _) = discover_cluster(&cluster.entry_point_info.gossip, 2).unwrap();
|
2019-03-09 16:25:38 -08:00
|
|
|
assert!(cluster_nodes.len() >= 2);
|
|
|
|
|
2019-05-23 23:20:04 -07:00
|
|
|
let leader_pubkey = cluster.entry_point_info.id;
|
2019-03-09 16:25:38 -08:00
|
|
|
|
2019-05-23 23:20:04 -07:00
|
|
|
let validator_info = cluster_nodes
|
|
|
|
.iter()
|
|
|
|
.find(|c| c.id != leader_pubkey)
|
|
|
|
.unwrap();
|
2019-03-09 16:25:38 -08:00
|
|
|
|
2019-03-10 17:33:01 -07:00
|
|
|
// Confirm that transactions were forwarded to and processed by the leader.
|
2019-08-21 23:59:11 -07:00
|
|
|
cluster_tests::send_many_transactions(&validator_info, &cluster.funding_keypair, 10, 20);
|
2019-03-09 16:25:38 -08:00
|
|
|
}
|
2019-03-23 19:19:55 -07:00
|
|
|
|
|
|
|
#[test]
|
2019-07-02 17:35:03 -07:00
|
|
|
#[serial]
|
2019-03-23 19:19:55 -07:00
|
|
|
fn test_restart_node() {
|
2019-07-30 13:48:46 -07:00
|
|
|
solana_logger::setup();
|
|
|
|
error!("test_restart_node");
|
2020-01-03 09:30:36 -08:00
|
|
|
let slots_per_epoch = MINIMUM_SLOTS_PER_EPOCH * 2 as u64;
|
2019-03-23 19:19:55 -07:00
|
|
|
let ticks_per_slot = 16;
|
2019-08-21 23:59:11 -07:00
|
|
|
let validator_config = ValidatorConfig::default();
|
2019-04-15 15:27:45 -07:00
|
|
|
let mut cluster = LocalCluster::new(&ClusterConfig {
|
2020-01-03 09:30:36 -08:00
|
|
|
node_stakes: vec![100; 1],
|
2019-04-15 15:27:45 -07:00
|
|
|
cluster_lamports: 100,
|
2019-08-21 23:59:11 -07:00
|
|
|
validator_configs: vec![validator_config.clone()],
|
2019-03-23 19:19:55 -07:00
|
|
|
ticks_per_slot,
|
|
|
|
slots_per_epoch,
|
2020-01-03 09:30:36 -08:00
|
|
|
stakers_slot_offset: slots_per_epoch,
|
2019-04-15 15:27:45 -07:00
|
|
|
..ClusterConfig::default()
|
|
|
|
});
|
2019-05-23 23:20:04 -07:00
|
|
|
let nodes = cluster.get_node_pubkeys();
|
2019-03-23 19:19:55 -07:00
|
|
|
cluster_tests::sleep_n_epochs(
|
|
|
|
1.0,
|
2019-11-08 20:56:57 -08:00
|
|
|
&cluster.genesis_config.poh_config,
|
2019-09-06 14:30:56 -07:00
|
|
|
clock::DEFAULT_TICKS_PER_SLOT,
|
2019-03-23 19:19:55 -07:00
|
|
|
slots_per_epoch,
|
|
|
|
);
|
2019-09-08 17:53:34 -07:00
|
|
|
cluster.exit_restart_node(&nodes[0], validator_config);
|
2019-03-23 19:19:55 -07:00
|
|
|
cluster_tests::sleep_n_epochs(
|
|
|
|
0.5,
|
2019-11-08 20:56:57 -08:00
|
|
|
&cluster.genesis_config.poh_config,
|
2019-09-06 14:30:56 -07:00
|
|
|
clock::DEFAULT_TICKS_PER_SLOT,
|
2019-03-23 19:19:55 -07:00
|
|
|
slots_per_epoch,
|
|
|
|
);
|
2019-08-21 23:59:11 -07:00
|
|
|
cluster_tests::send_many_transactions(
|
|
|
|
&cluster.entry_point_info,
|
|
|
|
&cluster.funding_keypair,
|
|
|
|
10,
|
|
|
|
1,
|
|
|
|
);
|
2019-03-23 19:19:55 -07:00
|
|
|
}
|
2019-04-15 15:27:45 -07:00
|
|
|
|
|
|
|
#[test]
|
2019-07-02 17:35:03 -07:00
|
|
|
#[serial]
|
2019-04-15 15:27:45 -07:00
|
|
|
fn test_listener_startup() {
|
|
|
|
let config = ClusterConfig {
|
|
|
|
node_stakes: vec![100; 1],
|
|
|
|
cluster_lamports: 1_000,
|
|
|
|
num_listeners: 3,
|
2019-06-19 00:13:19 -07:00
|
|
|
validator_configs: vec![ValidatorConfig::default(); 1],
|
2019-04-15 15:27:45 -07:00
|
|
|
..ClusterConfig::default()
|
|
|
|
};
|
|
|
|
let cluster = LocalCluster::new(&config);
|
2019-05-16 07:14:58 -07:00
|
|
|
let (cluster_nodes, _) = discover_cluster(&cluster.entry_point_info.gossip, 4).unwrap();
|
2019-04-15 15:27:45 -07:00
|
|
|
assert_eq!(cluster_nodes.len(), 4);
|
|
|
|
}
|
2019-05-24 19:20:09 -07:00
|
|
|
|
2019-10-23 11:50:10 -07:00
|
|
|
#[test]
|
|
|
|
#[serial]
|
|
|
|
fn test_softlaunch_operating_mode() {
|
|
|
|
solana_logger::setup();
|
|
|
|
|
|
|
|
let config = ClusterConfig {
|
|
|
|
operating_mode: OperatingMode::SoftLaunch,
|
|
|
|
node_stakes: vec![100; 1],
|
|
|
|
cluster_lamports: 1_000,
|
|
|
|
validator_configs: vec![ValidatorConfig::default(); 1],
|
|
|
|
..ClusterConfig::default()
|
|
|
|
};
|
|
|
|
let cluster = LocalCluster::new(&config);
|
|
|
|
let (cluster_nodes, _) = discover_cluster(&cluster.entry_point_info.gossip, 1).unwrap();
|
|
|
|
assert_eq!(cluster_nodes.len(), 1);
|
|
|
|
|
|
|
|
let client = create_client(
|
|
|
|
cluster.entry_point_info.client_facing_addr(),
|
|
|
|
solana_core::cluster_info::VALIDATOR_PORT_RANGE,
|
|
|
|
);
|
|
|
|
|
2019-12-06 22:16:28 -08:00
|
|
|
// Programs that are available at soft launch epoch 0
|
|
|
|
for program_id in [
|
2020-01-16 23:05:33 -08:00
|
|
|
&solana_config_program::id(),
|
2019-12-06 22:16:28 -08:00
|
|
|
&solana_sdk::system_program::id(),
|
|
|
|
&solana_stake_program::id(),
|
2020-01-16 23:05:33 -08:00
|
|
|
&solana_vote_program::id(),
|
2019-12-06 22:16:28 -08:00
|
|
|
]
|
|
|
|
.iter()
|
|
|
|
{
|
|
|
|
assert_matches!(
|
|
|
|
(
|
|
|
|
program_id,
|
|
|
|
client
|
|
|
|
.get_account_with_commitment(program_id, CommitmentConfig::recent())
|
|
|
|
.unwrap()
|
|
|
|
),
|
|
|
|
(_program_id, Some(_))
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Programs that are not available at soft launch epoch 0
|
2019-10-23 11:50:10 -07:00
|
|
|
for program_id in [
|
|
|
|
&solana_sdk::bpf_loader::id(),
|
2019-12-06 22:16:28 -08:00
|
|
|
&solana_storage_program::id(),
|
2019-11-20 10:12:43 -08:00
|
|
|
&solana_vest_program::id(),
|
2019-10-23 11:50:10 -07:00
|
|
|
]
|
|
|
|
.iter()
|
|
|
|
{
|
|
|
|
assert_eq!(
|
2019-11-06 13:15:00 -08:00
|
|
|
(
|
|
|
|
program_id,
|
|
|
|
client
|
|
|
|
.get_account_with_commitment(program_id, CommitmentConfig::recent())
|
|
|
|
.unwrap()
|
|
|
|
),
|
2019-10-23 11:50:10 -07:00
|
|
|
(program_id, None)
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-09-10 13:58:27 -07:00
|
|
|
#[allow(unused_attributes)]
|
2019-09-08 17:53:34 -07:00
|
|
|
#[test]
|
|
|
|
#[serial]
|
2019-09-26 13:29:05 -07:00
|
|
|
fn test_snapshot_restart_tower() {
|
2019-09-08 17:53:34 -07:00
|
|
|
// First set up the cluster with 2 nodes
|
|
|
|
let snapshot_interval_slots = 10;
|
|
|
|
let num_account_paths = 4;
|
|
|
|
|
|
|
|
let leader_snapshot_test_config =
|
|
|
|
setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
|
|
|
|
let validator_snapshot_test_config =
|
|
|
|
setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
|
|
|
|
|
|
|
|
let config = ClusterConfig {
|
2019-09-10 13:58:27 -07:00
|
|
|
node_stakes: vec![10000, 10],
|
2019-09-08 17:53:34 -07:00
|
|
|
cluster_lamports: 100000,
|
2019-09-10 13:58:27 -07:00
|
|
|
validator_configs: vec![
|
|
|
|
leader_snapshot_test_config.validator_config.clone(),
|
|
|
|
validator_snapshot_test_config.validator_config.clone(),
|
|
|
|
],
|
2019-09-08 17:53:34 -07:00
|
|
|
..ClusterConfig::default()
|
|
|
|
};
|
|
|
|
|
|
|
|
let mut cluster = LocalCluster::new(&config);
|
|
|
|
|
|
|
|
// Let the nodes run for a while, then stop one of the validators
|
|
|
|
sleep(Duration::from_millis(5000));
|
|
|
|
let all_pubkeys = cluster.get_node_pubkeys();
|
|
|
|
let validator_id = all_pubkeys
|
|
|
|
.into_iter()
|
|
|
|
.find(|x| *x != cluster.entry_point_info.id)
|
|
|
|
.unwrap();
|
2019-09-10 13:58:27 -07:00
|
|
|
let validator_info = cluster.exit_node(&validator_id);
|
2019-09-08 17:53:34 -07:00
|
|
|
|
2019-09-10 13:58:27 -07:00
|
|
|
// Get slot after which this was generated
|
|
|
|
let snapshot_package_output_path = &leader_snapshot_test_config
|
|
|
|
.validator_config
|
|
|
|
.snapshot_config
|
|
|
|
.as_ref()
|
|
|
|
.unwrap()
|
|
|
|
.snapshot_package_output_path;
|
2020-01-23 10:22:37 -08:00
|
|
|
let tar = snapshot_utils::get_snapshot_archive_path(&snapshot_package_output_path);
|
2019-09-10 13:58:27 -07:00
|
|
|
wait_for_next_snapshot(&cluster, &tar);
|
2019-09-08 17:53:34 -07:00
|
|
|
|
2019-09-10 13:58:27 -07:00
|
|
|
// Copy tar to validator's snapshot output directory
|
2020-01-23 10:22:37 -08:00
|
|
|
let validator_tar_path = snapshot_utils::get_snapshot_archive_path(
|
|
|
|
&validator_snapshot_test_config.snapshot_output_path,
|
|
|
|
);
|
2019-09-10 13:58:27 -07:00
|
|
|
fs::hard_link(tar, &validator_tar_path).unwrap();
|
2019-09-08 17:53:34 -07:00
|
|
|
|
2019-09-26 13:29:05 -07:00
|
|
|
// Restart validator from snapshot, the validator's tower state in this snapshot
|
2019-09-10 13:58:27 -07:00
|
|
|
// will contain slots < the root bank of the snapshot. Validator should not panic.
|
|
|
|
cluster.restart_node(&validator_id, validator_info);
|
2019-09-08 17:53:34 -07:00
|
|
|
|
2019-09-26 13:29:05 -07:00
|
|
|
// Test cluster can still make progress and get confirmations in tower
|
2020-01-30 21:51:11 -08:00
|
|
|
// Use the restarted node as the discovery point so that we get updated
|
|
|
|
// validator's ContactInfo
|
|
|
|
let restarted_node_info = cluster.get_contact_info(&validator_id).unwrap();
|
2019-09-10 13:58:27 -07:00
|
|
|
cluster_tests::spend_and_verify_all_nodes(
|
2020-01-30 21:51:11 -08:00
|
|
|
&restarted_node_info,
|
2019-09-10 13:58:27 -07:00
|
|
|
&cluster.funding_keypair,
|
|
|
|
1,
|
|
|
|
HashSet::new(),
|
|
|
|
);
|
|
|
|
}
|
2019-09-08 17:53:34 -07:00
|
|
|
|
2019-08-27 15:09:41 -07:00
|
|
|
#[test]
|
|
|
|
#[serial]
|
2020-01-13 13:13:52 -08:00
|
|
|
fn test_snapshots_blockstore_floor() {
|
2019-08-27 15:09:41 -07:00
|
|
|
// First set up the cluster with 1 snapshotting leader
|
|
|
|
let snapshot_interval_slots = 10;
|
|
|
|
let num_account_paths = 4;
|
|
|
|
|
|
|
|
let leader_snapshot_test_config =
|
|
|
|
setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
|
|
|
|
let validator_snapshot_test_config =
|
|
|
|
setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
|
|
|
|
|
|
|
|
let snapshot_package_output_path = &leader_snapshot_test_config
|
|
|
|
.validator_config
|
|
|
|
.snapshot_config
|
|
|
|
.as_ref()
|
|
|
|
.unwrap()
|
|
|
|
.snapshot_package_output_path;
|
|
|
|
|
|
|
|
let config = ClusterConfig {
|
|
|
|
node_stakes: vec![10000],
|
|
|
|
cluster_lamports: 100000,
|
|
|
|
validator_configs: vec![leader_snapshot_test_config.validator_config.clone()],
|
|
|
|
..ClusterConfig::default()
|
|
|
|
};
|
|
|
|
|
|
|
|
let mut cluster = LocalCluster::new(&config);
|
|
|
|
|
|
|
|
trace!("Waiting for snapshot tar to be generated with slot",);
|
|
|
|
|
2020-01-23 10:22:37 -08:00
|
|
|
let tar = snapshot_utils::get_snapshot_archive_path(&snapshot_package_output_path);
|
2019-08-27 15:09:41 -07:00
|
|
|
loop {
|
|
|
|
if tar.exists() {
|
|
|
|
trace!("snapshot tar exists");
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
sleep(Duration::from_millis(5000));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Copy tar to validator's snapshot output directory
|
2020-01-23 10:22:37 -08:00
|
|
|
let validator_tar_path = snapshot_utils::get_snapshot_archive_path(
|
|
|
|
&validator_snapshot_test_config.snapshot_output_path,
|
|
|
|
);
|
2019-08-27 15:09:41 -07:00
|
|
|
fs::hard_link(tar, &validator_tar_path).unwrap();
|
|
|
|
let slot_floor = snapshot_utils::bank_slot_from_archive(&validator_tar_path).unwrap();
|
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
// Start up a new node from a snapshot
|
2019-08-27 15:09:41 -07:00
|
|
|
let validator_stake = 5;
|
|
|
|
cluster.add_validator(
|
|
|
|
&validator_snapshot_test_config.validator_config,
|
|
|
|
validator_stake,
|
2019-12-03 16:31:59 -08:00
|
|
|
Arc::new(Keypair::new()),
|
2019-08-27 15:09:41 -07:00
|
|
|
);
|
|
|
|
let all_pubkeys = cluster.get_node_pubkeys();
|
|
|
|
let validator_id = all_pubkeys
|
|
|
|
.into_iter()
|
|
|
|
.find(|x| *x != cluster.entry_point_info.id)
|
|
|
|
.unwrap();
|
|
|
|
let validator_client = cluster.get_validator_client(&validator_id).unwrap();
|
|
|
|
let mut current_slot = 0;
|
|
|
|
|
2019-09-04 17:14:42 -07:00
|
|
|
// Let this validator run a while with repair
|
2019-08-27 15:09:41 -07:00
|
|
|
let target_slot = slot_floor + 40;
|
|
|
|
while current_slot <= target_slot {
|
|
|
|
trace!("current_slot: {}", current_slot);
|
2019-11-06 13:15:00 -08:00
|
|
|
if let Ok(slot) = validator_client.get_slot_with_commitment(CommitmentConfig::recent()) {
|
2019-08-27 15:09:41 -07:00
|
|
|
current_slot = slot;
|
|
|
|
} else {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
sleep(Duration::from_secs(1));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check the validator ledger doesn't contain any slots < slot_floor
|
|
|
|
cluster.close_preserve_ledgers();
|
2019-12-03 16:31:59 -08:00
|
|
|
let validator_ledger_path = &cluster.validators[&validator_id];
|
2020-01-13 13:13:52 -08:00
|
|
|
let blockstore = Blockstore::open(&validator_ledger_path.info.ledger_path).unwrap();
|
2019-08-27 15:09:41 -07:00
|
|
|
|
2020-01-13 13:13:52 -08:00
|
|
|
// Skip the zeroth slot in blockstore that the ledger is initialized with
|
|
|
|
let (first_slot, _) = blockstore.slot_meta_iterator(1).unwrap().next().unwrap();
|
2019-09-04 17:14:42 -07:00
|
|
|
|
2019-08-27 15:09:41 -07:00
|
|
|
assert_eq!(first_slot, slot_floor);
|
|
|
|
}
|
|
|
|
|
2019-08-21 23:59:11 -07:00
|
|
|
#[test]
|
|
|
|
#[serial]
|
|
|
|
fn test_snapshots_restart_validity() {
|
2019-08-25 21:33:41 -07:00
|
|
|
solana_logger::setup();
|
|
|
|
let snapshot_interval_slots = 10;
|
2019-08-27 15:09:41 -07:00
|
|
|
let num_account_paths = 4;
|
|
|
|
let mut snapshot_test_config =
|
|
|
|
setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
|
|
|
|
let snapshot_package_output_path = &snapshot_test_config
|
|
|
|
.validator_config
|
|
|
|
.snapshot_config
|
|
|
|
.as_ref()
|
|
|
|
.unwrap()
|
|
|
|
.snapshot_package_output_path;
|
2019-08-21 23:59:11 -07:00
|
|
|
|
|
|
|
// Set up the cluster with 1 snapshotting validator
|
2019-08-27 15:09:41 -07:00
|
|
|
let mut all_account_storage_dirs = vec![vec![]];
|
|
|
|
|
|
|
|
std::mem::swap(
|
|
|
|
&mut all_account_storage_dirs[0],
|
|
|
|
&mut snapshot_test_config.account_storage_dirs,
|
|
|
|
);
|
2019-08-21 23:59:11 -07:00
|
|
|
|
|
|
|
let config = ClusterConfig {
|
|
|
|
node_stakes: vec![10000],
|
|
|
|
cluster_lamports: 100000,
|
2019-08-27 15:09:41 -07:00
|
|
|
validator_configs: vec![snapshot_test_config.validator_config.clone()],
|
2019-08-21 23:59:11 -07:00
|
|
|
..ClusterConfig::default()
|
|
|
|
};
|
|
|
|
|
|
|
|
// Create and reboot the node from snapshot `num_runs` times
|
|
|
|
let num_runs = 3;
|
|
|
|
let mut expected_balances = HashMap::new();
|
|
|
|
let mut cluster = LocalCluster::new(&config);
|
2019-08-25 21:33:41 -07:00
|
|
|
for i in 1..num_runs {
|
|
|
|
info!("run {}", i);
|
2019-08-21 23:59:11 -07:00
|
|
|
// Push transactions to one of the nodes and confirm that transactions were
|
|
|
|
// forwarded to and processed.
|
|
|
|
trace!("Sending transactions");
|
|
|
|
let new_balances = cluster_tests::send_many_transactions(
|
|
|
|
&cluster.entry_point_info,
|
|
|
|
&cluster.funding_keypair,
|
|
|
|
10,
|
|
|
|
10,
|
|
|
|
);
|
|
|
|
|
|
|
|
expected_balances.extend(new_balances);
|
|
|
|
|
2020-01-23 10:22:37 -08:00
|
|
|
let tar = snapshot_utils::get_snapshot_archive_path(&snapshot_package_output_path);
|
2019-09-10 13:58:27 -07:00
|
|
|
wait_for_next_snapshot(&cluster, &tar);
|
2019-08-21 23:59:11 -07:00
|
|
|
|
2019-10-11 12:30:52 -07:00
|
|
|
// Create new account paths since validator exit is not guaranteed to cleanup RPC threads,
|
2019-08-21 23:59:11 -07:00
|
|
|
// which may delete the old accounts on exit at any point
|
|
|
|
let (new_account_storage_dirs, new_account_storage_paths) =
|
|
|
|
generate_account_paths(num_account_paths);
|
|
|
|
all_account_storage_dirs.push(new_account_storage_dirs);
|
2019-12-05 18:41:29 -08:00
|
|
|
snapshot_test_config.validator_config.account_paths = new_account_storage_paths;
|
2019-08-21 23:59:11 -07:00
|
|
|
|
2019-09-10 13:58:27 -07:00
|
|
|
// Restart node
|
2019-08-21 23:59:11 -07:00
|
|
|
trace!("Restarting cluster from snapshot");
|
|
|
|
let nodes = cluster.get_node_pubkeys();
|
2019-09-08 17:53:34 -07:00
|
|
|
cluster.exit_restart_node(&nodes[0], snapshot_test_config.validator_config.clone());
|
2019-08-21 23:59:11 -07:00
|
|
|
|
|
|
|
// Verify account balances on validator
|
|
|
|
trace!("Verifying balances");
|
|
|
|
cluster_tests::verify_balances(expected_balances.clone(), &cluster.entry_point_info);
|
|
|
|
|
|
|
|
// Check that we can still push transactions
|
|
|
|
trace!("Spending and verifying");
|
|
|
|
cluster_tests::spend_and_verify_all_nodes(
|
|
|
|
&cluster.entry_point_info,
|
|
|
|
&cluster.funding_keypair,
|
|
|
|
1,
|
|
|
|
HashSet::new(),
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-06-19 00:13:19 -07:00
|
|
|
#[test]
|
2019-07-02 17:35:03 -07:00
|
|
|
#[serial]
|
2019-10-24 21:16:17 -07:00
|
|
|
#[allow(unused_attributes)]
|
|
|
|
#[ignore]
|
2019-06-19 00:13:19 -07:00
|
|
|
fn test_fail_entry_verification_leader() {
|
2019-07-08 18:21:52 -07:00
|
|
|
test_faulty_node(BroadcastStageType::FailEntryVerification);
|
|
|
|
}
|
2019-06-19 00:13:19 -07:00
|
|
|
|
2019-07-01 17:54:03 -07:00
|
|
|
#[test]
|
2019-09-12 11:39:39 -07:00
|
|
|
#[allow(unused_attributes)]
|
2019-07-01 17:54:03 -07:00
|
|
|
#[ignore]
|
2019-11-14 11:49:31 -08:00
|
|
|
fn test_fake_shreds_broadcast_leader() {
|
|
|
|
test_faulty_node(BroadcastStageType::BroadcastFakeShreds);
|
2019-07-08 18:21:52 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
fn test_faulty_node(faulty_node_type: BroadcastStageType) {
|
2019-07-01 17:54:03 -07:00
|
|
|
solana_logger::setup();
|
|
|
|
let num_nodes = 4;
|
|
|
|
let validator_config = ValidatorConfig::default();
|
|
|
|
let mut error_validator_config = ValidatorConfig::default();
|
2019-07-08 18:21:52 -07:00
|
|
|
error_validator_config.broadcast_stage_type = faulty_node_type.clone();
|
2019-07-01 17:54:03 -07:00
|
|
|
let mut validator_configs = vec![validator_config; num_nodes - 1];
|
|
|
|
validator_configs.push(error_validator_config);
|
2019-07-08 18:21:52 -07:00
|
|
|
let mut node_stakes = vec![100; num_nodes - 1];
|
|
|
|
node_stakes.push(50);
|
2019-07-01 17:54:03 -07:00
|
|
|
let cluster_config = ClusterConfig {
|
|
|
|
cluster_lamports: 10_000,
|
2019-07-08 18:21:52 -07:00
|
|
|
node_stakes,
|
2019-07-01 17:54:03 -07:00
|
|
|
validator_configs: validator_configs,
|
|
|
|
slots_per_epoch: MINIMUM_SLOTS_PER_EPOCH * 2 as u64,
|
|
|
|
stakers_slot_offset: MINIMUM_SLOTS_PER_EPOCH * 2 as u64,
|
|
|
|
..ClusterConfig::default()
|
|
|
|
};
|
|
|
|
|
|
|
|
let cluster = LocalCluster::new(&cluster_config);
|
2019-10-08 22:34:26 -07:00
|
|
|
let epoch_schedule = EpochSchedule::custom(
|
2019-07-01 17:54:03 -07:00
|
|
|
cluster_config.slots_per_epoch,
|
|
|
|
cluster_config.stakers_slot_offset,
|
|
|
|
true,
|
|
|
|
);
|
2019-10-08 22:34:26 -07:00
|
|
|
let num_warmup_epochs = epoch_schedule.get_leader_schedule_epoch(0) + 1;
|
2019-07-01 17:54:03 -07:00
|
|
|
|
|
|
|
// Wait for the corrupted leader to be scheduled afer the warmup epochs expire
|
|
|
|
cluster_tests::sleep_n_epochs(
|
|
|
|
(num_warmup_epochs + 1) as f64,
|
2019-11-08 20:56:57 -08:00
|
|
|
&cluster.genesis_config.poh_config,
|
2019-07-01 17:54:03 -07:00
|
|
|
cluster_config.ticks_per_slot,
|
|
|
|
cluster_config.slots_per_epoch,
|
|
|
|
);
|
|
|
|
|
|
|
|
let corrupt_node = cluster
|
2019-12-03 16:31:59 -08:00
|
|
|
.validators
|
2019-07-01 17:54:03 -07:00
|
|
|
.iter()
|
2019-07-08 18:21:52 -07:00
|
|
|
.find(|(_, v)| v.config.broadcast_stage_type == faulty_node_type)
|
2019-07-01 17:54:03 -07:00
|
|
|
.unwrap()
|
|
|
|
.0;
|
|
|
|
let mut ignore = HashSet::new();
|
|
|
|
ignore.insert(*corrupt_node);
|
|
|
|
|
|
|
|
// Verify that we can still spend and verify even in the presence of corrupt nodes
|
|
|
|
cluster_tests::spend_and_verify_all_nodes(
|
|
|
|
&cluster.entry_point_info,
|
|
|
|
&cluster.funding_keypair,
|
|
|
|
num_nodes,
|
|
|
|
ignore,
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
2019-05-24 19:20:09 -07:00
|
|
|
#[test]
|
2019-11-06 01:02:26 -08:00
|
|
|
// Test that when a leader is leader for banks B_i..B_{i+n}, and B_i is not
|
|
|
|
// votable, then B_{i+1} still chains to B_i
|
|
|
|
fn test_no_voting() {
|
|
|
|
solana_logger::setup();
|
|
|
|
let mut validator_config = ValidatorConfig::default();
|
|
|
|
validator_config.rpc_config.enable_validator_exit = true;
|
|
|
|
validator_config.voting_disabled = true;
|
|
|
|
let config = ClusterConfig {
|
|
|
|
cluster_lamports: 10_000,
|
|
|
|
node_stakes: vec![100],
|
|
|
|
validator_configs: vec![validator_config.clone()],
|
|
|
|
..ClusterConfig::default()
|
|
|
|
};
|
|
|
|
let mut cluster = LocalCluster::new(&config);
|
|
|
|
let client = cluster
|
|
|
|
.get_validator_client(&cluster.entry_point_info.id)
|
|
|
|
.unwrap();
|
|
|
|
loop {
|
2019-11-07 09:41:58 -08:00
|
|
|
let last_slot = client
|
|
|
|
.get_slot_with_commitment(CommitmentConfig::recent())
|
|
|
|
.expect("Couldn't get slot");
|
2019-11-06 01:02:26 -08:00
|
|
|
if last_slot > 4 * VOTE_THRESHOLD_DEPTH as u64 {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
sleep(Duration::from_secs(1));
|
|
|
|
}
|
|
|
|
|
|
|
|
cluster.close_preserve_ledgers();
|
|
|
|
let leader_pubkey = cluster.entry_point_info.id;
|
2019-12-03 16:31:59 -08:00
|
|
|
let ledger_path = cluster.validators[&leader_pubkey].info.ledger_path.clone();
|
2020-01-13 13:13:52 -08:00
|
|
|
let ledger = Blockstore::open(&ledger_path).unwrap();
|
2019-11-06 01:02:26 -08:00
|
|
|
for i in 0..2 * VOTE_THRESHOLD_DEPTH {
|
|
|
|
let meta = ledger.meta(i as u64).unwrap().unwrap();
|
|
|
|
let parent = meta.parent_slot;
|
|
|
|
let expected_parent = i.saturating_sub(1);
|
|
|
|
assert_eq!(parent, expected_parent as u64);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-09-10 13:58:27 -07:00
|
|
|
fn wait_for_next_snapshot<P: AsRef<Path>>(cluster: &LocalCluster, tar: P) {
|
|
|
|
// Get slot after which this was generated
|
|
|
|
let client = cluster
|
|
|
|
.get_validator_client(&cluster.entry_point_info.id)
|
|
|
|
.unwrap();
|
2019-11-06 13:15:00 -08:00
|
|
|
let last_slot = client
|
|
|
|
.get_slot_with_commitment(CommitmentConfig::recent())
|
|
|
|
.expect("Couldn't get slot");
|
2019-09-10 13:58:27 -07:00
|
|
|
|
|
|
|
// Wait for a snapshot for a bank >= last_slot to be made so we know that the snapshot
|
|
|
|
// must include the transactions just pushed
|
|
|
|
trace!(
|
|
|
|
"Waiting for snapshot tar to be generated with slot > {}",
|
|
|
|
last_slot
|
|
|
|
);
|
|
|
|
loop {
|
|
|
|
if tar.as_ref().exists() {
|
|
|
|
trace!("snapshot tar exists");
|
|
|
|
let slot = snapshot_utils::bank_slot_from_archive(&tar).unwrap();
|
|
|
|
if slot >= last_slot {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
trace!("snapshot tar slot {} < last_slot {}", slot, last_slot);
|
|
|
|
}
|
|
|
|
sleep(Duration::from_millis(5000));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-12-05 18:41:29 -08:00
|
|
|
fn generate_account_paths(num_account_paths: usize) -> (Vec<TempDir>, Vec<PathBuf>) {
|
2019-08-21 23:59:11 -07:00
|
|
|
let account_storage_dirs: Vec<TempDir> = (0..num_account_paths)
|
|
|
|
.map(|_| TempDir::new().unwrap())
|
|
|
|
.collect();
|
|
|
|
let account_storage_paths: Vec<_> = account_storage_dirs
|
|
|
|
.iter()
|
2019-12-05 18:41:29 -08:00
|
|
|
.map(|a| a.path().to_path_buf())
|
2019-08-21 23:59:11 -07:00
|
|
|
.collect();
|
|
|
|
(account_storage_dirs, account_storage_paths)
|
|
|
|
}
|
2019-08-27 15:09:41 -07:00
|
|
|
|
|
|
|
struct SnapshotValidatorConfig {
|
|
|
|
_snapshot_dir: TempDir,
|
|
|
|
snapshot_output_path: TempDir,
|
|
|
|
account_storage_dirs: Vec<TempDir>,
|
|
|
|
validator_config: ValidatorConfig,
|
|
|
|
}
|
|
|
|
|
|
|
|
fn setup_snapshot_validator_config(
|
|
|
|
snapshot_interval_slots: usize,
|
|
|
|
num_account_paths: usize,
|
|
|
|
) -> SnapshotValidatorConfig {
|
|
|
|
// Create the snapshot config
|
|
|
|
let snapshot_dir = TempDir::new().unwrap();
|
|
|
|
let snapshot_output_path = TempDir::new().unwrap();
|
|
|
|
let snapshot_config = SnapshotConfig {
|
|
|
|
snapshot_interval_slots,
|
|
|
|
snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
|
|
|
|
snapshot_path: PathBuf::from(snapshot_dir.path()),
|
|
|
|
};
|
|
|
|
|
|
|
|
// Create the account paths
|
|
|
|
let (account_storage_dirs, account_storage_paths) = generate_account_paths(num_account_paths);
|
|
|
|
|
|
|
|
// Create the validator config
|
|
|
|
let mut validator_config = ValidatorConfig::default();
|
2019-10-11 12:30:52 -07:00
|
|
|
validator_config.rpc_config.enable_validator_exit = true;
|
2019-08-27 15:09:41 -07:00
|
|
|
validator_config.snapshot_config = Some(snapshot_config);
|
2019-12-05 18:41:29 -08:00
|
|
|
validator_config.account_paths = account_storage_paths;
|
2019-08-27 15:09:41 -07:00
|
|
|
|
|
|
|
SnapshotValidatorConfig {
|
|
|
|
_snapshot_dir: snapshot_dir,
|
|
|
|
snapshot_output_path,
|
|
|
|
account_storage_dirs,
|
|
|
|
validator_config,
|
|
|
|
}
|
|
|
|
}
|