From c5a74ada05a6a8b602841792303c2a82dd762f53 Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Tue, 5 Feb 2019 08:03:52 -0800 Subject: [PATCH] leader_scheduler: remove bootstrap_height --- benches/banking_stage.rs | 12 +- fullnode/src/main.rs | 17 +- genesis/src/main.rs | 4 +- src/bank.rs | 221 ++-- src/banking_stage.rs | 54 +- src/broadcast_service.rs | 40 +- src/compute_leader_confirmation_service.rs | 5 +- src/db_window.rs | 72 +- src/fullnode.rs | 394 ++++--- src/genesis_block.rs | 24 +- src/leader_scheduler.rs | 1185 +++++++++----------- src/poh_recorder.rs | 19 +- src/poh_service.rs | 15 +- src/replay_stage.rs | 227 ++-- src/replicator.rs | 6 +- src/rpc.rs | 3 +- src/thin_client.rs | 75 +- src/tpu.rs | 4 +- src/tvu.rs | 19 +- src/window.rs | 56 +- src/window_service.rs | 8 +- tests/multinode.rs | 844 ++++++-------- tests/replicator.rs | 55 +- tests/rpc.rs | 3 +- wallet/tests/deploy.rs | 5 +- wallet/tests/pay.rs | 9 +- wallet/tests/request_airdrop.rs | 4 +- 27 files changed, 1591 insertions(+), 1789 deletions(-) diff --git a/benches/banking_stage.rs b/benches/banking_stage.rs index ef6595d6a3..75e6d06231 100644 --- a/benches/banking_stage.rs +++ b/benches/banking_stage.rs @@ -12,7 +12,7 @@ use solana::last_id_queue::MAX_ENTRY_IDS; use solana::packet::to_packets_chunked; use solana_sdk::hash::hash; use solana_sdk::pubkey::Pubkey; -use solana_sdk::signature::{Keypair, KeypairUtil, Signature}; +use solana_sdk::signature::{KeypairUtil, Signature}; use solana_sdk::system_transaction::SystemTransaction; use std::iter; use std::sync::mpsc::{channel, Receiver}; @@ -48,7 +48,6 @@ fn bench_banking_stage_multi_accounts(bencher: &mut Bencher) { let (verified_sender, verified_receiver) = channel(); let bank = Arc::new(Bank::new(&genesis_block)); - let dummy_leader_id = Keypair::new().pubkey(); let dummy = SystemTransaction::new_move( &mint_keypair, mint_keypair.pubkey(), @@ -106,8 +105,8 @@ fn bench_banking_stage_multi_accounts(bencher: &mut Bencher) { verified_receiver, Default::default(), &genesis_block.last_id(), - None, - dummy_leader_id, + std::u64::MAX, + genesis_block.bootstrap_leader_id, &to_leader_sender, ); @@ -143,7 +142,6 @@ fn bench_banking_stage_multi_programs(bencher: &mut Bencher) { let (verified_sender, verified_receiver) = channel(); let bank = Arc::new(Bank::new(&genesis_block)); - let dummy_leader_id = Keypair::new().pubkey(); let dummy = SystemTransaction::new_move( &mint_keypair, mint_keypair.pubkey(), @@ -216,8 +214,8 @@ fn bench_banking_stage_multi_programs(bencher: &mut Bencher) { verified_receiver, Default::default(), &genesis_block.last_id(), - None, - dummy_leader_id, + std::u64::MAX, + genesis_block.bootstrap_leader_id, &to_leader_sender, ); diff --git a/fullnode/src/main.rs b/fullnode/src/main.rs index 75cd81037d..f2fd200e1e 100644 --- a/fullnode/src/main.rs +++ b/fullnode/src/main.rs @@ -3,7 +3,7 @@ use log::*; use solana::client::mk_client; use solana::cluster_info::{Node, NodeInfo, FULLNODE_PORT_RANGE}; use solana::fullnode::{Fullnode, FullnodeConfig}; -use solana::leader_scheduler::LeaderScheduler; +use solana::genesis_block::GenesisBlock; use solana::local_vote_signer_service::LocalVoteSignerService; use solana::socketaddr; use solana::thin_client::{poll_gossip_for_leader, ThinClient}; @@ -19,9 +19,6 @@ use std::net::{Ipv4Addr, SocketAddr}; use std::process::exit; use std::sync::mpsc::channel; use std::sync::Arc; -use std::sync::RwLock; -use std::thread::sleep; -use std::time::Duration; fn parse_identity(matches: &ArgMatches<'_>) -> (Keypair, SocketAddr) { if let Some(i) = matches.value_of("identity") { @@ -248,13 +245,16 @@ fn main() { node.info.rpc.set_port(rpc_port); node.info.rpc_pubsub.set_port(rpc_pubsub_port); - let mut leader_scheduler = LeaderScheduler::default(); - leader_scheduler.use_only_bootstrap_leader = use_only_bootstrap_leader; + let genesis_block = GenesisBlock::load(ledger_path).expect("Unable to load genesis block"); + if use_only_bootstrap_leader && node.info.id != genesis_block.bootstrap_leader_id { + fullnode_config.voting_disabled = true; + } let vote_signer: Box = if !no_signer { - info!("Signer service address: {:?}", signer_addr); + info!("Vote signer service address: {:?}", signer_addr); Box::new(RemoteVoteSigner::new(signer_addr)) } else { + info!("Node will not vote"); Box::new(LocalVoteSigner::default()) }; let vote_signer = VotingKeypair::new_with_signer(&keypair, vote_signer); @@ -266,7 +266,6 @@ fn main() { node, &keypair, ledger_path, - Arc::new(RwLock::new(leader_scheduler)), vote_signer, cluster_entrypoint .map(|i| NodeInfo::new_entry_point(&i)) @@ -277,7 +276,7 @@ fn main() { let (rotation_sender, rotation_receiver) = channel(); fullnode.run(Some(rotation_sender)); - if !no_signer { + if !fullnode_config.voting_disabled { let leader_node_info = loop { info!("Looking for leader..."); match poll_gossip_for_leader(gossip_addr, Some(10)) { diff --git a/genesis/src/main.rs b/genesis/src/main.rs index df1a27c62a..e7c8d0c8f4 100644 --- a/genesis/src/main.rs +++ b/genesis/src/main.rs @@ -3,7 +3,7 @@ use clap::{crate_version, value_t_or_exit, App, Arg}; use solana::db_ledger::create_new_ledger; use solana::genesis_block::GenesisBlock; -use solana_sdk::signature::{read_keypair, KeypairUtil}; +use solana_sdk::signature::{read_keypair, Keypair, KeypairUtil}; use std::error; /** @@ -66,11 +66,13 @@ fn main() -> Result<(), Box> { let bootstrap_leader_keypair = read_keypair(bootstrap_leader_keypair_file)?; let mint_keypair = read_keypair(mint_keypair_file)?; + let bootstrap_leader_vote_account_keypair = Keypair::new(); let genesis_block = GenesisBlock { mint_id: mint_keypair.pubkey(), tokens: num_tokens, bootstrap_leader_id: bootstrap_leader_keypair.pubkey(), bootstrap_leader_tokens: BOOTSTRAP_LEADER_TOKENS, + bootstrap_leader_vote_account_id: bootstrap_leader_vote_account_keypair.pubkey(), }; create_new_ledger(ledger_path, &genesis_block)?; diff --git a/src/bank.rs b/src/bank.rs index 6375459fe9..ad0f31dabc 100644 --- a/src/bank.rs +++ b/src/bank.rs @@ -9,7 +9,7 @@ use crate::entry::Entry; use crate::entry::EntrySlice; use crate::genesis_block::GenesisBlock; use crate::last_id_queue::{LastIdQueue, MAX_ENTRY_IDS}; -use crate::leader_scheduler::LeaderScheduler; +use crate::leader_scheduler::{LeaderScheduler, LeaderSchedulerConfig}; use crate::poh_recorder::{PohRecorder, PohRecorderError}; use crate::result::Error; use crate::runtime::{self, RuntimeError}; @@ -130,19 +130,30 @@ impl Default for Bank { last_id_queue: RwLock::new(LastIdQueue::default()), status_cache: RwLock::new(BankStatusCache::default()), confirmation_time: AtomicUsize::new(std::usize::MAX), - leader_scheduler: Arc::new(RwLock::new(LeaderScheduler::default())), + leader_scheduler: Arc::new(RwLock::new(Default::default())), subscriptions: RwLock::new(Box::new(Arc::new(LocalSubscriptions::default()))), } } } impl Bank { - pub fn new(genesis_block: &GenesisBlock) -> Self { - let bank = Self::default(); + pub fn new_with_leader_scheduler_config( + genesis_block: &GenesisBlock, + leader_scheduler_config_option: Option<&LeaderSchedulerConfig>, + ) -> Self { + let mut bank = Self::default(); + if let Some(leader_scheduler_config) = leader_scheduler_config_option { + bank.leader_scheduler = + Arc::new(RwLock::new(LeaderScheduler::new(leader_scheduler_config))); + } bank.process_genesis_block(genesis_block); bank.add_builtin_programs(); bank } + + pub fn new(genesis_block: &GenesisBlock) -> Self { + Self::new_with_leader_scheduler_config(genesis_block, None) + } pub fn set_subscriptions(&self, subscriptions: Box>) { let mut sub = self.subscriptions.write().unwrap(); *sub = subscriptions @@ -163,32 +174,60 @@ impl Bank { fn process_genesis_block(&self, genesis_block: &GenesisBlock) { assert!(genesis_block.mint_id != Pubkey::default()); + assert!(genesis_block.bootstrap_leader_id != Pubkey::default()); + assert!(genesis_block.bootstrap_leader_vote_account_id != Pubkey::default()); assert!(genesis_block.tokens >= genesis_block.bootstrap_leader_tokens); + assert!(genesis_block.bootstrap_leader_tokens >= 2); let mut mint_account = Account::default(); - let mut bootstrap_leader_account = Account::default(); - mint_account.tokens += genesis_block.tokens; - - if genesis_block.bootstrap_leader_id != Pubkey::default() { - mint_account.tokens -= genesis_block.bootstrap_leader_tokens; - bootstrap_leader_account.tokens += genesis_block.bootstrap_leader_tokens; - self.accounts.store_slow( - true, - &genesis_block.bootstrap_leader_id, - &bootstrap_leader_account, - ); - }; - + mint_account.tokens = genesis_block.tokens - genesis_block.bootstrap_leader_tokens; self.accounts .store_slow(true, &genesis_block.mint_id, &mint_account); + let mut bootstrap_leader_account = Account::default(); + bootstrap_leader_account.tokens = genesis_block.bootstrap_leader_tokens - 1; + self.accounts.store_slow( + true, + &genesis_block.bootstrap_leader_id, + &bootstrap_leader_account, + ); + + // Construct a vote account for the bootstrap_leader such that the leader_scheduler + // will be forced to select it as the leader for height 0 + let mut bootstrap_leader_vote_account = Account { + tokens: 1, + userdata: vec![0; vote_program::get_max_size() as usize], + owner: vote_program::id(), + executable: false, + loader: Pubkey::default(), + }; + + let mut vote_state = vote_program::VoteState::new( + genesis_block.bootstrap_leader_id, + genesis_block.bootstrap_leader_id, + ); + vote_state.votes.push_back(vote_program::Vote::new(0)); + vote_state + .serialize(&mut bootstrap_leader_vote_account.userdata) + .unwrap(); + + self.accounts.store_slow( + true, + &genesis_block.bootstrap_leader_vote_account_id, + &bootstrap_leader_vote_account, + ); + self.leader_scheduler + .write() + .unwrap() + .update_tick_height(0, self); + self.last_id_queue .write() .unwrap() .genesis_last_id(&genesis_block.last_id()); } - fn add_system_program(&self) { + fn add_builtin_programs(&self) { let system_program_account = Account { tokens: 1, owner: system_program::id(), @@ -198,10 +237,6 @@ impl Bank { }; self.accounts .store_slow(true, &system_program::id(), &system_program_account); - } - - fn add_builtin_programs(&self) { - self.add_system_program(); // Vote program let vote_program_account = Account { @@ -352,9 +387,16 @@ impl Bank { /// the oldest ones once its internal cache is full. Once boot, the /// bank will reject transactions using that `last_id`. pub fn register_tick(&self, last_id: &Hash) { - let mut last_id_queue = self.last_id_queue.write().unwrap(); - inc_new_counter_info!("bank-register_tick-registered", 1); - last_id_queue.register_tick(last_id) + let current_tick_height = { + let mut last_id_queue = self.last_id_queue.write().unwrap(); + inc_new_counter_info!("bank-register_tick-registered", 1); + last_id_queue.register_tick(last_id); + last_id_queue.tick_height + }; + self.leader_scheduler + .write() + .unwrap() + .update_tick_height(current_tick_height, self); } /// Process a Transaction. This is used for unit tests and simply calls the vector Bank::process_transactions method. @@ -664,10 +706,6 @@ impl Bank { } } else { self.register_tick(&entry.id); - self.leader_scheduler - .write() - .unwrap() - .update_height(self.tick_height(), self); } Ok(()) @@ -731,10 +769,6 @@ impl Bank { // if its a tick, execute the group and register the tick self.par_execute_entries(&mt_group)?; self.register_tick(&entry.id); - self.leader_scheduler - .write() - .unwrap() - .update_height(self.tick_height(), self); mt_group = vec![]; continue; } @@ -882,11 +916,12 @@ impl Bank { } } - pub fn get_current_leader(&self) -> Option<(Pubkey, u64)> { - self.leader_scheduler - .read() - .unwrap() - .get_scheduled_leader(self.tick_height() + 1) + #[cfg(test)] + fn get_current_leader(&self) -> Option { + let tick_height = self.tick_height(); + let leader_scheduler = self.leader_scheduler.read().unwrap(); + let slot = leader_scheduler.tick_height_to_slot(tick_height); + leader_scheduler.get_leader_for_slot(slot) } pub fn tick_height(&self) -> u64 { @@ -904,6 +939,7 @@ mod tests { use super::*; use crate::entry::{next_entries, next_entry, Entry}; use crate::gen_keys::GenKeys; + use crate::genesis_block::BOOTSTRAP_LEADER_TOKENS; use bincode::serialize; use hashbrown::HashSet; use solana_sdk::hash::hash; @@ -921,18 +957,28 @@ mod tests { fn test_bank_new() { let (genesis_block, _) = GenesisBlock::new(10_000); let bank = Bank::new(&genesis_block); - assert_eq!(bank.get_balance(&genesis_block.mint_id), 10_000); + assert_eq!( + bank.get_balance(&genesis_block.mint_id), + 10_000 - genesis_block.bootstrap_leader_tokens + ); } #[test] fn test_bank_new_with_leader() { let dummy_leader_id = Keypair::new().pubkey(); - let dummy_leader_tokens = 1; + let dummy_leader_tokens = BOOTSTRAP_LEADER_TOKENS; let (genesis_block, _) = GenesisBlock::new_with_leader(10_000, dummy_leader_id, dummy_leader_tokens); + assert_eq!(genesis_block.bootstrap_leader_tokens, dummy_leader_tokens); let bank = Bank::new(&genesis_block); - assert_eq!(bank.get_balance(&genesis_block.mint_id), 9999); - assert_eq!(bank.get_balance(&dummy_leader_id), 1); + assert_eq!( + bank.get_balance(&genesis_block.mint_id), + 10_000 - dummy_leader_tokens + ); + assert_eq!( + bank.get_balance(&dummy_leader_id), + dummy_leader_tokens - 1 /* 1 token goes to the vote account associated with dummy_leader_tokens */ + ); } #[test] @@ -954,7 +1000,7 @@ mod tests { #[test] fn test_one_source_two_tx_one_batch() { - let (genesis_block, mint_keypair) = GenesisBlock::new(1); + let (genesis_block, mint_keypair) = GenesisBlock::new(1 + BOOTSTRAP_LEADER_TOKENS); let key1 = Keypair::new().pubkey(); let key2 = Keypair::new().pubkey(); let bank = Bank::new(&genesis_block); @@ -979,7 +1025,7 @@ mod tests { #[test] fn test_one_tx_two_out_atomic_fail() { - let (genesis_block, mint_keypair) = GenesisBlock::new(1); + let (genesis_block, mint_keypair) = GenesisBlock::new(1 + BOOTSTRAP_LEADER_TOKENS); let key1 = Keypair::new().pubkey(); let key2 = Keypair::new().pubkey(); let bank = Bank::new(&genesis_block); @@ -1028,7 +1074,7 @@ mod tests { #[test] fn test_one_tx_two_out_atomic_pass() { - let (genesis_block, mint_keypair) = GenesisBlock::new(2); + let (genesis_block, mint_keypair) = GenesisBlock::new(2 + BOOTSTRAP_LEADER_TOKENS); let key1 = Keypair::new().pubkey(); let key2 = Keypair::new().pubkey(); let bank = Bank::new(&genesis_block); @@ -1051,7 +1097,7 @@ mod tests { // See github issue 1157 (https://github.com/solana-labs/solana/issues/1157) #[test] fn test_detect_failed_duplicate_transactions_issue_1157() { - let (genesis_block, mint_keypair) = GenesisBlock::new(1); + let (genesis_block, mint_keypair) = GenesisBlock::new(2 + BOOTSTRAP_LEADER_TOKENS); let bank = Bank::new(&genesis_block); let dest = Keypair::new(); @@ -1087,7 +1133,7 @@ mod tests { #[test] fn test_account_not_found() { - let (genesis_block, mint_keypair) = GenesisBlock::new(1); + let (genesis_block, mint_keypair) = GenesisBlock::new(BOOTSTRAP_LEADER_TOKENS); let bank = Bank::new(&genesis_block); let keypair = Keypair::new(); assert_eq!( @@ -1099,7 +1145,7 @@ mod tests { #[test] fn test_insufficient_funds() { - let (genesis_block, mint_keypair) = GenesisBlock::new(11_000); + let (genesis_block, mint_keypair) = GenesisBlock::new(11_000 + BOOTSTRAP_LEADER_TOKENS); let bank = Bank::new(&genesis_block); let pubkey = Keypair::new().pubkey(); bank.transfer(1_000, &mint_keypair, pubkey, genesis_block.last_id()) @@ -1132,7 +1178,7 @@ mod tests { #[test] fn test_debits_before_credits() { - let (genesis_block, mint_keypair) = GenesisBlock::new(2); + let (genesis_block, mint_keypair) = GenesisBlock::new(2 + BOOTSTRAP_LEADER_TOKENS); let bank = Bank::new(&genesis_block); let keypair = Keypair::new(); let tx0 = SystemTransaction::new_account( @@ -1159,7 +1205,7 @@ mod tests { #[test] fn test_process_empty_entry_is_registered() { - let (genesis_block, mint_keypair) = GenesisBlock::new(1); + let (genesis_block, mint_keypair) = GenesisBlock::new(2 + BOOTSTRAP_LEADER_TOKENS); let bank = Bank::new(&genesis_block); let keypair = Keypair::new(); let entry = next_entry(&genesis_block.last_id(), 1, vec![]); @@ -1178,22 +1224,15 @@ mod tests { #[test] fn test_process_genesis() { + solana_logger::setup(); let dummy_leader_id = Keypair::new().pubkey(); - let dummy_leader_tokens = 1; + let dummy_leader_tokens = 2; let (genesis_block, _) = GenesisBlock::new_with_leader(5, dummy_leader_id, dummy_leader_tokens); - let bank = Bank::default(); - bank.process_genesis_block(&genesis_block); - assert_eq!(bank.get_balance(&genesis_block.mint_id), 4); + let bank = Bank::new(&genesis_block); + assert_eq!(bank.get_balance(&genesis_block.mint_id), 3); assert_eq!(bank.get_balance(&dummy_leader_id), 1); - // TODO: Restore next assert_eq() once leader scheduler configuration is stored in the - // genesis block - /* - assert_eq!( - bank.leader_scheduler.read().unwrap().bootstrap_leader, - dummy_leader_id - ); - */ + assert_eq!(bank.get_current_leader(), Some(dummy_leader_id)); } fn create_sample_block_with_next_entries_using_keypairs( @@ -1228,11 +1267,10 @@ mod tests { fn create_sample_block_with_ticks( genesis_block: &GenesisBlock, mint_keypair: &Keypair, - num_entries: usize, + num_one_token_transfers: usize, tick_interval: usize, ) -> impl Iterator { - assert!(num_entries > 0); - let mut entries = Vec::with_capacity(num_entries); + let mut entries = vec![]; // Start off the ledger with a tick linked to the genesis block let tick = Entry::new(&genesis_block.last_id(), 0, 1, vec![]); @@ -1240,11 +1278,11 @@ mod tests { let mut last_id = tick.id; entries.push(tick); - let num_hashes = 1; - for i in 1..num_entries { + for i in 0..num_one_token_transfers { + // Transfer one token from the mint to a random account let keypair = Keypair::new(); let tx = SystemTransaction::new_account(mint_keypair, keypair.pubkey(), 1, last_id, 0); - let entry = Entry::new(&hash, 0, num_hashes, vec![tx]); + let entry = Entry::new(&hash, 0, 1, vec![tx]); hash = entry.id; entries.push(entry); @@ -1252,12 +1290,12 @@ mod tests { // ProgramError<0, ResultWithNegativeTokens> error when processed let keypair2 = Keypair::new(); let tx = SystemTransaction::new_account(&keypair, keypair2.pubkey(), 42, last_id, 0); - let entry = Entry::new(&hash, 0, num_hashes, vec![tx]); + let entry = Entry::new(&hash, 0, 1, vec![tx]); hash = entry.id; entries.push(entry); if (i + 1) % tick_interval == 0 { - let tick = Entry::new(&hash, 0, num_hashes, vec![]); + let tick = Entry::new(&hash, 0, 1, vec![]); hash = tick.id; last_id = hash; entries.push(tick); @@ -1268,30 +1306,42 @@ mod tests { fn create_sample_ledger( tokens: u64, - num_entries: usize, + num_one_token_transfers: usize, ) -> (GenesisBlock, Keypair, impl Iterator) { let mint_keypair = Keypair::new(); + let bootstrap_leader_vote_account_keypair = Keypair::new(); let genesis_block = GenesisBlock { bootstrap_leader_id: Keypair::new().pubkey(), - bootstrap_leader_tokens: 1, + bootstrap_leader_tokens: BOOTSTRAP_LEADER_TOKENS, + bootstrap_leader_vote_account_id: bootstrap_leader_vote_account_keypair.pubkey(), mint_id: mint_keypair.pubkey(), tokens, }; - let block = - create_sample_block_with_ticks(&genesis_block, &mint_keypair, num_entries, num_entries); + let block = create_sample_block_with_ticks( + &genesis_block, + &mint_keypair, + num_one_token_transfers, + num_one_token_transfers, + ); (genesis_block, mint_keypair, block) } #[test] fn test_process_ledger_simple() { - let (genesis_block, mint_keypair, ledger) = create_sample_ledger(100, 2); + let (genesis_block, mint_keypair, ledger) = + create_sample_ledger(100 + BOOTSTRAP_LEADER_TOKENS, 3); let mut bank = Bank::default(); + bank.add_builtin_programs(); bank.process_genesis_block(&genesis_block); assert_eq!(bank.tick_height(), 0); - bank.add_system_program(); + assert_eq!(bank.get_balance(&mint_keypair.pubkey()), 100); + assert_eq!( + bank.get_current_leader(), + Some(genesis_block.bootstrap_leader_id) + ); let (ledger_height, last_id) = bank.process_ledger(ledger).unwrap(); - assert_eq!(bank.get_balance(&mint_keypair.pubkey()), 98); - assert_eq!(ledger_height, 4); + assert_eq!(bank.get_balance(&mint_keypair.pubkey()), 100 - 3); + assert_eq!(ledger_height, 8); assert_eq!(bank.tick_height(), 2); assert_eq!(bank.last_id(), last_id); } @@ -1299,9 +1349,11 @@ mod tests { #[test] fn test_hash_internal_state() { let mint_keypair = Keypair::new(); + let bootstrap_leader_vote_account_keypair = Keypair::new(); let genesis_block = GenesisBlock { bootstrap_leader_id: Keypair::new().pubkey(), - bootstrap_leader_tokens: 1, + bootstrap_leader_tokens: BOOTSTRAP_LEADER_TOKENS, + bootstrap_leader_vote_account_id: bootstrap_leader_vote_account_keypair.pubkey(), mint_id: mint_keypair.pubkey(), tokens: 2_000, }; @@ -1320,11 +1372,11 @@ mod tests { ); let mut bank0 = Bank::default(); - bank0.add_system_program(); + bank0.add_builtin_programs(); bank0.process_genesis_block(&genesis_block); bank0.process_ledger(ledger0).unwrap(); let mut bank1 = Bank::default(); - bank1.add_system_program(); + bank1.add_builtin_programs(); bank1.process_genesis_block(&genesis_block); bank1.process_ledger(ledger1).unwrap(); @@ -1351,7 +1403,7 @@ mod tests { } #[test] fn test_interleaving_locks() { - let (genesis_block, mint_keypair) = GenesisBlock::new(3); + let (genesis_block, mint_keypair) = GenesisBlock::new(3 + BOOTSTRAP_LEADER_TOKENS); let bank = Bank::new(&genesis_block); let alice = Keypair::new(); let bob = Keypair::new(); @@ -1643,7 +1695,8 @@ mod tests { let (genesis_block, mint_keypair) = GenesisBlock::new(10_000); let bank = Arc::new(Bank::new(&genesis_block)); let (entry_sender, entry_receiver) = channel(); - let poh_recorder = PohRecorder::new(bank.clone(), entry_sender, bank.last_id(), None); + let poh_recorder = + PohRecorder::new(bank.clone(), entry_sender, bank.last_id(), std::u64::MAX); let pubkey = Keypair::new().pubkey(); let transactions = vec![ @@ -1764,7 +1817,7 @@ mod tests { bank.clone(), entry_sender, bank.last_id(), - Some(bank.tick_height() + 1), + bank.tick_height() + 1, ); bank.process_and_record_transactions(&transactions, &poh_recorder) diff --git a/src/banking_stage.rs b/src/banking_stage.rs index df78c2aaae..9d5d79ae5d 100644 --- a/src/banking_stage.rs +++ b/src/banking_stage.rs @@ -45,7 +45,7 @@ pub struct BankingStage { bank_thread_hdls: Vec>>, poh_service: PohService, compute_confirmation_service: ComputeLeaderConfirmationService, - max_tick_height: Option, + max_tick_height: u64, } impl BankingStage { @@ -56,7 +56,7 @@ impl BankingStage { verified_receiver: Receiver, config: Config, last_entry_id: &Hash, - max_tick_height: Option, + max_tick_height: u64, leader_id: Pubkey, to_validator_sender: &TpuRotationSender, ) -> (Self, Receiver>) { @@ -116,8 +116,6 @@ impl BankingStage { break Some(BankingStageReturnType::RecordFailure); } Error::PohRecorderError(PohRecorderError::MaxHeightReached) => { - assert!(max_tick_height.is_some()); - let max_tick_height = max_tick_height.unwrap(); if !thread_did_notify_rotation.load(Ordering::Relaxed) { // Leader rotation should only happen if a max_tick_height was specified let _ = thread_sender.send( @@ -279,9 +277,7 @@ impl Service for BankingStage { match poh_return_value { Ok(_) => (), Err(Error::PohRecorderError(PohRecorderError::MaxHeightReached)) => { - return_value = Some(BankingStageReturnType::LeaderRotation( - self.max_tick_height.unwrap(), - )); + return_value = Some(BankingStageReturnType::LeaderRotation(self.max_tick_height)); } Err(Error::SendError) => { return_value = Some(BankingStageReturnType::ChannelDisconnected); @@ -299,7 +295,7 @@ mod tests { use crate::bank::Bank; use crate::banking_stage::BankingStageReturnType; use crate::entry::EntrySlice; - use crate::genesis_block::GenesisBlock; + use crate::genesis_block::{GenesisBlock, BOOTSTRAP_LEADER_TOKENS}; use crate::packet::to_packets; use solana_sdk::signature::{Keypair, KeypairUtil}; use solana_sdk::system_transaction::SystemTransaction; @@ -307,9 +303,8 @@ mod tests { #[test] fn test_banking_stage_shutdown1() { - let (genesis_block, _mint_keypair) = GenesisBlock::new(2); + let (genesis_block, _mint_keypair) = GenesisBlock::new(2 + BOOTSTRAP_LEADER_TOKENS); let bank = Arc::new(Bank::new(&genesis_block)); - let dummy_leader_id = Keypair::new().pubkey(); let (verified_sender, verified_receiver) = channel(); let (to_validator_sender, _) = channel(); let (banking_stage, _entry_receiver) = BankingStage::new( @@ -317,8 +312,8 @@ mod tests { verified_receiver, Default::default(), &bank.last_id(), - None, - dummy_leader_id, + std::u64::MAX, + genesis_block.bootstrap_leader_id, &to_validator_sender, ); drop(verified_sender); @@ -330,9 +325,8 @@ mod tests { #[test] fn test_banking_stage_shutdown2() { - let (genesis_block, _mint_keypair) = GenesisBlock::new(2); + let (genesis_block, _mint_keypair) = GenesisBlock::new(2 + BOOTSTRAP_LEADER_TOKENS); let bank = Arc::new(Bank::new(&genesis_block)); - let dummy_leader_id = Keypair::new().pubkey(); let (_verified_sender, verified_receiver) = channel(); let (to_validator_sender, _) = channel(); let (banking_stage, entry_receiver) = BankingStage::new( @@ -340,8 +334,8 @@ mod tests { verified_receiver, Default::default(), &bank.last_id(), - None, - dummy_leader_id, + std::u64::MAX, + genesis_block.bootstrap_leader_id, &to_validator_sender, ); drop(entry_receiver); @@ -353,9 +347,8 @@ mod tests { #[test] fn test_banking_stage_tick() { - let (genesis_block, _mint_keypair) = GenesisBlock::new(2); + let (genesis_block, _mint_keypair) = GenesisBlock::new(2 + BOOTSTRAP_LEADER_TOKENS); let bank = Arc::new(Bank::new(&genesis_block)); - let dummy_leader_id = Keypair::new().pubkey(); let start_hash = bank.last_id(); let (verified_sender, verified_receiver) = channel(); let (to_validator_sender, _) = channel(); @@ -364,8 +357,8 @@ mod tests { verified_receiver, Config::Sleep(Duration::from_millis(1)), &bank.last_id(), - None, - dummy_leader_id, + std::u64::MAX, + genesis_block.bootstrap_leader_id, &to_validator_sender, ); sleep(Duration::from_millis(500)); @@ -383,9 +376,8 @@ mod tests { #[test] fn test_banking_stage_entries_only() { - let (genesis_block, mint_keypair) = GenesisBlock::new(2); + let (genesis_block, mint_keypair) = GenesisBlock::new(2 + BOOTSTRAP_LEADER_TOKENS); let bank = Arc::new(Bank::new(&genesis_block)); - let dummy_leader_id = Keypair::new().pubkey(); let start_hash = bank.last_id(); let (verified_sender, verified_receiver) = channel(); let (to_validator_sender, _) = channel(); @@ -394,8 +386,8 @@ mod tests { verified_receiver, Default::default(), &bank.last_id(), - None, - dummy_leader_id, + std::u64::MAX, + genesis_block.bootstrap_leader_id, &to_validator_sender, ); @@ -443,9 +435,8 @@ mod tests { // In this attack we'll demonstrate that a verifier can interpret the ledger // differently if either the server doesn't signal the ledger to add an // Entry OR if the verifier tries to parallelize across multiple Entries. - let (genesis_block, mint_keypair) = GenesisBlock::new(2); + let (genesis_block, mint_keypair) = GenesisBlock::new(2 + BOOTSTRAP_LEADER_TOKENS); let bank = Arc::new(Bank::new(&genesis_block)); - let dummy_leader_id = Keypair::new().pubkey(); let (verified_sender, verified_receiver) = channel(); let (to_validator_sender, _) = channel(); let (banking_stage, entry_receiver) = BankingStage::new( @@ -453,8 +444,8 @@ mod tests { verified_receiver, Default::default(), &bank.last_id(), - None, - dummy_leader_id, + std::u64::MAX, + genesis_block.bootstrap_leader_id, &to_validator_sender, ); @@ -512,9 +503,8 @@ mod tests { // with reason BankingStageReturnType::LeaderRotation #[test] fn test_max_tick_height_shutdown() { - let (genesis_block, _mint_keypair) = GenesisBlock::new(2); + let (genesis_block, _mint_keypair) = GenesisBlock::new(2 + BOOTSTRAP_LEADER_TOKENS); let bank = Arc::new(Bank::new(&genesis_block)); - let dummy_leader_id = Keypair::new().pubkey(); let (_verified_sender_, verified_receiver) = channel(); let (to_validator_sender, _to_validator_receiver) = channel(); let max_tick_height = 10; @@ -523,8 +513,8 @@ mod tests { verified_receiver, Default::default(), &bank.last_id(), - Some(max_tick_height), - dummy_leader_id, + max_tick_height, + genesis_block.bootstrap_leader_id, &to_validator_sender, ); assert_eq!( diff --git a/src/broadcast_service.rs b/src/broadcast_service.rs index 75bf26cf17..555fb86f84 100644 --- a/src/broadcast_service.rs +++ b/src/broadcast_service.rs @@ -33,7 +33,7 @@ pub enum BroadcastServiceReturnType { struct Broadcast { id: Pubkey, - max_tick_height: Option, + max_tick_height: u64, blob_index: u64, #[cfg(feature = "erasure")] @@ -60,15 +60,12 @@ impl Broadcast { num_entries += entries.len(); ventries.push(entries); } - let last_tick = match self.max_tick_height { - Some(max_tick_height) => { - if let Some(Some(last)) = ventries.last().map(|entries| entries.last()) { - last.tick_height == max_tick_height - } else { - false - } + let last_tick = { + if let Some(Some(last)) = ventries.last().map(|entries| entries.last()) { + last.tick_height == self.max_tick_height + } else { + false } - None => false, }; inc_new_counter_info!("broadcast_service-entries_received", num_entries); @@ -151,10 +148,7 @@ fn generate_slots( } else { e.tick_height + 1 }; - let (_, slot) = r_leader_scheduler - .get_scheduled_leader(tick_height) - .expect("Leader schedule should never be unknown while indexing blobs"); - slot + r_leader_scheduler.tick_height_to_slot(tick_height) }) .collect(); @@ -193,7 +187,7 @@ impl BroadcastService { entry_height: u64, leader_scheduler: &Arc>, receiver: &Receiver>, - max_tick_height: Option, + max_tick_height: u64, exit_signal: &Arc, blob_sender: &BlobSender, ) -> BroadcastServiceReturnType { @@ -259,7 +253,7 @@ impl BroadcastService { entry_height: u64, leader_scheduler: Arc>, receiver: Receiver>, - max_tick_height: Option, + max_tick_height: u64, exit_sender: Arc, blob_sender: &BlobSender, ) -> Self { @@ -352,7 +346,7 @@ mod test { entry_height, leader_scheduler, entry_receiver, - Some(max_tick_height), + max_tick_height, exit_sender, &blob_fetch_sender, ); @@ -371,15 +365,12 @@ mod test { { // Create the leader scheduler let leader_keypair = Keypair::new(); - let mut leader_scheduler = - LeaderScheduler::from_bootstrap_leader(leader_keypair.pubkey()); + let mut leader_scheduler = LeaderScheduler::default(); // Mock the tick height to look like the tick height right after a leader transition - leader_scheduler.last_seed_height = Some(leader_scheduler.bootstrap_height); leader_scheduler.set_leader_schedule(vec![leader_keypair.pubkey()]); - leader_scheduler.use_only_bootstrap_leader = false; - let start_tick_height = leader_scheduler.bootstrap_height; - let max_tick_height = start_tick_height + leader_scheduler.last_seed_height.unwrap(); + let start_tick_height = 0; + let max_tick_height = start_tick_height + leader_scheduler.seed_rotation_interval; let entry_height = 2 * start_tick_height; let leader_scheduler = Arc::new(RwLock::new(leader_scheduler)); @@ -405,11 +396,10 @@ mod test { sleep(Duration::from_millis(2000)); let db_ledger = broadcast_service.db_ledger; for i in 0..max_tick_height - start_tick_height { - let (_, slot) = leader_scheduler + let slot = leader_scheduler .read() .unwrap() - .get_scheduled_leader(start_tick_height + i + 1) - .expect("Leader should exist"); + .tick_height_to_slot(start_tick_height + i + 1); let result = db_ledger.get_data_blob(slot, entry_height + i).unwrap(); assert!(result.is_some()); diff --git a/src/compute_leader_confirmation_service.rs b/src/compute_leader_confirmation_service.rs index 47888b303b..ce3bb83d75 100644 --- a/src/compute_leader_confirmation_service.rs +++ b/src/compute_leader_confirmation_service.rs @@ -176,7 +176,6 @@ pub mod tests { solana_logger::setup(); let (genesis_block, mint_keypair) = GenesisBlock::new(1234); - let dummy_leader_id = Keypair::new().pubkey(); let bank = Arc::new(Bank::new(&genesis_block)); // generate 10 validators, but only vote for the first 6 validators let ids: Vec<_> = (0..10) @@ -216,7 +215,7 @@ pub mod tests { let mut last_confirmation_time = 0; ComputeLeaderConfirmationService::compute_confirmation( &bank, - dummy_leader_id, + genesis_block.bootstrap_leader_id, &mut last_confirmation_time, ); assert_eq!(bank.confirmation_time(), std::usize::MAX); @@ -228,7 +227,7 @@ pub mod tests { ComputeLeaderConfirmationService::compute_confirmation( &bank, - dummy_leader_id, + genesis_block.bootstrap_leader_id, &mut last_confirmation_time, ); assert!(bank.confirmation_time() != std::usize::MAX); diff --git a/src/db_window.rs b/src/db_window.rs index 63a309eff5..6e7c3e6c5e 100644 --- a/src/db_window.rs +++ b/src/db_window.rs @@ -31,7 +31,6 @@ pub fn repair( leader_scheduler_option: &Arc>, ) -> Result)>> { let rcluster_info = cluster_info.read().unwrap(); - let mut is_next_leader = false; let meta = db_ledger.meta()?; if meta.is_none() { return Ok(vec![]); @@ -43,35 +42,33 @@ pub fn repair( // Repair should only be called when received > consumed, enforced in window_service assert!(received > consumed); - { - let ls_lock = leader_scheduler_option.read().unwrap(); - if !ls_lock.use_only_bootstrap_leader { - // Calculate the next leader rotation height and check if we are the leader - if let Some(next_leader_rotation_height) = ls_lock.max_height_for_leader(tick_height) { - match ls_lock.get_scheduled_leader(next_leader_rotation_height) { - Some((leader_id, _)) if leader_id == *id => is_next_leader = true, - // In the case that we are not in the current scope of the leader schedule - // window then either: - // - // 1) The replay stage hasn't caught up to the "consumed" entries we sent, - // in which case it will eventually catch up - // - // 2) We are on the border between seed_rotation_intervals, so the - // schedule won't be known until the entry on that cusp is received - // by the replay stage (which comes after this stage). Hence, the next - // leader at the beginning of that next epoch will not know they are the - // leader until they receive that last "cusp" entry. The leader also won't ask for repairs - // for that entry because "is_next_leader" won't be set here. In this case, - // everybody will be blocking waiting for that "cusp" entry instead of repairing, - // until the leader hits "times" >= the max times in calculate_max_repair_entry_height(). - // The impact of this, along with the similar problem from broadcast for the transitioning - // leader, can be observed in the multinode test, test_full_leader_validator_network(), - None => (), - _ => (), - } - } + + // Check if we are the next next slot leader + let is_next_leader = { + let leader_scheduler = leader_scheduler_option.read().unwrap(); + let next_slot = leader_scheduler.tick_height_to_slot(tick_height) + 1; + match leader_scheduler.get_leader_for_slot(next_slot) { + Some(leader_id) if leader_id == *id => true, + // In the case that we are not in the current scope of the leader schedule + // window then either: + // + // 1) The replay stage hasn't caught up to the "consumed" entries we sent, + // in which case it will eventually catch up + // + // 2) We are on the border between seed_rotation_intervals, so the + // schedule won't be known until the entry on that cusp is received + // by the replay stage (which comes after this stage). Hence, the next + // leader at the beginning of that next epoch will not know they are the + // leader until they receive that last "cusp" entry. The leader also won't ask for repairs + // for that entry because "is_next_leader" won't be set here. In this case, + // everybody will be blocking waiting for that "cusp" entry instead of repairing, + // until the leader hits "times" >= the max times in calculate_max_repair_entry_height(). + // The impact of this, along with the similar problem from broadcast for the transitioning + // leader, can be observed in the multinode test, test_full_leader_validator_network(), + None => false, + _ => false, } - } + }; let num_peers = rcluster_info.repair_peers().len() as u64; @@ -195,7 +192,8 @@ pub fn process_blob( // TODO: Once the original leader signature is added to the blob, make sure that // the blob was originally generated by the expected leader for this slot if leader.is_none() { - return Ok(()); + warn!("No leader for slot {}, blob dropped", slot); + return Ok(()); // Occurs as a leader is rotating into a validator } // Insert the new blob into the window @@ -393,8 +391,9 @@ mod test { pub fn test_retransmit() { let leader = Keypair::new().pubkey(); let nonleader = Keypair::new().pubkey(); - let leader_scheduler = - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader(leader))); + let mut leader_scheduler = LeaderScheduler::default(); + leader_scheduler.set_leader_schedule(vec![leader]); + let leader_scheduler = Arc::new(RwLock::new(leader_scheduler)); let blob = SharedBlob::default(); let (blob_sender, blob_receiver) = channel(); @@ -655,17 +654,12 @@ mod test { #[test] fn test_process_blob() { - // Create the leader scheduler - let leader_keypair = Keypair::new(); - let mut leader_scheduler = LeaderScheduler::from_bootstrap_leader(leader_keypair.pubkey()); + let mut leader_scheduler = LeaderScheduler::default(); + leader_scheduler.set_leader_schedule(vec![Keypair::new().pubkey()]); let db_ledger_path = get_tmp_ledger_path("test_process_blob"); let db_ledger = Arc::new(DbLedger::open(&db_ledger_path).unwrap()); - // Mock the tick height to look like the tick height right after a leader transition - leader_scheduler.last_seed_height = None; - leader_scheduler.use_only_bootstrap_leader = false; - let leader_scheduler = Arc::new(RwLock::new(leader_scheduler)); let num_entries = 10; let original_entries = make_tiny_test_entries(num_entries); diff --git a/src/fullnode.rs b/src/fullnode.rs index e15aefad0b..9853e6492d 100644 --- a/src/fullnode.rs +++ b/src/fullnode.rs @@ -6,7 +6,7 @@ use crate::counter::Counter; use crate::db_ledger::DbLedger; use crate::genesis_block::GenesisBlock; use crate::gossip_service::GossipService; -use crate::leader_scheduler::LeaderScheduler; +use crate::leader_scheduler::{LeaderScheduler, LeaderSchedulerConfig}; use crate::rpc::JsonRpcService; use crate::rpc_pubsub::PubSubService; use crate::service::Service; @@ -25,8 +25,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::mpsc::{channel, Receiver, Sender, SyncSender}; use std::sync::{Arc, RwLock}; -use std::thread::{sleep, spawn, Result}; -use std::time::Duration; +use std::thread::{spawn, Result}; use std::time::Instant; pub type TvuRotationSender = Sender; @@ -73,6 +72,7 @@ pub struct FullnodeConfig { pub voting_disabled: bool, pub entry_stream: Option, pub storage_rotate_count: u64, + pub leader_scheduler_config: LeaderSchedulerConfig, } impl Default for FullnodeConfig { fn default() -> Self { @@ -85,6 +85,7 @@ impl Default for FullnodeConfig { voting_disabled: false, entry_stream: None, storage_rotate_count: NUM_HASHES_FOR_STORAGE_ROTATE, + leader_scheduler_config: Default::default(), } } } @@ -110,7 +111,6 @@ impl Fullnode { mut node: Node, keypair: &Arc, ledger_path: &str, - leader_scheduler: Arc>, voting_keypair: VotingKeypair, entrypoint_info_option: Option<&NodeInfo>, config: &FullnodeConfig, @@ -118,9 +118,11 @@ impl Fullnode { let id = keypair.pubkey(); let (genesis_block, db_ledger, ledger_signal_sender, ledger_signal_receiver) = Self::make_db_ledger(ledger_path); - let (bank, entry_height, last_entry_id) = - Self::new_bank_from_db_ledger(&genesis_block, &db_ledger, leader_scheduler); - + let (bank, entry_height, last_entry_id) = Self::new_bank_from_db_ledger( + &genesis_block, + &db_ledger, + Some(&config.leader_scheduler_config), + ); info!("node info: {:?}", node.info); info!("node entrypoint_info: {:?}", entrypoint_info_option); info!( @@ -186,10 +188,24 @@ impl Fullnode { } // Get the scheduled leader - let (scheduled_leader, _) = bank - .get_current_leader() - .expect("Leader not known after processing bank"); + let (scheduled_leader, max_tpu_tick_height) = { + let tick_height = bank.tick_height(); + let leader_scheduler = bank.leader_scheduler.read().unwrap(); + let slot = leader_scheduler.tick_height_to_slot(tick_height); + ( + leader_scheduler + .get_leader_for_slot(slot) + .expect("Leader not known after processing bank"), + tick_height + leader_scheduler.num_ticks_left_in_slot(tick_height), + ) + }; + + trace!( + "scheduled_leader: {} until tick_height {}", + scheduled_leader, + max_tpu_tick_height + ); cluster_info.write().unwrap().set_leader(scheduled_leader); // TODO: always start leader and validator, keep leader side switching between tpu @@ -238,11 +254,6 @@ impl Fullnode { ledger_signal_sender, ledger_signal_receiver, ); - let max_tick_height = { - let ls_lock = bank.leader_scheduler.read().unwrap(); - ls_lock.max_height_for_leader(bank.tick_height() + 1) - }; - let tpu = Tpu::new( &Arc::new(bank.copy_for_tpu()), Default::default(), @@ -258,7 +269,7 @@ impl Fullnode { cluster_info.clone(), entry_height, config.sigverify_disabled, - max_tick_height, + max_tpu_tick_height, &last_entry_id, id, scheduled_leader == id, @@ -286,30 +297,35 @@ impl Fullnode { } pub fn leader_to_validator(&mut self, tick_height: u64) -> FullnodeReturnType { - trace!("leader_to_validator: tick_height={}", tick_height); + trace!( + "leader_to_validator({:?}): tick_height={}", + self.id, + tick_height, + ); - while self.bank.tick_height() < tick_height { - sleep(Duration::from_millis(10)); - } + let scheduled_leader = { + let mut leader_scheduler = self.bank.leader_scheduler.write().unwrap(); - let (scheduled_leader, _) = self - .bank - .leader_scheduler - .read() - .unwrap() - .get_scheduled_leader(tick_height + 1) - .unwrap(); + // A transition is only permitted on the final tick of a slot + assert_eq!(leader_scheduler.num_ticks_left_in_slot(tick_height), 0); + let first_tick_of_next_slot = tick_height + 1; + leader_scheduler.update_tick_height(first_tick_of_next_slot, &self.bank); + let slot = leader_scheduler.tick_height_to_slot(first_tick_of_next_slot); + leader_scheduler.get_leader_for_slot(slot).unwrap() + }; self.cluster_info .write() .unwrap() .set_leader(scheduled_leader); if scheduled_leader == self.id { + debug!("node is still the leader"); let (last_entry_id, entry_height) = self.node_services.tvu.get_state(); self.validator_to_leader(tick_height, entry_height, last_entry_id); FullnodeReturnType::LeaderToLeaderRotation } else { + debug!("new leader is {}", scheduled_leader); self.node_services.tpu.switch_to_forwarder( self.tpu_sockets .iter() @@ -321,14 +337,43 @@ impl Fullnode { } } - pub fn validator_to_leader(&mut self, tick_height: u64, entry_height: u64, last_id: Hash) { - trace!("validator_to_leader"); + pub fn validator_to_leader( + &mut self, + tick_height: u64, + entry_height: u64, + last_entry_id: Hash, + ) { + trace!( + "validator_to_leader({:?}): tick_height={} entry_height={} last_entry_id={}", + self.id, + tick_height, + entry_height, + last_entry_id, + ); + + let (scheduled_leader, max_tick_height) = { + let mut leader_scheduler = self.bank.leader_scheduler.write().unwrap(); + + // A transition is only permitted on the final tick of a slot + assert_eq!(leader_scheduler.num_ticks_left_in_slot(tick_height), 0); + let first_tick_of_next_slot = tick_height + 1; + + leader_scheduler.update_tick_height(first_tick_of_next_slot, &self.bank); + let slot = leader_scheduler.tick_height_to_slot(first_tick_of_next_slot); + ( + leader_scheduler.get_leader_for_slot(slot).unwrap(), + first_tick_of_next_slot + + leader_scheduler.num_ticks_left_in_slot(first_tick_of_next_slot), + ) + }; + assert_eq!(scheduled_leader, self.id, "node is not the leader"); self.cluster_info.write().unwrap().set_leader(self.id); - let max_tick_height = { - let ls_lock = self.bank.leader_scheduler.read().unwrap(); - ls_lock.max_height_for_leader(tick_height + 1) - }; + debug!( + "node scheduled as leader for ticks [{}, {})", + tick_height + 1, + max_tick_height + ); let (to_validator_sender, to_validator_receiver) = channel(); self.role_notifiers.1 = to_validator_receiver; @@ -346,14 +391,14 @@ impl Fullnode { self.sigverify_disabled, max_tick_height, entry_height, - &last_id, + &last_entry_id, self.id, &to_validator_sender, &self.blob_sender, ) } - pub fn handle_role_transition(&mut self) -> Option { + fn handle_role_transition(&mut self) -> Option<(FullnodeReturnType, u64)> { loop { if self.exit.load(Ordering::Relaxed) { return None; @@ -363,11 +408,14 @@ impl Fullnode { match should_be_leader { Ok(TvuReturnType::LeaderRotation(tick_height, entry_height, last_entry_id)) => { self.validator_to_leader(tick_height, entry_height, last_entry_id); - return Some(FullnodeReturnType::ValidatorToLeaderRotation); + return Some(( + FullnodeReturnType::ValidatorToLeaderRotation, + tick_height + 1, + )); } _ => match should_be_forwarder { Ok(TpuReturnType::LeaderRotation(tick_height)) => { - return Some(self.leader_to_validator(tick_height)) + return Some((self.leader_to_validator(tick_height), tick_height + 1)) } _ => { continue; @@ -379,7 +427,10 @@ impl Fullnode { // Runs a thread to manage node role transitions. The returned closure can be used to signal the // node to exit. - pub fn run(mut self, rotation_notifier: Option>) -> impl FnOnce() { + pub fn run( + mut self, + rotation_notifier: Option>, + ) -> impl FnOnce() { let (sender, receiver) = channel(); let exit = self.exit.clone(); spawn(move || loop { @@ -426,11 +477,10 @@ impl Fullnode { pub fn new_bank_from_db_ledger( genesis_block: &GenesisBlock, db_ledger: &DbLedger, - leader_scheduler: Arc>, + leader_scheduler_config: Option<&LeaderSchedulerConfig>, ) -> (Bank, u64, Hash) { - let mut bank = Bank::new(genesis_block); - leader_scheduler.write().unwrap().bootstrap_leader = genesis_block.bootstrap_leader_id; - bank.leader_scheduler = leader_scheduler; + let mut bank = + Bank::new_with_leader_scheduler_config(genesis_block, leader_scheduler_config); let now = Instant::now(); let entries = db_ledger.read_ledger().expect("opening ledger"); @@ -440,19 +490,20 @@ impl Fullnode { // entry_height is the network-wide agreed height of the ledger. // initialize it from the input ledger info!( - "processed {} ledger entries in {}ms...", + "processed {} ledger entries in {}ms, tick_height={}...", entry_height, - duration_as_ms(&now.elapsed()) + duration_as_ms(&now.elapsed()), + bank.tick_height() ); (bank, entry_height, last_entry_id) } pub fn new_bank_from_ledger( ledger_path: &str, - leader_scheduler: Arc>, + leader_scheduler_config: Option<&LeaderSchedulerConfig>, ) -> (Bank, u64, Hash) { let (genesis_block, db_ledger, _, _) = Self::make_db_ledger(ledger_path); - Self::new_bank_from_db_ledger(&genesis_block, &db_ledger, leader_scheduler) + Self::new_bank_from_db_ledger(&genesis_block, &db_ledger, leader_scheduler_config) } pub fn get_leader_scheduler(&self) -> &Arc> { @@ -498,22 +549,16 @@ mod tests { use crate::cluster_info::Node; use crate::db_ledger::*; use crate::entry::make_consecutive_blobs; - use crate::leader_scheduler::{ - make_active_set_entries, LeaderScheduler, LeaderSchedulerConfig, - }; - use crate::poh_service::NUM_TICKS_PER_SECOND; + use crate::leader_scheduler::{make_active_set_entries, LeaderSchedulerConfig}; use crate::service::Service; use crate::streamer::responder; - use crate::tpu::TpuReturnType; - use crate::tvu::TvuReturnType; use crate::voting_keypair::VotingKeypair; use solana_sdk::hash::Hash; use solana_sdk::signature::{Keypair, KeypairUtil}; - use std::cmp; use std::fs::remove_dir_all; use std::net::UdpSocket; use std::sync::mpsc::channel; - use std::sync::{Arc, RwLock}; + use std::sync::Arc; #[test] fn validator_exit() { @@ -529,7 +574,6 @@ mod tests { validator_node, &Arc::new(validator_keypair), &validator_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::new(&Default::default()))), VotingKeypair::new(), Some(&leader_node.info), &FullnodeConfig::default(), @@ -556,12 +600,10 @@ mod tests { 1000, ); ledger_paths.push(validator_ledger_path.clone()); - Fullnode::new( validator_node, &Arc::new(validator_keypair), &validator_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::new(&Default::default()))), VotingKeypair::new(), Some(&leader_node.info), &FullnodeConfig::default(), @@ -584,13 +626,12 @@ mod tests { #[test] fn test_leader_to_leader_transition() { - // Create the leader node information + solana_logger::setup(); + let bootstrap_leader_keypair = Keypair::new(); let bootstrap_leader_node = Node::new_localhost_with_pubkey(bootstrap_leader_keypair.pubkey()); - let bootstrap_leader_info = bootstrap_leader_node.info.clone(); - // Make a mint and a genesis entries for leader ledger let (_mint_keypair, bootstrap_leader_ledger_path, _genesis_entry_height, _last_id) = create_tmp_sample_ledger( "test_leader_to_leader_transition", @@ -600,19 +641,14 @@ mod tests { 500, ); - // Create the common leader scheduling configuration - let num_slots_per_epoch = 3; + // Once the bootstrap leader hits the second epoch, because there are no other choices in + // the active set, this leader will remain the leader in the second epoch. In the second + // epoch, check that the same leader knows to shut down and restart as a leader again. let leader_rotation_interval = 5; + let num_slots_per_epoch = 2; let seed_rotation_interval = num_slots_per_epoch * leader_rotation_interval; - let active_window_length = 5; - - // Set the bootstrap height to be bigger than the initial tick height. - // Once the leader hits the bootstrap height ticks, because there are no other - // choices in the active set, this leader will remain the leader in the next - // epoch. In the next epoch, check that the same leader knows to shut down and - // restart as a leader again. + let active_window_length = 10 * seed_rotation_interval; let leader_scheduler_config = LeaderSchedulerConfig::new( - 2, leader_rotation_interval, seed_rotation_interval, active_window_length, @@ -620,15 +656,16 @@ mod tests { let bootstrap_leader_keypair = Arc::new(bootstrap_leader_keypair); let voting_keypair = VotingKeypair::new_local(&bootstrap_leader_keypair); - // Start up the leader + // Start the bootstrap leader + let mut fullnode_config = FullnodeConfig::default(); + fullnode_config.leader_scheduler_config = leader_scheduler_config; let bootstrap_leader = Fullnode::new( bootstrap_leader_node, &bootstrap_leader_keypair, &bootstrap_leader_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::new(&leader_scheduler_config))), voting_keypair, - Some(&bootstrap_leader_info), - &FullnodeConfig::default(), + None, + &fullnode_config, ); let (rotation_sender, rotation_receiver) = channel(); @@ -638,7 +675,10 @@ mod tests { // cluster it will continue to be the leader assert_eq!( rotation_receiver.recv().unwrap(), - FullnodeReturnType::LeaderToLeaderRotation + ( + FullnodeReturnType::LeaderToLeaderRotation, + leader_rotation_interval + ) ); bootstrap_leader_exit(); } @@ -647,6 +687,14 @@ mod tests { fn test_wrong_role_transition() { solana_logger::setup(); + let mut fullnode_config = FullnodeConfig::default(); + let leader_rotation_interval = 16; + fullnode_config.leader_scheduler_config = LeaderSchedulerConfig::new( + leader_rotation_interval, + leader_rotation_interval * 2, + leader_rotation_interval * 2, + ); + // Create the leader and validator nodes let bootstrap_leader_keypair = Arc::new(Keypair::new()); let validator_keypair = Arc::new(Keypair::new()); @@ -655,7 +703,9 @@ mod tests { &bootstrap_leader_keypair, &validator_keypair, 0, - 10, + // Generate enough ticks for two epochs to flush the bootstrap_leader's vote at + // tick_height = 0 from the leader scheduler's active window + leader_rotation_interval * 4, "test_wrong_role_transition", ); let bootstrap_leader_info = bootstrap_leader_node.info.clone(); @@ -668,29 +718,15 @@ mod tests { validator_ledger_path.clone(), ]; - // Create the common leader scheduling configuration - let leader_rotation_interval = 3; - - // Set the bootstrap height exactly the current tick height, so that we can - // test if the bootstrap leader knows to immediately transition to a validator - // after parsing the ledger during startup - let leader_scheduler_config = LeaderSchedulerConfig::new( - 1, - leader_rotation_interval, - leader_rotation_interval, - leader_rotation_interval * 10, - ); - { // Test that a node knows to transition to a validator based on parsing the ledger let bootstrap_leader = Fullnode::new( bootstrap_leader_node, &bootstrap_leader_keypair, &bootstrap_leader_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::new(&leader_scheduler_config))), VotingKeypair::new(), Some(&bootstrap_leader_info), - &FullnodeConfig::default(), + &fullnode_config, ); assert!(!bootstrap_leader.node_services.tpu.is_leader()); @@ -700,10 +736,9 @@ mod tests { validator_node, &validator_keypair, &validator_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::new(&leader_scheduler_config))), VotingKeypair::new(), Some(&bootstrap_leader_info), - &FullnodeConfig::default(), + &fullnode_config, ); assert!(validator.node_services.tpu.is_leader()); @@ -720,15 +755,15 @@ mod tests { #[test] fn test_validator_to_leader_transition() { + solana_logger::setup(); // Make leader and validator node let leader_keypair = Arc::new(Keypair::new()); let validator_keypair = Arc::new(Keypair::new()); - let num_genesis_ticks = 1; let (leader_node, validator_node, validator_ledger_path, ledger_initial_len, last_id) = setup_leader_validator( &leader_keypair, &validator_keypair, - num_genesis_ticks, + 0, 0, "test_validator_to_leader_transition", ); @@ -736,16 +771,17 @@ mod tests { let leader_id = leader_keypair.pubkey(); let validator_info = validator_node.info.clone(); - // Set the leader scheduler for the validator - let leader_rotation_interval = 16; - let num_bootstrap_slots = 2; - let bootstrap_height = num_bootstrap_slots * leader_rotation_interval; + info!("leader: {:?}", leader_id); + info!("validator: {:?}", validator_info.id); - let leader_scheduler_config = LeaderSchedulerConfig::new( - bootstrap_height, + // Set the leader scheduler for the validator + let leader_rotation_interval = 10; + + let mut fullnode_config = FullnodeConfig::default(); + fullnode_config.leader_scheduler_config = LeaderSchedulerConfig::new( leader_rotation_interval, - leader_rotation_interval * 2, - bootstrap_height, + leader_rotation_interval * 4, + leader_rotation_interval * 4, ); let voting_keypair = VotingKeypair::new_local(&validator_keypair); @@ -754,12 +790,18 @@ mod tests { validator_node, &validator_keypair, &validator_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::new(&leader_scheduler_config))), voting_keypair, Some(&leader_node.info), - &FullnodeConfig::default(), + &fullnode_config, ); + let blobs_to_send = fullnode_config + .leader_scheduler_config + .seed_rotation_interval + + fullnode_config + .leader_scheduler_config + .leader_rotation_interval; + // Send blobs to the validator from our mock leader let t_responder = { let (s_responder, r_responder) = channel(); @@ -771,14 +813,11 @@ mod tests { r_responder, ); - // Send the blobs out of order, in reverse. Also send an extra - // "extra_blobs" number of blobs to make sure the window stops in the right place. - let extra_blobs = cmp::max(leader_rotation_interval / 3, 1); - let total_blobs_to_send = bootstrap_height + extra_blobs; let tvu_address = &validator_info.tvu; + let msgs = make_consecutive_blobs( &leader_id, - total_blobs_to_send, + blobs_to_send, ledger_initial_len, last_id, &tvu_address, @@ -790,37 +829,28 @@ mod tests { t_responder }; - assert_ne!(validator.bank.get_current_leader().unwrap().0, validator.id); - loop { - let should_be_forwarder = validator.role_notifiers.1.try_recv(); - let should_be_leader = validator.role_notifiers.0.try_recv(); - match should_be_leader { - Ok(TvuReturnType::LeaderRotation(tick_height, entry_height, _)) => { - assert_eq!(validator.node_services.tvu.get_state().1, entry_height); - assert_eq!(validator.bank.tick_height(), tick_height); - assert_eq!(tick_height, bootstrap_height); - break; - } - _ => match should_be_forwarder { - Ok(TpuReturnType::LeaderRotation(_)) => { - panic!("shouldn't be rotating to forwarder") - } - _ => continue, - }, - } - } - - //close the validator so that rocksdb has locks available - validator.close().unwrap(); - let (bank, entry_height, _) = Fullnode::new_bank_from_ledger( - &validator_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::new(&leader_scheduler_config))), + info!("waiting for validator to rotate into the leader role"); + let (rotation_sender, rotation_receiver) = channel(); + let validator_exit = validator.run(Some(rotation_sender)); + let rotation = rotation_receiver.recv().unwrap(); + assert_eq!( + rotation, + (FullnodeReturnType::ValidatorToLeaderRotation, blobs_to_send) ); - assert!(bank.tick_height() >= bootstrap_height); - // Only the first genesis entry has num_hashes = 0, every other entry - // had num_hashes = 1 - assert!(entry_height >= bootstrap_height + ledger_initial_len - num_genesis_ticks); + // Close the validator so that rocksdb has locks available + // validator.close().unwrap(); + validator_exit(); + let (bank, entry_height, _) = Fullnode::new_bank_from_ledger(&validator_ledger_path, None); + + assert!( + bank.tick_height() + >= fullnode_config + .leader_scheduler_config + .seed_rotation_interval + ); + + assert!(entry_height >= ledger_initial_len); // Shut down t_responder.join().expect("responder thread join"); @@ -830,44 +860,48 @@ mod tests { } #[test] + #[ignore] // TODO: Make this test less hacky fn test_tvu_behind() { + solana_logger::setup(); + // Make leader node let leader_keypair = Arc::new(Keypair::new()); let validator_keypair = Arc::new(Keypair::new()); + info!("leader: {:?}", leader_keypair.pubkey()); + info!("validator: {:?}", validator_keypair.pubkey()); + let (leader_node, _, leader_ledger_path, _, _) = setup_leader_validator(&leader_keypair, &validator_keypair, 1, 0, "test_tvu_behind"); let leader_node_info = leader_node.info.clone(); // Set the leader scheduler for the validator - let leader_rotation_interval = NUM_TICKS_PER_SECOND as u64 * 5; - let bootstrap_height = leader_rotation_interval; + let leader_rotation_interval = 5; - let leader_scheduler_config = LeaderSchedulerConfig::new( - bootstrap_height, + let mut fullnode_config = FullnodeConfig::default(); + fullnode_config.leader_scheduler_config = LeaderSchedulerConfig::new( leader_rotation_interval, leader_rotation_interval * 2, - bootstrap_height, + leader_rotation_interval * 2, ); let voting_keypair = VotingKeypair::new_local(&leader_keypair); - // Start the bootstrap leader + info!("Start the bootstrap leader"); let mut leader = Fullnode::new( leader_node, &leader_keypair, &leader_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::new(&leader_scheduler_config))), voting_keypair, Some(&leader_node_info), - &FullnodeConfig::default(), + &fullnode_config, ); - // Hold Tvu bank lock to prevent tvu from making progress + info!("Hold Tvu bank lock to prevent tvu from making progress"); { let w_last_ids = leader.bank.last_ids().write().unwrap(); - // Wait for leader -> validator transition + info!("Wait for leader -> validator transition"); let signal = leader .role_notifiers .1 @@ -877,26 +911,42 @@ mod tests { rn_sender.send(signal).expect("send"); leader.role_notifiers = (leader.role_notifiers.0, rn_receiver); - // Make sure the tvu bank is behind - assert!(w_last_ids.tick_height < bootstrap_height); + info!("Make sure the tvu bank is behind"); + assert_eq!(w_last_ids.tick_height, 2); } - // Release tvu bank lock, tvu should start making progress again and - // handle_role_transition should successfully rotate the leader to a validator - assert_eq!( - leader.handle_role_transition().unwrap(), - FullnodeReturnType::LeaderToValidatorRotation - ); - assert_eq!( - leader.cluster_info.read().unwrap().leader_id(), - validator_keypair.pubkey(), - ); - assert!(!leader.node_services.tpu.is_leader()); - // Confirm the bank actually made progress - assert_eq!(leader.bank.tick_height(), bootstrap_height); + // Release tvu bank lock, tvu should start making progress again and should signal a + // rotate. After rotation it will still be the slot leader as a new leader schedule has + // not been computed yet (still in epoch 0) + info!("Release tvu bank lock"); + let (rotation_sender, rotation_receiver) = channel(); + let leader_exit = leader.run(Some(rotation_sender)); + let expected_rotations = vec![ + ( + FullnodeReturnType::LeaderToLeaderRotation, + leader_rotation_interval, + ), + ( + FullnodeReturnType::LeaderToLeaderRotation, + 2 * leader_rotation_interval, + ), + ( + FullnodeReturnType::LeaderToValidatorRotation, + 3 * leader_rotation_interval, + ), + ]; - // Shut down - leader.close().expect("leader shutdown"); + for expected_rotation in expected_rotations { + loop { + let transition = rotation_receiver.recv().unwrap(); + info!("leader transition: {:?}", transition); + assert_eq!(expected_rotation, transition); + break; + } + } + + info!("Shut down"); + leader_exit(); DbLedger::destroy(&leader_ledger_path).expect("Expected successful database destruction"); let _ignored = remove_dir_all(&leader_ledger_path).unwrap(); } @@ -910,11 +960,15 @@ mod tests { ) -> (Node, Node, String, u64, Hash) { // Make a leader identity let leader_node = Node::new_localhost_with_pubkey(leader_keypair.pubkey()); - let leader_id = leader_node.info.id; // Create validator identity - let (mint_keypair, ledger_path, genesis_entry_height, last_id) = - create_tmp_sample_ledger(test_name, 10_000, num_genesis_ticks, leader_id, 500); + let (mint_keypair, ledger_path, genesis_entry_height, last_id) = create_tmp_sample_ledger( + test_name, + 10_000, + num_genesis_ticks, + leader_node.info.id, + 500, + ); let validator_node = Node::new_localhost_with_pubkey(validator_keypair.pubkey()); @@ -928,6 +982,8 @@ mod tests { let (active_set_entries, _) = make_active_set_entries( validator_keypair, &mint_keypair, + 10, + 1, &last_id, &last_id, num_ending_ticks, diff --git a/src/genesis_block.rs b/src/genesis_block.rs index 53cefb77fc..ef8e5b2acd 100644 --- a/src/genesis_block.rs +++ b/src/genesis_block.rs @@ -7,10 +7,16 @@ use std::fs::File; use std::io::Write; use std::path::Path; +// The default (and minimal) amount of tokens given to the bootstrap leader: +// * 1 token for the bootstrap leader ID account +// * 1 token for the bootstrap leader vote account +pub const BOOTSTRAP_LEADER_TOKENS: u64 = 2; + #[derive(Serialize, Deserialize, Debug)] pub struct GenesisBlock { pub bootstrap_leader_id: Pubkey, pub bootstrap_leader_tokens: u64, + pub bootstrap_leader_vote_account_id: Pubkey, pub mint_id: Pubkey, pub tokens: u64, } @@ -18,11 +24,15 @@ pub struct GenesisBlock { impl GenesisBlock { #[allow(clippy::new_ret_no_self)] pub fn new(tokens: u64) -> (Self, Keypair) { + assert!(tokens >= 2); let mint_keypair = Keypair::new(); + let bootstrap_leader_keypair = Keypair::new(); + let bootstrap_leader_vote_account_keypair = Keypair::new(); ( Self { - bootstrap_leader_id: Pubkey::default(), - bootstrap_leader_tokens: 0, + bootstrap_leader_id: bootstrap_leader_keypair.pubkey(), + bootstrap_leader_tokens: BOOTSTRAP_LEADER_TOKENS, + bootstrap_leader_vote_account_id: bootstrap_leader_vote_account_keypair.pubkey(), mint_id: mint_keypair.pubkey(), tokens, }, @@ -36,10 +46,12 @@ impl GenesisBlock { bootstrap_leader_tokens: u64, ) -> (Self, Keypair) { let mint_keypair = Keypair::new(); + let bootstrap_leader_vote_account_keypair = Keypair::new(); ( Self { bootstrap_leader_id, bootstrap_leader_tokens, + bootstrap_leader_vote_account_id: bootstrap_leader_vote_account_keypair.pubkey(), mint_id: mint_keypair.pubkey(), tokens, }, @@ -74,8 +86,12 @@ mod tests { let (genesis_block, mint) = GenesisBlock::new(10_000); assert_eq!(genesis_block.tokens, 10_000); assert_eq!(genesis_block.mint_id, mint.pubkey()); - assert_eq!(genesis_block.bootstrap_leader_id, Pubkey::default()); - assert_eq!(genesis_block.bootstrap_leader_tokens, 0); + assert!(genesis_block.bootstrap_leader_id != Pubkey::default()); + assert!(genesis_block.bootstrap_leader_vote_account_id != Pubkey::default()); + assert_eq!( + genesis_block.bootstrap_leader_tokens, + BOOTSTRAP_LEADER_TOKENS + ); } #[test] diff --git a/src/leader_scheduler.rs b/src/leader_scheduler.rs index 1ea1321379..c9c450286b 100644 --- a/src/leader_scheduler.rs +++ b/src/leader_scheduler.rs @@ -2,7 +2,6 @@ //! managing the schedule for leader rotation use crate::bank::Bank; - use crate::entry::{create_ticks, Entry}; use crate::voting_keypair::VotingKeypair; use bincode::serialize; @@ -20,12 +19,11 @@ use std::sync::Arc; // At 10 ticks/s, 8 ticks per slot implies that leader rotation and voting will happen // every 800 ms. A fast voting cadence ensures faster finality and convergence pub const DEFAULT_TICKS_PER_SLOT: u64 = 8; -// Bootstrap height lasts for ~100 seconds -pub const DEFAULT_BOOTSTRAP_HEIGHT: u64 = 1024; pub const DEFAULT_SLOTS_PER_EPOCH: u64 = 64; pub const DEFAULT_SEED_ROTATION_INTERVAL: u64 = DEFAULT_SLOTS_PER_EPOCH * DEFAULT_TICKS_PER_SLOT; pub const DEFAULT_ACTIVE_WINDOW_LENGTH: u64 = DEFAULT_SEED_ROTATION_INTERVAL; +#[derive(Clone)] pub struct LeaderSchedulerConfig { // The interval at which to rotate the leader, should be much less than // seed_rotation_interval @@ -34,10 +32,6 @@ pub struct LeaderSchedulerConfig { // The interval at which to generate the seed used for ranking the validators pub seed_rotation_interval: u64, - // The last height at which the bootstrap_leader will be in power before - // the leader rotation process begins to pick future leaders - pub bootstrap_height: u64, - // The length of the acceptable window for determining live validators pub active_window_length: u64, } @@ -46,13 +40,11 @@ pub struct LeaderSchedulerConfig { // need leader rotation don't break impl LeaderSchedulerConfig { pub fn new( - bootstrap_height: u64, leader_rotation_interval: u64, seed_rotation_interval: u64, active_window_length: u64, ) -> Self { LeaderSchedulerConfig { - bootstrap_height, leader_rotation_interval, seed_rotation_interval, active_window_length, @@ -63,7 +55,6 @@ impl LeaderSchedulerConfig { impl Default for LeaderSchedulerConfig { fn default() -> Self { Self { - bootstrap_height: DEFAULT_BOOTSTRAP_HEIGHT, leader_rotation_interval: DEFAULT_TICKS_PER_SLOT, seed_rotation_interval: DEFAULT_SEED_ROTATION_INTERVAL, active_window_length: DEFAULT_ACTIVE_WINDOW_LENGTH, @@ -73,33 +64,23 @@ impl Default for LeaderSchedulerConfig { #[derive(Clone, Debug)] pub struct LeaderScheduler { - // Set to true if we want the default implementation of the LeaderScheduler, - // where ony the bootstrap leader is used - pub use_only_bootstrap_leader: bool, - - // The interval at which to rotate the leader, should be much less than - // seed_rotation_interval + // A leader slot duration in ticks pub leader_rotation_interval: u64, - // The interval at which to generate the seed used for ranking the validators + // Duration of an epoch (one or more slots) in ticks. + // This value must be divisible by leader_rotation_interval pub seed_rotation_interval: u64, - // The first leader who will bootstrap the network - pub bootstrap_leader: Pubkey, - - // The last height at which the bootstrap_leader will be in power before - // the leader rotation process begins to pick future leaders - pub bootstrap_height: u64, - - // The last height at which the seed + schedule was generated - pub last_seed_height: Option, - // The length of time in ticks for which a vote qualifies a candidate for leader // selection pub active_window_length: u64, - // Round-robin ordering for the validators - leader_schedule: Vec, + // Round-robin ordering of the validators for the current epoch at epoch_schedule[0], and the + // previous epoch at epoch_schedule[1] + epoch_schedule: [Vec; 2], + + // The epoch for epoch_schedule[0] + current_epoch: u64, // The seed used to determine the round robin order of leaders seed: u64, @@ -107,10 +88,7 @@ pub struct LeaderScheduler { // The LeaderScheduler implements a schedule for leaders as follows: // -// 1) During the bootstrapping period of bootstrap_height PoH counts, the -// leader is hard-coded to the bootstrap_leader that is read from the genesis block. -// -// 2) After the first seed is generated, this signals the beginning of actual leader rotation. +// 1) After the first seed is generated, this signals the beginning of actual leader rotation. // From this point on, every seed_rotation_interval PoH counts we generate the seed based // on the PoH height, and use it to do a weighted sample from the set // of validators based on current stake weight. This gets you the bootstrap leader A for @@ -120,211 +98,115 @@ pub struct LeaderScheduler { // PoH counts based on this fixed ordering, so the next // seed_rotation_interval / leader_rotation_interval leaders are determined. // -// 3) When we we hit the next seed rotation PoH height, step 2) is executed again to +// 2) When we we hit the next seed rotation PoH height, step 1) is executed again to // calculate the leader schedule for the upcoming seed_rotation_interval PoH counts. impl LeaderScheduler { - pub fn from_bootstrap_leader(bootstrap_leader: Pubkey) -> Self { - let config = LeaderSchedulerConfig::default(); - let mut leader_scheduler = LeaderScheduler::new(&config); - leader_scheduler.use_only_bootstrap_leader = true; - leader_scheduler.bootstrap_leader = bootstrap_leader; - leader_scheduler - } - pub fn new(config: &LeaderSchedulerConfig) -> Self { - let bootstrap_height = config.bootstrap_height; let leader_rotation_interval = config.leader_rotation_interval; let seed_rotation_interval = config.seed_rotation_interval; let active_window_length = config.active_window_length; // Enforced invariants assert!(seed_rotation_interval >= leader_rotation_interval); - assert!(bootstrap_height > 0); assert!(seed_rotation_interval % leader_rotation_interval == 0); LeaderScheduler { - use_only_bootstrap_leader: false, leader_rotation_interval, seed_rotation_interval, - leader_schedule: Vec::new(), - last_seed_height: None, - bootstrap_leader: Pubkey::default(), - bootstrap_height, active_window_length, seed: 0, + epoch_schedule: [Vec::new(), Vec::new()], + current_epoch: 0, } } - // Returns true if the given height is the first tick of a slot - pub fn is_first_slot_tick(&self, height: u64) -> bool { - if self.use_only_bootstrap_leader { - return false; - } - - if height < self.bootstrap_height { - return false; - } - - (height - self.bootstrap_height) % self.leader_rotation_interval == 1 + pub fn tick_height_to_slot(&self, tick_height: u64) -> u64 { + tick_height / self.leader_rotation_interval } - // Returns the number of ticks until the last tick the current slot - pub fn num_ticks_left_in_slot(&self, height: u64) -> Option { - if self.use_only_bootstrap_leader { - return None; - } - - if height <= self.bootstrap_height { - Some(self.bootstrap_height - height) - } else if (height - self.bootstrap_height) % self.leader_rotation_interval == 0 { - Some(0) - } else { - Some( - self.leader_rotation_interval - - ((height - self.bootstrap_height) % self.leader_rotation_interval), - ) - } + fn tick_height_to_epoch(&self, tick_height: u64) -> u64 { + tick_height / self.seed_rotation_interval } - // Returns the last tick height for a given slot index - pub fn max_tick_height_for_slot(&self, slot_index: u64) -> u64 { - if self.use_only_bootstrap_leader { - std::u64::MAX - } else { - slot_index * self.leader_rotation_interval + self.bootstrap_height - } + // Returns the number of ticks remaining from the specified tick_height to the end of the + // current slot + pub fn num_ticks_left_in_slot(&self, tick_height: u64) -> u64 { + self.leader_rotation_interval - tick_height % self.leader_rotation_interval - 1 } - // Let Leader X be the leader at the input tick height. This function returns the - // the PoH height at which Leader X's slot ends. - pub fn max_height_for_leader(&self, height: u64) -> Option { - if self.use_only_bootstrap_leader || self.get_scheduled_leader(height).is_none() { - return None; - } - - let result = { - if height <= self.bootstrap_height || self.leader_schedule.len() > 1 { - // Two cases to consider: - // - // 1) If height is less than the bootstrap height, then the current leader's - // slot ends when PoH height = bootstrap_height - // - // 2) Otherwise, if height >= bootstrap height, then we have generated a schedule. - // If this leader is not the only one in the schedule, then they will - // only be leader until the end of this slot (someone else is then guaranteed - // to take over) - // - // Both above cases are calculated by the function: - // num_ticks_left_in_slot() + height - self.num_ticks_left_in_slot(height).expect( - "Should return some value when not using default implementation - of LeaderScheduler", - ) + height - } else { - // If the height is greater than bootstrap_height and this leader is - // the only leader in the schedule, then that leader will be in power - // for every slot until the next epoch, which is seed_rotation_interval - // PoH counts from the beginning of the last epoch. - self.last_seed_height.expect( - "If height >= bootstrap height, then we expect - a seed has been generated", - ) + self.seed_rotation_interval - } - }; - - Some(result) - } - - pub fn reset(&mut self) { - self.last_seed_height = None; - } - - pub fn update_height(&mut self, height: u64, bank: &Bank) { - if self.use_only_bootstrap_leader { + // Inform the leader scheduler about the current tick height of the cluster. It may generate a + // new schedule as a side-effect. + pub fn update_tick_height(&mut self, tick_height: u64, bank: &Bank) { + let epoch = self.tick_height_to_epoch(tick_height); + trace!( + "update_tick_height: tick_height={} (epoch={})", + tick_height, + epoch, + ); + if epoch < self.current_epoch { return; } - if height < self.bootstrap_height { - return; - } - - if let Some(last_seed_height) = self.last_seed_height { - if height <= last_seed_height { - return; - } - } - - if (height - self.bootstrap_height) % self.seed_rotation_interval == 0 { - self.generate_schedule(height, bank); + // Leader schedule is computed at tick 0 (for bootstrap) and then on the second tick 1 for + // the current slot, so that generally the schedule applies to the range [slot N tick 1, + // slot N+1 tick 0). The schedule is shifted right 1 tick from the slot rotation interval so that + // the next leader is always known *before* a rotation occurs + if tick_height == 0 || tick_height % self.seed_rotation_interval == 1 { + self.generate_schedule(tick_height, bank); } } - // Uses the schedule generated by the last call to generate_schedule() to return the - // leader for a given PoH height in round-robin fashion - pub fn get_scheduled_leader(&self, height: u64) -> Option<(Pubkey, u64)> { - if self.use_only_bootstrap_leader { - return Some((self.bootstrap_leader, 0)); - } + // Returns the leader for the requested slot, or None if the slot is out of the schedule bounds + pub fn get_leader_for_slot(&self, slot: u64) -> Option { + trace!("get_leader_for_slot: slot {}", slot); + let tick_height = slot * self.leader_rotation_interval; + let epoch = self.tick_height_to_epoch(tick_height); + trace!( + "get_leader_for_slot: tick_height={} slot={} epoch={} (ce={})", + tick_height, + slot, + epoch, + self.current_epoch + ); - // This covers cases where the schedule isn't yet generated. - if self.last_seed_height == None { - if height <= self.bootstrap_height { - return Some((self.bootstrap_leader, 0)); - } else { - // If there's been no schedule generated yet before we reach the end of the - // bootstrapping period, then the leader is unknown - return None; - } - } - - // If we have a schedule, then just check that we are within the bounds of that - // schedule (last_seed_height, last_seed_height + seed_rotation_interval]. - // Leaders outside of this bound are undefined. - let last_seed_height = self.last_seed_height.unwrap(); - - if height > last_seed_height + self.seed_rotation_interval || height <= last_seed_height { - return None; - } - - // Find index into the leader_schedule that this PoH height maps to. Note by the time - // we reach here, last_seed_height is not None, which implies: - // - // last_seed_height >= self.bootstrap_height - // - // Also we know from the recent range check that height > last_seed_height, so - // height - self.bootstrap_height >= 1, so the below logic is safe. - let leader_slot = (height - self.bootstrap_height - 1) / self.leader_rotation_interval + 1; - let index = (height - last_seed_height - 1) / self.leader_rotation_interval; - let validator_index = index as usize % self.leader_schedule.len(); - Some((self.leader_schedule[validator_index], leader_slot)) - } - - pub fn get_leader_for_slot(&self, slot_height: u64) -> Option { - let tick_height = self.slot_height_to_first_tick_height(slot_height); - self.get_scheduled_leader(tick_height).map(|(id, _)| id) - } - - #[cfg(test)] - pub fn set_leader_schedule(&mut self, schedule: Vec) { - self.leader_schedule = schedule; - } - - // Maps the nth slot (where n == slot_height) to the tick height of - // the first tick for that slot - fn slot_height_to_first_tick_height(&self, slot_height: u64) -> u64 { - if slot_height == 0 { - 0 + if epoch > self.current_epoch { + warn!( + "get_leader_for_slot: leader unknown for epoch {}, which is larger than {}", + epoch, self.current_epoch + ); + None + } else if epoch < self.current_epoch.saturating_sub(1) { + warn!( + "get_leader_for_slot: leader unknown for epoch {}, which is less than {}", + epoch, + self.current_epoch.saturating_sub(1) + ); + None } else { - (slot_height - 1) * self.leader_rotation_interval + self.bootstrap_height + 1 + let schedule = &self.epoch_schedule[(self.current_epoch - epoch) as usize]; + if schedule.is_empty() { + panic!("leader_schedule is empty"); // Should never happen + } + + let first_tick_in_epoch = epoch * self.seed_rotation_interval; + let slot_index = (tick_height - first_tick_in_epoch) / self.leader_rotation_interval; + + // Round robin through each node in the schedule + Some(schedule[slot_index as usize % schedule.len()]) } } // TODO: We use a HashSet for now because a single validator could potentially register // multiple vote account. Once that is no longer possible (see the TODO in vote_program.rs, // process_transaction(), case VoteInstruction::RegisterAccount), we can use a vector. - fn get_active_set(&mut self, height: u64, bank: &Bank) -> HashSet { - let upper_bound = height; - let lower_bound = height.saturating_sub(self.active_window_length); + fn get_active_set(&mut self, tick_height: u64, bank: &Bank) -> HashSet { + let upper_bound = tick_height; + let lower_bound = tick_height.saturating_sub(self.active_window_length); + trace!( + "get_active_set: vote bounds ({}, {})", + lower_bound, + upper_bound + ); { let accounts = bank.accounts.accounts_db.read().unwrap(); @@ -336,11 +218,12 @@ impl LeaderScheduler { .filter_map(|account| { if vote_program::check_id(&account.owner) { if let Ok(vote_state) = VoteState::deserialize(&account.userdata) { + trace!("get_active_set: account vote_state: {:?}", vote_state); return vote_state .votes .back() .filter(|vote| { - vote.tick_height > lower_bound + vote.tick_height >= lower_bound && vote.tick_height <= upper_bound }) .map(|_| vote_state.staker_id); @@ -353,70 +236,96 @@ impl LeaderScheduler { } } - // Called every seed_rotation_interval entries, generates the leader schedule - // for the range of entries: (height, height + seed_rotation_interval] - fn generate_schedule(&mut self, height: u64, bank: &Bank) { - assert!(height >= self.bootstrap_height); - assert!((height - self.bootstrap_height) % self.seed_rotation_interval == 0); - let seed = Self::calculate_seed(height); - self.seed = seed; - let active_set = self.get_active_set(height, &bank); + // Updates the leader schedule to include ticks from tick_height to the first tick of the next epoch + fn generate_schedule(&mut self, tick_height: u64, bank: &Bank) { + let epoch = if tick_height == 0 { + 0 + } else { + self.tick_height_to_epoch(tick_height) + 1 + }; + trace!( + "generate_schedule: tick_height={} (epoch={})", + tick_height, + epoch + ); + if epoch < self.current_epoch { + // Don't support going backwards for implementation convenience + panic!( + "Unable to generate the schedule for epoch < current_epoch ({} < {})", + epoch, self.current_epoch + ); + } else if epoch > self.current_epoch + 1 { + // Don't support skipping epochs going forwards for implementation convenience + panic!( + "Unable to generate the schedule for epoch > current_epoch + 1 ({} > {})", + epoch, + self.current_epoch + 1 + ); + } + + if epoch > self.current_epoch { + self.epoch_schedule[1] = self.epoch_schedule[0].clone(); + self.current_epoch = epoch; + } + + self.seed = Self::calculate_seed(tick_height); + let active_set = self.get_active_set(tick_height, &bank); let ranked_active_set = Self::rank_active_set(bank, active_set.iter()); - // Handle case where there are no active validators with - // non-zero stake. In this case, use the bootstrap leader for - // the upcoming rounds if ranked_active_set.is_empty() { - self.last_seed_height = Some(height); - self.leader_schedule = vec![self.bootstrap_leader]; - self.last_seed_height = Some(height); - return; - } + info!( + "generate_schedule: empty ranked_active_set at tick_height {}, using leader_schedule from previous epoch", + tick_height, + ); + } else { + let (mut validator_rankings, total_stake) = ranked_active_set.iter().fold( + (Vec::with_capacity(ranked_active_set.len()), 0), + |(mut ids, total_stake), (pubkey, stake)| { + ids.push(**pubkey); + (ids, total_stake + stake) + }, + ); - let (mut validator_rankings, total_stake) = ranked_active_set.iter().fold( - (Vec::with_capacity(ranked_active_set.len()), 0), - |(mut ids, total_stake), (pk, stake)| { - ids.push(**pk); - (ids, total_stake + stake) - }, - ); + // Choose a validator to be the first slot leader in the new schedule + let ordered_account_stake = ranked_active_set.into_iter().map(|(_, stake)| stake); + let start_index = Self::choose_account(ordered_account_stake, self.seed, total_stake); + validator_rankings.rotate_left(start_index); - // Choose the validator that will be the first to be the leader in this new - // schedule - let ordered_account_stake = ranked_active_set.into_iter().map(|(_, stake)| stake); - let start_index = Self::choose_account(ordered_account_stake, self.seed, total_stake); - validator_rankings.rotate_left(start_index); + // If possible try to avoid having the same slot leader twice in a row, but + // if there's only one leader to choose from then we have no other choice + if validator_rankings.len() > 1 && tick_height > 0 { + let last_slot_leader = self + .get_leader_for_slot(self.tick_height_to_slot(tick_height - 1)) + .expect("Previous leader schedule should still exist"); + let next_slot_leader = validator_rankings[0]; - // There are only seed_rotation_interval / self.leader_rotation_interval slots, so - // we only need to keep at most that many validators in the schedule - let slots_per_epoch = self.seed_rotation_interval / self.leader_rotation_interval; - - // If possible, try to avoid having the same leader twice in a row, but - // if there's only one leader to choose from, then we have no other choice - if validator_rankings.len() > 1 { - let (old_epoch_last_leader, _) = self - .get_scheduled_leader(height - 1) - .expect("Previous leader schedule should still exist"); - let new_epoch_start_leader = validator_rankings[0]; - - if old_epoch_last_leader == new_epoch_start_leader { - if slots_per_epoch == 1 { - // If there is only one slot per epoch, and the same leader as the last slot - // of the previous epoch was chosen, then pick the next leader in the - // rankings instead - validator_rankings[0] = validator_rankings[1]; - } else { - // If there is more than one leader in the schedule, truncate and set the most - // recent leader to the back of the line. This way that node will still remain - // in the rotation, just at a later slot. - validator_rankings.truncate(slots_per_epoch as usize); - validator_rankings.rotate_left(1); + if last_slot_leader == next_slot_leader { + let slots_per_epoch = + self.seed_rotation_interval / self.leader_rotation_interval; + if slots_per_epoch == 1 { + // If there is only one slot per epoch, and the same leader as the last slot + // of the previous epoch was chosen, then pick the next leader in the + // rankings instead + validator_rankings[0] = validator_rankings[1]; + } else { + // If there is more than one leader in the schedule, truncate and set the most + // recent leader to the back of the line. This way that node will still remain + // in the rotation, just at a later slot. + validator_rankings.truncate(slots_per_epoch as usize); + validator_rankings.rotate_left(1); + } } } + self.epoch_schedule[0] = validator_rankings; } - self.leader_schedule = validator_rankings; - self.last_seed_height = Some(height); + assert!(!self.epoch_schedule[0].is_empty()); + trace!( + "generate_schedule: schedule for ticks ({}, {}): {:?} ", + tick_height, + tick_height + self.seed_rotation_interval, + self.epoch_schedule[0] + ); } fn rank_active_set<'a, I>(bank: &Bank, active: I) -> Vec<(&'a Pubkey, u64)> @@ -424,30 +333,28 @@ impl LeaderScheduler { I: Iterator, { let mut active_accounts: Vec<(&'a Pubkey, u64)> = active - .filter_map(|pk| { - let stake = bank.get_balance(pk); + .filter_map(|pubkey| { + let stake = bank.get_balance(pubkey); if stake > 0 { - Some((pk, stake as u64)) + Some((pubkey, stake as u64)) } else { None } }) .collect(); - active_accounts.sort_by( - |(pk1, t1), (pk2, t2)| { - if t1 == t2 { - pk1.cmp(&pk2) - } else { - t1.cmp(&t2) - } - }, - ); + active_accounts.sort_by(|(pubkey1, stake1), (pubkey2, stake2)| { + if stake1 == stake2 { + pubkey1.cmp(&pubkey2) + } else { + stake1.cmp(&stake2) + } + }); active_accounts } - fn calculate_seed(height: u64) -> u64 { - let hash = hash(&serialize(&height).unwrap()); + fn calculate_seed(tick_height: u64) -> u64 { + let hash = hash(&serialize(&tick_height).unwrap()); let bytes = hash.as_ref(); let mut rdr = Cursor::new(bytes); rdr.read_u64::().unwrap() @@ -473,34 +380,55 @@ impl LeaderScheduler { chosen_account } -} -impl Default for LeaderScheduler { - // Create a dummy leader scheduler - fn default() -> Self { - let id = Pubkey::default(); - Self::from_bootstrap_leader(id) + #[cfg(test)] + pub fn reset(&mut self) { + self.current_epoch = 0; + self.epoch_schedule = [Vec::new(), Vec::new()]; + } + + /// Force a schedule for the first epoch + #[cfg(test)] + pub fn set_leader_schedule(&mut self, schedule: Vec) { + assert!(!schedule.is_empty()); + self.current_epoch = 0; + self.epoch_schedule[0] = schedule; + self.epoch_schedule[1] = Vec::new(); } } -// Create two entries so that the node with keypair == active_keypair -// is in the active set for leader selection: +impl Default for LeaderScheduler { + fn default() -> Self { + let config = Default::default(); + Self::new(&config) + } +} + +// Create entries such the node identified by active_keypair +// will be added to the active set for leader selection: // 1) Give the node a nonzero number of tokens, // 2) A vote from the validator pub fn make_active_set_entries( active_keypair: &Arc, token_source: &Keypair, + stake: u64, + tick_height_to_vote_on: u64, last_entry_id: &Hash, last_tick_id: &Hash, num_ending_ticks: u64, ) -> (Vec, VotingKeypair) { - // 1) Create transfer token entry - let transfer_tx = - SystemTransaction::new_account(&token_source, active_keypair.pubkey(), 3, *last_tick_id, 0); + // 1) Assume the active_keypair node has no tokens staked + let transfer_tx = SystemTransaction::new_account( + &token_source, + active_keypair.pubkey(), + stake, + *last_tick_id, + 0, + ); let transfer_entry = Entry::new(last_entry_id, 0, 1, vec![transfer_tx]); let mut last_entry_id = transfer_entry.id; - // 2) Create and register the vote account + // 2) Create and register a vote account for active_keypair let voting_keypair = VotingKeypair::new_local(active_keypair); let vote_account_id = voting_keypair.pubkey(); @@ -510,7 +438,8 @@ pub fn make_active_set_entries( last_entry_id = new_vote_account_entry.id; // 3) Create vote entry - let vote_tx = VoteTransaction::new_vote(&voting_keypair, 1, *last_tick_id, 0); + let vote_tx = + VoteTransaction::new_vote(&voting_keypair, tick_height_to_vote_on, *last_tick_id, 0); let vote_entry = Entry::new(&last_entry_id, 0, 1, vec![vote_tx]); last_entry_id = vote_entry.id; @@ -525,10 +454,10 @@ pub fn make_active_set_entries( pub mod tests { use super::*; use crate::bank::Bank; - use crate::genesis_block::GenesisBlock; + use crate::genesis_block::{GenesisBlock, BOOTSTRAP_LEADER_TOKENS}; use crate::leader_scheduler::{ - LeaderScheduler, LeaderSchedulerConfig, DEFAULT_BOOTSTRAP_HEIGHT, - DEFAULT_SEED_ROTATION_INTERVAL, DEFAULT_TICKS_PER_SLOT, + LeaderScheduler, LeaderSchedulerConfig, DEFAULT_SEED_ROTATION_INTERVAL, + DEFAULT_TICKS_PER_SLOT, }; use crate::voting_keypair::VotingKeypair; use hashbrown::HashSet; @@ -563,39 +492,36 @@ pub mod tests { bank.process_transaction(&tx).unwrap(); } - fn push_vote(voting_keypair: &VotingKeypair, bank: &Bank, height: u64, last_id: Hash) { - let new_vote_tx = VoteTransaction::new_vote(voting_keypair, height, last_id, 0); + fn push_vote(voting_keypair: &VotingKeypair, bank: &Bank, tick_height: u64, last_id: Hash) { + let new_vote_tx = VoteTransaction::new_vote(voting_keypair, tick_height, last_id, 0); bank.process_transaction(&new_vote_tx).unwrap(); } fn run_scheduler_test( num_validators: usize, - bootstrap_height: u64, leader_rotation_interval: u64, seed_rotation_interval: u64, ) { + info!( + "run_scheduler_test({}, {}, {})", + num_validators, leader_rotation_interval, seed_rotation_interval + ); // Allow the validators to be in the active window for the entire test - let active_window_length = seed_rotation_interval + bootstrap_height; + let active_window_length = seed_rotation_interval; // Set up the LeaderScheduler struct - let bootstrap_leader_id = Keypair::new().pubkey(); let leader_scheduler_config = LeaderSchedulerConfig::new( - bootstrap_height, leader_rotation_interval, seed_rotation_interval, active_window_length, ); - let mut leader_scheduler = LeaderScheduler::new(&leader_scheduler_config); - leader_scheduler.bootstrap_leader = bootstrap_leader_id; - // Create the bank and validators, which are inserted in order of account balance let num_vote_account_tokens = 1; - let (genesis_block, mint_keypair) = GenesisBlock::new( - (((num_validators + 1) / 2) * (num_validators + 1) - + num_vote_account_tokens * num_validators) as u64, - ); - let bank = Bank::new(&genesis_block); + let (genesis_block, mint_keypair) = GenesisBlock::new(10_000); + info!("bootstrap_leader_id: {}", genesis_block.bootstrap_leader_id); + let bank = + Bank::new_with_leader_scheduler_config(&genesis_block, Some(&leader_scheduler_config)); let mut validators = vec![]; let last_id = genesis_block.last_id(); for i in 0..num_validators { @@ -603,14 +529,11 @@ pub mod tests { let new_pubkey = new_validator.pubkey(); let voting_keypair = VotingKeypair::new_local(&new_validator); validators.push(new_pubkey); + let stake = (i + 42) as u64; + info!("validator {}: stake={} pubkey={}", i, stake, new_pubkey); // Give the validator some tokens - bank.transfer( - (i + 1 + num_vote_account_tokens) as u64, - &mint_keypair, - new_pubkey, - last_id, - ) - .unwrap(); + bank.transfer(stake, &mint_keypair, new_pubkey, last_id) + .unwrap(); // Create a vote account new_vote_account( @@ -623,63 +546,58 @@ pub mod tests { // Vote to make the validator part of the active set for the entire test // (we made the active_window_length large enough at the beginning of the test) - push_vote(&voting_keypair, &bank, 1, genesis_block.last_id()); + push_vote( + &voting_keypair, + &bank, + seed_rotation_interval, + genesis_block.last_id(), + ); } - // The scheduled leader during the bootstrapping period (assuming a seed + schedule - // haven't been generated, otherwise that schedule takes precendent) should always - // be the bootstrap leader + let mut leader_scheduler = LeaderScheduler::new(&leader_scheduler_config); + // Generate the schedule for first epoch, bootstrap_leader will be the only leader + leader_scheduler.generate_schedule(0, &bank); + + // The leader outside of the newly generated schedule window: + // (0, seed_rotation_interval] + info!("yyy"); assert_eq!( - leader_scheduler.get_scheduled_leader(0), - Some((bootstrap_leader_id, 0)) + leader_scheduler.get_leader_for_slot(0), + Some(genesis_block.bootstrap_leader_id) ); + info!("xxxx"); assert_eq!( - leader_scheduler.get_scheduled_leader(bootstrap_height - 1), - Some((bootstrap_leader_id, 0)) - ); - assert_eq!( - leader_scheduler.get_scheduled_leader(bootstrap_height), - Some((bootstrap_leader_id, 0)) - ); - assert_eq!( - leader_scheduler.get_scheduled_leader(bootstrap_height + 1), + leader_scheduler + .get_leader_for_slot(leader_scheduler.tick_height_to_slot(seed_rotation_interval)), None ); - // Generate the schedule at the end of the bootstrapping period, should be the - // same leader for the next leader_rotation_interval entries - leader_scheduler.generate_schedule(bootstrap_height, &bank); + // Generate schedule for second epoch. This schedule won't be used but the schedule for + // the third epoch cannot be generated without an existing schedule for the second epoch + leader_scheduler.generate_schedule(1, &bank); - // The leader outside of the newly generated schedule window: - // (bootstrap_height, bootstrap_height + seed_rotation_interval] - // should be undefined - assert_eq!( - leader_scheduler.get_scheduled_leader(bootstrap_height), - None, - ); + // Generate schedule for third epoch to ensure the bootstrap leader will not be added to + // the schedule, as the bootstrap leader did not vote in the second epoch but all other + // validators did + leader_scheduler.generate_schedule(seed_rotation_interval + 1, &bank); - assert_eq!( - leader_scheduler.get_scheduled_leader(bootstrap_height + seed_rotation_interval + 1), - None, - ); - - // For the next seed_rotation_interval entries, call get_scheduled_leader every + // For the next seed_rotation_interval entries, call get_leader_for_slot every // leader_rotation_interval entries, and the next leader should be the next validator // in order of stake - - // Note: seed_rotation_interval must be divisible by leader_rotation_interval, enforced - // by the LeaderScheduler constructor - let num_rounds = seed_rotation_interval / leader_rotation_interval; + let num_slots = seed_rotation_interval / leader_rotation_interval; let mut start_leader_index = None; - for i in 0..num_rounds { - let begin_height = bootstrap_height + i * leader_rotation_interval + 1; - let (current_leader, slot) = leader_scheduler - .get_scheduled_leader(begin_height) + for i in 0..num_slots { + let tick_height = 2 * seed_rotation_interval + i * leader_rotation_interval; + info!("iteration {}: tick_height={}", i, tick_height); + let slot = leader_scheduler.tick_height_to_slot(tick_height); + let current_leader = leader_scheduler + .get_leader_for_slot(slot) .expect("Expected a leader from scheduler"); + info!("current_leader={} slot={}", current_leader, slot); // Note: The "validators" vector is already sorted by stake, so the expected order // for the leader schedule can be derived by just iterating through the vector - // in order. The only excpetion is for the bootstrap leader in the schedule, we need to + // in order. The only exception is for the bootstrap leader in the schedule, we need to // find the index into the "validators" vector where the schedule begins. if None == start_leader_index { start_leader_index = Some( @@ -693,27 +611,48 @@ pub mod tests { let expected_leader = validators[(start_leader_index.unwrap() + i as usize) % num_validators]; assert_eq!(current_leader, expected_leader); - assert_eq!(slot, i + 1); - // Check that the same leader is in power for the next leader_rotation_interval - 1 entries assert_eq!( - leader_scheduler.get_scheduled_leader(begin_height + leader_rotation_interval - 1), - Some((current_leader, slot)) + slot, + leader_scheduler.tick_height_to_slot(2 * seed_rotation_interval) + i + ); + assert_eq!( + slot, + leader_scheduler.tick_height_to_slot(tick_height + leader_rotation_interval - 1) + ); + assert_eq!( + leader_scheduler.get_leader_for_slot(slot), + Some(current_leader) ); } } + #[test] + fn test_num_ticks_left_in_slot() { + let leader_scheduler = LeaderScheduler::new(&LeaderSchedulerConfig::new(10, 20, 0)); + + assert_eq!(leader_scheduler.num_ticks_left_in_slot(0), 9); + assert_eq!(leader_scheduler.num_ticks_left_in_slot(1), 8); + assert_eq!(leader_scheduler.num_ticks_left_in_slot(8), 1); + assert_eq!(leader_scheduler.num_ticks_left_in_slot(9), 0); + assert_eq!(leader_scheduler.num_ticks_left_in_slot(10), 9); + assert_eq!(leader_scheduler.num_ticks_left_in_slot(11), 8); + assert_eq!(leader_scheduler.num_ticks_left_in_slot(19), 0); + assert_eq!(leader_scheduler.num_ticks_left_in_slot(20), 9); + assert_eq!(leader_scheduler.num_ticks_left_in_slot(21), 8); + } + #[test] fn test_active_set() { + solana_logger::setup(); + let leader_id = Keypair::new().pubkey(); let active_window_length = 1000; + let leader_scheduler_config = LeaderSchedulerConfig::new(100, 100, active_window_length); let (genesis_block, mint_keypair) = GenesisBlock::new_with_leader(10000, leader_id, 500); - let bank = Bank::new(&genesis_block); + let bank = + Bank::new_with_leader_scheduler_config(&genesis_block, Some(&leader_scheduler_config)); - let leader_scheduler_config = - LeaderSchedulerConfig::new(100, 100, 100, active_window_length); - - let mut leader_scheduler = LeaderScheduler::new(&leader_scheduler_config); - leader_scheduler.bootstrap_leader = leader_id; + let bootstrap_ids = to_hashset_owned(&vec![genesis_block.bootstrap_leader_id]); // Insert a bunch of votes at height "start_height" let start_height = 3; @@ -771,25 +710,41 @@ pub mod tests { push_vote( &voting_keypair, &bank, - start_height + active_window_length, + start_height + active_window_length + 1, genesis_block.last_id(), ); } - // Queries for the active set + // Query for the active set at various heights + let mut leader_scheduler = bank.leader_scheduler.write().unwrap(); + + let result = leader_scheduler.get_active_set(0, &bank); + assert_eq!(result, bootstrap_ids); + + let result = leader_scheduler.get_active_set(start_height - 1, &bank); + assert_eq!(result, bootstrap_ids); + let result = leader_scheduler.get_active_set(active_window_length + start_height - 1, &bank); assert_eq!(result, old_ids); let result = leader_scheduler.get_active_set(active_window_length + start_height, &bank); - assert_eq!(result, new_ids); + assert_eq!(result, old_ids); let result = - leader_scheduler.get_active_set(2 * active_window_length + start_height - 1, &bank); + leader_scheduler.get_active_set(active_window_length + start_height + 1, &bank); assert_eq!(result, new_ids); let result = leader_scheduler.get_active_set(2 * active_window_length + start_height, &bank); + assert_eq!(result, new_ids); + + let result = + leader_scheduler.get_active_set(2 * active_window_length + start_height + 1, &bank); + assert_eq!(result, new_ids); + + let result = + leader_scheduler.get_active_set(2 * active_window_length + start_height + 2, &bank); assert!(result.is_empty()); } @@ -809,8 +764,9 @@ pub mod tests { fn test_rank_active_set() { let num_validators: usize = 101; // Give genesis_block sum(1..num_validators) tokens - let (genesis_block, mint_keypair) = - GenesisBlock::new((((num_validators + 1) / 2) * (num_validators + 1)) as u64); + let (genesis_block, mint_keypair) = GenesisBlock::new( + BOOTSTRAP_LEADER_TOKENS + (((num_validators + 1) / 2) * (num_validators + 1)) as u64, + ); let bank = Bank::new(&genesis_block); let mut validators = vec![]; let last_id = genesis_block.last_id(); @@ -827,15 +783,15 @@ pub mod tests { .unwrap(); } - let validators_pk: Vec = validators.iter().map(Keypair::pubkey).collect(); - let result = LeaderScheduler::rank_active_set(&bank, validators_pk.iter()); + let validators_pubkey: Vec = validators.iter().map(Keypair::pubkey).collect(); + let result = LeaderScheduler::rank_active_set(&bank, validators_pubkey.iter()); assert_eq!(result.len(), validators.len()); // Expect the result to be the reverse of the list we passed into the rank_active_set() - for (i, (pk, stake)) in result.into_iter().enumerate() { + for (i, (pubkey, stake)) in result.into_iter().enumerate() { assert_eq!(stake, i as u64 + 1); - assert_eq!(*pk, validators[num_validators - i - 1].pubkey()); + assert_eq!(*pubkey, validators[num_validators - i - 1].pubkey()); } // Transfer all the tokens to a new set of validators, old validators should now @@ -862,21 +818,23 @@ pub mod tests { let result = LeaderScheduler::rank_active_set(&bank, all_validators.iter()); assert_eq!(result.len(), new_validators.len()); - for (i, (pk, balance)) in result.into_iter().enumerate() { + for (i, (pubkey, balance)) in result.into_iter().enumerate() { assert_eq!(balance, i as u64 + 1); - assert_eq!(*pk, new_validators[num_validators - i - 1].pubkey()); + assert_eq!(*pubkey, new_validators[num_validators - i - 1].pubkey()); } // Break ties between validators with the same balances using public key - let (genesis_block, mint_keypair) = GenesisBlock::new(num_validators as u64); + let (genesis_block, mint_keypair) = + GenesisBlock::new(BOOTSTRAP_LEADER_TOKENS + (num_validators + 1) as u64); let bank = Bank::new(&genesis_block); let mut tied_validators_pk = vec![]; let last_id = genesis_block.last_id(); - for _ in 0..num_validators { + for _i in 0..num_validators { let new_validator = Keypair::new(); let new_pubkey = new_validator.pubkey(); tied_validators_pk.push(new_pubkey); + assert!(bank.get_balance(&mint_keypair.pubkey()) > 1); bank.transfer(1, &mint_keypair, new_pubkey, last_id) .unwrap(); } @@ -933,93 +891,85 @@ pub mod tests { } #[test] - fn test_scheduler() { + fn test_scheduler_basic() { + solana_logger::setup(); // Test when the number of validators equals // seed_rotation_interval / leader_rotation_interval, so each validator // is selected once let mut num_validators = 100; - let mut bootstrap_height = 500; let mut leader_rotation_interval = 100; - let mut seed_rotation_interval = leader_rotation_interval * num_validators; + let mut seed_rotation_interval = leader_rotation_interval * num_validators as u64; + run_scheduler_test( num_validators, - bootstrap_height, - leader_rotation_interval as u64, - seed_rotation_interval as u64, + leader_rotation_interval, + seed_rotation_interval, ); // Test when there are fewer validators than // seed_rotation_interval / leader_rotation_interval, so each validator // is selected multiple times num_validators = 3; - bootstrap_height = 500; leader_rotation_interval = 100; seed_rotation_interval = 1000; run_scheduler_test( num_validators, - bootstrap_height, - leader_rotation_interval as u64, - seed_rotation_interval as u64, + leader_rotation_interval, + seed_rotation_interval, ); // Test when there are fewer number of validators than // seed_rotation_interval / leader_rotation_interval, so each validator // may not be selected num_validators = 10; - bootstrap_height = 500; leader_rotation_interval = 100; seed_rotation_interval = 200; run_scheduler_test( num_validators, - bootstrap_height, - leader_rotation_interval as u64, - seed_rotation_interval as u64, + leader_rotation_interval, + seed_rotation_interval, ); // Test when seed_rotation_interval == leader_rotation_interval, // only one validator should be selected num_validators = 10; - bootstrap_height = 1; - leader_rotation_interval = 1 as usize; - seed_rotation_interval = 1 as usize; + leader_rotation_interval = 2; + seed_rotation_interval = 2; run_scheduler_test( num_validators, - bootstrap_height, - leader_rotation_interval as u64, - seed_rotation_interval as u64, + leader_rotation_interval, + seed_rotation_interval, ); } #[test] fn test_scheduler_active_window() { + solana_logger::setup(); + let num_validators = 10; let num_vote_account_tokens = 1; - // Set up the LeaderScheduler struct - let bootstrap_leader_id = Keypair::new().pubkey(); - let bootstrap_height = 500; - let leader_rotation_interval = 100; + // Make sure seed_rotation_interval is big enough so we select all the // validators as part of the schedule each time (we need to check the active window // is the cause of validators being truncated later) + let leader_rotation_interval = 100; let seed_rotation_interval = leader_rotation_interval * num_validators; let active_window_length = seed_rotation_interval; let leader_scheduler_config = LeaderSchedulerConfig::new( - bootstrap_height, leader_rotation_interval, seed_rotation_interval, active_window_length, ); - let mut leader_scheduler = LeaderScheduler::new(&leader_scheduler_config); - leader_scheduler.bootstrap_leader = bootstrap_leader_id; - // Create the bank and validators let (genesis_block, mint_keypair) = GenesisBlock::new( - ((((num_validators + 1) / 2) * (num_validators + 1)) - + (num_vote_account_tokens * num_validators)) as u64, + BOOTSTRAP_LEADER_TOKENS + + ((((num_validators + 1) / 2) * (num_validators + 1)) + + (num_vote_account_tokens * num_validators)) as u64, ); - let bank = Bank::new(&genesis_block); + let bank = + Bank::new_with_leader_scheduler_config(&genesis_block, Some(&leader_scheduler_config)); let mut validators = vec![]; let last_id = genesis_block.last_id(); for i in 0..num_validators { @@ -1045,11 +995,10 @@ pub mod tests { genesis_block.last_id(), ); - // Vote at height i * active_window_length for validator i push_vote( &voting_keypair, &bank, - i * active_window_length + bootstrap_height, + (i + 2) * active_window_length - 1, genesis_block.last_id(), ); } @@ -1057,16 +1006,29 @@ pub mod tests { // Generate schedule every active_window_length entries and check that // validators are falling out of the rotation as they fall out of the // active set + let mut leader_scheduler = bank.leader_scheduler.write().unwrap(); + trace!("bootstrap_leader_id: {}", genesis_block.bootstrap_leader_id); + for i in 0..num_validators { + trace!("validators[{}]: {}", i, validators[i as usize]); + } + assert_eq!(leader_scheduler.current_epoch, 0); + leader_scheduler.generate_schedule(1, &bank); + assert_eq!(leader_scheduler.current_epoch, 1); for i in 0..=num_validators { - leader_scheduler.generate_schedule(i * active_window_length + bootstrap_height, &bank); - let result = &leader_scheduler.leader_schedule; - let expected = if i == num_validators { - bootstrap_leader_id + info!("i === {}", i); + leader_scheduler.generate_schedule((i + 1) * active_window_length, &bank); + assert_eq!(leader_scheduler.current_epoch, i + 2); + if i == 0 { + assert_eq!( + vec![genesis_block.bootstrap_leader_id], + leader_scheduler.epoch_schedule[0], + ); } else { - validators[i as usize] + assert_eq!( + vec![validators[(i - 1) as usize]], + leader_scheduler.epoch_schedule[0], + ); }; - - assert_eq!(vec![expected], *result); } } @@ -1076,17 +1038,25 @@ pub mod tests { let leader_id = leader_keypair.pubkey(); let active_window_length = 1000; let (genesis_block, _mint_keypair) = GenesisBlock::new_with_leader(10000, leader_id, 500); - let bank = Bank::new(&genesis_block); + let leader_scheduler_config = LeaderSchedulerConfig::new(100, 100, active_window_length); + let bank = + Bank::new_with_leader_scheduler_config(&genesis_block, Some(&leader_scheduler_config)); - let leader_scheduler_config = - LeaderSchedulerConfig::new(100, 100, 100, active_window_length); + // Bootstrap leader should be in the active set even without explicit votes + { + let mut leader_scheduler = bank.leader_scheduler.write().unwrap(); + let result = leader_scheduler.get_active_set(0, &bank); + assert_eq!(result, to_hashset_owned(&vec![leader_id])); - let mut leader_scheduler = LeaderScheduler::new(&leader_scheduler_config); - leader_scheduler.bootstrap_leader = leader_id; + let result = leader_scheduler.get_active_set(active_window_length, &bank); + assert_eq!(result, to_hashset_owned(&vec![leader_id])); + + let result = leader_scheduler.get_active_set(active_window_length + 1, &bank); + assert!(result.is_empty()); + } // Check that a node that votes twice in a row will get included in the active // window - let initial_vote_height = 1; let voting_keypair = VotingKeypair::new_local(&leader_keypair); // Create a vote account @@ -1098,78 +1068,155 @@ pub mod tests { genesis_block.last_id(), ); - // Vote twice - push_vote( - &voting_keypair, - &bank, - initial_vote_height, - genesis_block.last_id(), - ); - push_vote( - &voting_keypair, - &bank, - initial_vote_height + 1, - genesis_block.last_id(), - ); + // Vote at tick_height 1 + push_vote(&voting_keypair, &bank, 1, genesis_block.last_id()); - let result = - leader_scheduler.get_active_set(initial_vote_height + active_window_length, &bank); - assert_eq!(result, to_hashset_owned(&vec![leader_id])); - let result = - leader_scheduler.get_active_set(initial_vote_height + active_window_length + 1, &bank); - assert!(result.is_empty()); + { + let mut leader_scheduler = bank.leader_scheduler.write().unwrap(); + let result = leader_scheduler.get_active_set(active_window_length + 1, &bank); + assert_eq!(result, to_hashset_owned(&vec![leader_id])); + + let result = leader_scheduler.get_active_set(active_window_length + 2, &bank); + assert!(result.is_empty()); + } + + // Vote at tick_height 2 + push_vote(&voting_keypair, &bank, 2, genesis_block.last_id()); + + { + let mut leader_scheduler = bank.leader_scheduler.write().unwrap(); + let result = leader_scheduler.get_active_set(active_window_length + 2, &bank); + assert_eq!(result, to_hashset_owned(&vec![leader_id])); + + let result = leader_scheduler.get_active_set(active_window_length + 3, &bank); + assert!(result.is_empty()); + } } #[test] - fn test_update_height() { - let bootstrap_leader_id = Keypair::new().pubkey(); - let bootstrap_height = 500; + fn test_update_tick_height() { + solana_logger::setup(); + let leader_rotation_interval = 100; - // Make sure seed_rotation_interval is big enough so we select all the - // validators as part of the schedule each time (we need to check the active window - // is the cause of validators being truncated later) - let seed_rotation_interval = leader_rotation_interval; + let seed_rotation_interval = 2 * leader_rotation_interval; let active_window_length = 1; let leader_scheduler_config = LeaderSchedulerConfig::new( - bootstrap_height, leader_rotation_interval, seed_rotation_interval, active_window_length, ); - let mut leader_scheduler = LeaderScheduler::new(&leader_scheduler_config); - leader_scheduler.bootstrap_leader = bootstrap_leader_id; - // Check that the generate_schedule() function is being called by the - // update_height() function at the correct entry heights. - let bank = Bank::default(); - leader_scheduler.update_height(bootstrap_height - 1, &bank); - assert_eq!(leader_scheduler.last_seed_height, None); - leader_scheduler.update_height(bootstrap_height, &bank); - assert_eq!(leader_scheduler.last_seed_height, Some(bootstrap_height)); - leader_scheduler.update_height(bootstrap_height + seed_rotation_interval - 1, &bank); - assert_eq!(leader_scheduler.last_seed_height, Some(bootstrap_height)); - leader_scheduler.update_height(bootstrap_height + seed_rotation_interval, &bank); - assert_eq!( - leader_scheduler.last_seed_height, - Some(bootstrap_height + seed_rotation_interval) + // update_tick_height() function at the correct entry heights. + let (genesis_block, _) = GenesisBlock::new(10_000); + let bank = + Bank::new_with_leader_scheduler_config(&genesis_block, Some(&leader_scheduler_config)); + let mut leader_scheduler = bank.leader_scheduler.write().unwrap(); + info!( + "bootstrap_leader_id: {:?}", + genesis_block.bootstrap_leader_id ); + assert_eq!(bank.tick_height(), 0); + + // + // tick_height == 0 is a special case + // + leader_scheduler.update_tick_height(0, &bank); + assert_eq!(leader_scheduler.current_epoch, 0); + // The schedule for epoch 0 is known + assert_eq!( + leader_scheduler.get_leader_for_slot(0), + Some(genesis_block.bootstrap_leader_id) + ); + assert_eq!( + leader_scheduler.get_leader_for_slot(1), + Some(genesis_block.bootstrap_leader_id) + ); + // The schedule for epoch 1 is unknown + assert_eq!(leader_scheduler.get_leader_for_slot(2), None,); + + // + // Check various tick heights in epoch 0, and tick 0 of epoch 1 + // + for tick_height in &[ + 1, + leader_rotation_interval, + leader_rotation_interval + 1, + seed_rotation_interval - 1, + seed_rotation_interval, + ] { + info!("Checking tick_height {}", *tick_height); + leader_scheduler.update_tick_height(*tick_height, &bank); + assert_eq!(leader_scheduler.current_epoch, 1); + // The schedule for epoch 0 is known + assert_eq!( + leader_scheduler.get_leader_for_slot(0), + Some(genesis_block.bootstrap_leader_id) + ); + assert_eq!( + leader_scheduler.get_leader_for_slot(1), + Some(genesis_block.bootstrap_leader_id) + ); + // The schedule for epoch 1 is known + assert_eq!( + leader_scheduler.get_leader_for_slot(2), + Some(genesis_block.bootstrap_leader_id) + ); + assert_eq!( + leader_scheduler.get_leader_for_slot(3), + Some(genesis_block.bootstrap_leader_id) + ); + // The schedule for epoch 2 is unknown + assert_eq!(leader_scheduler.get_leader_for_slot(4), None); + } + + // + // Check various tick heights in epoch 1, and tick 0 of epoch 2 + // + for tick_height in &[ + seed_rotation_interval + 1, + seed_rotation_interval + leader_rotation_interval, + seed_rotation_interval + leader_rotation_interval + 1, + seed_rotation_interval + seed_rotation_interval - 1, + seed_rotation_interval + seed_rotation_interval, + ] { + info!("Checking tick_height {}", *tick_height); + leader_scheduler.update_tick_height(*tick_height, &bank); + assert_eq!(leader_scheduler.current_epoch, 2); + // The schedule for epoch 0 is unknown + assert_eq!(leader_scheduler.get_leader_for_slot(0), None); + assert_eq!(leader_scheduler.get_leader_for_slot(1), None); + // The schedule for epoch 1 is known + assert_eq!( + leader_scheduler.get_leader_for_slot(2), + Some(genesis_block.bootstrap_leader_id) + ); + assert_eq!( + leader_scheduler.get_leader_for_slot(3), + Some(genesis_block.bootstrap_leader_id) + ); + // The schedule for epoch 2 is known + assert_eq!( + leader_scheduler.get_leader_for_slot(4), + Some(genesis_block.bootstrap_leader_id) + ); + assert_eq!( + leader_scheduler.get_leader_for_slot(5), + Some(genesis_block.bootstrap_leader_id) + ); + // The schedule for epoch 3 is unknown + assert_eq!(leader_scheduler.get_leader_for_slot(6), None); + } } #[test] fn test_constructors() { - let bootstrap_leader_id = Keypair::new().pubkey(); - // Check defaults for LeaderScheduler let leader_scheduler_config = LeaderSchedulerConfig::default(); let leader_scheduler = LeaderScheduler::new(&leader_scheduler_config); - assert_eq!(leader_scheduler.bootstrap_leader, Pubkey::default()); - - assert_eq!(leader_scheduler.bootstrap_height, DEFAULT_BOOTSTRAP_HEIGHT); - assert_eq!( leader_scheduler.leader_rotation_interval, DEFAULT_TICKS_PER_SLOT @@ -1180,22 +1227,17 @@ pub mod tests { ); // Check actual arguments for LeaderScheduler - let bootstrap_height = 500; let leader_rotation_interval = 100; let seed_rotation_interval = 200; let active_window_length = 1; let leader_scheduler_config = LeaderSchedulerConfig::new( - bootstrap_height, leader_rotation_interval, seed_rotation_interval, active_window_length, ); - let mut leader_scheduler = LeaderScheduler::new(&leader_scheduler_config); - leader_scheduler.bootstrap_leader = bootstrap_leader_id; - - assert_eq!(leader_scheduler.bootstrap_height, bootstrap_height); + let leader_scheduler = LeaderScheduler::new(&leader_scheduler_config); assert_eq!( leader_scheduler.leader_rotation_interval, @@ -1210,25 +1252,21 @@ pub mod tests { fn run_consecutive_leader_test(num_slots_per_epoch: u64, add_validator: bool) { let bootstrap_leader_keypair = Arc::new(Keypair::new()); let bootstrap_leader_id = bootstrap_leader_keypair.pubkey(); - let bootstrap_height = 500; let leader_rotation_interval = 100; let seed_rotation_interval = num_slots_per_epoch * leader_rotation_interval; - let active_window_length = bootstrap_height + seed_rotation_interval; + let active_window_length = seed_rotation_interval; let leader_scheduler_config = LeaderSchedulerConfig::new( - bootstrap_height, leader_rotation_interval, seed_rotation_interval, active_window_length, ); - let mut leader_scheduler = LeaderScheduler::new(&leader_scheduler_config); - leader_scheduler.bootstrap_leader = bootstrap_leader_id; - // Create mint and bank let (genesis_block, mint_keypair) = - GenesisBlock::new_with_leader(10000, bootstrap_leader_id, 0); - let bank = Bank::new(&genesis_block); + GenesisBlock::new_with_leader(10_000, bootstrap_leader_id, BOOTSTRAP_LEADER_TOKENS); + let bank = + Bank::new_with_leader_scheduler_config(&genesis_block, Some(&leader_scheduler_config)); let last_id = genesis_block.last_id(); let initial_vote_height = 1; @@ -1261,7 +1299,7 @@ pub mod tests { // validator stake is always 1, then the rankings will always be // [(validator, 1), (leader, leader_stake)]. Thus we just need to make sure that // seed % (leader_stake + 1) > 0 to make sure that the leader is picked again. - let seed = LeaderScheduler::calculate_seed(bootstrap_height); + let seed = LeaderScheduler::calculate_seed(0); let leader_stake = if seed % 3 == 0 { 3 } else { 2 }; let vote_account_tokens = 1; @@ -1291,14 +1329,19 @@ pub mod tests { genesis_block.last_id(), ); - leader_scheduler.generate_schedule(bootstrap_height, &bank); + let mut leader_scheduler = LeaderScheduler::default(); + leader_scheduler.generate_schedule(0, &bank); + assert_eq!(leader_scheduler.current_epoch, 0); + assert_eq!(leader_scheduler.epoch_schedule[0], [bootstrap_leader_id]); // Make sure the validator, not the leader is selected on the first slot of the // next epoch + leader_scheduler.generate_schedule(1, &bank); + assert_eq!(leader_scheduler.current_epoch, 1); if add_validator { - assert!(leader_scheduler.leader_schedule[0] == validator_id); + assert_eq!(leader_scheduler.epoch_schedule[0][0], validator_id); } else { - assert!(leader_scheduler.leader_schedule[0] == bootstrap_leader_id); + assert_eq!(leader_scheduler.epoch_schedule[0][0], bootstrap_leader_id); } } @@ -1314,184 +1357,4 @@ pub mod tests { run_consecutive_leader_test(2, false); run_consecutive_leader_test(10, false); } - - #[test] - fn test_max_height_for_leader() { - let bootstrap_leader_keypair = Arc::new(Keypair::new()); - let bootstrap_leader_id = bootstrap_leader_keypair.pubkey(); - let bootstrap_height = 500; - let leader_rotation_interval = 100; - let seed_rotation_interval = 2 * leader_rotation_interval; - let active_window_length = bootstrap_height + seed_rotation_interval; - - let leader_scheduler_config = LeaderSchedulerConfig::new( - bootstrap_height, - leader_rotation_interval, - seed_rotation_interval, - active_window_length, - ); - - let mut leader_scheduler = LeaderScheduler::new(&leader_scheduler_config); - leader_scheduler.bootstrap_leader = bootstrap_leader_id; - - // Create mint and bank - let (genesis_block, mint_keypair) = - GenesisBlock::new_with_leader(10000, bootstrap_leader_id, 0); - let bank = Bank::new(&genesis_block); - let last_id = genesis_block.last_id(); - let initial_vote_height = 1; - - // No schedule generated yet, so for all heights <= bootstrap height, the - // max height will be bootstrap height - assert_eq!( - leader_scheduler.max_height_for_leader(0), - Some(bootstrap_height) - ); - assert_eq!( - leader_scheduler.max_height_for_leader(bootstrap_height - 1), - Some(bootstrap_height) - ); - assert_eq!( - leader_scheduler.max_height_for_leader(bootstrap_height), - Some(bootstrap_height) - ); - assert_eq!( - leader_scheduler.max_height_for_leader(bootstrap_height + 1), - None - ); - - // Test when the active set == 1 node - - // Generate schedule where the bootstrap leader will be the only - // choice because the active set is empty. Thus if the schedule - // was generated on PoH height bootstrap_height + n * seed_rotation_interval, - // then the same leader will be in power until PoH height - // bootstrap_height + (n + 1) * seed_rotation_interval - leader_scheduler.generate_schedule(bootstrap_height, &bank); - assert_eq!( - leader_scheduler.max_height_for_leader(bootstrap_height), - None - ); - assert_eq!( - leader_scheduler.max_height_for_leader(bootstrap_height - 1), - None - ); - assert_eq!( - leader_scheduler.max_height_for_leader(bootstrap_height + 1), - Some(bootstrap_height + seed_rotation_interval) - ); - leader_scheduler.generate_schedule(bootstrap_height + seed_rotation_interval, &bank); - assert_eq!( - leader_scheduler.max_height_for_leader(bootstrap_height + seed_rotation_interval + 1), - Some(bootstrap_height + 2 * seed_rotation_interval) - ); - assert_eq!( - leader_scheduler.max_height_for_leader(bootstrap_height + seed_rotation_interval), - None - ); - - leader_scheduler.reset(); - - // Now test when the active set > 1 node - - // Create and add validator to the active set - let validator_keypair = Arc::new(Keypair::new()); - let validator_id = validator_keypair.pubkey(); - - // Create a vote account for the validator - bank.transfer(5, &mint_keypair, validator_id, last_id) - .unwrap(); - let voting_keypair = VotingKeypair::new_local(&validator_keypair); - new_vote_account( - &validator_keypair, - &voting_keypair, - &bank, - 1, - genesis_block.last_id(), - ); - - push_vote( - &voting_keypair, - &bank, - initial_vote_height, - genesis_block.last_id(), - ); - - // Create a vote account for the leader - bank.transfer(5, &mint_keypair, bootstrap_leader_id, last_id) - .unwrap(); - let voting_keypair = VotingKeypair::new_local(&bootstrap_leader_keypair); - new_vote_account( - &bootstrap_leader_keypair, - &voting_keypair, - &bank, - 1, - genesis_block.last_id(), - ); - - // Add leader to the active set - push_vote( - &voting_keypair, - &bank, - initial_vote_height, - genesis_block.last_id(), - ); - - // Generate the schedule - leader_scheduler.generate_schedule(bootstrap_height, &bank); - - assert_eq!( - leader_scheduler.max_height_for_leader(bootstrap_height), - None - ); - assert_eq!( - leader_scheduler.max_height_for_leader(bootstrap_height + 1), - Some(bootstrap_height + leader_rotation_interval) - ); - assert_eq!( - leader_scheduler.max_height_for_leader(bootstrap_height + leader_rotation_interval - 1), - Some(bootstrap_height + leader_rotation_interval) - ); - assert_eq!( - leader_scheduler.max_height_for_leader(bootstrap_height + leader_rotation_interval), - Some(bootstrap_height + leader_rotation_interval) - ); - assert_eq!( - leader_scheduler.max_height_for_leader(bootstrap_height + leader_rotation_interval + 1), - Some(bootstrap_height + 2 * leader_rotation_interval) - ); - assert_eq!( - leader_scheduler.max_height_for_leader(bootstrap_height + seed_rotation_interval - 1), - Some(bootstrap_height + seed_rotation_interval), - ); - assert_eq!( - leader_scheduler.max_height_for_leader(bootstrap_height + seed_rotation_interval), - Some(bootstrap_height + seed_rotation_interval), - ); - assert_eq!( - leader_scheduler.max_height_for_leader(bootstrap_height + seed_rotation_interval + 1), - None, - ); - - // Generate another schedule - leader_scheduler.generate_schedule(bootstrap_height + seed_rotation_interval, &bank); - - assert_eq!( - leader_scheduler.max_height_for_leader(bootstrap_height + seed_rotation_interval), - None - ); - assert_eq!( - leader_scheduler.max_height_for_leader(bootstrap_height + seed_rotation_interval + 1), - Some(bootstrap_height + seed_rotation_interval + leader_rotation_interval) - ); - assert_eq!( - leader_scheduler.max_height_for_leader(bootstrap_height + 2 * seed_rotation_interval), - Some(bootstrap_height + 2 * seed_rotation_interval) - ); - assert_eq!( - leader_scheduler - .max_height_for_leader(bootstrap_height + 2 * seed_rotation_interval + 1), - None - ); - } } diff --git a/src/poh_recorder.rs b/src/poh_recorder.rs index 39acce7dc2..26554ef897 100644 --- a/src/poh_recorder.rs +++ b/src/poh_recorder.rs @@ -21,11 +21,11 @@ pub struct PohRecorder { poh: Arc>, bank: Arc, sender: Sender>, - max_tick_height: Option, + max_tick_height: u64, } impl PohRecorder { - pub fn max_tick_height(&self) -> Option { + pub fn max_tick_height(&self) -> u64 { self.max_tick_height } @@ -69,7 +69,7 @@ impl PohRecorder { bank: Arc, sender: Sender>, last_entry_id: Hash, - max_tick_height: Option, + max_tick_height: u64, ) -> Self { let poh = Arc::new(Mutex::new(Poh::new(last_entry_id, bank.tick_height()))); PohRecorder { @@ -81,11 +81,10 @@ impl PohRecorder { } fn check_tick_height(&self, poh: &Poh) -> Result<()> { - match self.max_tick_height { - Some(max_tick_height) if poh.tick_height >= max_tick_height => { - Err(Error::PohRecorderError(PohRecorderError::MaxHeightReached)) - } - _ => Ok(()), + if poh.tick_height >= self.max_tick_height { + Err(Error::PohRecorderError(PohRecorderError::MaxHeightReached)) + } else { + Ok(()) } } @@ -127,11 +126,11 @@ mod tests { #[test] fn test_poh_recorder() { - let (genesis_block, _mint_keypair) = GenesisBlock::new(1); + let (genesis_block, _mint_keypair) = GenesisBlock::new(2); let bank = Arc::new(Bank::new(&genesis_block)); let prev_id = bank.last_id(); let (entry_sender, entry_receiver) = channel(); - let mut poh_recorder = PohRecorder::new(bank, entry_sender, prev_id, Some(2)); + let mut poh_recorder = PohRecorder::new(bank, entry_sender, prev_id, 2); //send some data let h1 = hash(b"hello world!"); diff --git a/src/poh_service.rs b/src/poh_service.rs index 22bfff6df7..b33f6b5516 100644 --- a/src/poh_service.rs +++ b/src/poh_service.rs @@ -91,11 +91,8 @@ impl PohService { let res = poh.hash(); if let Err(e) = res { if let Error::PohRecorderError(PohRecorderError::MaxHeightReached) = e { - // Leader rotation should only happen if a max_tick_height was specified - assert!(max_tick_height.is_some()); - to_validator_sender.send(TpuReturnType::LeaderRotation( - max_tick_height.unwrap(), - ))?; + to_validator_sender + .send(TpuReturnType::LeaderRotation(max_tick_height))?; } return Err(e); } @@ -109,9 +106,7 @@ impl PohService { if let Err(e) = res { if let Error::PohRecorderError(PohRecorderError::MaxHeightReached) = e { // Leader rotation should only happen if a max_tick_height was specified - assert!(max_tick_height.is_some()); - to_validator_sender - .send(TpuReturnType::LeaderRotation(max_tick_height.unwrap()))?; + to_validator_sender.send(TpuReturnType::LeaderRotation(max_tick_height))?; } return Err(e); } @@ -147,11 +142,11 @@ mod tests { #[test] fn test_poh_service() { - let (genesis_block, _mint_keypair) = GenesisBlock::new(1); + let (genesis_block, _mint_keypair) = GenesisBlock::new(2); let bank = Arc::new(Bank::new(&genesis_block)); let prev_id = bank.last_id(); let (entry_sender, entry_receiver) = channel(); - let poh_recorder = PohRecorder::new(bank, entry_sender, prev_id, None); + let poh_recorder = PohRecorder::new(bank, entry_sender, prev_id, std::u64::MAX); let exit = Arc::new(AtomicBool::new(false)); let entry_producer: JoinHandle> = { diff --git a/src/replay_stage.rs b/src/replay_stage.rs index d62d1cab22..16ea75f6e3 100644 --- a/src/replay_stage.rs +++ b/src/replay_stage.rs @@ -11,7 +11,6 @@ use crate::entry_stream::EntryStreamHandler; #[cfg(test)] use crate::entry_stream::MockEntryStream as EntryStream; use crate::fullnode::TvuRotationSender; -use crate::leader_scheduler::DEFAULT_TICKS_PER_SLOT; use crate::packet::BlobError; use crate::result::{Error, Result}; use crate::service::Service; @@ -73,7 +72,7 @@ impl ReplayStage { error!("Entry Stream error: {:?}, {:?}", e, stream.socket); }); } - //coalesce all the available entries into a single vote + // Coalesce all the available entries into a single vote submit( influxdb::Point::new("replicate-stage") .add_field("count", influxdb::Value::Integer(entries.len() as i64)) @@ -92,17 +91,22 @@ impl ReplayStage { duration_as_ms(&now.elapsed()) as usize ); - let (current_leader, _) = bank - .get_current_leader() - .expect("Scheduled leader should be calculated by this point"); + let mut num_ticks_to_next_vote = bank + .leader_scheduler + .read() + .unwrap() + .num_ticks_left_in_slot(bank.tick_height()); - // Next vote tick is ceiling of (current tick/ticks per block) - let mut num_ticks_to_next_vote = - DEFAULT_TICKS_PER_SLOT - (bank.tick_height() % DEFAULT_TICKS_PER_SLOT); - let mut start_entry_index = 0; for (i, entry) in entries.iter().enumerate() { inc_new_counter_info!("replicate-stage_bank-tick", bank.tick_height() as usize); if entry.is_tick() { + if num_ticks_to_next_vote == 0 { + num_ticks_to_next_vote = bank + .leader_scheduler + .read() + .unwrap() + .leader_rotation_interval; + } num_ticks_to_next_vote -= 1; } inc_new_counter_info!( @@ -113,7 +117,7 @@ impl ReplayStage { // If we don't process the entry now, the for loop will exit and the entry // will be dropped. if 0 == num_ticks_to_next_vote || (i + 1) == entries.len() { - res = bank.process_entries(&entries[start_entry_index..=i]); + res = bank.process_entries(&entries[0..=i]); if res.is_err() { // TODO: This will return early from the first entry that has an erroneous @@ -122,11 +126,7 @@ impl ReplayStage { // bank.process_entries() was used to process the entries, but doesn't solve the // issue that the bank state was still changed, leading to inconsistencies with the // leader as the leader currently should not be publishing erroneous transactions - inc_new_counter_info!( - "replicate-stage_failed_process_entries", - (i - start_entry_index) - ); - + inc_new_counter_info!("replicate-stage_failed_process_entries", i); break; } @@ -142,19 +142,8 @@ impl ReplayStage { cluster_info.write().unwrap().push_vote(vote); } } - let (scheduled_leader, _) = bank - .get_current_leader() - .expect("Scheduled leader should be calculated by this point"); - - // TODO: Remove this soon once we boot the leader from ClusterInfo - if scheduled_leader != current_leader { - cluster_info.write().unwrap().set_leader(scheduled_leader); - num_entries_to_write = i + 1; - break; - } - - start_entry_index = i + 1; - num_ticks_to_next_vote = DEFAULT_TICKS_PER_SLOT; + num_entries_to_write = i + 1; + break; } } @@ -206,24 +195,26 @@ impl ReplayStage { let (ledger_entry_sender, ledger_entry_receiver) = channel(); let mut entry_stream = entry_stream.cloned().map(EntryStream::new); - let (_, mut current_slot) = bank - .get_current_leader() - .expect("Scheduled leader should be calculated by this point"); - - let mut max_tick_height_for_slot = bank - .leader_scheduler - .read() - .unwrap() - .max_tick_height_for_slot(current_slot); - let exit_ = exit.clone(); let t_replay = Builder::new() .name("solana-replay-stage".to_string()) .spawn(move || { let _exit = Finalizer::new(exit_.clone()); - let (mut last_leader_id, _) = bank - .get_current_leader() - .expect("Scheduled leader should be calculated by this point"); + let mut last_leader_id = Self::get_leader_for_next_tick(&bank); + + let (mut current_slot, mut max_tick_height_for_slot) = { + let tick_height = bank.tick_height(); + let leader_scheduler = bank.leader_scheduler.read().unwrap(); + let current_slot = leader_scheduler.tick_height_to_slot(tick_height + 1); + let first_tick_in_current_slot = + current_slot * leader_scheduler.leader_rotation_interval; + ( + current_slot, + first_tick_in_current_slot + + leader_scheduler.num_ticks_left_in_slot(first_tick_in_current_slot), + ) + }; + // Loop through db_ledger MAX_ENTRY_RECV_PER_ITER entries at a time for each // relevant slot to see if there are any available updates loop { @@ -259,7 +250,7 @@ impl ReplayStage { &last_entry_id, entry_stream.as_mut(), ) { - error!("{:?}", e); + error!("process_entries failed: {:?}", e); } let current_tick_height = bank.tick_height(); @@ -268,11 +259,15 @@ impl ReplayStage { // for leader rotation if max_tick_height_for_slot == current_tick_height { // Check for leader rotation - let leader_id = Self::get_leader(&bank, &cluster_info); + let leader_id = Self::get_leader_for_next_tick(&bank); + + // TODO: Remove this soon once we boot the leader from ClusterInfo + cluster_info.write().unwrap().set_leader(leader_id); + if leader_id != last_leader_id && my_id == leader_id { to_leader_sender .send(TvuReturnType::LeaderRotation( - bank.tick_height(), + current_tick_height, *entry_height.read().unwrap(), *last_entry_id.read().unwrap(), )) @@ -280,11 +275,11 @@ impl ReplayStage { } current_slot += 1; - max_tick_height_for_slot = bank + max_tick_height_for_slot += bank .leader_scheduler .read() .unwrap() - .max_tick_height_for_slot(current_slot); + .leader_rotation_interval; last_leader_id = leader_id; } } @@ -319,15 +314,13 @@ impl ReplayStage { let _ = self.ledger_signal_sender.send(true); } - fn get_leader(bank: &Bank, cluster_info: &Arc>) -> Pubkey { - let (scheduled_leader, _) = bank - .get_current_leader() - .expect("Scheduled leader should be calculated by this point"); - - // TODO: Remove this soon once we boot the leader from ClusterInfo - cluster_info.write().unwrap().set_leader(scheduled_leader); - - scheduled_leader + fn get_leader_for_next_tick(bank: &Bank) -> Pubkey { + let tick_height = bank.tick_height(); + let leader_scheduler = bank.leader_scheduler.read().unwrap(); + let slot = leader_scheduler.tick_height_to_slot(tick_height + 1); + leader_scheduler + .get_leader_for_slot(slot) + .expect("Scheduled leader should be calculated by this point") } } @@ -350,9 +343,7 @@ mod test { use crate::entry::Entry; use crate::fullnode::Fullnode; use crate::genesis_block::GenesisBlock; - use crate::leader_scheduler::{ - make_active_set_entries, LeaderScheduler, LeaderSchedulerConfig, - }; + use crate::leader_scheduler::{make_active_set_entries, LeaderSchedulerConfig}; use crate::replay_stage::ReplayStage; use crate::tvu::TvuReturnType; use crate::voting_keypair::VotingKeypair; @@ -366,6 +357,7 @@ mod test { use std::sync::{Arc, RwLock}; #[test] + #[ignore] // TODO: Fix this test to not send all entries in slot 0 pub fn test_replay_stage_leader_rotation_exit() { solana_logger::setup(); @@ -379,43 +371,38 @@ mod test { let old_leader_id = Keypair::new().pubkey(); // Create a ledger - let num_ending_ticks = 3; let (mint_keypair, my_ledger_path, genesis_entry_height, mut last_id) = create_tmp_sample_ledger( "test_replay_stage_leader_rotation_exit", 10_000, - num_ending_ticks, + 0, old_leader_id, 500, ); + info!("my_id: {:?}", my_id); + info!("old_leader_id: {:?}", old_leader_id); + + // Set up the LeaderScheduler so that my_id becomes the leader for epoch 1 + let leader_rotation_interval = 16; + let leader_scheduler_config = LeaderSchedulerConfig::new( + leader_rotation_interval, + leader_rotation_interval, + leader_rotation_interval, + ); + let my_keypair = Arc::new(my_keypair); - // Write two entries to the ledger so that the validator is in the active set: - // 1) Give the validator a nonzero number of tokens 2) A vote from the validator. - // This will cause leader rotation after the bootstrap height - let (active_set_entries, voting_keypair) = - make_active_set_entries(&my_keypair, &mint_keypair, &last_id, &last_id, 0); + let (active_set_entries, voting_keypair) = make_active_set_entries( + &my_keypair, + &mint_keypair, + 100, + leader_rotation_interval, // add a vote for tick_height = leader_rotation_interval + &last_id, + &last_id, + 0, + ); last_id = active_set_entries.last().unwrap().id; - let initial_tick_height = genesis_entry_height; - let active_set_entries_len = active_set_entries.len() as u64; - let initial_non_tick_height = genesis_entry_height - initial_tick_height; - { - // Set up the LeaderScheduler so that this this node becomes the leader at - // bootstrap_height = num_bootstrap_slots * leader_rotation_interval - let leader_rotation_interval = 16; - let bootstrap_height = 2 * leader_rotation_interval; - assert!((num_ending_ticks as u64) < bootstrap_height); - let leader_scheduler_config = LeaderSchedulerConfig::new( - bootstrap_height, - leader_rotation_interval, - leader_rotation_interval * 2, - bootstrap_height, - ); - - let leader_scheduler = - Arc::new(RwLock::new(LeaderScheduler::new(&leader_scheduler_config))); - let (db_ledger, l_sender, l_receiver) = DbLedger::open_with_signal(&my_ledger_path).unwrap(); let db_ledger = Arc::new(db_ledger); @@ -431,18 +418,22 @@ mod test { .expect("Expected to successfully open genesis block"); // Set up the bank - let (bank, _, last_entry_id) = - Fullnode::new_bank_from_db_ledger(&genesis_block, &db_ledger, leader_scheduler); + let (bank, _, last_entry_id) = Fullnode::new_bank_from_db_ledger( + &genesis_block, + &db_ledger, + Some(&leader_scheduler_config), + ); // Set up the replay stage let (rotation_sender, rotation_receiver) = channel(); let meta = db_ledger.meta().unwrap().unwrap(); let exit = Arc::new(AtomicBool::new(false)); + let bank = Arc::new(bank); let (replay_stage, ledger_writer_recv) = ReplayStage::new( my_id, Some(Arc::new(voting_keypair)), db_ledger.clone(), - Arc::new(bank), + bank.clone(), Arc::new(RwLock::new(cluster_info_me)), exit.clone(), Arc::new(RwLock::new(meta.consumed)), @@ -453,36 +444,26 @@ mod test { l_receiver, ); - // Send enough ticks to trigger leader rotation - let extra_entries = leader_rotation_interval; - let total_entries_to_send = (bootstrap_height + extra_entries) as usize; - let num_hashes = 1; + let total_entries_to_send = 2 * leader_rotation_interval as usize - 2; let mut entries_to_send = vec![]; - while entries_to_send.len() < total_entries_to_send { - let entry = Entry::new(&mut last_id, 0, num_hashes, vec![]); + let entry = Entry::new(&mut last_id, 0, 1, vec![]); last_id = entry.id; entries_to_send.push(entry); } - assert!((num_ending_ticks as u64) < bootstrap_height); - - // Add on the only entries that weren't ticks to the bootstrap height to get the - // total expected entry length - let leader_rotation_index = (bootstrap_height - initial_tick_height) as usize; - let expected_entry_height = - bootstrap_height + initial_non_tick_height + active_set_entries_len; - let expected_last_id = entries_to_send[leader_rotation_index - 1].id; + let expected_entry_height = (active_set_entries.len() + total_entries_to_send) as u64; + let expected_last_id = entries_to_send.last().unwrap().id; // Write the entries to the ledger, replay_stage should get notified of changes db_ledger .write_entries(DEFAULT_SLOT_HEIGHT, meta.consumed, &entries_to_send) .unwrap(); - // Wait for replay_stage to exit and check return value is correct + info!("Wait for replay_stage to exit and check return value is correct"); assert_eq!( Some(TvuReturnType::LeaderRotation( - bootstrap_height, + 2 * leader_rotation_interval - 1, expected_entry_height, expected_last_id, )), @@ -495,24 +476,20 @@ mod test { } ); - // Check that the entries on the ledger writer channel are correct - + info!("Check that the entries on the ledger writer channel are correct"); let mut received_ticks = ledger_writer_recv .recv() - .expect("Expected to recieve an entry on the ledger writer receiver"); + .expect("Expected to receive an entry on the ledger writer receiver"); while let Ok(entries) = ledger_writer_recv.try_recv() { received_ticks.extend(entries); } + assert_eq!(&received_ticks[..], &entries_to_send[..]); - assert_eq!( - &received_ticks[..], - &entries_to_send[..leader_rotation_index] - ); - - //replay stage should continue running even after rotation has happened (tvu never goes down) + // Replay stage should continue running even after rotation has happened (tvu never goes down) assert_eq!(exit.load(Ordering::Relaxed), false); - //force exit + + info!("Close replay_stage"); replay_stage .close() .expect("Expect successful ReplayStage exit"); @@ -529,13 +506,11 @@ mod test { // Create keypair for the leader let leader_id = Keypair::new().pubkey(); - let leader_scheduler = Arc::new(RwLock::new(LeaderScheduler::default())); - let num_ending_ticks = 1; let (_, my_ledger_path, _, _) = create_tmp_sample_ledger( "test_vote_error_replay_stage_correctness", 10_000, - num_ending_ticks, + 1, leader_id, 500, ); @@ -556,7 +531,7 @@ mod test { let genesis_block = GenesisBlock::load(&my_ledger_path) .expect("Expected to successfully open genesis block"); let (bank, entry_height, last_entry_id) = - Fullnode::new_bank_from_db_ledger(&genesis_block, &db_ledger, leader_scheduler); + Fullnode::new_bank_from_db_ledger(&genesis_block, &db_ledger, None); let bank = Arc::new(bank); let (replay_stage, ledger_writer_recv) = ReplayStage::new( my_keypair.pubkey(), @@ -625,7 +600,7 @@ mod test { // 1) Give the validator a nonzero number of tokens 2) A vote from the validator. // This will cause leader rotation after the bootstrap height let (active_set_entries, voting_keypair) = - make_active_set_entries(&my_keypair, &mint_keypair, &last_id, &last_id, 0); + make_active_set_entries(&my_keypair, &mint_keypair, 100, 1, &last_id, &last_id, 0); let mut last_id = active_set_entries.last().unwrap().id; let initial_tick_height = genesis_entry_height; let active_set_entries_len = active_set_entries.len() as u64; @@ -639,15 +614,11 @@ mod test { let num_bootstrap_slots = 2; let bootstrap_height = num_bootstrap_slots * leader_rotation_interval; let leader_scheduler_config = LeaderSchedulerConfig::new( - bootstrap_height, leader_rotation_interval, leader_rotation_interval * 2, bootstrap_height, ); - let leader_scheduler = - Arc::new(RwLock::new(LeaderScheduler::new(&leader_scheduler_config))); - // Set up the cluster info let cluster_info_me = Arc::new(RwLock::new(ClusterInfo::new(my_node.info.clone()))); @@ -673,8 +644,11 @@ mod test { // Set up the bank let genesis_block = GenesisBlock::load(&my_ledger_path) .expect("Expected to successfully open genesis block"); - let (bank, _, last_entry_id) = - Fullnode::new_bank_from_db_ledger(&genesis_block, &db_ledger, leader_scheduler); + let (bank, _, last_entry_id) = Fullnode::new_bank_from_db_ledger( + &genesis_block, + &db_ledger, + Some(&leader_scheduler_config), + ); let voting_keypair = Arc::new(voting_keypair); let bank = Arc::new(bank); @@ -776,7 +750,7 @@ mod test { let voting_keypair = Arc::new(VotingKeypair::new_local(&my_keypair)); let res = ReplayStage::process_entries( entries.clone(), - &Arc::new(Bank::default()), + &Arc::new(Bank::new(&GenesisBlock::new(10_000).0)), &cluster_info_me, Some(&voting_keypair), &ledger_entry_sender, @@ -845,9 +819,10 @@ mod test { let my_keypair = Arc::new(my_keypair); let voting_keypair = Arc::new(VotingKeypair::new_local(&my_keypair)); + let bank = Bank::new(&GenesisBlock::new(123).0); ReplayStage::process_entries( entries.clone(), - &Arc::new(Bank::default()), + &Arc::new(bank), &cluster_info_me, Some(&voting_keypair), &ledger_entry_sender, diff --git a/src/replicator.rs b/src/replicator.rs index b0202dd1b1..0b01ee5d87 100644 --- a/src/replicator.rs +++ b/src/replicator.rs @@ -128,7 +128,7 @@ impl Replicator { { let mut cluster_info_w = cluster_info.write().unwrap(); cluster_info_w.insert_info(leader_info.clone()); - cluster_info_w.set_leader(leader_info.id); + cluster_info_w.set_leader(leader_pubkey); } // Create DbLedger, eventually will simply repurpose the input @@ -182,9 +182,7 @@ impl Replicator { blob_fetch_receiver, retransmit_sender, repair_socket, - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_pubkey, - ))), + Arc::new(RwLock::new(LeaderScheduler::default())), done.clone(), exit.clone(), ); diff --git a/src/rpc.rs b/src/rpc.rs index cac87e3a28..af9e73e703 100644 --- a/src/rpc.rs +++ b/src/rpc.rs @@ -493,7 +493,8 @@ mod tests { #[test] fn test_rpc_new() { - let (genesis_block, alice) = GenesisBlock::new(10_000); + let (genesis_block, alice) = + GenesisBlock::new(10_000 + crate::genesis_block::BOOTSTRAP_LEADER_TOKENS); let bank = Bank::new(&genesis_block); let cluster_info = Arc::new(RwLock::new(ClusterInfo::new(NodeInfo::default()))); let rpc_addr = SocketAddr::new( diff --git a/src/thin_client.rs b/src/thin_client.rs index 5b9f51d46c..098f795e0c 100644 --- a/src/thin_client.rs +++ b/src/thin_client.rs @@ -20,7 +20,7 @@ use solana_metrics::influxdb; use solana_sdk::account::Account; use solana_sdk::hash::Hash; use solana_sdk::pubkey::Pubkey; -use solana_sdk::signature::{Keypair, Signature}; +use solana_sdk::signature::{Keypair, KeypairUtil, Signature}; use solana_sdk::system_transaction::SystemTransaction; use solana_sdk::timing; use solana_sdk::transaction::Transaction; @@ -136,6 +136,13 @@ impl ThinClient { to: Pubkey, last_id: &Hash, ) -> io::Result { + debug!( + "transfer: n={} from={:?} to={:?} last_id={:?}", + n, + keypair.pubkey(), + to, + last_id + ); let now = Instant::now(); let tx = SystemTransaction::new_account(keypair, to, n, *last_id, 0); let result = self.transfer_signed(&tx); @@ -439,7 +446,7 @@ pub fn retry_get_balance( pub fn new_fullnode(ledger_name: &'static str) -> (Fullnode, NodeInfo, Keypair, String) { use crate::cluster_info::Node; use crate::db_ledger::create_tmp_sample_ledger; - use crate::leader_scheduler::LeaderScheduler; + use crate::fullnode::Fullnode; use crate::voting_keypair::VotingKeypair; use solana_sdk::signature::KeypairUtil; @@ -450,14 +457,12 @@ pub fn new_fullnode(ledger_name: &'static str) -> (Fullnode, NodeInfo, Keypair, let (mint_keypair, ledger_path, _, _) = create_tmp_sample_ledger(ledger_name, 10_000, 0, node_info.id, 42); - let leader_scheduler = LeaderScheduler::from_bootstrap_leader(node_info.id); let vote_account_keypair = Arc::new(Keypair::new()); let voting_keypair = VotingKeypair::new_local(&vote_account_keypair); let node = Fullnode::new( node, &node_keypair, &ledger_path, - Arc::new(RwLock::new(leader_scheduler)), voting_keypair, None, &FullnodeConfig::default(), @@ -478,41 +483,52 @@ mod tests { use std::fs::remove_dir_all; #[test] - fn test_thin_client() { + fn test_thin_client_basic() { solana_logger::setup(); let (server, leader_data, alice, ledger_path) = new_fullnode("thin_client"); + let server_exit = server.run(None); let bob_pubkey = Keypair::new().pubkey(); - sleep(Duration::from_millis(900)); + info!( + "found leader: {:?}", + poll_gossip_for_leader(leader_data.gossip, Some(5)).unwrap() + ); let mut client = mk_client(&leader_data); let transaction_count = client.transaction_count(); assert_eq!(transaction_count, 0); + let confirmation = client.get_confirmation_time(); assert_eq!(confirmation, 18446744073709551615); + let last_id = client.get_last_id(); + info!("test_thin_client last_id: {:?}", last_id); + let signature = client.transfer(500, &alice, bob_pubkey, &last_id).unwrap(); + info!("test_thin_client signature: {:?}", signature); client.poll_for_signature(&signature).unwrap(); + let balance = client.get_balance(&bob_pubkey); assert_eq!(balance.unwrap(), 500); + let transaction_count = client.transaction_count(); assert_eq!(transaction_count, 1); - server.close().unwrap(); + server_exit(); remove_dir_all(ledger_path).unwrap(); } - // sleep(Duration::from_millis(300)); is unstable #[test] #[ignore] fn test_bad_sig() { solana_logger::setup(); - let (server, leader_data, alice, ledger_path) = new_fullnode("bad_sig"); + let server_exit = server.run(None); let bob_pubkey = Keypair::new().pubkey(); - - //TODO: remove this sleep, or add a retry so CI is stable - sleep(Duration::from_millis(300)); + info!( + "found leader: {:?}", + poll_gossip_for_leader(leader_data.gossip, Some(5)).unwrap() + ); let mut client = mk_client(&leader_data); @@ -534,24 +550,8 @@ mod tests { client.poll_for_signature(&signature).unwrap(); let balance = client.get_balance(&bob_pubkey); - assert_eq!(balance.unwrap(), 500); - server.close().unwrap(); - remove_dir_all(ledger_path).unwrap(); - } - - #[test] - fn test_client_check_signature() { - solana_logger::setup(); - let (server, leader_data, alice, ledger_path) = new_fullnode("thin_client"); - let bob_pubkey = Keypair::new().pubkey(); - - let mut client = mk_client(&leader_data); - let last_id = client.get_last_id(); - let signature = client.transfer(500, &alice, bob_pubkey, &last_id).unwrap(); - - client.poll_for_signature(&signature).unwrap(); - - server.close().unwrap(); + assert_eq!(balance.unwrap(), 1001); + server_exit(); remove_dir_all(ledger_path).unwrap(); } @@ -559,6 +559,11 @@ mod tests { fn test_register_vote_account() { solana_logger::setup(); let (server, leader_data, alice, ledger_path) = new_fullnode("thin_client"); + let server_exit = server.run(None); + info!( + "found leader: {:?}", + poll_gossip_for_leader(leader_data.gossip, Some(5)).unwrap() + ); let mut client = mk_client(&leader_data); @@ -604,7 +609,7 @@ mod tests { sleep(Duration::from_millis(900)); } - server.close().unwrap(); + server_exit(); remove_dir_all(ledger_path).unwrap(); } @@ -623,8 +628,14 @@ mod tests { fn test_zero_balance_after_nonzero() { solana_logger::setup(); let (server, leader_data, alice, ledger_path) = new_fullnode("thin_client"); + let server_exit = server.run(None); let bob_keypair = Keypair::new(); + info!( + "found leader: {:?}", + poll_gossip_for_leader(leader_data.gossip, Some(5)).unwrap() + ); + let mut client = mk_client(&leader_data); let last_id = client.get_last_id(); @@ -651,7 +662,7 @@ mod tests { let balance = client.poll_get_balance(&bob_keypair.pubkey()); assert!(balance.is_err()); - server.close().unwrap(); + server_exit(); remove_dir_all(ledger_path).unwrap(); } } diff --git a/src/tpu.rs b/src/tpu.rs index e2e839680f..b400919b64 100644 --- a/src/tpu.rs +++ b/src/tpu.rs @@ -82,7 +82,7 @@ impl Tpu { cluster_info: Arc>, entry_height: u64, sigverify_disabled: bool, - max_tick_height: Option, + max_tick_height: u64, last_entry_id: &Hash, leader_id: Pubkey, is_leader: bool, @@ -171,7 +171,7 @@ impl Tpu { broadcast_socket: UdpSocket, cluster_info: Arc>, sigverify_disabled: bool, - max_tick_height: Option, + max_tick_height: u64, entry_height: u64, last_entry_id: &Hash, leader_id: Pubkey, diff --git a/src/tvu.rs b/src/tvu.rs index a2401dfa10..051049f00a 100644 --- a/src/tvu.rs +++ b/src/tvu.rs @@ -203,10 +203,8 @@ pub mod tests { use crate::db_ledger::get_tmp_ledger_path; use crate::db_ledger::DbLedger; use crate::entry::Entry; - use crate::gossip_service::GossipService; - use crate::leader_scheduler::LeaderScheduler; - use crate::genesis_block::GenesisBlock; + use crate::gossip_service::GossipService; use crate::packet::SharedBlob; use crate::service::Service; use crate::storage_stage::{StorageState, STORAGE_ROTATE_TEST_COUNT}; @@ -241,13 +239,7 @@ pub mod tests { let starting_balance = 10_000; let (genesis_block, _mint_keypair) = GenesisBlock::new(starting_balance); - let leader_id = leader.info.id; - let leader_scheduler = Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_id, - ))); - let mut bank = Bank::new(&genesis_block); - bank.leader_scheduler = leader_scheduler; - let bank = Arc::new(bank); + let bank = Arc::new(Bank::new(&genesis_block)); //start cluster_info1 let mut cluster_info1 = ClusterInfo::new(target1.info.clone()); @@ -332,12 +324,7 @@ pub mod tests { let starting_balance = 10_000; let (genesis_block, mint_keypair) = GenesisBlock::new(starting_balance); let tvu_addr = target1.info.tvu; - let leader_scheduler = Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_id, - ))); - let mut bank = Bank::new(&genesis_block); - bank.leader_scheduler = leader_scheduler; - let bank = Arc::new(bank); + let bank = Arc::new(Bank::new(&genesis_block)); //start cluster_info1 let mut cluster_info1 = ClusterInfo::new(target1.info.clone()); diff --git a/src/window.rs b/src/window.rs index 541ac7fa0b..ed6ae4103b 100644 --- a/src/window.rs +++ b/src/window.rs @@ -128,38 +128,32 @@ impl WindowUtil for Window { leader_scheduler_option: &Arc>, ) -> Vec<(SocketAddr, Vec)> { let rcluster_info = cluster_info.read().unwrap(); - let mut is_next_leader = false; - { - let ls_lock = leader_scheduler_option.read().unwrap(); - if !ls_lock.use_only_bootstrap_leader { - // Calculate the next leader rotation height and check if we are the leader - if let Some(next_leader_rotation_height) = - ls_lock.max_height_for_leader(tick_height) - { - match ls_lock.get_scheduled_leader(next_leader_rotation_height) { - Some((leader_id, _)) if leader_id == *id => is_next_leader = true, - // In the case that we are not in the current scope of the leader schedule - // window then either: - // - // 1) The replay stage hasn't caught up to the "consumed" entries we sent, - // in which case it will eventually catch up - // - // 2) We are on the border between seed_rotation_intervals, so the - // schedule won't be known until the entry on that cusp is received - // by the replay stage (which comes after this stage). Hence, the next - // leader at the beginning of that next epoch will not know they are the - // leader until they receive that last "cusp" entry. The leader also won't ask for repairs - // for that entry because "is_next_leader" won't be set here. In this case, - // everybody will be blocking waiting for that "cusp" entry instead of repairing, - // until the leader hits "times" >= the max times in calculate_max_repair(). - // The impact of this, along with the similar problem from broadcast for the transitioning - // leader, can be observed in the multinode test, test_full_leader_validator_network(), - None => (), - _ => (), - } - } + // Check if we are the next next slot leader + let is_next_leader = { + let leader_scheduler = leader_scheduler_option.read().unwrap(); + let next_slot = leader_scheduler.tick_height_to_slot(tick_height) + 1; + match leader_scheduler.get_leader_for_slot(next_slot) { + Some(leader_id) if leader_id == *id => true, + // In the case that we are not in the current scope of the leader schedule + // window then either: + // + // 1) The replay stage hasn't caught up to the "consumed" entries we sent, + // in which case it will eventually catch up + // + // 2) We are on the border between seed_rotation_intervals, so the + // schedule won't be known until the entry on that cusp is received + // by the replay stage (which comes after this stage). Hence, the next + // leader at the beginning of that next epoch will not know they are the + // leader until they receive that last "cusp" entry. The leader also won't ask for repairs + // for that entry because "is_next_leader" won't be set here. In this case, + // everybody will be blocking waiting for that "cusp" entry instead of repairing, + // until the leader hits "times" >= the max times in calculate_max_repair(). + // The impact of this, along with the similar problem from broadcast for the transitioning + // leader, can be observed in the multinode test, test_full_leader_validator_network(), + None => false, + _ => false, } - } + }; let num_peers = rcluster_info.repair_peers().len() as u64; let max_repair = if max_entry_height == 0 { diff --git a/src/window_service.rs b/src/window_service.rs index 1a2306dbbe..99ada7e8c7 100644 --- a/src/window_service.rs +++ b/src/window_service.rs @@ -244,6 +244,8 @@ mod test { let db_ledger = Arc::new( DbLedger::open(&db_ledger_path).expect("Expected to be able to open database ledger"), ); + let mut leader_schedule = LeaderScheduler::default(); + leader_schedule.set_leader_schedule(vec![me_id]); let t_window = window_service( db_ledger, subs, @@ -253,7 +255,7 @@ mod test { r_reader, s_retransmit, Arc::new(leader_node.sockets.repair), - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader(me_id))), + Arc::new(RwLock::new(leader_schedule)), done, exit.clone(), ); @@ -324,6 +326,8 @@ mod test { let db_ledger = Arc::new( DbLedger::open(&db_ledger_path).expect("Expected to be able to open database ledger"), ); + let mut leader_schedule = LeaderScheduler::default(); + leader_schedule.set_leader_schedule(vec![me_id]); let t_window = window_service( db_ledger, subs.clone(), @@ -333,7 +337,7 @@ mod test { r_reader, s_retransmit, Arc::new(leader_node.sockets.repair), - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader(me_id))), + Arc::new(RwLock::new(leader_schedule)), done, exit.clone(), ); diff --git a/tests/multinode.rs b/tests/multinode.rs index e070b801a8..f859f5906c 100644 --- a/tests/multinode.rs +++ b/tests/multinode.rs @@ -1,24 +1,20 @@ -#[macro_use] -extern crate log; - +use log::*; use solana::blob_fetch_stage::BlobFetchStage; +use solana::client::mk_client; use solana::cluster_info::{ClusterInfo, Node, NodeInfo}; -use solana::contact_info::ContactInfo; use solana::db_ledger::{create_tmp_sample_ledger, tmp_copy_ledger}; use solana::db_ledger::{DbLedger, DEFAULT_SLOT_HEIGHT}; use solana::entry::{reconstruct_entries_from_blobs, Entry}; use solana::fullnode::{Fullnode, FullnodeConfig, FullnodeReturnType}; use solana::gossip_service::GossipService; -use solana::leader_scheduler::{make_active_set_entries, LeaderScheduler, LeaderSchedulerConfig}; +use solana::leader_scheduler::{make_active_set_entries, LeaderSchedulerConfig}; use solana::packet::SharedBlob; -use solana::poh_service::NUM_TICKS_PER_SECOND; use solana::result; use solana::service::Service; -use solana::thin_client::{poll_gossip_for_leader, retry_get_balance, ThinClient}; +use solana::thin_client::{poll_gossip_for_leader, retry_get_balance}; use solana::tpu::TpuReturnType; use solana::tvu::TvuReturnType; use solana::voting_keypair::VotingKeypair; -use solana_sdk::hash::Hash; use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::{Keypair, KeypairUtil}; use solana_sdk::system_transaction::SystemTransaction; @@ -26,11 +22,10 @@ use solana_sdk::timing::duration_as_s; use std::collections::{HashSet, VecDeque}; use std::env; use std::fs::remove_dir_all; -use std::net::UdpSocket; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::channel; use std::sync::{Arc, RwLock}; -use std::thread::{sleep, Builder, JoinHandle}; +use std::thread::{sleep, Builder}; use std::time::{Duration, Instant}; fn read_ledger(ledger_path: &str) -> Vec { @@ -89,20 +84,21 @@ fn make_listening_node( (gossip_service, new_node_cluster_info_ref, new_node, id) } -fn converge(leader: &NodeInfo, num_nodes: usize) -> Vec { +fn converge(node: &NodeInfo, num_nodes: usize) -> Vec { + info!("Wait for convergence with {} nodes", num_nodes); // Let's spy on the network - let (gossip_service, spy_ref, id) = make_spy_node(leader); + let (gossip_service, spy_ref, id) = make_spy_node(node); trace!( "converge spy_node {} looking for at least {} nodes", id, num_nodes ); - // Wait for the network to converge + // Wait for the cluster to converge for _ in 0..15 { let rpc_peers = spy_ref.read().unwrap().rpc_peers(); - if rpc_peers.len() >= num_nodes { - debug!("converge found {} nodes", rpc_peers.len()); + if rpc_peers.len() == num_nodes { + debug!("converge found {} nodes: {:?}", rpc_peers.len(), rpc_peers); gossip_service.close().unwrap(); return rpc_peers; } @@ -116,26 +112,17 @@ fn converge(leader: &NodeInfo, num_nodes: usize) -> Vec { panic!("Failed to converge"); } -fn make_tiny_test_entries(start_hash: Hash, num: usize) -> Vec { - let mut id = start_hash; - let mut num_hashes = 0; - (0..num) - .map(|_| Entry::new_mut(&mut id, &mut num_hashes, vec![])) - .collect() -} - #[test] fn test_multi_node_ledger_window() -> result::Result<()> { solana_logger::setup(); let leader_keypair = Arc::new(Keypair::new()); - let leader_pubkey = leader_keypair.pubkey().clone(); let leader = Node::new_localhost_with_pubkey(leader_keypair.pubkey()); let leader_data = leader.info.clone(); let bob_pubkey = Keypair::new().pubkey(); let mut ledger_paths = Vec::new(); - let (alice, leader_ledger_path, last_entry_height, last_entry_id) = + let (alice, leader_ledger_path, mut last_entry_height, mut last_entry_id) = create_tmp_sample_ledger("multi_node_ledger_window", 10_000, 0, leader_data.id, 500); ledger_paths.push(leader_ledger_path.clone()); @@ -143,14 +130,39 @@ fn test_multi_node_ledger_window() -> result::Result<()> { let zero_ledger_path = tmp_copy_ledger(&leader_ledger_path, "multi_node_ledger_window"); ledger_paths.push(zero_ledger_path.clone()); - // write a bunch more ledger into leader's ledger, this should populate the leader's window + let fullnode_config = FullnodeConfig::default(); + info!( + "leader_rotation_interval: {}", + fullnode_config + .leader_scheduler_config + .leader_rotation_interval + ); + + // Write some into leader's ledger, this should populate the leader's window // and force it to respond to repair from the ledger window + // TODO: write out more than slot 0 { - let entries = make_tiny_test_entries(last_entry_id, 100); let db_ledger = DbLedger::open(&leader_ledger_path).unwrap(); + + let entries = solana::entry::create_ticks( + fullnode_config + .leader_scheduler_config + .leader_rotation_interval + - last_entry_height + - 2, + last_entry_id, + ); db_ledger - .write_entries(DEFAULT_SLOT_HEIGHT, last_entry_height, &entries) + .write_entries(0, last_entry_height, &entries) .unwrap(); + + last_entry_height += entries.len() as u64; + last_entry_id = entries.last().unwrap().id; + + info!( + "Final last_entry_height: {}, last_entry_id: {:?}", + last_entry_height, last_entry_id + ); } let voting_keypair = VotingKeypair::new_local(&leader_keypair); @@ -158,18 +170,22 @@ fn test_multi_node_ledger_window() -> result::Result<()> { leader, &leader_keypair, &leader_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_pubkey, - ))), voting_keypair, None, - &FullnodeConfig::default(), + &fullnode_config, ); + let leader_exit = leader.run(None); - // start up another validator from zero, converge and then check - // balances + // Give validator some tokens for voting let keypair = Arc::new(Keypair::new()); let validator_pubkey = keypair.pubkey().clone(); + info!("validator id: {:?}", validator_pubkey); + let validator_balance = + send_tx_and_retry_get_balance(&leader_data, &alice, &validator_pubkey, 500, None).unwrap(); + info!("validator balance {}", validator_balance); + + // Start up another validator from zero, converge and then check + // balances let validator = Node::new_localhost_with_pubkey(keypair.pubkey()); let validator_data = validator.info.clone(); let voting_keypair = VotingKeypair::new_local(&keypair); @@ -177,34 +193,31 @@ fn test_multi_node_ledger_window() -> result::Result<()> { validator, &keypair, &zero_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_pubkey, - ))), voting_keypair, Some(&leader_data), &FullnodeConfig::default(), ); + let validator_exit = validator.run(None); - // Send validator some tokens to vote - let validator_balance = - send_tx_and_retry_get_balance(&leader_data, &alice, &validator_pubkey, 500, None).unwrap(); - info!("validator balance {}", validator_balance); + converge(&leader_data, 2); - // contains the leader and new node - info!("converging...."); - let _servers = converge(&leader_data, 2); - info!("converged."); - - // another transaction with leader + // Another transaction with leader let bob_balance = send_tx_and_retry_get_balance(&leader_data, &alice, &bob_pubkey, 1, None).unwrap(); info!("bob balance on leader {}", bob_balance); let mut checks = 1; loop { - let mut client = mk_client(&validator_data); - let bal = client.poll_get_balance(&bob_pubkey); + let mut leader_client = mk_client(&leader_data); + let bal = leader_client.poll_get_balance(&bob_pubkey); info!( - "bob balance on validator {:?} after {} checks...", + "Bob balance on leader is {:?} after {} checks...", + bal, checks + ); + + let mut validator_client = mk_client(&validator_data); + let bal = validator_client.poll_get_balance(&bob_pubkey); + info!( + "Bob balance on validator is {:?} after {} checks...", bal, checks ); if bal.unwrap_or(0) == bob_balance { @@ -212,10 +225,10 @@ fn test_multi_node_ledger_window() -> result::Result<()> { } checks += 1; } - info!("done!"); - validator.close()?; - leader.close()?; + info!("Done!"); + validator_exit(); + leader_exit(); for path in ledger_paths { remove_dir_all(path).unwrap(); @@ -230,7 +243,6 @@ fn test_multi_node_validator_catchup_from_zero() -> result::Result<()> { const N: usize = 2; trace!("test_multi_node_validator_catchup_from_zero"); let leader_keypair = Arc::new(Keypair::new()); - let leader_pubkey = leader_keypair.pubkey().clone(); let leader = Node::new_localhost_with_pubkey(leader_keypair.pubkey()); let leader_data = leader.info.clone(); let bob_pubkey = Keypair::new().pubkey(); @@ -261,15 +273,12 @@ fn test_multi_node_validator_catchup_from_zero() -> result::Result<()> { leader, &leader_keypair, &leader_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_pubkey, - ))), voting_keypair, None, &FullnodeConfig::default(), ); - let mut nodes = vec![server]; + let mut node_exits = vec![server.run(None)]; for _ in 0..N { let keypair = Arc::new(Keypair::new()); let validator_pubkey = keypair.pubkey().clone(); @@ -290,21 +299,17 @@ fn test_multi_node_validator_catchup_from_zero() -> result::Result<()> { ); let voting_keypair = VotingKeypair::new_local(&keypair); - let val = Fullnode::new( + let validator = Fullnode::new( validator, &keypair, &ledger_path, - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_pubkey, - ))), voting_keypair, Some(&leader_data), &FullnodeConfig::default(), ); - nodes.push(val); + node_exits.push(validator.run(None)); } - let servers = converge(&leader_data, N + 1); // contains the leader addr as well - assert_eq!(servers.len(), N + 1); + let nodes = converge(&leader_data, N + 1); // contains the leader addr as well // Verify leader can transfer from alice to bob let leader_balance = @@ -313,7 +318,7 @@ fn test_multi_node_validator_catchup_from_zero() -> result::Result<()> { // Verify validators all have the same balance for bob let mut success = 0usize; - for server in servers.iter() { + for server in nodes.iter() { let id = server.id; info!("0server: {}", id); let mut client = mk_client(server); @@ -340,7 +345,7 @@ fn test_multi_node_validator_catchup_from_zero() -> result::Result<()> { } assert!(found); } - assert_eq!(success, servers.len()); + assert_eq!(success, nodes.len()); success = 0; @@ -352,19 +357,17 @@ fn test_multi_node_validator_catchup_from_zero() -> result::Result<()> { let voting_keypair = VotingKeypair::new_local(&keypair); info!("created start from zero validator {:?}", validator_pubkey); - let val = Fullnode::new( + let validator = Fullnode::new( validator, &keypair, &zero_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_pubkey, - ))), voting_keypair, Some(&leader_data), &FullnodeConfig::default(), ); - nodes.push(val); - let servers = converge(&leader_data, N + 2); // contains the leader and new node + + node_exits.push(validator.run(None)); + let nodes = converge(&leader_data, N + 2); // contains the leader and new node // Transfer a little more from alice to bob let mut leader_balance = @@ -380,7 +383,7 @@ fn test_multi_node_validator_catchup_from_zero() -> result::Result<()> { } assert_eq!(leader_balance, 456); - for server in servers.iter() { + for server in nodes.iter() { let id = server.id; info!("1server: {}", id); let mut client = mk_client(server); @@ -402,16 +405,16 @@ fn test_multi_node_validator_catchup_from_zero() -> result::Result<()> { id, i, result ); } - sleep(Duration::new(1, 0)); + sleep(Duration::new(2, 0)); } assert!(found); } - assert_eq!(success, servers.len()); + assert_eq!(success, nodes.len()); trace!("done!"); - for node in nodes { - node.close()?; + for node_exit in node_exits { + node_exit(); } for path in ledger_paths { @@ -426,8 +429,9 @@ fn test_multi_node_basic() { solana_logger::setup(); const N: usize = 5; trace!("test_multi_node_basic"); + + let fullnode_config = FullnodeConfig::default(); let leader_keypair = Arc::new(Keypair::new()); - let leader_pubkey = leader_keypair.pubkey().clone(); let leader = Node::new_localhost_with_pubkey(leader_keypair.pubkey()); let leader_data = leader.info.clone(); let bob_pubkey = Keypair::new().pubkey(); @@ -445,16 +449,13 @@ fn test_multi_node_basic() { leader, &leader_keypair, &leader_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_pubkey, - ))), voting_keypair, None, - &FullnodeConfig::default(), + &fullnode_config, ); - let mut nodes = vec![server]; - for _ in 0..N { + let mut exit_signals = vec![server.run(None)]; + for i in 0..N { let keypair = Arc::new(Keypair::new()); let validator_pubkey = keypair.pubkey().clone(); let validator = Node::new_localhost_with_pubkey(keypair.pubkey()); @@ -466,25 +467,21 @@ fn test_multi_node_basic() { send_tx_and_retry_get_balance(&leader_data, &alice, &validator_pubkey, 500, None) .unwrap(); info!( - "validator {}, balance {}", - validator_pubkey, validator_balance + "validator #{} - {}, balance {}", + i, validator_pubkey, validator_balance ); let voting_keypair = VotingKeypair::new_local(&keypair); let val = Fullnode::new( validator, &keypair, &ledger_path, - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_pubkey, - ))), voting_keypair, Some(&leader_data), - &FullnodeConfig::default(), + &fullnode_config, ); - nodes.push(val); + exit_signals.push(val.run(None)); } - let servers = converge(&leader_data, N + 1); - assert_eq!(servers.len(), N + 1); // contains the leader as well + let nodes = converge(&leader_data, N + 1); // Verify leader can do transfer from alice to bob let leader_bob_balance = @@ -493,7 +490,7 @@ fn test_multi_node_basic() { // Verify validators all have the same balance for bob let mut success = 0usize; - for server in servers.iter() { + for server in nodes.iter() { let id = server.id; info!("mk_client for {}", id); let mut client = mk_client(server); @@ -519,11 +516,11 @@ fn test_multi_node_basic() { } assert!(found); } - assert_eq!(success, servers.len()); + assert_eq!(success, nodes.len()); trace!("done!"); - for node in nodes { - node.close().unwrap(); + for exit_signal in exit_signals { + exit_signal() } for path in ledger_paths { @@ -553,9 +550,6 @@ fn test_boot_validator_from_file() -> result::Result<()> { leader, &leader_keypair, &leader_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_pubkey, - ))), voting_keypair, None, &FullnodeConfig::default(), @@ -577,9 +571,6 @@ fn test_boot_validator_from_file() -> result::Result<()> { validator, &keypair, &ledger_path, - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_pubkey, - ))), voting_keypair, Some(&leader_data), &FullnodeConfig::default(), @@ -609,9 +600,6 @@ fn create_leader( leader, &leader_keypair, &ledger_path, - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_data.id, - ))), voting_keypair, None, &FullnodeConfig::default(), @@ -687,9 +675,6 @@ fn test_leader_restart_validator_start_from_old_ledger() -> result::Result<()> { validator, &keypair, &stale_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_data.id, - ))), voting_keypair, Some(&leader_data), &FullnodeConfig::default(), @@ -760,34 +745,32 @@ fn test_multi_node_dynamic_network() { leader, &leader_keypair, &leader_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_pubkey, - ))), voting_keypair, None, &FullnodeConfig::default(), ); + let server_exit = server.run(None); info!( "found leader: {:?}", poll_gossip_for_leader(leader_data.gossip, Some(5)).unwrap() ); - let leader_balance = retry_send_tx_and_retry_get_balance( + let bob_balance = retry_send_tx_and_retry_get_balance( &leader_data, &alice_arc.read().unwrap(), &bob_pubkey, Some(500), ) .unwrap(); - assert_eq!(leader_balance, 500); - let leader_balance = retry_send_tx_and_retry_get_balance( + assert_eq!(bob_balance, 500); + let bob_balance = retry_send_tx_and_retry_get_balance( &leader_data, &alice_arc.read().unwrap(), &bob_pubkey, Some(1000), ) .unwrap(); - assert_eq!(leader_balance, 1000); + assert_eq!(bob_balance, 1000); let t1: Vec<_> = (0..num_nodes) .into_iter() @@ -826,22 +809,20 @@ fn test_multi_node_dynamic_network() { .name("validator-launch-thread".to_string()) .spawn(move || { let validator = Node::new_localhost_with_pubkey(keypair.pubkey()); - let rd = validator.info.clone(); - info!("starting {} {}", keypair.pubkey(), rd.id); + let validator_info = validator.info.clone(); + info!("starting {}", keypair.pubkey()); let keypair = Arc::new(keypair); let voting_keypair = VotingKeypair::new_local(&keypair); - let val = Fullnode::new( + let validator = Fullnode::new( validator, &keypair, &ledger_path, - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_pubkey, - ))), voting_keypair, Some(&leader_data), &FullnodeConfig::default(), ); - (rd, val) + let validator_exit = validator.run(None); + (validator_info, validator_exit) }) .unwrap() }) @@ -852,10 +833,10 @@ fn test_multi_node_dynamic_network() { let mut client = mk_client(&leader_data); let start = Instant::now(); let mut consecutive_success = 0; - let mut expected_balance = leader_balance; + let mut expected_balance = bob_balance; let mut last_id = client.get_last_id(); for i in 0..std::cmp::max(20, num_nodes) { - trace!("Getting last_id..."); + trace!("Getting last_id (iteration {})...", i); let mut retries = 30; loop { let new_last_id = client.get_last_id(); @@ -872,29 +853,15 @@ fn test_multi_node_dynamic_network() { } debug!("last_id: {}", last_id); trace!("Executing leader transfer of 100"); + + let mut transaction = + SystemTransaction::new_move(&alice_arc.read().unwrap(), bob_pubkey, 100, last_id, 0); let sig = client - .transfer(100, &alice_arc.read().unwrap(), bob_pubkey, &last_id) + .retry_transfer(&alice_arc.read().unwrap(), &mut transaction, 5) .unwrap(); + trace!("transfer sig: {:?}", sig); expected_balance += 100; - - let mut retries = 30; - loop { - let e = client.poll_for_signature(&sig); - if e.is_ok() { - break; - } - debug!( - "signature not found yet: {}: {:?}, retries={}", - sig, e, retries - ); - retries -= 1; - if retries == 0 { - assert!(e.is_ok(), "err: {:?}", e); - } - sleep(Duration::from_millis(100)); - } - let mut retries = 30; loop { let balance = retry_get_balance(&mut client, &bob_pubkey, Some(expected_balance)); @@ -928,7 +895,6 @@ fn test_multi_node_dynamic_network() { let mut client = mk_client(&server.0); trace!("{} checking signature", server.0.id); num_nodes_behind += if client.check_signature(&sig) { 0 } else { 1 }; - server.1.exit(); true }); @@ -943,14 +909,10 @@ fn test_multi_node_dynamic_network() { info!("done!"); assert_eq!(consecutive_success, 10); - for (_, node) in &validators { - node.exit(); + for (_, validator_exit) in validators { + validator_exit(); } - server.exit(); - for (_, node) in validators { - node.join().unwrap(); - } - server.join().unwrap(); + server_exit(); for path in ledger_paths { remove_dir_all(path).unwrap(); @@ -960,7 +922,6 @@ fn test_multi_node_dynamic_network() { #[test] fn test_leader_to_validator_transition() { solana_logger::setup(); - let leader_rotation_interval = 20; // Make a dummy validator id to be the next leader let validator_keypair = Arc::new(Keypair::new()); @@ -970,22 +931,39 @@ fn test_leader_to_validator_transition() { let leader_node = Node::new_localhost_with_pubkey(leader_keypair.pubkey()); let leader_info = leader_node.info.clone(); + let mut fullnode_config = FullnodeConfig::default(); + let leader_rotation_interval = 5; + fullnode_config.leader_scheduler_config = LeaderSchedulerConfig::new( + leader_rotation_interval, + leader_rotation_interval, + // Setup window length to exclude the genesis bootstrap leader vote at tick height 0, so + // that when the leader schedule is recomputed for epoch 1 only the validator vote at tick + // height 1 will be considered. + leader_rotation_interval, + ); + // Initialize the leader ledger. Make a mint and a genesis entry // in the leader ledger - let num_ending_ticks = 1; let (mint_keypair, leader_ledger_path, genesis_entry_height, last_id) = create_tmp_sample_ledger( "test_leader_to_validator_transition", 10_000, - num_ending_ticks, + 0, leader_info.id, 500, ); - // Write the bootstrap entries to the ledger that will cause leader rotation - // after the bootstrap height - let (bootstrap_entries, _) = - make_active_set_entries(&validator_keypair, &mint_keypair, &last_id, &last_id, 0); + // Write the votes entries to the ledger that will cause leader rotation + // to validator_keypair at slot 2 + let (bootstrap_entries, _) = make_active_set_entries( + &validator_keypair, + &mint_keypair, + 100, + leader_rotation_interval, + &last_id, + &last_id, + 0, + ); { let db_ledger = DbLedger::open(&leader_ledger_path).unwrap(); db_ledger @@ -996,111 +974,64 @@ fn test_leader_to_validator_transition() { ) .unwrap(); } + info!("leader id: {}", leader_keypair.pubkey()); + info!("validator id: {}", validator_keypair.pubkey()); // Start the leader node - let bootstrap_height = leader_rotation_interval; - let leader_scheduler_config = LeaderSchedulerConfig::new( - bootstrap_height, - leader_rotation_interval, - leader_rotation_interval * 2, - bootstrap_height, - ); - let voting_keypair = VotingKeypair::new_local(&leader_keypair); - let mut leader = Fullnode::new( + let leader = Fullnode::new( leader_node, &leader_keypair, &leader_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::new(&leader_scheduler_config))), voting_keypair, Some(&leader_info), - &FullnodeConfig::default(), + &fullnode_config, ); + let (rotation_sender, rotation_receiver) = channel(); + let leader_exit = leader.run(Some(rotation_sender)); - // Make an extra node for our leader to broadcast to, - // who won't vote and mess with our leader's entry count - let (gossip_service, spy_node, _) = make_spy_node(&leader_info); + // There will be two rotations: + // slot 0 -> slot 1: bootstrap leader remains the leader + // slot 1 -> slot 2: bootstrap leader to the validator + let expected_rotations = vec![ + ( + FullnodeReturnType::LeaderToLeaderRotation, + leader_rotation_interval, + ), + ( + FullnodeReturnType::LeaderToValidatorRotation, + 2 * leader_rotation_interval, + ), + ]; - // Wait for the leader to see the spy node - let mut converged = false; - for _ in 0..30 { - let num = spy_node.read().unwrap().convergence(); - let v: Vec = spy_node.read().unwrap().rpc_peers(); - // There's only one person excluding the spy node (the leader) who should see - // two nodes on the network - if num >= 2 && v.len() >= 1 { - converged = true; + for expected_rotation in expected_rotations { + loop { + let transition = rotation_receiver.recv().unwrap(); + info!("leader transition: {:?}", transition); + assert_eq!(transition, expected_rotation); break; } - sleep(Duration::new(1, 0)); } - assert!(converged); + info!("Shut down..."); + leader_exit(); - // Account that will be the sink for all the test's transactions - let bob_pubkey = Keypair::new().pubkey(); + info!("Check the ledger to make sure it's the right height..."); + let (bank, _, _) = Fullnode::new_bank_from_ledger(&leader_ledger_path, None); - // Push transactions until we detect an exit - let mut i = 1; - loop { - // Poll to see that the bank state is updated after every transaction - // to ensure that each transaction is packaged as a single entry, - // so that we can be sure leader rotation is triggered - let result = send_tx_and_retry_get_balance( - &leader_info, - &mint_keypair, - &bob_pubkey, - 1, - Some(i as u64), - ); - - // If the transaction wasn't reflected in the node, then we assume - // the node has transitioned already - if result != Some(i as u64) { - break; - } - - i += 1; - } - - // Wait for leader to shut down tpu and restart tvu - match leader.handle_role_transition() { - Some(FullnodeReturnType::LeaderToValidatorRotation) => (), - _ => panic!("Expected reason for exit to be leader rotation"), - } - - // Query newly transitioned validator to make sure that they have the proper balances in - // the after the transitions - let mut leader_client = mk_client(&leader_info); - - // Leader could have executed transactions in bank but not recorded them, so - // we only have an upper bound on the balance - if let Ok(bal) = leader_client.poll_get_balance(&bob_pubkey) { - assert!(bal <= i); - } - - // Shut down - gossip_service.close().unwrap(); - leader.close().unwrap(); - - // Check the ledger to make sure it's the right height, we should've - // transitioned after tick_height == bootstrap_height - let (bank, _, _) = Fullnode::new_bank_from_ledger( - &leader_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::default())), + assert_eq!( + bank.tick_height(), + 2 * fullnode_config + .leader_scheduler_config + .leader_rotation_interval + - 1 ); - - assert_eq!(bank.tick_height(), bootstrap_height); remove_dir_all(leader_ledger_path).unwrap(); } #[test] fn test_leader_validator_basic() { solana_logger::setup(); - let leader_rotation_interval = 10; - - // Account that will be the sink for all the test's transactions - let bob_pubkey = Keypair::new().pubkey(); // Create the leader node information let leader_keypair = Arc::new(Keypair::new()); @@ -1111,28 +1042,29 @@ fn test_leader_validator_basic() { let validator_keypair = Arc::new(Keypair::new()); let validator_node = Node::new_localhost_with_pubkey(validator_keypair.pubkey()); + info!("leader id: {}", leader_keypair.pubkey()); + info!("validator id: {}", validator_keypair.pubkey()); + // Make a common mint and a genesis entry for both leader + validator ledgers - let num_ending_ticks = 1; let (mint_keypair, leader_ledger_path, genesis_entry_height, last_id) = create_tmp_sample_ledger( "test_leader_validator_basic", 10_000, - num_ending_ticks, + 0, leader_info.id, 500, ); - let validator_ledger_path = tmp_copy_ledger(&leader_ledger_path, "test_leader_validator_basic"); - - // Initialize both leader + validator ledger - let mut ledger_paths = Vec::new(); - ledger_paths.push(leader_ledger_path.clone()); - ledger_paths.push(validator_ledger_path.clone()); - - // Write the bootstrap entries to the ledger that will cause leader rotation - // after the bootstrap height - let (active_set_entries, _) = - make_active_set_entries(&validator_keypair, &mint_keypair, &last_id, &last_id, 0); + // Add validator vote on tick height 1 + let (active_set_entries, _) = make_active_set_entries( + &validator_keypair, + &mint_keypair, + 100, + 1, + &last_id, + &last_id, + 0, + ); { let db_ledger = DbLedger::open(&leader_ledger_path).unwrap(); db_ledger @@ -1144,92 +1076,79 @@ fn test_leader_validator_basic() { .unwrap(); } + // Initialize both leader + validator ledger + let mut ledger_paths = Vec::new(); + ledger_paths.push(leader_ledger_path.clone()); + let validator_ledger_path = tmp_copy_ledger(&leader_ledger_path, "test_leader_validator_basic"); + ledger_paths.push(validator_ledger_path.clone()); + // Create the leader scheduler config - let num_bootstrap_slots = 2; - let bootstrap_height = num_bootstrap_slots * leader_rotation_interval; - let leader_scheduler_config = LeaderSchedulerConfig::new( - bootstrap_height, + let mut fullnode_config = FullnodeConfig::default(); + let leader_rotation_interval = 5; + fullnode_config.leader_scheduler_config = LeaderSchedulerConfig::new( + leader_rotation_interval, + leader_rotation_interval, // 1 slot per epoch leader_rotation_interval, - leader_rotation_interval * 2, - bootstrap_height, ); // Start the validator node let voting_keypair = VotingKeypair::new_local(&validator_keypair); - let mut validator = Fullnode::new( + let validator = Fullnode::new( validator_node, &validator_keypair, &validator_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::new(&leader_scheduler_config))), voting_keypair, Some(&leader_info), - &FullnodeConfig::default(), + &fullnode_config, ); + let (validator_rotation_sender, validator_rotation_receiver) = channel(); + let validator_exit = validator.run(Some(validator_rotation_sender)); // Start the leader fullnode let voting_keypair = VotingKeypair::new_local(&leader_keypair); - let mut leader = Fullnode::new( + let leader = Fullnode::new( leader_node, &leader_keypair, &leader_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::new(&leader_scheduler_config))), voting_keypair, Some(&leader_info), - &FullnodeConfig::default(), + &fullnode_config, + ); + let (leader_rotation_sender, leader_rotation_receiver) = channel(); + let leader_exit = leader.run(Some(leader_rotation_sender)); + + converge(&leader_info, 2); + + info!("Waiting for slot 0 -> slot 1: bootstrap leader will remain the leader"); + assert_eq!( + leader_rotation_receiver.recv().unwrap(), + ( + FullnodeReturnType::LeaderToLeaderRotation, + leader_rotation_interval, + ) ); - // Wait for convergence - let servers = converge(&leader_info, 2); - assert_eq!(servers.len(), 2); + info!("Waiting for slot 1 -> slot 2: bootstrap leader becomes a validator"); + assert_eq!( + leader_rotation_receiver.recv().unwrap(), + ( + FullnodeReturnType::LeaderToValidatorRotation, + leader_rotation_interval * 2, + ) + ); - // Push transactions until we detect the nodes exit - let mut i = 1; - loop { - // Poll to see that the bank state is updated after every transaction - // to ensure that each transaction is packaged as a single entry, - // so that we can be sure leader rotation is triggered - let result = - send_tx_and_retry_get_balance(&leader_info, &mint_keypair, &bob_pubkey, 1, None); + info!("Waiting for slot 1 -> slot 2: validator becomes the leader"); + assert_eq!( + validator_rotation_receiver.recv().unwrap(), + ( + FullnodeReturnType::ValidatorToLeaderRotation, + leader_rotation_interval * 2, + ) + ); - // If the transaction wasn't reflected in the node, then we assume - // the node has transitioned already - if result != Some(i as u64) { - break; - } - - i += 1; - } - - // Wait for validator to shut down tvu and restart tpu - match validator.handle_role_transition() { - Some(FullnodeReturnType::ValidatorToLeaderRotation) => (), - _ => panic!("Expected reason for exit to be leader rotation"), - } - - // Wait for the leader to shut down tpu and restart tvu - match leader.handle_role_transition() { - Some(FullnodeReturnType::LeaderToValidatorRotation) => (), - _ => panic!("Expected reason for exit to be leader rotation"), - } - - // Query newly transitioned validator to make sure they have the proper balances - // in the bank after the transitions - let mut leader_client = mk_client(&leader_info); - - // Leader could have executed transactions in bank but not recorded them, so - // we only have an upper bound on the balance - if let Ok(bal) = leader_client.poll_get_balance(&bob_pubkey) { - assert!(bal <= i); - } - - // Shut down - // stop the leader first so no more ticks/txs are created - leader.exit(); - validator.exit(); - leader.join().expect("Expected successful leader close"); - validator - .join() - .expect("Expected successful validator close"); + info!("Shut down"); + validator_exit(); + leader_exit(); // Check the ledger of the validator to make sure the entry height is correct // and that the old leader and the new leader's ledgers agree up to the point @@ -1237,8 +1156,7 @@ fn test_leader_validator_basic() { let validator_entries: Vec = read_ledger(&validator_ledger_path); let leader_entries = read_ledger(&leader_ledger_path); - - assert!(leader_entries.len() as u64 >= bootstrap_height); + assert!(leader_entries.len() as u64 >= leader_rotation_interval); for (v, l) in validator_entries.iter().zip(leader_entries) { assert_eq!(*v, l); @@ -1251,33 +1169,6 @@ fn test_leader_validator_basic() { } } -fn run_node(id: Pubkey, mut fullnode: Fullnode, should_exit: Arc) -> JoinHandle<()> { - Builder::new() - .name(format!("run_node-{:?}", id).to_string()) - .spawn(move || loop { - if should_exit.load(Ordering::Relaxed) { - fullnode.close().expect("failed to close"); - return; - } - let should_be_fwdr = fullnode.role_notifiers.1.try_recv(); - let should_be_leader = fullnode.role_notifiers.0.try_recv(); - match should_be_leader { - Ok(TvuReturnType::LeaderRotation(tick_height, entry_height, last_entry_id)) => { - fullnode.validator_to_leader(tick_height, entry_height, last_entry_id); - } - Err(_) => match should_be_fwdr { - Ok(TpuReturnType::LeaderRotation(tick_height)) => { - fullnode.leader_to_validator(tick_height); - } - Err(_) => { - sleep(Duration::new(1, 0)); - } - }, - } - }) - .unwrap() -} - #[test] fn test_dropped_handoff_recovery() { solana_logger::setup(); @@ -1291,6 +1182,17 @@ fn test_dropped_handoff_recovery() { let bootstrap_leader_node = Node::new_localhost_with_pubkey(bootstrap_leader_keypair.pubkey()); let bootstrap_leader_info = bootstrap_leader_node.info.clone(); + // Create the common leader scheduling configuration + let num_slots_per_epoch = (N + 1) as u64; + let leader_rotation_interval = 5; + let seed_rotation_interval = num_slots_per_epoch * leader_rotation_interval; + let mut fullnode_config = FullnodeConfig::default(); + fullnode_config.leader_scheduler_config = LeaderSchedulerConfig::new( + leader_rotation_interval, + seed_rotation_interval, + seed_rotation_interval, + ); + // Make a common mint and a genesis entry for both leader + validator's ledgers let num_ending_ticks = 1; let (mint_keypair, genesis_ledger_path, genesis_entry_height, last_id) = @@ -1313,8 +1215,15 @@ fn test_dropped_handoff_recovery() { // Make the entries to give the next_leader validator some stake so that they will be in // leader election active set - let (active_set_entries, _) = - make_active_set_entries(&next_leader_keypair, &mint_keypair, &last_id, &last_id, 0); + let (active_set_entries, _) = make_active_set_entries( + &next_leader_keypair, + &mint_keypair, + 100, + leader_rotation_interval, + &last_id, + &last_id, + 0, + ); // Write the entries { @@ -1332,18 +1241,6 @@ fn test_dropped_handoff_recovery() { tmp_copy_ledger(&genesis_ledger_path, "test_dropped_handoff_recovery"); ledger_paths.push(next_leader_ledger_path.clone()); - // Create the common leader scheduling configuration - let initial_tick_height = genesis_entry_height; - let num_slots_per_epoch = (N + 1) as u64; - let leader_rotation_interval = 5; - let seed_rotation_interval = num_slots_per_epoch * leader_rotation_interval; - let bootstrap_height = initial_tick_height + 1; - let leader_scheduler_config = LeaderSchedulerConfig::new( - bootstrap_height, - leader_rotation_interval, - seed_rotation_interval, - leader_rotation_interval, - ); info!("bootstrap_leader: {}", bootstrap_leader_keypair.pubkey()); info!("'next leader': {}", next_leader_keypair.pubkey()); @@ -1352,17 +1249,18 @@ fn test_dropped_handoff_recovery() { let bootstrap_leader_ledger_path = tmp_copy_ledger(&genesis_ledger_path, "test_dropped_handoff_recovery"); ledger_paths.push(bootstrap_leader_ledger_path.clone()); + let bootstrap_leader = Fullnode::new( bootstrap_leader_node, &bootstrap_leader_keypair, &bootstrap_leader_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::new(&leader_scheduler_config))), voting_keypair, Some(&bootstrap_leader_info), - &FullnodeConfig::default(), + &fullnode_config, ); - let mut nodes = vec![bootstrap_leader]; + let (rotation_sender, rotation_receiver) = channel(); + let mut node_exits = vec![bootstrap_leader.run(Some(rotation_sender))]; // Start up the validators other than the "next_leader" validator for i in 0..(N - 1) { @@ -1378,52 +1276,55 @@ fn test_dropped_handoff_recovery() { validator_node, &keypair, &validator_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::new(&leader_scheduler_config))), voting_keypair, Some(&bootstrap_leader_info), - &FullnodeConfig::default(), + &fullnode_config, ); - nodes.push(validator); + node_exits.push(validator.run(None)); } - // Wait for convergence - let num_converged = converge(&bootstrap_leader_info, N).len(); - assert_eq!(num_converged, N); + converge(&bootstrap_leader_info, N); info!("Wait for bootstrap_leader to transition to a validator",); - match nodes[0].handle_role_transition() { - Some(FullnodeReturnType::LeaderToValidatorRotation) => (), - _ => panic!("Expected reason for exit to be leader rotation"), + loop { + let transition = rotation_receiver.recv().unwrap(); + info!("bootstrap leader transition event: {:?}", transition); + if transition.0 == FullnodeReturnType::LeaderToValidatorRotation { + break; + } } - info!("Starting the 'next leader' node"); + info!("Starting the 'next leader' node *after* rotation has occurred"); let next_leader_node = Node::new_localhost_with_pubkey(next_leader_keypair.pubkey()); let voting_keypair = VotingKeypair::new_local(&next_leader_keypair); let next_leader = Fullnode::new( next_leader_node, &next_leader_keypair, &next_leader_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::new(&leader_scheduler_config))), voting_keypair, Some(&bootstrap_leader_info), &FullnodeConfig::default(), ); + let (rotation_sender, _rotation_receiver) = channel(); + node_exits.push(next_leader.run(Some(rotation_sender))); info!("Wait for 'next leader' to assume leader role"); error!("TODO: FIX https://github.com/solana-labs/solana/issues/2482"); // TODO: Once fixed restore the commented out code below /* - match next_leader.handle_role_transition() { - Some(FullnodeReturnType::ValidatorToLeaderRotation) => (), - _ => panic!("Expected reason for exit to be leader rotation"), + loop { + let transition = _rotation_receiver.recv().unwrap(); + info!("next leader transition event: {:?}", transition); + if transition == FullnodeReturnType::ValidatorToLeaderRotation { + break; + } } */ - nodes.push(next_leader); info!("done!"); - for node in nodes { - node.close().unwrap(); + for exit in node_exits { + exit(); } for path in ledger_paths { @@ -1435,16 +1336,15 @@ fn test_dropped_handoff_recovery() { fn test_full_leader_validator_network() { solana_logger::setup(); // The number of validators - const N: usize = 5; - solana_logger::setup(); + const N: usize = 2; // Create the bootstrap leader node information - let bootstrap_leader_keypair = Keypair::new(); + let bootstrap_leader_keypair = Arc::new(Keypair::new()); + info!("bootstrap leader: {:?}", bootstrap_leader_keypair.pubkey()); let bootstrap_leader_node = Node::new_localhost_with_pubkey(bootstrap_leader_keypair.pubkey()); let bootstrap_leader_info = bootstrap_leader_node.info.clone(); let mut node_keypairs = VecDeque::new(); - node_keypairs.push_back(Arc::new(bootstrap_leader_keypair)); // Create the validator keypairs for _ in 0..N { @@ -1467,20 +1367,19 @@ fn test_full_leader_validator_network() { let mut last_entry_id = last_id; // Create a common ledger with entries in the beginnging that will add all the validators - // to the active set for leader election. TODO: Leader rotation does not support dynamic - // stakes for safe leader -> validator transitions due to the unpredictability of - // bank state due to transactions being in-flight during leader seed calculation in - // write stage. + // to the active set for leader election. let mut ledger_paths = Vec::new(); ledger_paths.push(bootstrap_leader_ledger_path.clone()); let mut index = genesis_entry_height; for node_keypair in node_keypairs.iter() { - // Make entries to give each node some stake so that they will be in the + // Make entries to give each validator node some stake so that they will be in the // leader election active set let (bootstrap_entries, _) = make_active_set_entries( node_keypair, &mint_keypair, + 100, + 1, &last_entry_id, &last_tick_id, 0, @@ -1502,27 +1401,16 @@ fn test_full_leader_validator_network() { // Create the common leader scheduling configuration let num_slots_per_epoch = (N + 1) as u64; - let num_bootstrap_slots = 2; let leader_rotation_interval = 5; let seed_rotation_interval = num_slots_per_epoch * leader_rotation_interval; - let bootstrap_height = num_bootstrap_slots * leader_rotation_interval; - let leader_scheduler_config = LeaderSchedulerConfig::new( - bootstrap_height, + let mut fullnode_config = FullnodeConfig::default(); + fullnode_config.leader_scheduler_config = LeaderSchedulerConfig::new( leader_rotation_interval, seed_rotation_interval, - 100, + seed_rotation_interval, ); - let exit = Arc::new(AtomicBool::new(false)); - - // Postpone starting the leader until after the validators are up and running - // to avoid - // 1) Scenario where leader rotates before validators can start up - // 2) Modifying the leader ledger which validators are going to be copying - // during startup - let leader_keypair = node_keypairs.pop_front().unwrap(); - let mut schedules: Vec>> = vec![]; - let mut t_nodes = vec![]; + let mut nodes = vec![]; info!("Start up the validators"); for kp in node_keypairs.into_iter() { @@ -1536,77 +1424,66 @@ fn test_full_leader_validator_network() { let validator_id = kp.pubkey(); let validator_node = Node::new_localhost_with_pubkey(validator_id); let voting_keypair = VotingKeypair::new_local(&kp); - let leader_scheduler = - Arc::new(RwLock::new(LeaderScheduler::new(&leader_scheduler_config))); + info!("validator: {:?}", validator_id); let validator = Fullnode::new( validator_node, &kp, &validator_ledger_path, - leader_scheduler.clone(), voting_keypair, Some(&bootstrap_leader_info), - &FullnodeConfig::default(), + &fullnode_config, ); - schedules.push(leader_scheduler); - t_nodes.push(run_node(validator_id, validator, exit.clone())); + let (rotation_sender, rotation_receiver) = channel(); + nodes.push(( + validator_id, + validator.run(Some(rotation_sender)), + rotation_receiver, + )); } info!("Start up the bootstrap leader"); - let voting_keypair = VotingKeypair::new_local(&leader_keypair); - let leader_scheduler = Arc::new(RwLock::new(LeaderScheduler::new(&leader_scheduler_config))); + let voting_keypair = VotingKeypair::new_local(&bootstrap_leader_keypair); let bootstrap_leader = Fullnode::new( bootstrap_leader_node, - &leader_keypair, + &bootstrap_leader_keypair, &bootstrap_leader_ledger_path, - leader_scheduler.clone(), voting_keypair, Some(&bootstrap_leader_info), - &FullnodeConfig::default(), + &fullnode_config, ); + let (bootstrap_leader_rotation_sender, bootstrap_leader_rotation_receiver) = channel(); + let bootstrap_leader_exit = bootstrap_leader.run(Some(bootstrap_leader_rotation_sender)); - schedules.push(leader_scheduler); - t_nodes.push(run_node( - bootstrap_leader_info.id, - bootstrap_leader, - exit.clone(), - )); + converge(&bootstrap_leader_info, N + 1); - info!("Wait for convergence"); - let num_converged = converge(&bootstrap_leader_info, N + 1).len(); - assert_eq!(num_converged, N + 1); - - // Wait for each node to hit a specific target height in the leader schedule. - // Once all the nodes hit that height, exit them all together. They must - // only quit once they've all confirmed to reach that specific target height. - // Otherwise, some nodes may never reach the target height if a critical - // next leader node exits first, and stops generating entries. (We don't - // have a timeout mechanism). - let target_height = bootstrap_height + seed_rotation_interval; - let mut num_reached_target_height = 0; - - while num_reached_target_height != N + 1 { - num_reached_target_height = 0; - for n in schedules.iter() { - let ls_lock = n.read().unwrap().last_seed_height; - if let Some(sh) = ls_lock { - if sh >= target_height { - num_reached_target_height += 1; - } - } - drop(ls_lock); + // Wait for the bootstrap_leader to transition to a validator + loop { + let transition = bootstrap_leader_rotation_receiver.recv().unwrap(); + info!("bootstrap leader transition event: {:?}", transition); + if transition.0 == FullnodeReturnType::LeaderToValidatorRotation { + break; } - - sleep(Duration::new(1, 0)); } - exit.store(true, Ordering::Relaxed); - - info!("Wait for threads running the nodes to exit"); - for t in t_nodes { - t.join().unwrap(); + // Ensure each node in the cluster rotates into the leader role + for (id, _, rotation_receiver) in &nodes { + info!("Waiting for {:?} to become the leader", id); + loop { + let transition = rotation_receiver.recv().unwrap(); + info!("node {:?} transition event: {:?}", id, transition); + if transition.0 == FullnodeReturnType::ValidatorToLeaderRotation { + break; + } + } } + info!("Exit all nodes"); + for node in nodes { + node.1(); + } + bootstrap_leader_exit(); + let mut node_entries = vec![]; info!("Check that all the ledgers match"); for ledger_path in ledger_paths.iter() { @@ -1661,9 +1538,15 @@ fn test_full_leader_validator_network() { length += 1; } - assert!(shortest.unwrap() >= target_height); + let shortest = shortest.unwrap(); + assert!( + shortest + >= fullnode_config + .leader_scheduler_config + .leader_rotation_interval + * 3, + ); - info!("done!"); for path in ledger_paths { DbLedger::destroy(&path).expect("Expected successful database destruction"); remove_dir_all(path).unwrap(); @@ -1687,7 +1570,7 @@ fn test_broadcast_last_tick() { create_tmp_sample_ledger( "test_broadcast_last_tick", 10_000, - 1, + 0, bootstrap_leader_info.id, 500, ); @@ -1716,33 +1599,29 @@ fn test_broadcast_last_tick() { }) .collect(); - // Create fullnode, should take 10 seconds to reach end of bootstrap period - let bootstrap_height = (NUM_TICKS_PER_SECOND * 10) as u64; - let leader_rotation_interval = 100; - let seed_rotation_interval = 200; - let leader_scheduler_config = LeaderSchedulerConfig::new( - bootstrap_height, - leader_rotation_interval, - seed_rotation_interval, - leader_rotation_interval, - ); + let leader_rotation_interval = 40; + let seed_rotation_interval = 2 * leader_rotation_interval; // Start up the bootstrap leader fullnode let bootstrap_leader_keypair = Arc::new(bootstrap_leader_keypair); let voting_keypair = VotingKeypair::new_local(&bootstrap_leader_keypair); + let mut fullnode_config = FullnodeConfig::default(); + fullnode_config.leader_scheduler_config = LeaderSchedulerConfig::new( + leader_rotation_interval, + seed_rotation_interval, + seed_rotation_interval, + ); let bootstrap_leader = Fullnode::new( bootstrap_leader_node, &bootstrap_leader_keypair, &bootstrap_leader_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::new(&leader_scheduler_config))), voting_keypair, Some(&bootstrap_leader_info), - &FullnodeConfig::default(), + &fullnode_config, ); // Wait for convergence - let servers = converge(&bootstrap_leader_info, N + 1); - assert_eq!(servers.len(), N + 1); + converge(&bootstrap_leader_info, N + 1); info!("Waiting for leader rotation..."); // manually intercept the role transition to accurately arrest tick generation @@ -1754,7 +1633,8 @@ fn test_broadcast_last_tick() { panic!("Expected rotation to validator"); } _ => match should_be_forwarder { - Ok(TpuReturnType::LeaderRotation(_)) => { + Ok(TpuReturnType::LeaderRotation(tick_height)) => { + info!("rotation occurred at tick_height {:?}", tick_height); break; } _ => continue, @@ -1765,16 +1645,11 @@ fn test_broadcast_last_tick() { info!("Shutting down the leader..."); bootstrap_leader.close().unwrap(); - //index of the last tick is always bootstrap_height - 1 - let last_tick_entry_index = bootstrap_height - 1; + // Index of the last tick is always leader_rotation_interval - 1 + let last_tick_entry_index = leader_rotation_interval as usize - 2; let entries = read_ledger(&bootstrap_leader_ledger_path); - assert!( - entries.len() >= bootstrap_height as usize, - "entries: {:?} less than last_tick_entry_index: {:?}", - entries.len(), - last_tick_entry_index as usize - ); - let expected_last_tick = &entries[last_tick_entry_index as usize]; + assert_eq!(entries.len(), last_tick_entry_index + 1); + let expected_last_tick = &entries[last_tick_entry_index]; debug!("last_tick_entry_index: {:?}", last_tick_entry_index); debug!("expected_last_tick: {:?}", expected_last_tick); @@ -1785,7 +1660,7 @@ fn test_broadcast_last_tick() { while let Ok(new_blobs) = receiver.try_recv() { let last_blob = new_blobs .into_iter() - .find(|b| b.read().unwrap().index() == last_tick_entry_index); + .find(|b| b.read().unwrap().index() == last_tick_entry_index as u64); if let Some(last_blob) = last_blob { last_tick_blob = last_blob; break; @@ -1814,12 +1689,6 @@ fn test_broadcast_last_tick() { remove_dir_all(bootstrap_leader_ledger_path).unwrap(); } -fn mk_client(leader: &NodeInfo) -> ThinClient { - let transactions_socket = UdpSocket::bind("0.0.0.0:0").unwrap(); - assert!(ContactInfo::is_valid_address(&leader.tpu)); - ThinClient::new(leader.rpc, leader.tpu, transactions_socket) -} - fn send_tx_and_retry_get_balance( leader: &NodeInfo, alice: &Keypair, @@ -1837,8 +1706,11 @@ fn send_tx_and_retry_get_balance( alice.pubkey(), *bob_pubkey ); - let _res = client.retry_transfer(&alice, &mut tx, 30); - retry_get_balance(&mut client, bob_pubkey, expected) + if client.retry_transfer(&alice, &mut tx, 5).is_err() { + None + } else { + retry_get_balance(&mut client, bob_pubkey, expected) + } } fn retry_send_tx_and_retry_get_balance( diff --git a/tests/replicator.rs b/tests/replicator.rs index 29887ce8c7..e88103632d 100644 --- a/tests/replicator.rs +++ b/tests/replicator.rs @@ -12,7 +12,6 @@ use solana::db_ledger::DbLedger; use solana::db_ledger::{create_tmp_sample_ledger, get_tmp_ledger_path, tmp_copy_ledger}; use solana::entry::Entry; use solana::fullnode::{Fullnode, FullnodeConfig}; -use solana::leader_scheduler::LeaderScheduler; use solana::replicator::Replicator; use solana::storage_stage::STORAGE_ROTATE_TEST_COUNT; use solana::streamer::blob_receiver; @@ -23,13 +22,12 @@ use solana_sdk::system_transaction::SystemTransaction; use std::fs::remove_dir_all; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::channel; -use std::sync::{Arc, RwLock}; -use std::thread::sleep; +use std::sync::Arc; use std::time::Duration; #[test] #[ignore] -fn test_replicator_startup() { +fn test_replicator_startup_basic() { solana_logger::setup(); info!("starting replicator test"); let replicator_ledger_path = &get_tmp_ledger_path("replicator_test_replicator_ledger"); @@ -41,7 +39,7 @@ fn test_replicator_startup() { let leader_ledger_path = "replicator_test_leader_ledger"; let (mint_keypair, leader_ledger_path, _last_entry_height, _last_entry_id) = - create_tmp_sample_ledger(leader_ledger_path, 1_000_000_000, 0, leader_info.id, 1); + create_tmp_sample_ledger(leader_ledger_path, 1_000_000_000, 0, leader_info.id, 42); let validator_ledger_path = tmp_copy_ledger(&leader_ledger_path, "replicator_test_validator_ledger"); @@ -55,21 +53,23 @@ fn test_replicator_startup() { leader_node, &leader_keypair, &leader_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_info.id.clone(), - ))), voting_keypair, None, &fullnode_config, ); + let leader_exit = leader.run(None); + + debug!( + "leader: {:?}", + solana::thin_client::poll_gossip_for_leader(leader_info.gossip, Some(5)).unwrap() + ); let validator_keypair = Arc::new(Keypair::new()); let voting_keypair = VotingKeypair::new_local(&validator_keypair); let mut leader_client = mk_client(&leader_info); - let last_id = leader_client.get_last_id(); - let mut leader_client = mk_client(&leader_info); + debug!("last_id: {:?}", last_id); leader_client .transfer(10, &mint_keypair, validator_keypair.pubkey(), &last_id) @@ -83,24 +83,29 @@ fn test_replicator_startup() { validator_node, &validator_keypair, &validator_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_info.id, - ))), voting_keypair, Some(&leader_info), &fullnode_config, ); + let validator_exit = validator.run(None); let bob = Keypair::new(); info!("starting transfers.."); - - for _ in 0..64 { + for i in 0..64 { + debug!("transfer {}", i); let last_id = leader_client.get_last_id(); + let mut transaction = + SystemTransaction::new_account(&mint_keypair, bob.pubkey(), 1, last_id, 0); leader_client - .transfer(1, &mint_keypair, bob.pubkey(), &last_id) + .retry_transfer(&mint_keypair, &mut transaction, 5) .unwrap(); - sleep(Duration::from_millis(200)); + debug!( + "transfer {}: mint balance={:?}, bob balance={:?}", + i, + leader_client.get_balance(&mint_keypair.pubkey()), + leader_client.get_balance(&bob.pubkey()), + ); } let replicator_keypair = Keypair::new(); @@ -109,11 +114,10 @@ fn test_replicator_startup() { let last_id = leader_client.get_last_id(); // Give the replicator some tokens - let amount = 1; let mut tx = SystemTransaction::new_account( &mint_keypair, replicator_keypair.pubkey(), - amount, + 1, last_id, 0, ); @@ -209,10 +213,11 @@ fn test_replicator_startup() { } replicator.close(); - validator.exit(); - leader.close().expect("Expected successful node closure"); + validator_exit(); + leader_exit(); } + info!("cleanup"); DbLedger::destroy(&leader_ledger_path).expect("Expected successful database destruction"); DbLedger::destroy(&replicator_ledger_path).expect("Expected successful database destruction"); let _ignored = remove_dir_all(&leader_ledger_path); @@ -271,7 +276,7 @@ fn test_replicator_startup_ledger_hang() { let leader_ledger_path = "replicator_test_leader_ledger"; let (_mint_keypair, leader_ledger_path, _last_entry_height, _last_entry_id) = - create_tmp_sample_ledger(leader_ledger_path, 100, 0, leader_info.id, 1); + create_tmp_sample_ledger(leader_ledger_path, 100, 0, leader_info.id, 42); let validator_ledger_path = tmp_copy_ledger(&leader_ledger_path, "replicator_test_validator_ledger"); @@ -283,9 +288,6 @@ fn test_replicator_startup_ledger_hang() { leader_node, &leader_keypair, &leader_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_info.id.clone(), - ))), voting_keypair, None, &FullnodeConfig::default(), @@ -299,9 +301,6 @@ fn test_replicator_startup_ledger_hang() { validator_node, &validator_keypair, &validator_ledger_path, - Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader( - leader_info.id, - ))), voting_keypair, Some(&leader_info), &FullnodeConfig::default(), diff --git a/tests/rpc.rs b/tests/rpc.rs index a8fcb5d04d..932a24c17d 100644 --- a/tests/rpc.rs +++ b/tests/rpc.rs @@ -17,6 +17,7 @@ fn test_rpc_send_tx() { solana_logger::setup(); let (server, leader_data, alice, ledger_path) = new_fullnode("test_rpc_send_tx"); + let server_exit = server.run(None); let bob_pubkey = Keypair::new().pubkey(); let client = reqwest::Client::new(); @@ -92,6 +93,6 @@ fn test_rpc_send_tx() { assert_eq!(confirmed_tx, true); - server.close().unwrap(); + server_exit(); remove_dir_all(ledger_path).unwrap(); } diff --git a/wallet/tests/deploy.rs b/wallet/tests/deploy.rs index 1f3a9233ce..14e64bfd8b 100644 --- a/wallet/tests/deploy.rs +++ b/wallet/tests/deploy.rs @@ -11,6 +11,8 @@ use std::sync::mpsc::channel; #[test] fn test_wallet_deploy_program() { + solana_logger::setup(); + let mut pathbuf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); pathbuf.push("tests"); pathbuf.push("fixtures"); @@ -18,6 +20,7 @@ fn test_wallet_deploy_program() { pathbuf.set_extension("so"); let (server, leader_data, alice, ledger_path) = new_fullnode("test_wallet_deploy_program"); + let server_exit = server.run(None); let (sender, receiver) = channel(); run_local_drone(alice, sender); @@ -73,6 +76,6 @@ fn test_wallet_deploy_program() { &elf ); - server.close().unwrap(); + server_exit(); remove_dir_all(ledger_path).unwrap(); } diff --git a/wallet/tests/pay.rs b/wallet/tests/pay.rs index 1d65b3c25a..f83e316735 100644 --- a/wallet/tests/pay.rs +++ b/wallet/tests/pay.rs @@ -25,6 +25,7 @@ fn check_balance(expected_balance: u64, client: &RpcClient, params: Value) { #[test] fn test_wallet_timestamp_tx() { let (server, leader_data, alice, ledger_path) = new_fullnode("test_wallet_timestamp_tx"); + let server_exit = server.run(None); let bob_pubkey = Keypair::new().pubkey(); let (sender, receiver) = channel(); @@ -85,13 +86,14 @@ fn test_wallet_timestamp_tx() { let params = json!([format!("{}", bob_pubkey)]); check_balance(10, &rpc_client, params); // recipient balance - server.close().unwrap(); + server_exit(); remove_dir_all(ledger_path).unwrap(); } #[test] fn test_wallet_witness_tx() { let (server, leader_data, alice, ledger_path) = new_fullnode("test_wallet_witness_tx"); + let server_exit = server.run(None); let bob_pubkey = Keypair::new().pubkey(); let (sender, receiver) = channel(); @@ -148,13 +150,14 @@ fn test_wallet_witness_tx() { let params = json!([format!("{}", bob_pubkey)]); check_balance(10, &rpc_client, params); // recipient balance - server.close().unwrap(); + server_exit(); remove_dir_all(ledger_path).unwrap(); } #[test] fn test_wallet_cancel_tx() { let (server, leader_data, alice, ledger_path) = new_fullnode("test_wallet_cancel_tx"); + let server_exit = server.run(None); let bob_pubkey = Keypair::new().pubkey(); let (sender, receiver) = channel(); @@ -211,6 +214,6 @@ fn test_wallet_cancel_tx() { let params = json!([format!("{}", bob_pubkey)]); check_balance(0, &rpc_client, params); // recipient balance - server.close().unwrap(); + server_exit(); remove_dir_all(ledger_path).unwrap(); } diff --git a/wallet/tests/request_airdrop.rs b/wallet/tests/request_airdrop.rs index 54ff73d61d..16fe50530d 100644 --- a/wallet/tests/request_airdrop.rs +++ b/wallet/tests/request_airdrop.rs @@ -10,7 +10,7 @@ use std::sync::mpsc::channel; #[test] fn test_wallet_request_airdrop() { let (server, leader_data, alice, ledger_path) = new_fullnode("test_wallet_request_airdrop"); - + let server_exit = server.run(None); let (sender, receiver) = channel(); run_local_drone(alice, sender); let drone_addr = receiver.recv().unwrap(); @@ -33,6 +33,6 @@ fn test_wallet_request_airdrop() { .unwrap(); assert_eq!(balance, 50); - server.close().unwrap(); + server_exit(); remove_dir_all(ledger_path).unwrap(); }