have banks save vote_state by epoch to support stable leader schedules (#3019)

have banks save vote_state by epoch to support stable leader schedules

runtime/src/bank.rs

@@ -97,7 +97,7 @@ pub struct Bank {
     /// Hash of this Bank's parent's state
     parent_hash: Hash,
 
-    /// Bank fork id
+    /// Bank fork (i.e. slot, i.e. block)
     slot: u64,
 
     /// The number of ticks in each slot.
@@ -111,6 +111,10 @@ pub struct Bank {
 
     /// The pubkey to send transactions fees to.
     collector_id: Pubkey,
+
+    /// staked nodes on epoch boundaries, saved off when a bank.slot() is at
+    ///   a leader schedule boundary
+    epoch_vote_states: HashMap<u64, HashMap<Pubkey, VoteState>>,
 }
 
 impl Bank {
@@ -123,6 +127,14 @@ impl Bank {
         bank.accounts = Some(Arc::new(Accounts::new(bank.slot, paths)));
         bank.process_genesis_block(genesis_block);
         bank.add_builtin_programs();
+
+        // genesis needs stakes for all epochs up to the epoch implied by
+        //  slot = 0 and genesis configuration
+        let vote_states = bank.vote_states(|_, _| true);
+        for i in 0..=bank.epoch_from_stakers_slot_offset() {
+            bank.epoch_vote_states.insert(i, vote_states.clone());
+        }
+
         bank
     }
 
@@ -143,6 +155,19 @@ impl Bank {
         bank.accounts = Some(parent.accounts());
         bank.accounts().new_from_parent(bank.slot, parent.slot);
 
+        bank.epoch_vote_states = {
+            let mut epoch_vote_states = parent.epoch_vote_states.clone();
+            let epoch = bank.epoch_from_stakers_slot_offset();
+
+            // update epoch_vote_states cache
+            //  if my parent didn't populate for this epoch, we've
+            //  crossed a boundary
+            if epoch_vote_states.get(&epoch).is_none() {
+                epoch_vote_states.insert(epoch, bank.vote_states(|_, _| true));
+            }
+            epoch_vote_states
+        };
+
         bank
     }
 
@@ -715,6 +740,30 @@ impl Bank {
         self.slots_per_epoch
     }
 
+    /// returns the epoch for which this bank's stakers_slot_offset and slot would
+    ///  need to cache stakers
+    fn epoch_from_stakers_slot_offset(&self) -> u64 {
+        (self.slot + self.stakers_slot_offset) / self.slots_per_epoch
+    }
+
+    pub fn epoch_vote_states<F>(&self, epoch: u64, cond: F) -> Option<HashMap<Pubkey, VoteState>>
+    where
+        F: Fn(&Pubkey, &VoteState) -> bool,
+    {
+        self.epoch_vote_states.get(&epoch).map(|vote_states| {
+            vote_states
+                .iter()
+                .filter_map(|(p, vote_state)| {
+                    if cond(p, vote_state) {
+                        Some((*p, vote_state.clone()))
+                    } else {
+                        None
+                    }
+                })
+                .collect()
+        })
+    }
+
     pub fn vote_states<F>(&self, cond: F) -> HashMap<Pubkey, VoteState>
     where
         F: Fn(&Pubkey, &VoteState) -> bool,
@@ -1402,4 +1451,47 @@ mod tests {
         bank.squash();
         assert_eq!(parent.get_balance(&key1.pubkey()), 1);
     }
+
+    #[test]
+    fn test_bank_epoch_vote_states() {
+        let leader_id = Keypair::new().pubkey();
+        let leader_tokens = 2;
+        let (mut genesis_block, _) = GenesisBlock::new_with_leader(5, leader_id, leader_tokens);
+
+        // set this up weird, forces:
+        //  1. genesis bank to cover epochs 0, 1, *and* 2
+        //  2. child banks to cover epochs in their future
+        //
+        const SLOTS_PER_EPOCH: u64 = 8;
+        const STAKERS_SLOT_OFFSET: u64 = 21;
+        genesis_block.slots_per_epoch = SLOTS_PER_EPOCH;
+        genesis_block.stakers_slot_offset = STAKERS_SLOT_OFFSET;
+
+        let parent = Arc::new(Bank::new(&genesis_block));
+
+        let vote_states0 =
+            parent.epoch_vote_states(0, |_, vote_state| vote_state.delegate_id == leader_id);
+        assert!(vote_states0.is_some());
+        assert!(vote_states0.iter().len() != 0);
+
+        assert!(parent.epoch_vote_states(1, |_, _| true).is_some());
+        assert!(parent.epoch_vote_states(2, |_, _| true).is_some());
+
+        // child crosses epoch boundary and is the first slot in the epoch
+        let child = Bank::new_from_parent_and_id(
+            &parent,
+            leader_id,
+            SLOTS_PER_EPOCH - (STAKERS_SLOT_OFFSET % SLOTS_PER_EPOCH),
+        );
+        assert!(child.epoch_vote_states(3, |_, _| true).is_some());
+
+        // child crosses epoch boundary but isn't the first slot in the epoch
+        let child = Bank::new_from_parent_and_id(
+            &parent,
+            leader_id,
+            SLOTS_PER_EPOCH - (STAKERS_SLOT_OFFSET % SLOTS_PER_EPOCH) + 1,
+        );
+        assert!(child.epoch_vote_states(3, |_, _| true).is_some());
+    }
+
 }

sdk/src/vote_program.rs

@@ -102,7 +102,7 @@ impl VoteInstruction {
     }
 }
 
-#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq, Clone)]
 pub struct VoteState {
     pub votes: VecDeque<Lockout>,
     pub delegate_id: Pubkey,