stake-pool: Optimizations for 4k validators (#2041)

* In-place serde

* Add cleanup instruction

* Add BigVec tests, clarify lifetimes

stake-pool/cli/src/main.rs

@@ -697,7 +697,7 @@ fn command_list(config: &Config, stake_pool_address: &Pubkey) -> CommandResult {
     );
     println!(
         "Max number of validators: {}",
-        validator_list.max_validators
+        validator_list.header.max_validators
     );
 
     if config.verbose {
@@ -755,7 +755,7 @@ fn command_update(
 
     let validator_list = get_validator_list(&config.rpc_client, &stake_pool.validator_list)?;
 
-    let (mut update_list_instructions, update_balance_instruction) =
+    let (mut update_list_instructions, final_instructions) =
         spl_stake_pool::instruction::update_stake_pool(
             &spl_stake_pool::id(),
             &stake_pool,
@@ -787,7 +787,7 @@ fn command_update(
     }
     let transaction = checked_transaction_with_signers(
         config,
-        &[update_balance_instruction],
+        &final_instructions,
         &[config.fee_payer.as_ref()],
     )?;
     send_transaction(config, transaction)?;

stake-pool/program/src/big_vec.rs

@@ -0,0 +1,375 @@
+//! Big vector type, used with vectors that can't be serde'd
+
+use {
+    arrayref::array_ref,
+    borsh::{BorshDeserialize, BorshSerialize},
+    solana_program::{
+        program_error::ProgramError, program_memory::sol_memmove, program_pack::Pack,
+    },
+    std::marker::PhantomData,
+};
+
+/// Contains easy to use utilities for a big vector of Borsh-compatible types,
+/// to avoid managing the entire struct on-chain and blow through stack limits.
+pub struct BigVec<'data> {
+    /// Underlying data buffer, pieces of which are serialized
+    pub data: &'data mut [u8],
+}
+
+const VEC_SIZE_BYTES: usize = 4;
+
+impl<'data> BigVec<'data> {
+    /// Get the length of the vector
+    pub fn len(&self) -> u32 {
+        let vec_len = array_ref![self.data, 0, VEC_SIZE_BYTES];
+        u32::from_le_bytes(*vec_len)
+    }
+
+    /// Find out if the vector has no contents (as demanded by clippy)
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    /// Retain all elements that match the provided function, discard all others
+    pub fn retain<T: Pack>(&mut self, predicate: fn(&[u8]) -> bool) -> Result<(), ProgramError> {
+        let mut vec_len = self.len();
+        let mut removals_found = 0;
+        let mut dst_start_index = 0;
+
+        let data_start_index = VEC_SIZE_BYTES;
+        let data_end_index =
+            data_start_index.saturating_add((vec_len as usize).saturating_mul(T::LEN));
+        for start_index in (data_start_index..data_end_index).step_by(T::LEN) {
+            let end_index = start_index + T::LEN;
+            let slice = &self.data[start_index..end_index];
+            if !predicate(slice) {
+                let gap = removals_found * T::LEN;
+                if removals_found > 0 {
+                    // In case the compute budget is ever bumped up, allowing us
+                    // to use this safe code instead:
+                    // self.data.copy_within(dst_start_index + gap..start_index, dst_start_index);
+                    unsafe {
+                        sol_memmove(
+                            self.data[dst_start_index..start_index - gap].as_mut_ptr(),
+                            self.data[dst_start_index + gap..start_index].as_mut_ptr(),
+                            start_index - gap - dst_start_index,
+                        );
+                    }
+                }
+                dst_start_index = start_index - gap;
+                removals_found += 1;
+                vec_len -= 1;
+            }
+        }
+
+        // final memmove
+        if removals_found > 0 {
+            let gap = removals_found * T::LEN;
+            // In case the compute budget is ever bumped up, allowing us
+            // to use this safe code instead:
+            //self.data.copy_within(dst_start_index + gap..data_end_index, dst_start_index);
+            unsafe {
+                sol_memmove(
+                    self.data[dst_start_index..data_end_index - gap].as_mut_ptr(),
+                    self.data[dst_start_index + gap..data_end_index].as_mut_ptr(),
+                    data_end_index - gap - dst_start_index,
+                );
+            }
+        }
+
+        let mut vec_len_ref = &mut self.data[0..VEC_SIZE_BYTES];
+        vec_len.serialize(&mut vec_len_ref)?;
+
+        Ok(())
+    }
+
+    /// Extracts a slice of the data types
+    pub fn deserialize_mut_slice<T: Pack>(
+        &mut self,
+        skip: usize,
+        len: usize,
+    ) -> Result<Vec<&'data mut T>, ProgramError> {
+        let vec_len = self.len();
+        if skip + len > vec_len as usize {
+            return Err(ProgramError::AccountDataTooSmall);
+        }
+
+        let start_index = VEC_SIZE_BYTES.saturating_add(skip.saturating_mul(T::LEN));
+        let end_index = start_index.saturating_add(len.saturating_mul(T::LEN));
+        let mut deserialized = vec![];
+        for slice in self.data[start_index..end_index].chunks_exact_mut(T::LEN) {
+            deserialized.push(unsafe { &mut *(slice.as_ptr() as *mut T) });
+        }
+        Ok(deserialized)
+    }
+
+    /// Add new element to the end
+    pub fn push<T: Pack>(&mut self, element: T) -> Result<(), ProgramError> {
+        let mut vec_len_ref = &mut self.data[0..VEC_SIZE_BYTES];
+        let mut vec_len = u32::try_from_slice(vec_len_ref)?;
+
+        let start_index = VEC_SIZE_BYTES + vec_len as usize * T::LEN;
+        let end_index = start_index + T::LEN;
+
+        vec_len += 1;
+        vec_len.serialize(&mut vec_len_ref)?;
+
+        if self.data.len() < end_index {
+            return Err(ProgramError::AccountDataTooSmall);
+        }
+        let mut element_ref = &mut self.data[start_index..start_index + T::LEN];
+        element.pack_into_slice(&mut element_ref);
+        Ok(())
+    }
+
+    /// Get an iterator for the type provided
+    pub fn iter<'vec, T: Pack>(&'vec self) -> Iter<'data, 'vec, T> {
+        Iter {
+            len: self.len() as usize,
+            current: 0,
+            current_index: VEC_SIZE_BYTES,
+            inner: self,
+            phantom: PhantomData,
+        }
+    }
+
+    /// Get a mutable iterator for the type provided
+    pub fn iter_mut<'vec, T: Pack>(&'vec mut self) -> IterMut<'data, 'vec, T> {
+        IterMut {
+            len: self.len() as usize,
+            current: 0,
+            current_index: VEC_SIZE_BYTES,
+            inner: self,
+            phantom: PhantomData,
+        }
+    }
+
+    /// Find matching data in the array
+    pub fn find<T: Pack>(&self, data: &[u8], predicate: fn(&[u8], &[u8]) -> bool) -> Option<&T> {
+        let len = self.len() as usize;
+        let mut current = 0;
+        let mut current_index = VEC_SIZE_BYTES;
+        while current != len {
+            let end_index = current_index + T::LEN;
+            let current_slice = &self.data[current_index..end_index];
+            if predicate(current_slice, data) {
+                return Some(unsafe { &*(current_slice.as_ptr() as *const T) });
+            }
+            current_index = end_index;
+            current += 1;
+        }
+        None
+    }
+
+    /// Find matching data in the array
+    pub fn find_mut<T: Pack>(
+        &mut self,
+        data: &[u8],
+        predicate: fn(&[u8], &[u8]) -> bool,
+    ) -> Option<&mut T> {
+        let len = self.len() as usize;
+        let mut current = 0;
+        let mut current_index = VEC_SIZE_BYTES;
+        while current != len {
+            let end_index = current_index + T::LEN;
+            let current_slice = &self.data[current_index..end_index];
+            if predicate(current_slice, data) {
+                return Some(unsafe { &mut *(current_slice.as_ptr() as *mut T) });
+            }
+            current_index = end_index;
+            current += 1;
+        }
+        None
+    }
+}
+
+/// Iterator wrapper over a BigVec
+pub struct Iter<'data, 'vec, T> {
+    len: usize,
+    current: usize,
+    current_index: usize,
+    inner: &'vec BigVec<'data>,
+    phantom: PhantomData<T>,
+}
+
+impl<'data, 'vec, T: Pack + 'data> Iterator for Iter<'data, 'vec, T> {
+    type Item = &'data T;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.current == self.len {
+            None
+        } else {
+            let end_index = self.current_index + T::LEN;
+            let value = Some(unsafe {
+                &*(self.inner.data[self.current_index..end_index].as_ptr() as *const T)
+            });
+            self.current += 1;
+            self.current_index = end_index;
+            value
+        }
+    }
+}
+
+/// Iterator wrapper over a BigVec
+pub struct IterMut<'data, 'vec, T> {
+    len: usize,
+    current: usize,
+    current_index: usize,
+    inner: &'vec mut BigVec<'data>,
+    phantom: PhantomData<T>,
+}
+
+impl<'data, 'vec, T: Pack + 'data> Iterator for IterMut<'data, 'vec, T> {
+    type Item = &'data mut T;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.current == self.len {
+            None
+        } else {
+            let end_index = self.current_index + T::LEN;
+            let value = Some(unsafe {
+                &mut *(self.inner.data[self.current_index..end_index].as_ptr() as *mut T)
+            });
+            self.current += 1;
+            self.current_index = end_index;
+            value
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use {
+        super::*,
+        solana_program::{program_memory::sol_memcmp, program_pack::Sealed},
+    };
+
+    #[derive(Debug, PartialEq)]
+    struct TestStruct {
+        value: u64,
+    }
+
+    impl Sealed for TestStruct {}
+
+    impl Pack for TestStruct {
+        const LEN: usize = 8;
+        fn pack_into_slice(&self, data: &mut [u8]) {
+            let mut data = data;
+            self.value.serialize(&mut data).unwrap();
+        }
+        fn unpack_from_slice(src: &[u8]) -> Result<Self, ProgramError> {
+            Ok(TestStruct {
+                value: u64::try_from_slice(src).unwrap(),
+            })
+        }
+    }
+
+    impl TestStruct {
+        fn new(value: u64) -> Self {
+            Self { value }
+        }
+    }
+
+    fn from_slice<'data, 'other>(data: &'data mut [u8], vec: &'other [u64]) -> BigVec<'data> {
+        let mut big_vec = BigVec { data };
+        for element in vec {
+            big_vec.push(TestStruct::new(*element)).unwrap();
+        }
+        big_vec
+    }
+
+    fn check_big_vec_eq(big_vec: &BigVec, slice: &[u64]) {
+        assert!(big_vec
+            .iter::<TestStruct>()
+            .map(|x| &x.value)
+            .zip(slice.iter())
+            .all(|(a, b)| a == b));
+    }
+
+    #[test]
+    fn push() {
+        let mut data = [0u8; 4 + 8 * 3];
+        let mut v = BigVec { data: &mut data };
+        v.push(TestStruct::new(1)).unwrap();
+        check_big_vec_eq(&v, &[1]);
+        v.push(TestStruct::new(2)).unwrap();
+        check_big_vec_eq(&v, &[1, 2]);
+        v.push(TestStruct::new(3)).unwrap();
+        check_big_vec_eq(&v, &[1, 2, 3]);
+        assert_eq!(
+            v.push(TestStruct::new(4)).unwrap_err(),
+            ProgramError::AccountDataTooSmall
+        );
+    }
+
+    #[test]
+    fn retain() {
+        fn mod_2_predicate(data: &[u8]) -> bool {
+            u64::try_from_slice(data).unwrap() % 2 == 0
+        }
+
+        let mut data = [0u8; 4 + 8 * 4];
+        let mut v = from_slice(&mut data, &[1, 2, 3, 4]);
+        v.retain::<TestStruct>(mod_2_predicate).unwrap();
+        check_big_vec_eq(&v, &[2, 4]);
+    }
+
+    fn find_predicate(a: &[u8], b: &[u8]) -> bool {
+        if a.len() != b.len() {
+            false
+        } else {
+            sol_memcmp(a, b, a.len()) == 0
+        }
+    }
+
+    #[test]
+    fn find() {
+        let mut data = [0u8; 4 + 8 * 4];
+        let v = from_slice(&mut data, &[1, 2, 3, 4]);
+        assert_eq!(
+            v.find::<TestStruct>(&1u64.to_le_bytes(), find_predicate),
+            Some(&TestStruct::new(1))
+        );
+        assert_eq!(
+            v.find::<TestStruct>(&4u64.to_le_bytes(), find_predicate),
+            Some(&TestStruct::new(4))
+        );
+        assert_eq!(
+            v.find::<TestStruct>(&5u64.to_le_bytes(), find_predicate),
+            None
+        );
+    }
+
+    #[test]
+    fn find_mut() {
+        let mut data = [0u8; 4 + 8 * 4];
+        let mut v = from_slice(&mut data, &[1, 2, 3, 4]);
+        let mut test_struct = v
+            .find_mut::<TestStruct>(&1u64.to_le_bytes(), find_predicate)
+            .unwrap();
+        test_struct.value = 0;
+        check_big_vec_eq(&v, &[0, 2, 3, 4]);
+        assert_eq!(
+            v.find_mut::<TestStruct>(&5u64.to_le_bytes(), find_predicate),
+            None
+        );
+    }
+
+    #[test]
+    fn deserialize_mut_slice() {
+        let mut data = [0u8; 4 + 8 * 4];
+        let mut v = from_slice(&mut data, &[1, 2, 3, 4]);
+        let mut slice = v.deserialize_mut_slice::<TestStruct>(1, 2).unwrap();
+        slice[0].value = 10;
+        slice[1].value = 11;
+        check_big_vec_eq(&v, &[1, 10, 11, 4]);
+        assert_eq!(
+            v.deserialize_mut_slice::<TestStruct>(1, 4).unwrap_err(),
+            ProgramError::AccountDataTooSmall
+        );
+        assert_eq!(
+            v.deserialize_mut_slice::<TestStruct>(4, 1).unwrap_err(),
+            ProgramError::AccountDataTooSmall
+        );
+    }
+}

stake-pool/program/src/instruction.rs

@@ -176,9 +176,9 @@ pub enum StakePoolInstruction {
     /// between SOL staked on different validators, the staker can force all
     /// deposits and/or withdraws to go to one chosen account, or unset that account.
     ///
-    /// 0. `[]` Stake pool
+    /// 0. `[w]` Stake pool
     /// 1. `[s]` Stake pool staker
-    /// 2. `[w]` Validator list
+    /// 2. `[]` Validator list
     ///
     /// Fails if the validator is not part of the stake pool.
     SetPreferredValidator {
@@ -231,6 +231,12 @@ pub enum StakePoolInstruction {
     ///   7. `[]` Pool token program
     UpdateStakePoolBalance,
 
+    ///   Cleans up validator stake account entries marked as `ReadyForRemoval`
+    ///
+    ///   0. `[]` Stake pool
+    ///   1. `[w]` Validator stake list storage account
+    CleanupRemovedValidatorEntries,
+
     ///   Deposit some stake into the pool.  The output is a "pool" token representing ownership
     ///   into the pool. Inputs are converted to the current ratio.
     ///
@@ -515,9 +521,9 @@ pub fn set_preferred_validator(
     Instruction {
         program_id: *program_id,
         accounts: vec![
-            AccountMeta::new_readonly(*stake_pool_address, false),
+            AccountMeta::new(*stake_pool_address, false),
             AccountMeta::new_readonly(*staker, true),
-            AccountMeta::new(*validator_list_address, false),
+            AccountMeta::new_readonly(*validator_list_address, false),
         ],
         data: StakePoolInstruction::SetPreferredValidator {
             validator_type,
@@ -730,6 +736,25 @@ pub fn update_stake_pool_balance(
     }
 }
 
+/// Creates `CleanupRemovedValidatorEntries` instruction (removes entries from the validator list)
+pub fn cleanup_removed_validator_entries(
+    program_id: &Pubkey,
+    stake_pool: &Pubkey,
+    validator_list_storage: &Pubkey,
+) -> Instruction {
+    let accounts = vec![
+        AccountMeta::new_readonly(*stake_pool, false),
+        AccountMeta::new(*validator_list_storage, false),
+    ];
+    Instruction {
+        program_id: *program_id,
+        accounts,
+        data: StakePoolInstruction::CleanupRemovedValidatorEntries
+            .try_to_vec()
+            .unwrap(),
+    }
+}
+
 /// Creates all `UpdateValidatorListBalance` and `UpdateStakePoolBalance`
 /// instructions for fully updating a stake pool each epoch
 pub fn update_stake_pool(
@@ -738,7 +763,7 @@ pub fn update_stake_pool(
     validator_list: &ValidatorList,
     stake_pool_address: &Pubkey,
     no_merge: bool,
-) -> (Vec<Instruction>, Instruction) {
+) -> (Vec<Instruction>, Vec<Instruction>) {
     let vote_accounts: Vec<Pubkey> = validator_list
         .validators
         .iter()
@@ -764,7 +789,8 @@ pub fn update_stake_pool(
         start_index += MAX_VALIDATORS_TO_UPDATE as u32;
     }
 
-    let update_balance_instruction = update_stake_pool_balance(
+    let final_instructions = vec![
+        update_stake_pool_balance(
             program_id,
             stake_pool_address,
             &withdraw_authority,
@@ -772,8 +798,14 @@ pub fn update_stake_pool(
             &stake_pool.reserve_stake,
             &stake_pool.manager_fee_account,
             &stake_pool.pool_mint,
-    );
+        ),
-    (update_list_instructions, update_balance_instruction)
+        cleanup_removed_validator_entries(
+            program_id,
+            stake_pool_address,
+            &stake_pool.validator_list,
+        ),
+    ];
+    (update_list_instructions, final_instructions)
 }
 
 /// Creates instructions required to deposit into a stake pool, given a stake

stake-pool/program/src/lib.rs

@@ -2,6 +2,7 @@
 
 //! A program for creating and managing pools of stake
 
+pub mod big_vec;
 pub mod error;
 pub mod instruction;
 pub mod processor;

stake-pool/program/src/processor.rs

@@ -6,7 +6,10 @@ use {
         find_deposit_authority_program_address,
         instruction::{PreferredValidatorType, StakePoolInstruction},
         minimum_reserve_lamports, minimum_stake_lamports, stake_program,
-        state::{AccountType, Fee, StakePool, StakeStatus, ValidatorList, ValidatorStakeInfo},
+        state::{
+            AccountType, Fee, StakePool, StakeStatus, ValidatorList, ValidatorListHeader,
+            ValidatorStakeInfo,
+        },
         AUTHORITY_DEPOSIT, AUTHORITY_WITHDRAW, MINIMUM_ACTIVE_STAKE, TRANSIENT_STAKE_SEED,
     },
     borsh::{BorshDeserialize, BorshSerialize},
@@ -480,7 +483,7 @@ impl Processor {
         check_account_owner(validator_list_info, program_id)?;
         let mut validator_list =
             try_from_slice_unchecked::<ValidatorList>(&validator_list_info.data.borrow())?;
-        if !validator_list.is_uninitialized() {
+        if !validator_list.header.is_uninitialized() {
             msg!("Provided validator list already in use");
             return Err(StakePoolError::AlreadyInUse.into());
         }
@@ -495,11 +498,9 @@ impl Processor {
             );
             return Err(StakePoolError::UnexpectedValidatorListAccountSize.into());
         }
-        validator_list.account_type = AccountType::ValidatorList;
+        validator_list.header.account_type = AccountType::ValidatorList;
-        validator_list.preferred_deposit_validator_vote_address = None;
+        validator_list.header.max_validators = max_validators;
-        validator_list.preferred_withdraw_validator_vote_address = None;
         validator_list.validators.clear();
-        validator_list.max_validators = max_validators;
 
         if !rent.is_exempt(stake_pool_info.lamports(), stake_pool_info.data_len()) {
             msg!("Stake pool not rent-exempt");
@@ -610,6 +611,8 @@ impl Processor {
         stake_pool.total_stake_lamports = total_stake_lamports;
         stake_pool.fee = fee;
         stake_pool.next_epoch_fee = None;
+        stake_pool.preferred_deposit_validator_vote_address = None;
+        stake_pool.preferred_withdraw_validator_vote_address = None;
 
         stake_pool
             .serialize(&mut *stake_pool_info.data.borrow_mut())
@@ -746,12 +749,13 @@ impl Processor {
         }
 
         check_account_owner(validator_list_info, program_id)?;
-        let mut validator_list =
+        let mut validator_list_data = validator_list_info.data.borrow_mut();
-            try_from_slice_unchecked::<ValidatorList>(&validator_list_info.data.borrow())?;
+        let (header, mut validator_list) =
-        if !validator_list.is_valid() {
+            ValidatorListHeader::deserialize_vec(&mut validator_list_data)?;
+        if !header.is_valid() {
             return Err(StakePoolError::InvalidState.into());
         }
-        if validator_list.max_validators as usize == validator_list.validators.len() {
+        if header.max_validators == validator_list.len() {
             return Err(ProgramError::AccountDataTooSmall);
         }
 
@@ -769,7 +773,11 @@ impl Processor {
             return Err(StakePoolError::WrongStakeState.into());
         }
 
-        if validator_list.contains(&vote_account_address) {
+        let maybe_validator_stake_info = validator_list.find::<ValidatorStakeInfo>(
+            vote_account_address.as_ref(),
+            ValidatorStakeInfo::memcmp_pubkey,
+        );
+        if maybe_validator_stake_info.is_some() {
             return Err(StakePoolError::ValidatorAlreadyAdded.into());
         }
 
@@ -797,14 +805,13 @@ impl Processor {
             stake_program_info.clone(),
         )?;
 
-        validator_list.validators.push(ValidatorStakeInfo {
+        validator_list.push(ValidatorStakeInfo {
             status: StakeStatus::Active,
             vote_account_address,
             active_stake_lamports: stake_lamports.saturating_sub(minimum_lamport_amount),
             transient_stake_lamports: 0,
             last_update_epoch: clock.epoch,
-        });
+        })?;
-        validator_list.serialize(&mut *validator_list_info.data.borrow_mut())?;
 
         Ok(())
     }
@@ -829,7 +836,7 @@ impl Processor {
         check_stake_program(stake_program_info.key)?;
         check_account_owner(stake_pool_info, program_id)?;
 
-        let stake_pool = try_from_slice_unchecked::<StakePool>(&stake_pool_info.data.borrow())?;
+        let mut stake_pool = try_from_slice_unchecked::<StakePool>(&stake_pool_info.data.borrow())?;
         if !stake_pool.is_valid() {
             return Err(StakePoolError::InvalidState.into());
         }
@@ -848,9 +855,10 @@ impl Processor {
         stake_pool.check_validator_list(validator_list_info)?;
 
         check_account_owner(validator_list_info, program_id)?;
-        let mut validator_list =
+        let mut validator_list_data = validator_list_info.data.borrow_mut();
-            try_from_slice_unchecked::<ValidatorList>(&validator_list_info.data.borrow())?;
+        let (header, mut validator_list) =
-        if !validator_list.is_valid() {
+            ValidatorListHeader::deserialize_vec(&mut validator_list_data)?;
+        if !header.is_valid() {
             return Err(StakePoolError::InvalidState.into());
         }
 
@@ -869,15 +877,17 @@ impl Processor {
             &vote_account_address,
         )?;
 
-        let maybe_validator_list_entry = validator_list.find_mut(&vote_account_address);
+        let maybe_validator_stake_info = validator_list.find_mut::<ValidatorStakeInfo>(
-        if maybe_validator_list_entry.is_none() {
+            vote_account_address.as_ref(),
+            ValidatorStakeInfo::memcmp_pubkey,
+        );
+        if maybe_validator_stake_info.is_none() {
             msg!(
                 "Vote account {} not found in stake pool",
                 vote_account_address
             );
             return Err(StakePoolError::ValidatorNotFound.into());
         }
-        let mut validator_list_entry = maybe_validator_list_entry.unwrap();
 
         let stake_lamports = **stake_account_info.lamports.borrow();
         let required_lamports = minimum_stake_lamports(&meta);
@@ -919,21 +929,16 @@ impl Processor {
             stake_program_info.clone(),
         )?;
 
-        match new_status {
+        let mut validator_stake_info = maybe_validator_stake_info.unwrap();
-            StakeStatus::DeactivatingTransient => validator_list_entry.status = new_status,
+        validator_stake_info.status = new_status;
-            StakeStatus::ReadyForRemoval => validator_list
-                .validators
-                .retain(|item| item.vote_account_address != vote_account_address),
-            _ => unreachable!(),
-        }
 
-        if validator_list.preferred_deposit_validator_vote_address == Some(vote_account_address) {
+        if stake_pool.preferred_deposit_validator_vote_address == Some(vote_account_address) {
-            validator_list.preferred_deposit_validator_vote_address = None;
+            stake_pool.preferred_deposit_validator_vote_address = None;
         }
-        if validator_list.preferred_withdraw_validator_vote_address == Some(vote_account_address) {
+        if stake_pool.preferred_withdraw_validator_vote_address == Some(vote_account_address) {
-            validator_list.preferred_withdraw_validator_vote_address = None;
+            stake_pool.preferred_withdraw_validator_vote_address = None;
         }
-        validator_list.serialize(&mut *validator_list_info.data.borrow_mut())?;
+        stake_pool.serialize(&mut *stake_pool_info.data.borrow_mut())?;
 
         Ok(())
     }
@@ -981,9 +986,10 @@ impl Processor {
 
         stake_pool.check_validator_list(validator_list_info)?;
         check_account_owner(validator_list_info, program_id)?;
-        let mut validator_list =
+        let validator_list_data = &mut *validator_list_info.data.borrow_mut();
-            try_from_slice_unchecked::<ValidatorList>(&validator_list_info.data.borrow())?;
+        let (validator_list_header, mut validator_list) =
-        if !validator_list.is_valid() {
+            ValidatorListHeader::deserialize_vec(validator_list_data)?;
+        if !validator_list_header.is_valid() {
             return Err(StakePoolError::InvalidState.into());
         }
 
@@ -1009,15 +1015,18 @@ impl Processor {
             &[transient_stake_bump_seed],
         ];
 
-        let maybe_validator_list_entry = validator_list.find_mut(&vote_account_address);
+        let maybe_validator_stake_info = validator_list.find_mut::<ValidatorStakeInfo>(
-        if maybe_validator_list_entry.is_none() {
+            vote_account_address.as_ref(),
+            ValidatorStakeInfo::memcmp_pubkey,
+        );
+        if maybe_validator_stake_info.is_none() {
             msg!(
                 "Vote account {} not found in stake pool",
                 vote_account_address
             );
             return Err(StakePoolError::ValidatorNotFound.into());
         }
-        let mut validator_list_entry = maybe_validator_list_entry.unwrap();
+        let mut validator_stake_info = maybe_validator_stake_info.unwrap();
 
         let stake_rent = rent.minimum_balance(std::mem::size_of::<stake_program::StakeState>());
         if lamports <= stake_rent {
@@ -1056,12 +1065,11 @@ impl Processor {
             stake_pool.withdraw_bump_seed,
         )?;
 
-        validator_list_entry.active_stake_lamports = validator_list_entry
+        validator_stake_info.active_stake_lamports = validator_stake_info
             .active_stake_lamports
             .checked_sub(lamports)
             .ok_or(StakePoolError::CalculationFailure)?;
-        validator_list_entry.transient_stake_lamports = lamports;
+        validator_stake_info.transient_stake_lamports = lamports;
-        validator_list.serialize(&mut *validator_list_info.data.borrow_mut())?;
 
         Ok(())
     }
@@ -1114,9 +1122,10 @@ impl Processor {
         stake_pool.check_reserve_stake(reserve_stake_account_info)?;
         check_account_owner(validator_list_info, program_id)?;
 
-        let mut validator_list =
+        let mut validator_list_data = validator_list_info.data.borrow_mut();
-            try_from_slice_unchecked::<ValidatorList>(&validator_list_info.data.borrow())?;
+        let (header, mut validator_list) =
-        if !validator_list.is_valid() {
+            ValidatorListHeader::deserialize_vec(&mut validator_list_data)?;
+        if !header.is_valid() {
             return Err(StakePoolError::InvalidState.into());
         }
 
@@ -1134,17 +1143,20 @@ impl Processor {
             &[transient_stake_bump_seed],
         ];
 
-        let maybe_validator_list_entry = validator_list.find_mut(vote_account_address);
+        let maybe_validator_stake_info = validator_list.find_mut::<ValidatorStakeInfo>(
-        if maybe_validator_list_entry.is_none() {
+            vote_account_address.as_ref(),
+            ValidatorStakeInfo::memcmp_pubkey,
+        );
+        if maybe_validator_stake_info.is_none() {
             msg!(
                 "Vote account {} not found in stake pool",
                 vote_account_address
             );
             return Err(StakePoolError::ValidatorNotFound.into());
         }
-        let mut validator_list_entry = maybe_validator_list_entry.unwrap();
+        let mut validator_stake_info = maybe_validator_stake_info.unwrap();
 
-        if validator_list_entry.status != StakeStatus::Active {
+        if validator_stake_info.status != StakeStatus::Active {
             msg!("Validator is marked for removal and no longer allows increases");
             return Err(StakePoolError::ValidatorNotFound.into());
         }
@@ -1208,8 +1220,7 @@ impl Processor {
             stake_pool.withdraw_bump_seed,
         )?;
 
-        validator_list_entry.transient_stake_lamports = total_lamports;
+        validator_stake_info.transient_stake_lamports = total_lamports;
-        validator_list.serialize(&mut *validator_list_info.data.borrow_mut())?;
 
         Ok(())
     }
@@ -1229,7 +1240,7 @@ impl Processor {
         check_account_owner(stake_pool_info, program_id)?;
         check_account_owner(validator_list_info, program_id)?;
 
-        let stake_pool = try_from_slice_unchecked::<StakePool>(&stake_pool_info.data.borrow())?;
+        let mut stake_pool = try_from_slice_unchecked::<StakePool>(&stake_pool_info.data.borrow())?;
         if !stake_pool.is_valid() {
             msg!("Expected valid stake pool");
             return Err(StakePoolError::InvalidState.into());
@@ -1238,14 +1249,19 @@ impl Processor {
         stake_pool.check_staker(staker_info)?;
         stake_pool.check_validator_list(validator_list_info)?;
 
-        let mut validator_list =
+        let mut validator_list_data = validator_list_info.data.borrow_mut();
-            try_from_slice_unchecked::<ValidatorList>(&validator_list_info.data.borrow())?;
+        let (header, validator_list) =
-        if !validator_list.is_valid() {
+            ValidatorListHeader::deserialize_vec(&mut validator_list_data)?;
+        if !header.is_valid() {
             return Err(StakePoolError::InvalidState.into());
         }
 
         if let Some(vote_account_address) = vote_account_address {
-            if !validator_list.contains(&vote_account_address) {
+            let maybe_validator_stake_info = validator_list.find::<ValidatorStakeInfo>(
+                vote_account_address.as_ref(),
+                ValidatorStakeInfo::memcmp_pubkey,
+            );
+            if maybe_validator_stake_info.is_none() {
                 msg!("Validator for {} not present in the stake pool, cannot set as preferred deposit account");
                 return Err(StakePoolError::ValidatorNotFound.into());
             }
@@ -1253,13 +1269,13 @@ impl Processor {
 
         match validator_type {
             PreferredValidatorType::Deposit => {
-                validator_list.preferred_deposit_validator_vote_address = vote_account_address
+                stake_pool.preferred_deposit_validator_vote_address = vote_account_address
             }
             PreferredValidatorType::Withdraw => {
-                validator_list.preferred_withdraw_validator_vote_address = vote_account_address
+                stake_pool.preferred_withdraw_validator_vote_address = vote_account_address
             }
         };
-        validator_list.serialize(&mut *validator_list_info.data.borrow_mut())?;
+        stake_pool.serialize(&mut *stake_pool_info.data.borrow_mut())?;
         Ok(())
     }
 
@@ -1296,17 +1312,20 @@ impl Processor {
         check_stake_program(stake_program_info.key)?;
 
         check_account_owner(validator_list_info, program_id)?;
-        let mut validator_list =
+        let mut validator_list_data = validator_list_info.data.borrow_mut();
-            try_from_slice_unchecked::<ValidatorList>(&validator_list_info.data.borrow())?;
+        let (validator_list_header, mut validator_slice) =
-        if !validator_list.is_valid() {
+            ValidatorListHeader::deserialize_mut_slice(
+                &mut validator_list_data,
+                start_index as usize,
+                validator_stake_accounts.len() / 2,
+            )?;
+
+        if !validator_list_header.is_valid() {
             return Err(StakePoolError::InvalidState.into());
         }
 
-        let mut changes = false;
+        let validator_iter = &mut validator_slice
-        let validator_iter = &mut validator_list
-            .validators
             .iter_mut()
-            .skip(start_index as usize)
             .zip(validator_stake_accounts.chunks_exact(2));
         for (validator_stake_record, validator_stakes) in validator_iter {
             // chunks_exact means that we always get 2 elements, making this safe
@@ -1485,11 +1504,6 @@ impl Processor {
             validator_stake_record.last_update_epoch = clock.epoch;
             validator_stake_record.active_stake_lamports = active_stake_lamports;
             validator_stake_record.transient_stake_lamports = transient_stake_lamports;
-            changes = true;
-        }
-
-        if changes {
-            validator_list.serialize(&mut *validator_list_info.data.borrow_mut())?;
         }
 
         Ok(())
@@ -1531,9 +1545,10 @@ impl Processor {
         }
 
         check_account_owner(validator_list_info, program_id)?;
-        let mut validator_list =
+        let mut validator_list_data = validator_list_info.data.borrow_mut();
-            try_from_slice_unchecked::<ValidatorList>(&validator_list_info.data.borrow())?;
+        let (header, validator_list) =
-        if !validator_list.is_valid() {
+            ValidatorListHeader::deserialize_vec(&mut validator_list_data)?;
+        if !header.is_valid() {
             return Err(StakePoolError::InvalidState.into());
         }
 
@@ -1551,7 +1566,7 @@ impl Processor {
                 msg!("Reserve stake account in unknown state, aborting");
                 return Err(StakePoolError::WrongStakeState.into());
             };
-        for validator_stake_record in &validator_list.validators {
+        for validator_stake_record in validator_list.iter::<ValidatorStakeInfo>() {
             if validator_stake_record.last_update_epoch < clock.epoch {
                 return Err(StakePoolError::StakeListOutOfDate.into());
             }
@@ -1587,10 +1602,6 @@ impl Processor {
             stake_pool.fee = next_epoch_fee;
             stake_pool.next_epoch_fee = None;
         }
-        validator_list
-            .validators
-            .retain(|item| item.status != StakeStatus::ReadyForRemoval);
-        validator_list.serialize(&mut *validator_list_info.data.borrow_mut())?;
         stake_pool.total_stake_lamports = total_stake_lamports;
         stake_pool.last_update_epoch = clock.epoch;
         stake_pool.serialize(&mut *stake_pool_info.data.borrow_mut())?;
@@ -1598,6 +1609,35 @@ impl Processor {
         Ok(())
     }
 
+    /// Processes the `CleanupRemovedValidatorEntries` instruction
+    fn process_cleanup_removed_validator_entries(
+        program_id: &Pubkey,
+        accounts: &[AccountInfo],
+    ) -> ProgramResult {
+        let account_info_iter = &mut accounts.iter();
+        let stake_pool_info = next_account_info(account_info_iter)?;
+        let validator_list_info = next_account_info(account_info_iter)?;
+
+        check_account_owner(stake_pool_info, program_id)?;
+        let stake_pool = try_from_slice_unchecked::<StakePool>(&stake_pool_info.data.borrow())?;
+        if !stake_pool.is_valid() {
+            return Err(StakePoolError::InvalidState.into());
+        }
+        stake_pool.check_validator_list(validator_list_info)?;
+
+        check_account_owner(validator_list_info, program_id)?;
+        let mut validator_list_data = validator_list_info.data.borrow_mut();
+        let (header, mut validator_list) =
+            ValidatorListHeader::deserialize_vec(&mut validator_list_data)?;
+        if !header.is_valid() {
+            return Err(StakePoolError::InvalidState.into());
+        }
+
+        validator_list.retain::<ValidatorStakeInfo>(ValidatorStakeInfo::is_not_removed)?;
+
+        Ok(())
+    }
+
     /// Check stake activation status
     #[allow(clippy::unnecessary_wraps)]
     fn _check_stake_activation(
@@ -1673,9 +1713,10 @@ impl Processor {
         }
 
         check_account_owner(validator_list_info, program_id)?;
-        let mut validator_list =
+        let mut validator_list_data = validator_list_info.data.borrow_mut();
-            try_from_slice_unchecked::<ValidatorList>(&validator_list_info.data.borrow())?;
+        let (header, mut validator_list) =
-        if !validator_list.is_valid() {
+            ValidatorListHeader::deserialize_vec(&mut validator_list_data)?;
+        if !header.is_valid() {
             return Err(StakePoolError::InvalidState.into());
         }
 
@@ -1688,17 +1729,25 @@ impl Processor {
             validator_stake_account_info.key,
             &vote_account_address,
         )?;
-        if let Some(preferred_deposit) = validator_list.preferred_deposit_validator_vote_address {
+        if let Some(preferred_deposit) = stake_pool.preferred_deposit_validator_vote_address {
             if preferred_deposit != vote_account_address {
+                msg!(
+                    "Incorrect deposit address, expected {}, received {}",
+                    preferred_deposit,
+                    vote_account_address
+                );
                 return Err(StakePoolError::IncorrectDepositVoteAddress.into());
             }
         }
 
-        let validator_list_item = validator_list
+        let mut validator_stake_info = validator_list
-            .find_mut(&vote_account_address)
+            .find_mut::<ValidatorStakeInfo>(
+                vote_account_address.as_ref(),
+                ValidatorStakeInfo::memcmp_pubkey,
+            )
             .ok_or(StakePoolError::ValidatorNotFound)?;
 
-        if validator_list_item.status != StakeStatus::Active {
+        if validator_stake_info.status != StakeStatus::Active {
             msg!("Validator is marked for removal and no longer accepting deposits");
             return Err(StakePoolError::ValidatorNotFound.into());
         }
@@ -1795,12 +1844,11 @@ impl Processor {
             .ok_or(StakePoolError::CalculationFailure)?;
         stake_pool.serialize(&mut *stake_pool_info.data.borrow_mut())?;
 
-        validator_list_item.active_stake_lamports = post_validator_stake
+        validator_stake_info.active_stake_lamports = post_validator_stake
             .delegation
             .stake
             .checked_sub(MINIMUM_ACTIVE_STAKE)
             .ok_or(StakePoolError::CalculationFailure)?;
-        validator_list.serialize(&mut *validator_list_info.data.borrow_mut())?;
 
         Ok(())
     }
@@ -1850,9 +1898,10 @@ impl Processor {
         }
 
         check_account_owner(validator_list_info, program_id)?;
-        let mut validator_list =
+        let mut validator_list_data = validator_list_info.data.borrow_mut();
-            try_from_slice_unchecked::<ValidatorList>(&validator_list_info.data.borrow())?;
+        let (header, mut validator_list) =
-        if !validator_list.is_valid() {
+            ValidatorListHeader::deserialize_vec(&mut validator_list_data)?;
+        if !header.is_valid() {
             return Err(StakePoolError::InvalidState.into());
         }
 
@@ -1860,19 +1909,23 @@ impl Processor {
             .calc_lamports_withdraw_amount(pool_tokens)
             .ok_or(StakePoolError::CalculationFailure)?;
 
+        let has_active_stake = validator_list
+            .find::<ValidatorStakeInfo>(
+                &0u64.to_le_bytes(),
+                ValidatorStakeInfo::memcmp_active_lamports,
+            )
+            .is_some();
+
         let validator_list_item_info = if *stake_split_from.key == stake_pool.reserve_stake {
             // check that the validator stake accounts have no withdrawable stake
-            if let Some(withdrawable_entry) = validator_list
+            let has_transient_stake = validator_list
-                .validators
+                .find::<ValidatorStakeInfo>(
-                .iter()
+                    &0u64.to_le_bytes(),
-                .find(|&&x| x.stake_lamports() != 0)
+                    ValidatorStakeInfo::memcmp_transient_lamports,
-            {
+                )
-                let (validator_stake_address, _) = crate::find_stake_program_address(
+                .is_some();
-                    program_id,
+            if has_transient_stake || has_active_stake {
-                    &withdrawable_entry.vote_account_address,
+                msg!("Error withdrawing from reserve: validator stake accounts have lamports available, please use those first.");
-                    stake_pool_info.key,
-                );
-                msg!("Error withdrawing from reserve: validator stake account {} has {} lamports available, please use that first.", validator_stake_address, withdrawable_entry.stake_lamports());
                 return Err(StakePoolError::StakeLamportsNotEqualToMinimum.into());
             }
 
@@ -1891,10 +1944,13 @@ impl Processor {
             let vote_account_address = stake.delegation.voter_pubkey;
 
             if let Some(preferred_withdraw_validator) =
-                validator_list.preferred_withdraw_validator_vote_address
+                stake_pool.preferred_withdraw_validator_vote_address
             {
                 let preferred_validator_info = validator_list
-                    .find(&preferred_withdraw_validator)
+                    .find::<ValidatorStakeInfo>(
+                        preferred_withdraw_validator.as_ref(),
+                        ValidatorStakeInfo::memcmp_pubkey,
+                    )
                     .ok_or(StakePoolError::ValidatorNotFound)?;
                 if preferred_withdraw_validator != vote_account_address
                     && preferred_validator_info.active_stake_lamports > 0
@@ -1906,7 +1962,7 @@ impl Processor {
 
             // if there's any active stake, we must withdraw from an active
             // stake account
-            let withdrawing_from_transient_stake = if validator_list.has_active_stake() {
+            let withdrawing_from_transient_stake = if has_active_stake {
                 check_validator_stake_address(
                     program_id,
                     stake_pool_info.key,
@@ -1924,11 +1980,14 @@ impl Processor {
                 true
             };
 
-            let validator_list_item = validator_list
+            let validator_stake_info = validator_list
-                .find_mut(&vote_account_address)
+                .find_mut::<ValidatorStakeInfo>(
+                    vote_account_address.as_ref(),
+                    ValidatorStakeInfo::memcmp_pubkey,
+                )
                 .ok_or(StakePoolError::ValidatorNotFound)?;
 
-            if validator_list_item.status != StakeStatus::Active {
+            if validator_stake_info.status != StakeStatus::Active {
                 msg!("Validator is marked for removal and no longer allowing withdrawals");
                 return Err(StakePoolError::ValidatorNotFound.into());
             }
@@ -1938,7 +1997,7 @@ impl Processor {
                 msg!("Attempting to withdraw {} lamports from validator account with {} stake lamports, {} must remain", withdraw_lamports, stake.delegation.stake, MINIMUM_ACTIVE_STAKE);
                 return Err(StakePoolError::StakeLamportsNotEqualToMinimum.into());
             }
-            Some((validator_list_item, withdrawing_from_transient_stake))
+            Some((validator_stake_info, withdrawing_from_transient_stake))
         };
 
         Self::token_burn(
@@ -1994,7 +2053,6 @@ impl Processor {
                     .checked_sub(withdraw_lamports)
                     .ok_or(StakePoolError::CalculationFailure)?;
             }
-            validator_list.serialize(&mut *validator_list_info.data.borrow_mut())?;
         }
 
         Ok(())
@@ -2146,6 +2204,10 @@ impl Processor {
                 msg!("Instruction: UpdateStakePoolBalance");
                 Self::process_update_stake_pool_balance(program_id, accounts)
             }
+            StakePoolInstruction::CleanupRemovedValidatorEntries => {
+                msg!("Instruction: CleanupRemovedValidatorEntries");
+                Self::process_cleanup_removed_validator_entries(program_id, accounts)
+            }
             StakePoolInstruction::Deposit => {
                 msg!("Instruction: Deposit");
                 Self::process_deposit(program_id, accounts)

stake-pool/program/src/state.rs

@@ -1,9 +1,19 @@
 //! State transition types
 
 use {
-    crate::{error::StakePoolError, stake_program::Lockup},
+    crate::{big_vec::BigVec, error::StakePoolError, stake_program::Lockup},
     borsh::{BorshDeserialize, BorshSchema, BorshSerialize},
-    solana_program::{account_info::AccountInfo, msg, program_error::ProgramError, pubkey::Pubkey},
+    num_derive::FromPrimitive,
+    num_traits::FromPrimitive,
+    solana_program::{
+        account_info::AccountInfo,
+        borsh::get_instance_packed_len,
+        msg,
+        program_error::ProgramError,
+        program_memory::sol_memcmp,
+        program_pack::{Pack, Sealed},
+        pubkey::{Pubkey, PUBKEY_BYTES},
+    },
     std::convert::TryFrom,
 };
 
@@ -87,6 +97,12 @@ pub struct StakePool {
 
     /// Fee for next epoch
     pub next_epoch_fee: Option<Fee>,
+
+    /// Preferred deposit validator vote account pubkey
+    pub preferred_deposit_validator_vote_address: Option<Pubkey>,
+
+    /// Preferred withdraw validator vote account pubkey
+    pub preferred_withdraw_validator_vote_address: Option<Pubkey>,
 }
 impl StakePool {
     /// calculate the pool tokens that should be minted for a deposit of `stake_lamports`
@@ -281,24 +297,28 @@ impl StakePool {
 #[repr(C)]
 #[derive(Clone, Debug, Default, PartialEq, BorshDeserialize, BorshSerialize, BorshSchema)]
 pub struct ValidatorList {
-    /// Account type, must be ValidatorList currently
+    /// Data outside of the validator list, separated out for cheaper deserializations
-    pub account_type: AccountType,
+    pub header: ValidatorListHeader,
-
-    /// Preferred deposit validator vote account pubkey
-    pub preferred_deposit_validator_vote_address: Option<Pubkey>,
-
-    /// Preferred withdraw validator vote account pubkey
-    pub preferred_withdraw_validator_vote_address: Option<Pubkey>,
-
-    /// Maximum allowable number of validators
-    pub max_validators: u32,
 
     /// List of stake info for each validator in the pool
     pub validators: Vec<ValidatorStakeInfo>,
 }
 
+/// Helper type to deserialize just the start of a ValidatorList
+#[repr(C)]
+#[derive(Clone, Debug, Default, PartialEq, BorshDeserialize, BorshSerialize, BorshSchema)]
+pub struct ValidatorListHeader {
+    /// Account type, must be ValidatorList currently
+    pub account_type: AccountType,
+
+    /// Maximum allowable number of validators
+    pub max_validators: u32,
+}
+
 /// Status of the stake account in the validator list, for accounting
-#[derive(Copy, Clone, Debug, PartialEq, BorshDeserialize, BorshSerialize, BorshSchema)]
+#[derive(
+    FromPrimitive, Copy, Clone, Debug, PartialEq, BorshDeserialize, BorshSerialize, BorshSchema,
+)]
 pub enum StakeStatus {
     /// Stake account is active, there may be a transient stake as well
     Active,
@@ -316,10 +336,10 @@ impl Default for StakeStatus {
     }
 }
 
-/// Information about the singe validator stake account
+/// Packed version of the validator stake info, for use with pointer casts
-#[repr(C)]
+#[repr(packed)]
-#[derive(Clone, Copy, Debug, Default, PartialEq, BorshDeserialize, BorshSerialize, BorshSchema)]
+#[derive(Clone, Copy, Debug, Default, PartialEq)]
-pub struct ValidatorStakeInfo {
+pub struct ValidatorStakeInfoPacked {
     /// Status of the validator stake account
     pub status: StakeStatus,
 
@@ -340,6 +360,37 @@ pub struct ValidatorStakeInfo {
     pub last_update_epoch: u64,
 }
 
+/// Information about a validator in the pool
+///
+/// NOTE: ORDER IS VERY IMPORTANT HERE, PLEASE DO NOT RE-ORDER THE FIELDS UNLESS
+/// THERE'S AN EXTREMELY GOOD REASON.
+///
+/// To save on BPF instructions, the serialized bytes are reinterpreted with an
+/// unsafe pointer cast, which means that this structure cannot have any
+/// undeclared alignment-padding in its representation.
+#[repr(C)]
+#[derive(Clone, Copy, Debug, Default, PartialEq, BorshDeserialize, BorshSerialize, BorshSchema)]
+pub struct ValidatorStakeInfo {
+    /// Amount of active stake delegated to this validator
+    /// Note that if `last_update_epoch` does not match the current epoch then
+    /// this field may not be accurate
+    pub active_stake_lamports: u64,
+
+    /// Amount of transient stake delegated to this validator
+    /// Note that if `last_update_epoch` does not match the current epoch then
+    /// this field may not be accurate
+    pub transient_stake_lamports: u64,
+
+    /// Last epoch the active and transient stake lamports fields were updated
+    pub last_update_epoch: u64,
+
+    /// Status of the validator stake account
+    pub status: StakeStatus,
+
+    /// Validator vote account address
+    pub vote_account_address: Pubkey,
+}
+
 impl ValidatorStakeInfo {
     /// Get the total lamports delegated to this validator (active and transient)
     pub fn stake_lamports(&self) -> u64 {
@@ -347,24 +398,65 @@ impl ValidatorStakeInfo {
             .checked_add(self.transient_stake_lamports)
             .unwrap()
     }
+
+    /// Performs a very cheap comparison, for checking if this validator stake
+    /// info matches the vote account address
+    pub fn memcmp_pubkey(data: &[u8], vote_address_bytes: &[u8]) -> bool {
+        sol_memcmp(
+            &data[25..25 + PUBKEY_BYTES],
+            vote_address_bytes,
+            PUBKEY_BYTES,
+        ) == 0
+    }
+
+    /// Performs a very cheap comparison, for checking if this validator stake
+    /// info has active lamports equal to the given bytes
+    pub fn memcmp_active_lamports(data: &[u8], lamports_le_bytes: &[u8]) -> bool {
+        sol_memcmp(&data[0..8], lamports_le_bytes, 8) != 0
+    }
+
+    /// Performs a very cheap comparison, for checking if this validator stake
+    /// info has lamports equal to the given bytes
+    pub fn memcmp_transient_lamports(data: &[u8], lamports_le_bytes: &[u8]) -> bool {
+        sol_memcmp(&data[8..16], lamports_le_bytes, 8) != 0
+    }
+
+    /// Check that the validator stake info is valid
+    pub fn is_not_removed(data: &[u8]) -> bool {
+        FromPrimitive::from_u8(data[24]) != Some(StakeStatus::ReadyForRemoval)
+    }
+}
+
+impl Sealed for ValidatorStakeInfo {}
+
+impl Pack for ValidatorStakeInfo {
+    const LEN: usize = 57;
+    fn pack_into_slice(&self, data: &mut [u8]) {
+        let mut data = data;
+        self.serialize(&mut data).unwrap();
+    }
+    fn unpack_from_slice(src: &[u8]) -> Result<Self, ProgramError> {
+        let unpacked = Self::try_from_slice(src)?;
+        Ok(unpacked)
+    }
 }
 
 impl ValidatorList {
     /// Create an empty instance containing space for `max_validators` and preferred validator keys
     pub fn new(max_validators: u32) -> Self {
         Self {
+            header: ValidatorListHeader {
                 account_type: AccountType::ValidatorList,
-            preferred_deposit_validator_vote_address: Some(Pubkey::default()),
-            preferred_withdraw_validator_vote_address: Some(Pubkey::default()),
                 max_validators,
+            },
             validators: vec![ValidatorStakeInfo::default(); max_validators as usize],
         }
     }
 
     /// Calculate the number of validator entries that fit in the provided length
     pub fn calculate_max_validators(buffer_length: usize) -> usize {
-        let header_size = 1 + 4 + 4 + 33 + 33;
+        let header_size = ValidatorListHeader::LEN + 4;
-        buffer_length.saturating_sub(header_size) / 57
+        buffer_length.saturating_sub(header_size) / ValidatorStakeInfo::LEN
     }
 
     /// Check if contains validator with particular pubkey
@@ -387,6 +479,15 @@ impl ValidatorList {
             .find(|x| x.vote_account_address == *vote_account_address)
     }
 
+    /// Check if the list has any active stake
+    pub fn has_active_stake(&self) -> bool {
+        self.validators.iter().any(|x| x.active_stake_lamports > 0)
+    }
+}
+
+impl ValidatorListHeader {
+    const LEN: usize = 1 + 4;
+
     /// Check if validator stake list is actually initialized as a validator stake list
     pub fn is_valid(&self) -> bool {
         self.account_type == AccountType::ValidatorList
@@ -397,9 +498,28 @@ impl ValidatorList {
         self.account_type == AccountType::Uninitialized
     }
 
-    /// Check if the list has any active stake
+    /// Extracts a slice of ValidatorStakeInfo types from the vec part
-    pub fn has_active_stake(&self) -> bool {
+    /// of the ValidatorList
-        self.validators.iter().any(|x| x.active_stake_lamports > 0)
+    pub fn deserialize_mut_slice(
+        data: &mut [u8],
+        skip: usize,
+        len: usize,
+    ) -> Result<(Self, Vec<&mut ValidatorStakeInfo>), ProgramError> {
+        let (header, mut big_vec) = Self::deserialize_vec(data)?;
+        let validator_list = big_vec.deserialize_mut_slice::<ValidatorStakeInfo>(skip, len)?;
+        Ok((header, validator_list))
+    }
+
+    /// Extracts the validator list into its header and internal BigVec
+    pub fn deserialize_vec(data: &mut [u8]) -> Result<(Self, BigVec), ProgramError> {
+        let mut data_mut = &data[..];
+        let header = ValidatorListHeader::deserialize(&mut data_mut)?;
+        let length = get_instance_packed_len(&header)?;
+
+        let big_vec = BigVec {
+            data: &mut data[length..],
+        };
+        Ok((header, big_vec))
     }
 }
 
@@ -427,20 +547,20 @@ mod test {
 
     fn uninitialized_validator_list() -> ValidatorList {
         ValidatorList {
+            header: ValidatorListHeader {
                 account_type: AccountType::Uninitialized,
-            preferred_deposit_validator_vote_address: None,
-            preferred_withdraw_validator_vote_address: None,
                 max_validators: 0,
+            },
             validators: vec![],
         }
     }
 
     fn test_validator_list(max_validators: u32) -> ValidatorList {
         ValidatorList {
+            header: ValidatorListHeader {
                 account_type: AccountType::ValidatorList,
-            preferred_deposit_validator_vote_address: Some(Pubkey::new_unique()),
-            preferred_withdraw_validator_vote_address: Some(Pubkey::new_unique()),
                 max_validators,
+            },
             validators: vec![
                 ValidatorStakeInfo {
                     status: StakeStatus::Active,
@@ -480,10 +600,10 @@ mod test {
 
         // Empty, one preferred key
         let stake_list = ValidatorList {
+            header: ValidatorListHeader {
                 account_type: AccountType::ValidatorList,
-            preferred_deposit_validator_vote_address: Some(Pubkey::new_unique()),
-            preferred_withdraw_validator_vote_address: None,
                 max_validators: 0,
+            },
             validators: vec![],
         };
         let mut byte_vec = vec![0u8; size];
@@ -512,6 +632,64 @@ mod test {
         assert!(!validator_list.has_active_stake());
     }
 
+    #[test]
+    fn validator_list_deserialize_mut_slice() {
+        let max_validators = 10;
+        let stake_list = test_validator_list(max_validators);
+        let mut serialized = stake_list.try_to_vec().unwrap();
+        let (header, list) = ValidatorListHeader::deserialize_mut_slice(
+            &mut serialized,
+            0,
+            stake_list.validators.len(),
+        )
+        .unwrap();
+        assert_eq!(header.account_type, AccountType::ValidatorList);
+        assert_eq!(header.max_validators, max_validators);
+        assert!(list
+            .iter()
+            .zip(stake_list.validators.iter())
+            .all(|(a, b)| *a == b));
+
+        let (_, list) = ValidatorListHeader::deserialize_mut_slice(&mut serialized, 1, 2).unwrap();
+        assert!(list
+            .iter()
+            .zip(stake_list.validators[1..].iter())
+            .all(|(a, b)| *a == b));
+        let (_, list) = ValidatorListHeader::deserialize_mut_slice(&mut serialized, 2, 1).unwrap();
+        assert!(list
+            .iter()
+            .zip(stake_list.validators[2..].iter())
+            .all(|(a, b)| *a == b));
+        let (_, list) = ValidatorListHeader::deserialize_mut_slice(&mut serialized, 0, 2).unwrap();
+        assert!(list
+            .iter()
+            .zip(stake_list.validators[..2].iter())
+            .all(|(a, b)| *a == b));
+
+        assert_eq!(
+            ValidatorListHeader::deserialize_mut_slice(&mut serialized, 0, 4).unwrap_err(),
+            ProgramError::AccountDataTooSmall
+        );
+        assert_eq!(
+            ValidatorListHeader::deserialize_mut_slice(&mut serialized, 1, 3).unwrap_err(),
+            ProgramError::AccountDataTooSmall
+        );
+    }
+
+    #[test]
+    fn validator_list_iter() {
+        let max_validators = 10;
+        let stake_list = test_validator_list(max_validators);
+        let mut serialized = stake_list.try_to_vec().unwrap();
+        let (_, big_vec) = ValidatorListHeader::deserialize_vec(&mut serialized).unwrap();
+        for (a, b) in big_vec
+            .iter::<ValidatorStakeInfo>()
+            .zip(stake_list.validators.iter())
+        {
+            assert_eq!(a, b);
+        }
+    }
+
     proptest! {
         #[test]
         fn stake_list_size_calculation(test_amount in 0..=100_000_u32) {

stake-pool/program/tests/helpers/mod.rs

@@ -735,6 +735,25 @@ impl StakePoolAccounts {
         banks_client.process_transaction(transaction).await.err()
     }
 
+    pub async fn cleanup_removed_validator_entries(
+        &self,
+        banks_client: &mut BanksClient,
+        payer: &Keypair,
+        recent_blockhash: &Hash,
+    ) -> Option<TransportError> {
+        let transaction = Transaction::new_signed_with_payer(
+            &[instruction::cleanup_removed_validator_entries(
+                &id(),
+                &self.stake_pool.pubkey(),
+                &self.validator_list.pubkey(),
+            )],
+            Some(&payer.pubkey()),
+            &[payer],
+            *recent_blockhash,
+        );
+        banks_client.process_transaction(transaction).await.err()
+    }
+
     pub async fn update_all(
         &self,
         banks_client: &mut BanksClient,
@@ -764,6 +783,11 @@ impl StakePoolAccounts {
                     &self.pool_fee_account.pubkey(),
                     &self.pool_mint.pubkey(),
                 ),
+                instruction::cleanup_removed_validator_entries(
+                    &id(),
+                    &self.stake_pool.pubkey(),
+                    &self.validator_list.pubkey(),
+                ),
             ],
             Some(&payer.pubkey()),
             &[payer],

stake-pool/program/tests/huge_pool.rs

@@ -0,0 +1,704 @@
+#![cfg(feature = "test-bpf")]
+
+mod helpers;
+
+use {
+    bincode,
+    borsh::BorshSerialize,
+    helpers::*,
+    solana_program::{
+        borsh::try_from_slice_unchecked, program_option::COption, program_pack::Pack,
+        pubkey::Pubkey,
+    },
+    solana_program_test::*,
+    solana_sdk::{
+        account::{Account, WritableAccount},
+        clock::{Clock, Epoch},
+        signature::{Keypair, Signer},
+        transaction::Transaction,
+    },
+    solana_vote_program::{
+        self,
+        vote_state::{VoteInit, VoteState, VoteStateVersions},
+    },
+    spl_stake_pool::{
+        find_stake_program_address, find_transient_stake_program_address,
+        find_withdraw_authority_program_address, id,
+        instruction::{self, PreferredValidatorType},
+        stake_program,
+        state::{AccountType, StakePool, StakeStatus, ValidatorList, ValidatorStakeInfo},
+        MAX_VALIDATORS_TO_UPDATE, MINIMUM_ACTIVE_STAKE,
+    },
+    spl_token::state::{Account as SplAccount, AccountState as SplAccountState, Mint},
+};
+
+const HUGE_POOL_SIZE: u32 = 4_000;
+const ACCOUNT_RENT_EXEMPTION: u64 = 1_000_000_000; // go with something big to be safe
+const STAKE_AMOUNT: u64 = 200_000_000_000;
+const STAKE_ACCOUNT_RENT_EXEMPTION: u64 = 2_282_880;
+
+async fn setup(
+    max_validators: u32,
+    num_validators: u32,
+    stake_amount: u64,
+) -> (
+    ProgramTestContext,
+    StakePoolAccounts,
+    Vec<Pubkey>,
+    Pubkey,
+    Keypair,
+    Pubkey,
+    Pubkey,
+) {
+    let mut program_test = program_test();
+    let mut vote_account_pubkeys = vec![];
+    let mut stake_pool_accounts = StakePoolAccounts::new();
+    stake_pool_accounts.max_validators = max_validators;
+
+    let stake_pool_pubkey = stake_pool_accounts.stake_pool.pubkey();
+    let (_, withdraw_bump_seed) =
+        find_withdraw_authority_program_address(&id(), &stake_pool_pubkey);
+
+    let mut stake_pool = StakePool {
+        account_type: AccountType::StakePool,
+        manager: stake_pool_accounts.manager.pubkey(),
+        staker: stake_pool_accounts.staker.pubkey(),
+        deposit_authority: stake_pool_accounts.deposit_authority,
+        withdraw_bump_seed,
+        validator_list: stake_pool_accounts.validator_list.pubkey(),
+        reserve_stake: stake_pool_accounts.reserve_stake.pubkey(),
+        pool_mint: stake_pool_accounts.pool_mint.pubkey(),
+        manager_fee_account: stake_pool_accounts.pool_fee_account.pubkey(),
+        token_program_id: spl_token::id(),
+        total_stake_lamports: 0,
+        pool_token_supply: 0,
+        last_update_epoch: 0,
+        lockup: stake_program::Lockup::default(),
+        fee: stake_pool_accounts.fee,
+        next_epoch_fee: None,
+        preferred_deposit_validator_vote_address: None,
+        preferred_withdraw_validator_vote_address: None,
+    };
+
+    let mut validator_list = ValidatorList::new(max_validators);
+    validator_list.validators = vec![];
+
+    let authorized_voter = Pubkey::new_unique();
+    let authorized_withdrawer = Pubkey::new_unique();
+    let commission = 1;
+
+    let meta = stake_program::Meta {
+        rent_exempt_reserve: STAKE_ACCOUNT_RENT_EXEMPTION,
+        authorized: stake_program::Authorized {
+            staker: stake_pool_accounts.withdraw_authority,
+            withdrawer: stake_pool_accounts.withdraw_authority,
+        },
+        lockup: stake_program::Lockup::default(),
+    };
+
+    for _ in 0..max_validators {
+        // create vote account
+        let vote_pubkey = Pubkey::new_unique();
+        vote_account_pubkeys.push(vote_pubkey);
+        let node_pubkey = Pubkey::new_unique();
+        let vote_state = VoteStateVersions::new_current(VoteState::new(
+            &VoteInit {
+                node_pubkey,
+                authorized_voter,
+                authorized_withdrawer,
+                commission,
+            },
+            &Clock::default(),
+        ));
+        let vote_account = Account::create(
+            ACCOUNT_RENT_EXEMPTION,
+            bincode::serialize::<VoteStateVersions>(&vote_state).unwrap(),
+            solana_vote_program::id(),
+            false,
+            Epoch::default(),
+        );
+        program_test.add_account(vote_pubkey, vote_account);
+    }
+
+    for i in 0..num_validators as usize {
+        let vote_account_address = vote_account_pubkeys[i];
+
+        // create validator stake account
+        let stake = stake_program::Stake {
+            delegation: stake_program::Delegation {
+                voter_pubkey: vote_account_address,
+                stake: stake_amount,
+                activation_epoch: 0,
+                deactivation_epoch: u64::MAX,
+                warmup_cooldown_rate: 0.25, // default
+            },
+            credits_observed: 0,
+        };
+
+        let stake_account = Account::create(
+            stake_amount + STAKE_ACCOUNT_RENT_EXEMPTION,
+            bincode::serialize::<stake_program::StakeState>(&stake_program::StakeState::Stake(
+                meta, stake,
+            ))
+            .unwrap(),
+            stake_program::id(),
+            false,
+            Epoch::default(),
+        );
+
+        let (stake_address, _) =
+            find_stake_program_address(&id(), &vote_account_address, &stake_pool_pubkey);
+        program_test.add_account(stake_address, stake_account);
+        let active_stake_lamports = stake_amount - MINIMUM_ACTIVE_STAKE;
+        // add to validator list
+        validator_list.validators.push(ValidatorStakeInfo {
+            status: StakeStatus::Active,
+            vote_account_address,
+            active_stake_lamports,
+            transient_stake_lamports: 0,
+            last_update_epoch: 0,
+        });
+
+        stake_pool.total_stake_lamports += active_stake_lamports;
+        stake_pool.pool_token_supply += active_stake_lamports;
+    }
+
+    let mut validator_list_bytes = validator_list.try_to_vec().unwrap();
+
+    // add extra room if needed
+    for _ in num_validators..max_validators {
+        validator_list_bytes.append(&mut ValidatorStakeInfo::default().try_to_vec().unwrap());
+    }
+
+    let reserve_stake_account = Account::create(
+        stake_amount + STAKE_ACCOUNT_RENT_EXEMPTION,
+        bincode::serialize::<stake_program::StakeState>(&stake_program::StakeState::Initialized(
+            meta,
+        ))
+        .unwrap(),
+        stake_program::id(),
+        false,
+        Epoch::default(),
+    );
+    program_test.add_account(
+        stake_pool_accounts.reserve_stake.pubkey(),
+        reserve_stake_account,
+    );
+
+    let mut stake_pool_bytes = stake_pool.try_to_vec().unwrap();
+    // more room for optionals
+    stake_pool_bytes.extend_from_slice(&Pubkey::default().to_bytes());
+    stake_pool_bytes.extend_from_slice(&Pubkey::default().to_bytes());
+    let stake_pool_account = Account::create(
+        ACCOUNT_RENT_EXEMPTION,
+        stake_pool_bytes,
+        id(),
+        false,
+        Epoch::default(),
+    );
+    program_test.add_account(stake_pool_pubkey, stake_pool_account);
+
+    let validator_list_account = Account::create(
+        ACCOUNT_RENT_EXEMPTION,
+        validator_list_bytes,
+        id(),
+        false,
+        Epoch::default(),
+    );
+    program_test.add_account(
+        stake_pool_accounts.validator_list.pubkey(),
+        validator_list_account,
+    );
+
+    let mut mint_vec = vec![0u8; Mint::LEN];
+    let mint = Mint {
+        mint_authority: COption::Some(stake_pool_accounts.withdraw_authority),
+        supply: stake_pool.pool_token_supply,
+        decimals: 9,
+        is_initialized: true,
+        freeze_authority: COption::None,
+    };
+    Pack::pack(mint, &mut mint_vec).unwrap();
+    let stake_pool_mint = Account::create(
+        ACCOUNT_RENT_EXEMPTION,
+        mint_vec,
+        spl_token::id(),
+        false,
+        Epoch::default(),
+    );
+    program_test.add_account(stake_pool_accounts.pool_mint.pubkey(), stake_pool_mint);
+
+    let mut fee_account_vec = vec![0u8; SplAccount::LEN];
+    let fee_account_data = SplAccount {
+        mint: stake_pool_accounts.pool_mint.pubkey(),
+        owner: stake_pool_accounts.manager.pubkey(),
+        amount: 0,
+        delegate: COption::None,
+        state: SplAccountState::Initialized,
+        is_native: COption::None,
+        delegated_amount: 0,
+        close_authority: COption::None,
+    };
+    Pack::pack(fee_account_data, &mut fee_account_vec).unwrap();
+    let fee_account = Account::create(
+        ACCOUNT_RENT_EXEMPTION,
+        fee_account_vec,
+        spl_token::id(),
+        false,
+        Epoch::default(),
+    );
+    program_test.add_account(stake_pool_accounts.pool_fee_account.pubkey(), fee_account);
+
+    let mut context = program_test.start_with_context().await;
+
+    let vote_pubkey = vote_account_pubkeys[HUGE_POOL_SIZE as usize - 1];
+    // make stake account
+    let user = Keypair::new();
+    let deposit_stake = Keypair::new();
+    let lockup = stake_program::Lockup::default();
+
+    let authorized = stake_program::Authorized {
+        staker: user.pubkey(),
+        withdrawer: user.pubkey(),
+    };
+
+    let _stake_lamports = create_independent_stake_account(
+        &mut context.banks_client,
+        &context.payer,
+        &context.last_blockhash,
+        &deposit_stake,
+        &authorized,
+        &lockup,
+        stake_amount,
+    )
+    .await;
+
+    delegate_stake_account(
+        &mut context.banks_client,
+        &context.payer,
+        &context.last_blockhash,
+        &deposit_stake.pubkey(),
+        &user,
+        &vote_pubkey,
+    )
+    .await;
+
+    // make pool token account
+    let pool_token_account = Keypair::new();
+    create_token_account(
+        &mut context.banks_client,
+        &context.payer,
+        &context.last_blockhash,
+        &pool_token_account,
+        &stake_pool_accounts.pool_mint.pubkey(),
+        &user.pubkey(),
+    )
+    .await
+    .unwrap();
+
+    (
+        context,
+        stake_pool_accounts,
+        vote_account_pubkeys,
+        vote_pubkey,
+        user,
+        deposit_stake.pubkey(),
+        pool_token_account.pubkey(),
+    )
+}
+
+#[tokio::test]
+async fn update() {
+    let (mut context, stake_pool_accounts, vote_account_pubkeys, _, _, _, _) =
+        setup(HUGE_POOL_SIZE, HUGE_POOL_SIZE, STAKE_AMOUNT).await;
+
+    let transaction = Transaction::new_signed_with_payer(
+        &[
+            instruction::update_validator_list_balance(
+                &id(),
+                &stake_pool_accounts.stake_pool.pubkey(),
+                &stake_pool_accounts.withdraw_authority,
+                &stake_pool_accounts.validator_list.pubkey(),
+                &stake_pool_accounts.reserve_stake.pubkey(),
+                &vote_account_pubkeys[0..MAX_VALIDATORS_TO_UPDATE],
+                0,
+                /* no_merge = */ false,
+            ),
+            instruction::update_stake_pool_balance(
+                &id(),
+                &stake_pool_accounts.stake_pool.pubkey(),
+                &stake_pool_accounts.withdraw_authority,
+                &stake_pool_accounts.validator_list.pubkey(),
+                &stake_pool_accounts.reserve_stake.pubkey(),
+                &stake_pool_accounts.pool_fee_account.pubkey(),
+                &stake_pool_accounts.pool_mint.pubkey(),
+            ),
+            instruction::cleanup_removed_validator_entries(
+                &id(),
+                &stake_pool_accounts.stake_pool.pubkey(),
+                &stake_pool_accounts.validator_list.pubkey(),
+            ),
+        ],
+        Some(&context.payer.pubkey()),
+        &[&context.payer],
+        context.last_blockhash,
+    );
+    let error = context
+        .banks_client
+        .process_transaction(transaction)
+        .await
+        .err();
+    assert!(error.is_none());
+}
+
+#[tokio::test]
+async fn remove_validator_from_pool() {
+    let (mut context, stake_pool_accounts, vote_account_pubkeys, _, _, _, _) =
+        setup(HUGE_POOL_SIZE, HUGE_POOL_SIZE, MINIMUM_ACTIVE_STAKE).await;
+
+    let first_vote = vote_account_pubkeys[0];
+    let (stake_address, _) =
+        find_stake_program_address(&id(), &first_vote, &stake_pool_accounts.stake_pool.pubkey());
+    let (transient_stake_address, _) = find_transient_stake_program_address(
+        &id(),
+        &first_vote,
+        &stake_pool_accounts.stake_pool.pubkey(),
+    );
+
+    let new_authority = Pubkey::new_unique();
+    let error = stake_pool_accounts
+        .remove_validator_from_pool(
+            &mut context.banks_client,
+            &context.payer,
+            &context.last_blockhash,
+            &new_authority,
+            &stake_address,
+            &transient_stake_address,
+        )
+        .await;
+    assert!(error.is_none());
+
+    let middle_index = HUGE_POOL_SIZE as usize / 2;
+    let middle_vote = vote_account_pubkeys[middle_index];
+    let (stake_address, _) = find_stake_program_address(
+        &id(),
+        &middle_vote,
+        &stake_pool_accounts.stake_pool.pubkey(),
+    );
+    let (transient_stake_address, _) = find_transient_stake_program_address(
+        &id(),
+        &middle_vote,
+        &stake_pool_accounts.stake_pool.pubkey(),
+    );
+
+    let new_authority = Pubkey::new_unique();
+    let error = stake_pool_accounts
+        .remove_validator_from_pool(
+            &mut context.banks_client,
+            &context.payer,
+            &context.last_blockhash,
+            &new_authority,
+            &stake_address,
+            &transient_stake_address,
+        )
+        .await;
+    assert!(error.is_none());
+
+    let last_index = HUGE_POOL_SIZE as usize - 1;
+    let last_vote = vote_account_pubkeys[last_index];
+    let (stake_address, _) =
+        find_stake_program_address(&id(), &last_vote, &stake_pool_accounts.stake_pool.pubkey());
+    let (transient_stake_address, _) = find_transient_stake_program_address(
+        &id(),
+        &last_vote,
+        &stake_pool_accounts.stake_pool.pubkey(),
+    );
+
+    let new_authority = Pubkey::new_unique();
+    let error = stake_pool_accounts
+        .remove_validator_from_pool(
+            &mut context.banks_client,
+            &context.payer,
+            &context.last_blockhash,
+            &new_authority,
+            &stake_address,
+            &transient_stake_address,
+        )
+        .await;
+    assert!(error.is_none());
+
+    let validator_list = get_account(
+        &mut context.banks_client,
+        &stake_pool_accounts.validator_list.pubkey(),
+    )
+    .await;
+    let validator_list =
+        try_from_slice_unchecked::<ValidatorList>(validator_list.data.as_slice()).unwrap();
+    let first_element = &validator_list.validators[0];
+    assert_eq!(first_element.status, StakeStatus::ReadyForRemoval);
+    assert_eq!(first_element.active_stake_lamports, 0);
+    assert_eq!(first_element.transient_stake_lamports, 0);
+
+    let middle_element = &validator_list.validators[middle_index];
+    assert_eq!(middle_element.status, StakeStatus::ReadyForRemoval);
+    assert_eq!(middle_element.active_stake_lamports, 0);
+    assert_eq!(middle_element.transient_stake_lamports, 0);
+
+    let last_element = &validator_list.validators[last_index];
+    assert_eq!(last_element.status, StakeStatus::ReadyForRemoval);
+    assert_eq!(last_element.active_stake_lamports, 0);
+    assert_eq!(last_element.transient_stake_lamports, 0);
+
+    let transaction = Transaction::new_signed_with_payer(
+        &[instruction::cleanup_removed_validator_entries(
+            &id(),
+            &stake_pool_accounts.stake_pool.pubkey(),
+            &stake_pool_accounts.validator_list.pubkey(),
+        )],
+        Some(&context.payer.pubkey()),
+        &[&context.payer],
+        context.last_blockhash,
+    );
+
+    let error = context
+        .banks_client
+        .process_transaction(transaction)
+        .await
+        .err();
+    assert!(error.is_none());
+
+    let validator_list = get_account(
+        &mut context.banks_client,
+        &stake_pool_accounts.validator_list.pubkey(),
+    )
+    .await;
+    let validator_list =
+        try_from_slice_unchecked::<ValidatorList>(validator_list.data.as_slice()).unwrap();
+    assert_eq!(validator_list.validators.len() as u32, HUGE_POOL_SIZE - 3);
+    // assert they're gone
+    assert!(!validator_list
+        .validators
+        .iter()
+        .any(|x| x.vote_account_address == first_vote));
+    assert!(!validator_list
+        .validators
+        .iter()
+        .any(|x| x.vote_account_address == middle_vote));
+    assert!(!validator_list
+        .validators
+        .iter()
+        .any(|x| x.vote_account_address == last_vote));
+
+    // but that we didn't remove too many
+    assert!(validator_list
+        .validators
+        .iter()
+        .any(|x| x.vote_account_address == vote_account_pubkeys[1]));
+    assert!(validator_list
+        .validators
+        .iter()
+        .any(|x| x.vote_account_address == vote_account_pubkeys[middle_index - 1]));
+    assert!(validator_list
+        .validators
+        .iter()
+        .any(|x| x.vote_account_address == vote_account_pubkeys[middle_index + 1]));
+    assert!(validator_list
+        .validators
+        .iter()
+        .any(|x| x.vote_account_address == vote_account_pubkeys[last_index - 1]));
+}
+
+#[tokio::test]
+async fn add_validator_to_pool() {
+    let (mut context, stake_pool_accounts, _, test_vote_address, _, _, _) =
+        setup(HUGE_POOL_SIZE, HUGE_POOL_SIZE - 1, STAKE_AMOUNT).await;
+
+    let last_index = HUGE_POOL_SIZE as usize - 1;
+    let stake_pool_pubkey = stake_pool_accounts.stake_pool.pubkey();
+    let (stake_address, _) =
+        find_stake_program_address(&id(), &test_vote_address, &stake_pool_pubkey);
+
+    create_validator_stake_account(
+        &mut context.banks_client,
+        &context.payer,
+        &context.last_blockhash,
+        &stake_pool_pubkey,
+        &stake_pool_accounts.staker,
+        &stake_address,
+        &test_vote_address,
+    )
+    .await;
+
+    let error = stake_pool_accounts
+        .add_validator_to_pool(
+            &mut context.banks_client,
+            &context.payer,
+            &context.last_blockhash,
+            &stake_address,
+        )
+        .await;
+    assert!(error.is_none());
+
+    let validator_list = get_account(
+        &mut context.banks_client,
+        &stake_pool_accounts.validator_list.pubkey(),
+    )
+    .await;
+    let validator_list =
+        try_from_slice_unchecked::<ValidatorList>(validator_list.data.as_slice()).unwrap();
+    assert_eq!(validator_list.validators.len(), last_index + 1);
+    let last_element = validator_list.validators[last_index];
+    assert_eq!(last_element.status, StakeStatus::Active);
+    assert_eq!(last_element.active_stake_lamports, 0);
+    assert_eq!(last_element.transient_stake_lamports, 0);
+    assert_eq!(last_element.vote_account_address, test_vote_address);
+
+    let (transient_stake_address, _) =
+        find_transient_stake_program_address(&id(), &test_vote_address, &stake_pool_pubkey);
+    let increase_amount = MINIMUM_ACTIVE_STAKE;
+    let error = stake_pool_accounts
+        .increase_validator_stake(
+            &mut context.banks_client,
+            &context.payer,
+            &context.last_blockhash,
+            &transient_stake_address,
+            &test_vote_address,
+            increase_amount,
+        )
+        .await;
+    assert!(error.is_none());
+
+    let validator_list = get_account(
+        &mut context.banks_client,
+        &stake_pool_accounts.validator_list.pubkey(),
+    )
+    .await;
+    let validator_list =
+        try_from_slice_unchecked::<ValidatorList>(validator_list.data.as_slice()).unwrap();
+    let last_element = validator_list.validators[last_index];
+    assert_eq!(last_element.status, StakeStatus::Active);
+    assert_eq!(last_element.active_stake_lamports, 0);
+    assert_eq!(
+        last_element.transient_stake_lamports,
+        increase_amount + STAKE_ACCOUNT_RENT_EXEMPTION
+    );
+    assert_eq!(last_element.vote_account_address, test_vote_address);
+}
+
+#[tokio::test]
+async fn set_preferred() {
+    let (mut context, stake_pool_accounts, _, vote_account_address, _, _, _) =
+        setup(HUGE_POOL_SIZE, HUGE_POOL_SIZE, STAKE_AMOUNT).await;
+
+    let error = stake_pool_accounts
+        .set_preferred_validator(
+            &mut context.banks_client,
+            &context.payer,
+            &context.last_blockhash,
+            PreferredValidatorType::Deposit,
+            Some(vote_account_address),
+        )
+        .await;
+    assert!(error.is_none());
+    let error = stake_pool_accounts
+        .set_preferred_validator(
+            &mut context.banks_client,
+            &context.payer,
+            &context.last_blockhash,
+            PreferredValidatorType::Withdraw,
+            Some(vote_account_address),
+        )
+        .await;
+    assert!(error.is_none());
+
+    let stake_pool = get_account(
+        &mut context.banks_client,
+        &stake_pool_accounts.stake_pool.pubkey(),
+    )
+    .await;
+    let stake_pool = try_from_slice_unchecked::<StakePool>(&stake_pool.data.as_slice()).unwrap();
+
+    assert_eq!(
+        stake_pool.preferred_deposit_validator_vote_address,
+        Some(vote_account_address)
+    );
+    assert_eq!(
+        stake_pool.preferred_withdraw_validator_vote_address,
+        Some(vote_account_address)
+    );
+}
+
+#[tokio::test]
+async fn deposit() {
+    let (mut context, stake_pool_accounts, _, vote_pubkey, user, stake_pubkey, pool_account_pubkey) =
+        setup(HUGE_POOL_SIZE, HUGE_POOL_SIZE, STAKE_AMOUNT).await;
+
+    let (stake_address, _) = find_stake_program_address(
+        &id(),
+        &vote_pubkey,
+        &stake_pool_accounts.stake_pool.pubkey(),
+    );
+
+    let error = stake_pool_accounts
+        .deposit_stake(
+            &mut context.banks_client,
+            &context.payer,
+            &context.last_blockhash,
+            &stake_pubkey,
+            &pool_account_pubkey,
+            &stake_address,
+            &user,
+        )
+        .await;
+    assert!(error.is_none());
+}
+
+#[tokio::test]
+async fn withdraw() {
+    let (mut context, stake_pool_accounts, _, vote_pubkey, user, stake_pubkey, pool_account_pubkey) =
+        setup(HUGE_POOL_SIZE, HUGE_POOL_SIZE, STAKE_AMOUNT).await;
+
+    let (stake_address, _) = find_stake_program_address(
+        &id(),
+        &vote_pubkey,
+        &stake_pool_accounts.stake_pool.pubkey(),
+    );
+
+    let error = stake_pool_accounts
+        .deposit_stake(
+            &mut context.banks_client,
+            &context.payer,
+            &context.last_blockhash,
+            &stake_pubkey,
+            &pool_account_pubkey,
+            &stake_address,
+            &user,
+        )
+        .await;
+    assert!(error.is_none());
+
+    // Create stake account to withdraw to
+    let user_stake_recipient = Keypair::new();
+    create_blank_stake_account(
+        &mut context.banks_client,
+        &context.payer,
+        &context.last_blockhash,
+        &user_stake_recipient,
+    )
+    .await;
+
+    let error = stake_pool_accounts
+        .withdraw_stake(
+            &mut context.banks_client,
+            &context.payer,
+            &context.last_blockhash,
+            &user_stake_recipient.pubkey(),
+            &user,
+            &pool_account_pubkey,
+            &stake_address,
+            &user.pubkey(),
+            STAKE_AMOUNT,
+        )
+        .await;
+    assert!(error.is_none());
+}

stake-pool/program/tests/initialize.rs

@@ -85,7 +85,7 @@ async fn success() {
     .await;
     let validator_list =
         try_from_slice_unchecked::<state::ValidatorList>(validator_list.data.as_slice()).unwrap();
-    assert!(validator_list.is_valid());
+    assert!(validator_list.header.is_valid());
 }
 
 #[tokio::test]

stake-pool/program/tests/set_preferred.rs

@@ -16,7 +16,7 @@ use {
     spl_stake_pool::{
         error, id,
         instruction::{self, PreferredValidatorType},
-        state::ValidatorList,
+        state::StakePool,
     },
 };
 
@@ -68,22 +68,14 @@ async fn success_deposit() {
         .await;
     assert!(error.is_none());
 
-    let validator_list = get_account(
+    let stake_pool = get_account(&mut banks_client, &stake_pool_accounts.stake_pool.pubkey()).await;
-        &mut banks_client,
+    let stake_pool = try_from_slice_unchecked::<StakePool>(&stake_pool.data.as_slice()).unwrap();
-        &stake_pool_accounts.validator_list.pubkey(),
-    )
-    .await;
-    let validator_list =
-        try_from_slice_unchecked::<ValidatorList>(&validator_list.data.as_slice()).unwrap();
 
     assert_eq!(
-        validator_list.preferred_deposit_validator_vote_address,
+        stake_pool.preferred_deposit_validator_vote_address,
         Some(vote_account_address)
     );
-    assert_eq!(
+    assert_eq!(stake_pool.preferred_withdraw_validator_vote_address, None);
-        validator_list.preferred_withdraw_validator_vote_address,
-        None
-    );
 }
 
 #[tokio::test]
@@ -104,20 +96,12 @@ async fn success_withdraw() {
         .await;
     assert!(error.is_none());
 
-    let validator_list = get_account(
+    let stake_pool = get_account(&mut banks_client, &stake_pool_accounts.stake_pool.pubkey()).await;
-        &mut banks_client,
+    let stake_pool = try_from_slice_unchecked::<StakePool>(&stake_pool.data.as_slice()).unwrap();
-        &stake_pool_accounts.validator_list.pubkey(),
-    )
-    .await;
-    let validator_list =
-        try_from_slice_unchecked::<ValidatorList>(&validator_list.data.as_slice()).unwrap();
 
+    assert_eq!(stake_pool.preferred_deposit_validator_vote_address, None);
     assert_eq!(
-        validator_list.preferred_deposit_validator_vote_address,
+        stake_pool.preferred_withdraw_validator_vote_address,
-        None
-    );
-    assert_eq!(
-        validator_list.preferred_withdraw_validator_vote_address,
         Some(vote_account_address)
     );
 }
@@ -139,16 +123,11 @@ async fn success_unset() {
         .await;
     assert!(error.is_none());
 
-    let validator_list = get_account(
+    let stake_pool = get_account(&mut banks_client, &stake_pool_accounts.stake_pool.pubkey()).await;
-        &mut banks_client,
+    let stake_pool = try_from_slice_unchecked::<StakePool>(&stake_pool.data.as_slice()).unwrap();
-        &stake_pool_accounts.validator_list.pubkey(),
-    )
-    .await;
-    let validator_list =
-        try_from_slice_unchecked::<ValidatorList>(&validator_list.data.as_slice()).unwrap();
 
     assert_eq!(
-        validator_list.preferred_withdraw_validator_vote_address,
+        stake_pool.preferred_withdraw_validator_vote_address,
         Some(vote_account_address)
     );
 
@@ -163,18 +142,10 @@ async fn success_unset() {
         .await;
     assert!(error.is_none());
 
-    let validator_list = get_account(
+    let stake_pool = get_account(&mut banks_client, &stake_pool_accounts.stake_pool.pubkey()).await;
-        &mut banks_client,
+    let stake_pool = try_from_slice_unchecked::<StakePool>(&stake_pool.data.as_slice()).unwrap();
-        &stake_pool_accounts.validator_list.pubkey(),
-    )
-    .await;
-    let validator_list =
-        try_from_slice_unchecked::<ValidatorList>(&validator_list.data.as_slice()).unwrap();
 
-    assert_eq!(
+    assert_eq!(stake_pool.preferred_withdraw_validator_vote_address, None);
-        validator_list.preferred_withdraw_validator_vote_address,
-        None
-    );
 }
 
 #[tokio::test]

stake-pool/program/tests/update_validator_list_balance.rs

@@ -82,11 +82,7 @@ async fn setup(
             &mut context.banks_client,
             &context.payer,
             &context.last_blockhash,
-            stake_accounts
+            &[],
-                .iter()
-                .map(|v| v.vote.pubkey())
-                .collect::<Vec<Pubkey>>()
-                .as_slice(),
             false,
         )
         .await;
@@ -229,7 +225,7 @@ async fn merge_into_reserve() {
         .unwrap();
     let pre_reserve_lamports = reserve_stake.lamports;
 
-    // Decrease from all validators
+    println!("Decrease from all validators");
     for stake_account in &stake_accounts {
         let error = stake_pool_accounts
             .decrease_validator_stake(
@@ -244,7 +240,7 @@ async fn merge_into_reserve() {
         assert!(error.is_none());
     }
 
-    // Update, should not change, no merges yet
+    println!("Update, should not change, no merges yet");
     stake_pool_accounts
         .update_all(
             &mut context.banks_client,
@@ -275,7 +271,7 @@ async fn merge_into_reserve() {
     let stake_pool = try_from_slice_unchecked::<StakePool>(&stake_pool_info.data).unwrap();
     assert_eq!(expected_lamports, stake_pool.total_stake_lamports);
 
-    // Warp one more epoch so the stakes deactivate
+    println!("Warp one more epoch so the stakes deactivate");
     let slots_per_epoch = context.genesis_config().epoch_schedule.slots_per_epoch;
     slot += slots_per_epoch;
     context.warp_to_slot(slot).unwrap();
@@ -565,7 +561,7 @@ async fn merge_transient_stake_after_remove() {
         reserve_lamports + deactivated_lamports + 2 * stake_rent + 1
     );
 
-    // Update stake pool balance, should be gone
+    // Update stake pool balance and cleanup, should be gone
     let error = stake_pool_accounts
         .update_stake_pool_balance(
             &mut context.banks_client,
@@ -575,6 +571,15 @@ async fn merge_transient_stake_after_remove() {
         .await;
     assert!(error.is_none());
 
+    let error = stake_pool_accounts
+        .cleanup_removed_validator_entries(
+            &mut context.banks_client,
+            &context.payer,
+            &context.last_blockhash,
+        )
+        .await;
+    assert!(error.is_none());
+
     let validator_list = get_account(
         &mut context.banks_client,
         &stake_pool_accounts.validator_list.pubkey(),

stake-pool/program/tests/vsa_add.rs

@@ -81,10 +81,10 @@ async fn success() {
     assert_eq!(
         validator_list,
         state::ValidatorList {
+            header: state::ValidatorListHeader {
                 account_type: state::AccountType::ValidatorList,
-            preferred_deposit_validator_vote_address: None,
-            preferred_withdraw_validator_vote_address: None,
                 max_validators: stake_pool_accounts.max_validators,
+            },
             validators: vec![state::ValidatorStakeInfo {
                 status: state::StakeStatus::Active,
                 vote_account_address: user_stake.vote.pubkey(),

stake-pool/program/tests/vsa_remove.rs

@@ -83,6 +83,11 @@ async fn success() {
         .await;
     assert!(error.is_none());
 
+    let error = stake_pool_accounts
+        .cleanup_removed_validator_entries(&mut banks_client, &payer, &recent_blockhash)
+        .await;
+    assert!(error.is_none());
+
     // Check if account was removed from the list of stake accounts
     let validator_list = get_account(
         &mut banks_client,
@@ -94,10 +99,10 @@ async fn success() {
     assert_eq!(
         validator_list,
         state::ValidatorList {
+            header: state::ValidatorListHeader {
                 account_type: state::AccountType::ValidatorList,
-            preferred_deposit_validator_vote_address: None,
-            preferred_withdraw_validator_vote_address: None,
                 max_validators: stake_pool_accounts.max_validators,
+            },
             validators: vec![]
         }
     );
@@ -254,6 +259,11 @@ async fn fail_double_remove() {
         .await;
     assert!(error.is_none());
 
+    let error = stake_pool_accounts
+        .cleanup_removed_validator_entries(&mut banks_client, &payer, &recent_blockhash)
+        .await;
+    assert!(error.is_none());
+
     let latest_blockhash = banks_client.get_recent_blockhash().await.unwrap();
 
     let transaction_error = stake_pool_accounts
@@ -519,10 +529,10 @@ async fn success_with_deactivating_transient_stake() {
     let validator_list =
         try_from_slice_unchecked::<state::ValidatorList>(validator_list.data.as_slice()).unwrap();
     let expected_list = state::ValidatorList {
+        header: state::ValidatorListHeader {
             account_type: state::AccountType::ValidatorList,
-        preferred_deposit_validator_vote_address: None,
-        preferred_withdraw_validator_vote_address: None,
             max_validators: stake_pool_accounts.max_validators,
+        },
         validators: vec![state::ValidatorStakeInfo {
             status: state::StakeStatus::DeactivatingTransient,
             vote_account_address: validator_stake.vote.pubkey(),
@@ -592,6 +602,11 @@ async fn success_resets_preferred_validator() {
         .await;
     assert!(error.is_none());
 
+    let error = stake_pool_accounts
+        .cleanup_removed_validator_entries(&mut banks_client, &payer, &recent_blockhash)
+        .await;
+    assert!(error.is_none());
+
     // Check if account was removed from the list of stake accounts
     let validator_list = get_account(
         &mut banks_client,
@@ -603,10 +618,10 @@ async fn success_resets_preferred_validator() {
     assert_eq!(
         validator_list,
         state::ValidatorList {
+            header: state::ValidatorListHeader {
                 account_type: state::AccountType::ValidatorList,
-            preferred_deposit_validator_vote_address: None,
-            preferred_withdraw_validator_vote_address: None,
                 max_validators: stake_pool_accounts.max_validators,
+            },
             validators: vec![]
         }
     );