Add ability to abort scan (#21314)

ledger-tool/src/main.rs

@@ -28,7 +28,7 @@ use solana_ledger::{
 use solana_measure::measure::Measure;
 use solana_runtime::{
     accounts_db::AccountsDbConfig,
-    accounts_index::AccountsIndexConfig,
+    accounts_index::{AccountsIndexConfig, ScanConfig},
     bank::{Bank, RewardCalculationEvent},
     bank_forks::BankForks,
     cost_model::CostModel,
@@ -2251,7 +2251,7 @@ fn main() {
 
                     if remove_stake_accounts {
                         for (address, mut account) in bank
-                            .get_program_accounts(&stake::program::id())
+                            .get_program_accounts(&stake::program::id(), ScanConfig::default())
                             .unwrap()
                             .into_iter()
                         {
@@ -2275,7 +2275,7 @@ fn main() {
 
                     if !vote_accounts_to_destake.is_empty() {
                         for (address, mut account) in bank
-                            .get_program_accounts(&stake::program::id())
+                            .get_program_accounts(&stake::program::id(), ScanConfig::default())
                             .unwrap()
                             .into_iter()
                         {
@@ -2313,7 +2313,7 @@ fn main() {
 
                         // Delete existing vote accounts
                         for (address, mut account) in bank
-                            .get_program_accounts(&solana_vote_program::id())
+                            .get_program_accounts(&solana_vote_program::id(), ScanConfig::default())
                             .unwrap()
                             .into_iter()
                         {

rpc/src/rpc.rs

@@ -39,7 +39,7 @@ use {
     solana_perf::packet::PACKET_DATA_SIZE,
     solana_runtime::{
         accounts::AccountAddressFilter,
-        accounts_index::{AccountIndex, AccountSecondaryIndexes, IndexKey},
+        accounts_index::{AccountIndex, AccountSecondaryIndexes, IndexKey, ScanConfig},
         bank::{Bank, TransactionSimulationResult},
         bank_forks::BankForks,
         commitment::{BlockCommitmentArray, BlockCommitmentCache, CommitmentSlots},
@@ -1830,20 +1830,24 @@ impl JsonRpcRequestProcessor {
                 });
             }
             Ok(bank
-                .get_filtered_indexed_accounts(&IndexKey::ProgramId(*program_id), |account| {
+                .get_filtered_indexed_accounts(
+                    &IndexKey::ProgramId(*program_id),
+                    |account| {
                         // The program-id account index checks for Account owner on inclusion. However, due
                         // to the current AccountsDb implementation, an account may remain in storage as a
                         // zero-lamport AccountSharedData::Default() after being wiped and reinitialized in later
                         // updates. We include the redundant filters here to avoid returning these
                         // accounts.
                         account.owner() == program_id && filter_closure(account)
-                })
+                    },
+                    ScanConfig::default(),
+                )
                 .map_err(|e| RpcCustomError::ScanError {
                     message: e.to_string(),
                 })?)
         } else {
             Ok(bank
-                .get_filtered_program_accounts(program_id, filter_closure)
+                .get_filtered_program_accounts(program_id, filter_closure, ScanConfig::default())
                 .map_err(|e| RpcCustomError::ScanError {
                     message: e.to_string(),
                 })?)
@@ -1884,13 +1888,21 @@ impl JsonRpcRequestProcessor {
                 });
             }
             Ok(bank
-                .get_filtered_indexed_accounts(&IndexKey::SplTokenOwner(*owner_key), |account| {
+                .get_filtered_indexed_accounts(
+                    &IndexKey::SplTokenOwner(*owner_key),
+                    |account| {
                         account.owner() == &spl_token_id_v2_0()
                             && filters.iter().all(|filter_type| match filter_type {
-                            RpcFilterType::DataSize(size) => account.data().len() as u64 == *size,
+                                RpcFilterType::DataSize(size) => {
-                            RpcFilterType::Memcmp(compare) => compare.bytes_match(account.data()),
+                                    account.data().len() as u64 == *size
-                        })
+                                }
+                                RpcFilterType::Memcmp(compare) => {
+                                    compare.bytes_match(account.data())
+                                }
                             })
+                    },
+                    ScanConfig::default(),
+                )
                 .map_err(|e| RpcCustomError::ScanError {
                     message: e.to_string(),
                 })?)
@@ -1932,13 +1944,21 @@ impl JsonRpcRequestProcessor {
                 });
             }
             Ok(bank
-                .get_filtered_indexed_accounts(&IndexKey::SplTokenMint(*mint_key), |account| {
+                .get_filtered_indexed_accounts(
+                    &IndexKey::SplTokenMint(*mint_key),
+                    |account| {
                         account.owner() == &spl_token_id_v2_0()
                             && filters.iter().all(|filter_type| match filter_type {
-                            RpcFilterType::DataSize(size) => account.data().len() as u64 == *size,
+                                RpcFilterType::DataSize(size) => {
-                            RpcFilterType::Memcmp(compare) => compare.bytes_match(account.data()),
+                                    account.data().len() as u64 == *size
-                        })
+                                }
+                                RpcFilterType::Memcmp(compare) => {
+                                    compare.bytes_match(account.data())
+                                }
                             })
+                    },
+                    ScanConfig::default(),
+                )
                 .map_err(|e| RpcCustomError::ScanError {
                     message: e.to_string(),
                 })?)

runtime/benches/accounts.rs

@@ -9,7 +9,7 @@ use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
 use solana_runtime::{
     accounts::{create_test_accounts, AccountAddressFilter, Accounts},
     accounts_db::AccountShrinkThreshold,
-    accounts_index::AccountSecondaryIndexes,
+    accounts_index::{AccountSecondaryIndexes, ScanConfig},
     ancestors::Ancestors,
     bank::*,
 };
@@ -267,6 +267,7 @@ fn bench_concurrent_scan_write(bencher: &mut Bencher) {
                     &Ancestors::default(),
                     0,
                     AccountSharedData::default().owner(),
+                    ScanConfig::default(),
                 )
                 .unwrap(),
         );

runtime/src/accounts.rs

@@ -4,7 +4,7 @@ use crate::{
         LoadHint, LoadedAccount, ScanStorageResult, ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS,
         ACCOUNTS_DB_CONFIG_FOR_TESTING,
     },
-    accounts_index::{AccountSecondaryIndexes, IndexKey, ScanResult},
+    accounts_index::{AccountSecondaryIndexes, IndexKey, ScanConfig, ScanResult},
     accounts_update_notifier_interface::AccountsUpdateNotifier,
     ancestors::Ancestors,
     bank::{
@@ -667,6 +667,7 @@ impl Accounts {
                     collector.push(Reverse((account.lamports(), *pubkey)));
                 }
             },
+            ScanConfig::default(),
         )?;
         Ok(account_balances
             .into_sorted_vec()
@@ -744,6 +745,7 @@ impl Accounts {
         ancestors: &Ancestors,
         bank_id: BankId,
         program_id: &Pubkey,
+        config: ScanConfig,
     ) -> ScanResult<Vec<(Pubkey, AccountSharedData)>> {
         self.accounts_db.scan_accounts(
             ancestors,
@@ -753,6 +755,7 @@ impl Accounts {
                     account.owner() == program_id
                 })
             },
+            config,
         )
     }
 
@@ -762,6 +765,7 @@ impl Accounts {
         bank_id: BankId,
         program_id: &Pubkey,
         filter: F,
+        config: ScanConfig,
     ) -> ScanResult<Vec<(Pubkey, AccountSharedData)>> {
         self.accounts_db.scan_accounts(
             ancestors,
@@ -771,6 +775,7 @@ impl Accounts {
                     account.owner() == program_id && filter(account)
                 })
             },
+            config,
         )
     }
 
@@ -780,6 +785,7 @@ impl Accounts {
         bank_id: BankId,
         index_key: &IndexKey,
         filter: F,
+        config: ScanConfig,
     ) -> ScanResult<Vec<(Pubkey, AccountSharedData)>> {
         self.accounts_db
             .index_scan_accounts(
@@ -791,6 +797,7 @@ impl Accounts {
                         filter(account)
                     })
                 },
+                config,
             )
             .map(|result| result.0)
     }
@@ -814,6 +821,7 @@ impl Accounts {
                     collector.push((*pubkey, account, slot))
                 }
             },
+            ScanConfig::default(),
         )
     }
 
@@ -835,7 +843,7 @@ impl Accounts {
             "load_to_collect_rent_eagerly_scan_elapsed",
             ancestors,
             range,
-            true,
+            ScanConfig::new(true),
             |collector: &mut Vec<(Pubkey, AccountSharedData)>, option| {
                 Self::load_while_filtering(collector, option, |_| true)
             },

runtime/src/accounts_db.rs

@@ -24,8 +24,8 @@ use crate::{
     accounts_hash::{AccountsHash, CalculateHashIntermediate, HashStats, PreviousPass},
     accounts_index::{
         AccountIndexGetResult, AccountSecondaryIndexes, AccountsIndex, AccountsIndexConfig,
-        AccountsIndexRootsStats, IndexKey, IndexValue, IsCached, RefCount, ScanResult, SlotList,
+        AccountsIndexRootsStats, IndexKey, IndexValue, IsCached, RefCount, ScanConfig, ScanResult,
-        SlotSlice, ZeroLamport, ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS,
+        SlotList, SlotSlice, ZeroLamport, ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS,
         ACCOUNTS_INDEX_CONFIG_FOR_TESTING,
     },
     accounts_update_notifier_interface::AccountsUpdateNotifier,
@@ -3059,6 +3059,7 @@ impl AccountsDb {
         ancestors: &Ancestors,
         bank_id: BankId,
         scan_func: F,
+        config: ScanConfig,
     ) -> ScanResult<A>
     where
         F: Fn(&mut A, Option<(&Pubkey, AccountSharedData, Slot)>),
@@ -3067,14 +3068,18 @@ impl AccountsDb {
         let mut collector = A::default();
 
         // This can error out if the slots being scanned over are aborted
-        self.accounts_index
+        self.accounts_index.scan_accounts(
-            .scan_accounts(ancestors, bank_id, |pubkey, (account_info, slot)| {
+            ancestors,
+            bank_id,
+            |pubkey, (account_info, slot)| {
                 let account_slot = self
                     .get_account_accessor(slot, pubkey, account_info.store_id, account_info.offset)
                     .get_loaded_account()
                     .map(|loaded_account| (pubkey, loaded_account.take_account(), slot));
                 scan_func(&mut collector, account_slot)
-            })?;
+            },
+            config,
+        )?;
 
         Ok(collector)
     }
@@ -3084,7 +3089,7 @@ impl AccountsDb {
         metric_name: &'static str,
         ancestors: &Ancestors,
         scan_func: F,
-        collect_all_unsorted: bool,
+        config: ScanConfig,
     ) -> A
     where
         F: Fn(&mut A, (&Pubkey, LoadedAccount, Slot)),
@@ -3102,7 +3107,7 @@ impl AccountsDb {
                     scan_func(&mut collector, (pubkey, loaded_account, slot));
                 }
             },
-            collect_all_unsorted,
+            config,
         );
         collector
     }
@@ -3112,7 +3117,7 @@ impl AccountsDb {
         metric_name: &'static str,
         ancestors: &Ancestors,
         range: R,
-        collect_all_unsorted: bool,
+        config: ScanConfig,
         scan_func: F,
     ) -> A
     where
@@ -3125,7 +3130,7 @@ impl AccountsDb {
             metric_name,
             ancestors,
             range,
-            collect_all_unsorted,
+            config,
             |pubkey, (account_info, slot)| {
                 // unlike other scan fns, this is called from Bank::collect_rent_eagerly(),
                 // which is on-consensus processing in the banking/replaying stage.
@@ -3153,6 +3158,7 @@ impl AccountsDb {
         bank_id: BankId,
         index_key: IndexKey,
         scan_func: F,
+        config: ScanConfig,
     ) -> ScanResult<(A, bool)>
     where
         F: Fn(&mut A, Option<(&Pubkey, AccountSharedData, Slot)>),
@@ -3166,7 +3172,7 @@ impl AccountsDb {
         if !self.account_indexes.include_key(key) {
             // the requested key was not indexed in the secondary index, so do a normal scan
             let used_index = false;
-            let scan_result = self.scan_accounts(ancestors, bank_id, scan_func)?;
+            let scan_result = self.scan_accounts(ancestors, bank_id, scan_func, config)?;
             return Ok((scan_result, used_index));
         }
 
@@ -3182,6 +3188,7 @@ impl AccountsDb {
                     .map(|loaded_account| (pubkey, loaded_account.take_account(), slot));
                 scan_func(&mut collector, account_slot)
             },
+            config,
         )?;
         let used_index = true;
         Ok((collector, used_index))
@@ -8035,8 +8042,6 @@ pub mod tests {
         );
     }
 
-    const COLLECT_ALL_UNSORTED_FALSE: bool = false;
-
     #[test]
     fn test_accountsdb_latest_ancestor() {
         solana_logger::setup();
@@ -8067,7 +8072,7 @@ pub mod tests {
             |accounts: &mut Vec<AccountSharedData>, option| {
                 accounts.push(option.1.take_account());
             },
-            COLLECT_ALL_UNSORTED_FALSE,
+            ScanConfig::default(),
         );
         assert_eq!(accounts, vec![account1]);
     }
@@ -8970,9 +8975,15 @@ pub mod tests {
         let bank_id = 0;
         accounts
             .accounts_index
-            .index_scan_accounts(&Ancestors::default(), bank_id, index_key, |key, _| {
+            .index_scan_accounts(
+                &Ancestors::default(),
+                bank_id,
+                index_key,
+                |key, _| {
                     found_accounts.insert(*key);
-            })
+                },
+                ScanConfig::default(),
+            )
             .unwrap();
         assert_eq!(found_accounts.len(), 2);
         assert!(found_accounts.contains(&pubkey1));
@@ -8992,6 +9003,7 @@ pub mod tests {
                     |collection: &mut HashSet<Pubkey>, account| {
                         collection.insert(*account.unwrap().0);
                     },
+                    ScanConfig::default(),
                 )
                 .unwrap();
             assert!(!found_accounts.1);
@@ -9010,6 +9022,7 @@ pub mod tests {
                     |collection: &mut HashSet<Pubkey>, account| {
                         collection.insert(*account.unwrap().0);
                     },
+                    ScanConfig::default(),
                 )
                 .unwrap();
             assert!(found_accounts.1);
@@ -9043,6 +9056,7 @@ pub mod tests {
                 bank_id,
                 IndexKey::SplTokenMint(mint_key),
                 |key, _| found_accounts.push(*key),
+                ScanConfig::default(),
             )
             .unwrap();
         assert_eq!(found_accounts, vec![pubkey2]);
@@ -9598,7 +9612,7 @@ pub mod tests {
             |accounts: &mut Vec<AccountSharedData>, option| {
                 accounts.push(option.1.take_account());
             },
-            COLLECT_ALL_UNSORTED_FALSE,
+            ScanConfig::default(),
         );
         assert_eq!(accounts, vec![account0]);
 
@@ -9609,7 +9623,7 @@ pub mod tests {
             |accounts: &mut Vec<AccountSharedData>, option| {
                 accounts.push(option.1.take_account());
             },
-            COLLECT_ALL_UNSORTED_FALSE,
+            ScanConfig::default(),
         );
         assert_eq!(accounts.len(), 2);
     }
@@ -11783,6 +11797,7 @@ pub mod tests {
                             }
                         }
                     },
+                    ScanConfig::default(),
                 )
                 .unwrap();
             })

runtime/src/accounts_index.rs

@@ -59,6 +59,34 @@ pub type SlotSlice<'s, T> = &'s [(Slot, T)];
 pub type RefCount = u64;
 pub type AccountMap<V> = Arc<InMemAccountsIndex<V>>;
 
+#[derive(Debug, Default)]
+pub struct ScanConfig {
+    /// checked by the scan. When true, abort scan.
+    abort: Option<Arc<AtomicBool>>,
+
+    /// true to allow return of all matching items and allow them to be unsorted.
+    /// This is more efficient.
+    collect_all_unsorted: bool,
+}
+
+impl ScanConfig {
+    pub fn new(collect_all_unsorted: bool) -> Self {
+        Self {
+            collect_all_unsorted,
+            ..ScanConfig::default()
+        }
+    }
+
+    /// true if scan should abort
+    pub fn is_aborted(&self) -> bool {
+        if let Some(abort) = self.abort.as_ref() {
+            abort.load(Ordering::Relaxed)
+        } else {
+            false
+        }
+    }
+}
+
 pub(crate) type AccountMapEntry<T> = Arc<AccountMapEntryInner<T>>;
 
 pub trait IsCached:
@@ -837,7 +865,7 @@ impl<T: IndexValue> AccountsIndex<T> {
         scan_bank_id: BankId,
         func: F,
         scan_type: ScanTypes<R>,
-        collect_all_unsorted: bool,
+        config: ScanConfig,
     ) -> Result<(), ScanError>
     where
         F: FnMut(&Pubkey, (&T, Slot)),
@@ -990,14 +1018,7 @@ impl<T: IndexValue> AccountsIndex<T> {
         match scan_type {
             ScanTypes::Unindexed(range) => {
                 // Pass "" not to log metrics, so RPC doesn't get spammy
-                self.do_scan_accounts(
+                self.do_scan_accounts(metric_name, ancestors, func, range, Some(max_root), config);
-                    metric_name,
-                    ancestors,
-                    func,
-                    range,
-                    Some(max_root),
-                    collect_all_unsorted,
-                );
             }
             ScanTypes::Indexed(IndexKey::ProgramId(program_id)) => {
                 self.do_scan_secondary_index(
@@ -1006,6 +1027,7 @@ impl<T: IndexValue> AccountsIndex<T> {
                     &self.program_id_index,
                     &program_id,
                     Some(max_root),
+                    config,
                 );
             }
             ScanTypes::Indexed(IndexKey::SplTokenMint(mint_key)) => {
@@ -1015,6 +1037,7 @@ impl<T: IndexValue> AccountsIndex<T> {
                     &self.spl_token_mint_index,
                     &mint_key,
                     Some(max_root),
+                    config,
                 );
             }
             ScanTypes::Indexed(IndexKey::SplTokenOwner(owner_key)) => {
@@ -1024,6 +1047,7 @@ impl<T: IndexValue> AccountsIndex<T> {
                     &self.spl_token_owner_index,
                     &owner_key,
                     Some(max_root),
+                    config,
                 );
             }
         }
@@ -1061,19 +1085,12 @@ impl<T: IndexValue> AccountsIndex<T> {
         ancestors: &Ancestors,
         func: F,
         range: Option<R>,
-        collect_all_unsorted: bool,
+        config: ScanConfig,
     ) where
         F: FnMut(&Pubkey, (&T, Slot)),
         R: RangeBounds<Pubkey> + std::fmt::Debug,
     {
-        self.do_scan_accounts(
+        self.do_scan_accounts(metric_name, ancestors, func, range, None, config);
-            metric_name,
-            ancestors,
-            func,
-            range,
-            None,
-            collect_all_unsorted,
-        );
     }
 
     // Scan accounts and return latest version of each account that is either:
@@ -1086,7 +1103,7 @@ impl<T: IndexValue> AccountsIndex<T> {
         mut func: F,
         range: Option<R>,
         max_root: Option<Slot>,
-        collect_all_unsorted: bool,
+        config: ScanConfig,
     ) where
         F: FnMut(&Pubkey, (&T, Slot)),
         R: RangeBounds<Pubkey> + std::fmt::Debug,
@@ -1100,7 +1117,7 @@ impl<T: IndexValue> AccountsIndex<T> {
         let mut read_lock_elapsed = 0;
         let mut iterator_elapsed = 0;
         let mut iterator_timer = Measure::start("iterator_elapsed");
-        for pubkey_list in self.iter(range.as_ref(), collect_all_unsorted) {
+        for pubkey_list in self.iter(range.as_ref(), config.collect_all_unsorted) {
             iterator_timer.stop();
             iterator_elapsed += iterator_timer.as_us();
             for (pubkey, list) in pubkey_list {
@@ -1118,6 +1135,9 @@ impl<T: IndexValue> AccountsIndex<T> {
                     load_account_timer.stop();
                     load_account_elapsed += load_account_timer.as_us();
                 }
+                if config.is_aborted() {
+                    return;
+                }
             }
             iterator_timer = Measure::start("iterator_elapsed");
         }
@@ -1146,6 +1166,7 @@ impl<T: IndexValue> AccountsIndex<T> {
         index: &SecondaryIndex<SecondaryIndexEntryType>,
         index_key: &Pubkey,
         max_root: Option<Slot>,
+        config: ScanConfig,
     ) where
         F: FnMut(&Pubkey, (&T, Slot)),
     {
@@ -1160,6 +1181,9 @@ impl<T: IndexValue> AccountsIndex<T> {
                     (&list_r.slot_list()[index].1, list_r.slot_list()[index].0),
                 );
             }
+            if config.is_aborted() {
+                break;
+            }
         }
     }
 
@@ -1212,11 +1236,11 @@ impl<T: IndexValue> AccountsIndex<T> {
         ancestors: &Ancestors,
         scan_bank_id: BankId,
         func: F,
+        config: ScanConfig,
     ) -> Result<(), ScanError>
     where
         F: FnMut(&Pubkey, (&T, Slot)),
     {
-        let collect_all_unsorted = false;
         // Pass "" not to log metrics, so RPC doesn't get spammy
         self.do_checked_scan_accounts(
             "",
@@ -1224,7 +1248,7 @@ impl<T: IndexValue> AccountsIndex<T> {
             scan_bank_id,
             func,
             ScanTypes::Unindexed(None::<Range<Pubkey>>),
-            collect_all_unsorted,
+            config,
         )
     }
 
@@ -1233,7 +1257,7 @@ impl<T: IndexValue> AccountsIndex<T> {
         metric_name: &'static str,
         ancestors: &Ancestors,
         func: F,
-        collect_all_unsorted: bool,
+        config: ScanConfig,
     ) where
         F: FnMut(&Pubkey, (&T, Slot)),
     {
@@ -1242,7 +1266,7 @@ impl<T: IndexValue> AccountsIndex<T> {
             ancestors,
             func,
             None::<Range<Pubkey>>,
-            collect_all_unsorted,
+            config,
         );
     }
 
@@ -1252,20 +1276,14 @@ impl<T: IndexValue> AccountsIndex<T> {
         metric_name: &'static str,
         ancestors: &Ancestors,
         range: R,
-        collect_all_unsorted: bool,
+        config: ScanConfig,
         func: F,
     ) where
         F: FnMut(&Pubkey, (&T, Slot)),
         R: RangeBounds<Pubkey> + std::fmt::Debug,
     {
         // Only the rent logic should be calling this, which doesn't need the safety checks
-        self.do_unchecked_scan_accounts(
+        self.do_unchecked_scan_accounts(metric_name, ancestors, func, Some(range), config);
-            metric_name,
-            ancestors,
-            func,
-            Some(range),
-            collect_all_unsorted,
-        );
     }
 
     /// call func with every pubkey and index visible from a given set of ancestors
@@ -1275,12 +1293,11 @@ impl<T: IndexValue> AccountsIndex<T> {
         scan_bank_id: BankId,
         index_key: IndexKey,
         func: F,
+        config: ScanConfig,
     ) -> Result<(), ScanError>
     where
         F: FnMut(&Pubkey, (&T, Slot)),
     {
-        let collect_all_unsorted = false;
-
         // Pass "" not to log metrics, so RPC doesn't get spammy
         self.do_checked_scan_accounts(
             "",
@@ -1288,7 +1305,7 @@ impl<T: IndexValue> AccountsIndex<T> {
             scan_bank_id,
             func,
             ScanTypes::<Range<Pubkey>>::Indexed(index_key),
-            collect_all_unsorted,
+            config,
         )
     }
 
@@ -2676,7 +2693,7 @@ pub mod tests {
             "",
             &ancestors,
             |_pubkey, _index| num += 1,
-            COLLECT_ALL_UNSORTED_FALSE,
+            ScanConfig::default(),
         );
         assert_eq!(num, 0);
     }
@@ -2754,7 +2771,7 @@ pub mod tests {
             "",
             &ancestors,
             |_pubkey, _index| num += 1,
-            COLLECT_ALL_UNSORTED_FALSE,
+            ScanConfig::default(),
         );
         assert_eq!(num, 0);
     }
@@ -2793,7 +2810,7 @@ pub mod tests {
             "",
             &ancestors,
             |_pubkey, _index| num += 1,
-            COLLECT_ALL_UNSORTED_FALSE,
+            ScanConfig::default(),
         );
         assert_eq!(num, 0);
         ancestors.insert(slot, 0);
@@ -2803,7 +2820,7 @@ pub mod tests {
             "",
             &ancestors,
             |_pubkey, _index| num += 1,
-            COLLECT_ALL_UNSORTED_FALSE,
+            ScanConfig::default(),
         );
         assert_eq!(num, 1);
 
@@ -2822,7 +2839,7 @@ pub mod tests {
             "",
             &ancestors,
             |_pubkey, _index| num += 1,
-            COLLECT_ALL_UNSORTED_FALSE,
+            ScanConfig::default(),
         );
         assert_eq!(num, 0);
         ancestors.insert(slot, 0);
@@ -2832,7 +2849,7 @@ pub mod tests {
             "",
             &ancestors,
             |_pubkey, _index| num += 1,
-            COLLECT_ALL_UNSORTED_FALSE,
+            ScanConfig::default(),
         );
         assert_eq!(num, 1);
     }
@@ -3019,7 +3036,7 @@ pub mod tests {
             "",
             &ancestors,
             |_pubkey, _index| num += 1,
-            COLLECT_ALL_UNSORTED_FALSE,
+            ScanConfig::default(),
         );
         assert_eq!(num, 0);
         ancestors.insert(slot, 0);
@@ -3028,7 +3045,7 @@ pub mod tests {
             "",
             &ancestors,
             |_pubkey, _index| num += 1,
-            COLLECT_ALL_UNSORTED_FALSE,
+            ScanConfig::default(),
         );
         assert_eq!(num, 1);
     }
@@ -3058,7 +3075,7 @@ pub mod tests {
             "",
             &ancestors,
             |_pubkey, _index| num += 1,
-            COLLECT_ALL_UNSORTED_FALSE,
+            ScanConfig::default(),
         );
         assert_eq!(num, 0);
     }
@@ -3095,7 +3112,7 @@ pub mod tests {
                 };
                 num += 1
             },
-            COLLECT_ALL_UNSORTED_FALSE,
+            ScanConfig::default(),
         );
         assert_eq!(num, 1);
         assert!(found_key);
@@ -3168,7 +3185,7 @@ pub mod tests {
             "",
             &ancestors,
             pubkey_range,
-            COLLECT_ALL_UNSORTED_FALSE,
+            ScanConfig::default(),
             |pubkey, _index| {
                 scanned_keys.insert(*pubkey);
             },
@@ -3247,7 +3264,7 @@ pub mod tests {
             |pubkey, _index| {
                 scanned_keys.insert(*pubkey);
             },
-            COLLECT_ALL_UNSORTED_FALSE,
+            ScanConfig::default(),
         );
         assert_eq!(scanned_keys.len(), num_pubkeys);
     }
@@ -3553,7 +3570,7 @@ pub mod tests {
                 };
                 num += 1
             },
-            COLLECT_ALL_UNSORTED_FALSE,
+            ScanConfig::default(),
         );
         assert_eq!(num, 1);
         assert!(found_key);

runtime/src/bank.rs

@@ -39,7 +39,7 @@ use crate::{
         AccountShrinkThreshold, AccountsDbConfig, ErrorCounters, SnapshotStorages,
         ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS, ACCOUNTS_DB_CONFIG_FOR_TESTING,
     },
-    accounts_index::{AccountSecondaryIndexes, IndexKey, ScanResult},
+    accounts_index::{AccountSecondaryIndexes, IndexKey, ScanConfig, ScanResult},
     accounts_update_notifier_interface::AccountsUpdateNotifier,
     ancestors::{Ancestors, AncestorsForSerialization},
     blockhash_queue::BlockhashQueue,
@@ -5254,22 +5254,25 @@ impl Bank {
     pub fn get_program_accounts(
         &self,
         program_id: &Pubkey,
+        config: ScanConfig,
     ) -> ScanResult<Vec<(Pubkey, AccountSharedData)>> {
         self.rc
             .accounts
-            .load_by_program(&self.ancestors, self.bank_id, program_id)
+            .load_by_program(&self.ancestors, self.bank_id, program_id, config)
     }
 
     pub fn get_filtered_program_accounts<F: Fn(&AccountSharedData) -> bool>(
         &self,
         program_id: &Pubkey,
         filter: F,
+        config: ScanConfig,
     ) -> ScanResult<Vec<(Pubkey, AccountSharedData)>> {
         self.rc.accounts.load_by_program_with_filter(
             &self.ancestors,
             self.bank_id,
             program_id,
             filter,
+            config,
         )
     }
 
@@ -5277,12 +5280,14 @@ impl Bank {
         &self,
         index_key: &IndexKey,
         filter: F,
+        config: ScanConfig,
     ) -> ScanResult<Vec<(Pubkey, AccountSharedData)>> {
         self.rc.accounts.load_by_index_key_with_filter(
             &self.ancestors,
             self.bank_id,
             index_key,
             filter,
+            config,
         )
     }
 
@@ -10495,11 +10500,15 @@ pub(crate) mod tests {
         let bank1 = Arc::new(new_from_parent(&bank0));
         bank1.squash();
         assert_eq!(
-            bank0.get_program_accounts(&program_id).unwrap(),
+            bank0
+                .get_program_accounts(&program_id, ScanConfig::default(),)
+                .unwrap(),
             vec![(pubkey0, account0.clone())]
         );
         assert_eq!(
-            bank1.get_program_accounts(&program_id).unwrap(),
+            bank1
+                .get_program_accounts(&program_id, ScanConfig::default(),)
+                .unwrap(),
             vec![(pubkey0, account0)]
         );
         assert_eq!(
@@ -10518,8 +10527,20 @@ pub(crate) mod tests {
 
         let bank3 = Arc::new(new_from_parent(&bank2));
         bank3.squash();
-        assert_eq!(bank1.get_program_accounts(&program_id).unwrap().len(), 2);
+        assert_eq!(
-        assert_eq!(bank3.get_program_accounts(&program_id).unwrap().len(), 2);
+            bank1
+                .get_program_accounts(&program_id, ScanConfig::default(),)
+                .unwrap()
+                .len(),
+            2
+        );
+        assert_eq!(
+            bank3
+                .get_program_accounts(&program_id, ScanConfig::default(),)
+                .unwrap()
+                .len(),
+            2
+        );
     }
 
     #[test]
@@ -10540,7 +10561,11 @@ pub(crate) mod tests {
         bank.store_account(&address, &account);
 
         let indexed_accounts = bank
-            .get_filtered_indexed_accounts(&IndexKey::ProgramId(program_id), |_| true)
+            .get_filtered_indexed_accounts(
+                &IndexKey::ProgramId(program_id),
+                |_| true,
+                ScanConfig::default(),
+            )
             .unwrap();
         assert_eq!(indexed_accounts.len(), 1);
         assert_eq!(indexed_accounts[0], (address, account));
@@ -10553,27 +10578,39 @@ pub(crate) mod tests {
         let bank = Arc::new(new_from_parent(&bank));
         bank.store_account(&address, &new_account);
         let indexed_accounts = bank
-            .get_filtered_indexed_accounts(&IndexKey::ProgramId(program_id), |_| true)
+            .get_filtered_indexed_accounts(
+                &IndexKey::ProgramId(program_id),
+                |_| true,
+                ScanConfig::default(),
+            )
             .unwrap();
         assert_eq!(indexed_accounts.len(), 1);
         assert_eq!(indexed_accounts[0], (address, new_account.clone()));
         let indexed_accounts = bank
-            .get_filtered_indexed_accounts(&IndexKey::ProgramId(another_program_id), |_| true)
+            .get_filtered_indexed_accounts(
+                &IndexKey::ProgramId(another_program_id),
+                |_| true,
+                ScanConfig::default(),
+            )
             .unwrap();
         assert_eq!(indexed_accounts.len(), 1);
         assert_eq!(indexed_accounts[0], (address, new_account.clone()));
 
         // Post-processing filter
         let indexed_accounts = bank
-            .get_filtered_indexed_accounts(&IndexKey::ProgramId(program_id), |account| {
+            .get_filtered_indexed_accounts(
-                account.owner() == &program_id
+                &IndexKey::ProgramId(program_id),
-            })
+                |account| account.owner() == &program_id,
+                ScanConfig::default(),
+            )
             .unwrap();
         assert!(indexed_accounts.is_empty());
         let indexed_accounts = bank
-            .get_filtered_indexed_accounts(&IndexKey::ProgramId(another_program_id), |account| {
+            .get_filtered_indexed_accounts(
-                account.owner() == &another_program_id
+                &IndexKey::ProgramId(another_program_id),
-            })
+                |account| account.owner() == &another_program_id,
+                ScanConfig::default(),
+            )
             .unwrap();
         assert_eq!(indexed_accounts.len(), 1);
         assert_eq!(indexed_accounts[0], (address, new_account));
@@ -13206,13 +13243,25 @@ pub(crate) mod tests {
 
         let bank2 =
             Bank::new_from_parent(&bank1, &Pubkey::default(), bank1.first_slot_in_next_epoch());
-        assert_eq!(bank2.get_program_accounts(&sysvar::id()).unwrap().len(), 8);
+        assert_eq!(
+            bank2
+                .get_program_accounts(&sysvar::id(), ScanConfig::default(),)
+                .unwrap()
+                .len(),
+            8
+        );
 
         // force rent collection for sysvars
         bank2.collect_rent_in_partition((0, 0, 1)); // all range
 
         // no sysvar should be deleted due to rent
-        assert_eq!(bank2.get_program_accounts(&sysvar::id()).unwrap().len(), 8);
+        assert_eq!(
+            bank2
+                .get_program_accounts(&sysvar::id(), ScanConfig::default(),)
+                .unwrap()
+                .len(),
+            8
+        );
     }
 
     // this test can be removed after rent_for_sysvars activation on mainnet-beta
@@ -13245,7 +13294,9 @@ pub(crate) mod tests {
         );
 
         {
-            let sysvars = bank1.get_program_accounts(&sysvar::id()).unwrap();
+            let sysvars = bank1
+                .get_program_accounts(&sysvar::id(), ScanConfig::default())
+                .unwrap();
             assert_eq!(sysvars.len(), 8);
             assert!(sysvars
                 .iter()
@@ -13280,7 +13331,9 @@ pub(crate) mod tests {
         );
 
         {
-            let sysvars = bank2.get_program_accounts(&sysvar::id()).unwrap();
+            let sysvars = bank2
+                .get_program_accounts(&sysvar::id(), ScanConfig::default())
+                .unwrap();
             assert_eq!(sysvars.len(), 8);
             assert!(sysvars
                 .iter()
@@ -13357,7 +13410,9 @@ pub(crate) mod tests {
             &feature::create_account(&Feature { activated_at: None }, feature_balance),
         );
         {
-            let sysvars = bank1.get_program_accounts(&sysvar::id()).unwrap();
+            let sysvars = bank1
+                .get_program_accounts(&sysvar::id(), ScanConfig::default())
+                .unwrap();
             assert_eq!(sysvars.len(), 9);
             assert!(sysvars
                 .iter()
@@ -13392,7 +13447,9 @@ pub(crate) mod tests {
             },
         );
         {
-            let sysvars = bank2.get_program_accounts(&sysvar::id()).unwrap();
+            let sysvars = bank2
+                .get_program_accounts(&sysvar::id(), ScanConfig::default())
+                .unwrap();
             assert_eq!(sysvars.len(), 9);
             assert!(sysvars
                 .iter()
@@ -13761,7 +13818,8 @@ pub(crate) mod tests {
                             bank_to_scan_receiver.recv_timeout(Duration::from_millis(10))
                         {
                             info!("scanning program accounts for slot {}", bank_to_scan.slot());
-                            let accounts_result = bank_to_scan.get_program_accounts(&program_id);
+                            let accounts_result = bank_to_scan
+                                .get_program_accounts(&program_id, ScanConfig::default());
                             let _ = scan_finished_sender.send(bank_to_scan.bank_id());
                             num_banks_scanned.fetch_add(1, Relaxed);
                             match (&acceptable_scan_results, accounts_result.is_err()) {

runtime/src/non_circulating_supply.rs

@@ -1,6 +1,6 @@
 use {
     crate::{
-        accounts_index::{AccountIndex, IndexKey, ScanResult},
+        accounts_index::{AccountIndex, IndexKey, ScanConfig, ScanResult},
         bank::Bank,
     },
     log::*,
@@ -28,6 +28,7 @@ pub fn calculate_non_circulating_supply(bank: &Arc<Bank>) -> ScanResult<NonCircu
     let withdraw_authority_list = withdraw_authority();
 
     let clock = bank.clock();
+    let config = ScanConfig::default();
     let stake_accounts = if bank
         .rc
         .accounts
@@ -42,9 +43,10 @@ pub fn calculate_non_circulating_supply(bank: &Arc<Bank>) -> ScanResult<NonCircu
             // zero-lamport Account::Default() after being wiped and reinitialized in later
             // updates. We include the redundant filter here to avoid returning these accounts.
             |account| account.owner() == &stake::program::id(),
+            config,
         )?
     } else {
-        bank.get_program_accounts(&stake::program::id())?
+        bank.get_program_accounts(&stake::program::id(), config)?
     };
 
     for (pubkey, account) in stake_accounts.iter() {