2022-05-10 14:41:04 -07:00
|
|
|
//! helpers for squashing append vecs into ancient append vecs
|
|
|
|
//! an ancient append vec is:
|
|
|
|
//! 1. a slot that is older than an epoch old
|
|
|
|
//! 2. multiple 'slots' squashed into a single older (ie. ancient) slot for convenience and performance
|
|
|
|
//! Otherwise, an ancient append vec is the same as any other append vec
|
|
|
|
use {
|
2022-11-30 10:09:23 -08:00
|
|
|
crate::{accounts_db::FoundStoredAccount, append_vec::AppendVec},
|
2022-11-22 15:36:57 -08:00
|
|
|
solana_sdk::clock::Slot,
|
2022-05-10 14:41:04 -07:00
|
|
|
};
|
|
|
|
|
|
|
|
/// a set of accounts need to be stored.
|
|
|
|
/// If there are too many to fit in 'Primary', the rest are put in 'Overflow'
|
2022-05-22 18:00:42 -07:00
|
|
|
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
2022-05-10 14:41:04 -07:00
|
|
|
pub enum StorageSelector {
|
|
|
|
Primary,
|
|
|
|
Overflow,
|
|
|
|
}
|
|
|
|
|
|
|
|
/// reference a set of accounts to store
|
|
|
|
/// The accounts may have to be split between 2 storages (primary and overflow) if there is not enough room in the primary storage.
|
|
|
|
/// The 'store' functions need data stored in a slice of specific type.
|
|
|
|
/// We need 1-2 of these slices constructed based on available bytes and individual account sizes.
|
|
|
|
/// The slice arithmetic accross both hashes and account data gets messy. So, this struct abstracts that.
|
|
|
|
pub struct AccountsToStore<'a> {
|
2022-11-30 10:09:23 -08:00
|
|
|
accounts: &'a [&'a FoundStoredAccount<'a>],
|
2022-05-10 14:41:04 -07:00
|
|
|
/// if 'accounts' contains more items than can be contained in the primary storage, then we have to split these accounts.
|
|
|
|
/// 'index_first_item_overflow' specifies the index of the first item in 'accounts' that will go into the overflow storage
|
|
|
|
index_first_item_overflow: usize,
|
2022-11-18 10:15:41 -08:00
|
|
|
pub slot: Slot,
|
2022-05-10 14:41:04 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
impl<'a> AccountsToStore<'a> {
|
|
|
|
/// break 'stored_accounts' into primary and overflow
|
|
|
|
/// available_bytes: how many bytes remain in the primary storage. Excess accounts will be directed to an overflow storage
|
|
|
|
pub fn new(
|
|
|
|
mut available_bytes: u64,
|
2022-11-30 10:09:23 -08:00
|
|
|
accounts: &'a [&'a FoundStoredAccount<'a>],
|
2022-11-30 15:46:52 -08:00
|
|
|
alive_total_bytes: usize,
|
2022-05-10 14:41:04 -07:00
|
|
|
slot: Slot,
|
|
|
|
) -> Self {
|
2022-11-30 10:09:23 -08:00
|
|
|
let num_accounts = accounts.len();
|
2022-05-10 14:41:04 -07:00
|
|
|
// index of the first account that doesn't fit in the current append vec
|
|
|
|
let mut index_first_item_overflow = num_accounts; // assume all fit
|
2022-11-30 15:46:52 -08:00
|
|
|
if alive_total_bytes > available_bytes as usize {
|
|
|
|
// not all the alive bytes fit, so we have to find how many accounts fit within available_bytes
|
|
|
|
for (i, account) in accounts.iter().enumerate() {
|
|
|
|
let account_size = account.account.stored_size as u64;
|
|
|
|
if available_bytes >= account_size {
|
|
|
|
available_bytes = available_bytes.saturating_sub(account_size);
|
|
|
|
} else if index_first_item_overflow == num_accounts {
|
|
|
|
// the # of accounts we have so far seen is the most that will fit in the current ancient append vec
|
|
|
|
index_first_item_overflow = i;
|
|
|
|
break;
|
|
|
|
}
|
2022-05-10 14:41:04 -07:00
|
|
|
}
|
2022-11-30 10:09:23 -08:00
|
|
|
}
|
2022-05-10 14:41:04 -07:00
|
|
|
Self {
|
|
|
|
accounts,
|
|
|
|
index_first_item_overflow,
|
2022-11-18 10:15:41 -08:00
|
|
|
slot,
|
2022-05-10 14:41:04 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-12 08:14:27 -07:00
|
|
|
/// true if a request to 'get' 'Overflow' would return accounts & hashes
|
|
|
|
pub fn has_overflow(&self) -> bool {
|
|
|
|
self.index_first_item_overflow < self.accounts.len()
|
|
|
|
}
|
|
|
|
|
2022-11-22 15:36:57 -08:00
|
|
|
/// get the accounts to store in the given 'storage'
|
2022-11-30 10:09:23 -08:00
|
|
|
pub fn get(&self, storage: StorageSelector) -> &[&'a FoundStoredAccount<'a>] {
|
2022-05-10 14:41:04 -07:00
|
|
|
let range = match storage {
|
|
|
|
StorageSelector::Primary => 0..self.index_first_item_overflow,
|
|
|
|
StorageSelector::Overflow => self.index_first_item_overflow..self.accounts.len(),
|
|
|
|
};
|
2022-11-22 15:36:57 -08:00
|
|
|
&self.accounts[range]
|
2022-05-10 14:41:04 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-11 06:02:05 -07:00
|
|
|
/// capacity of an ancient append vec
|
|
|
|
pub fn get_ancient_append_vec_capacity() -> u64 {
|
|
|
|
use crate::append_vec::MAXIMUM_APPEND_VEC_FILE_SIZE;
|
|
|
|
// smaller than max by a bit just in case
|
|
|
|
// some functions add slop on allocation
|
2022-07-19 06:51:36 -07:00
|
|
|
// temporarily smaller to force ancient append vec operations to occur more often to flush out any bugs
|
|
|
|
MAXIMUM_APPEND_VEC_FILE_SIZE / 10 - 2048
|
2022-05-11 06:02:05 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/// is this a max-size append vec designed to be used as an ancient append vec?
|
|
|
|
pub fn is_ancient(storage: &AppendVec) -> bool {
|
|
|
|
storage.capacity() >= get_ancient_append_vec_capacity()
|
|
|
|
}
|
|
|
|
|
2022-05-10 14:41:04 -07:00
|
|
|
#[cfg(test)]
|
|
|
|
pub mod tests {
|
|
|
|
use {
|
|
|
|
super::*,
|
|
|
|
crate::{
|
2022-05-11 06:02:05 -07:00
|
|
|
accounts_db::{get_temp_accounts_paths, AppendVecId},
|
2022-11-30 10:09:23 -08:00
|
|
|
append_vec::{AccountMeta, StoredAccountMeta, StoredMeta},
|
2022-05-10 14:41:04 -07:00
|
|
|
},
|
2022-11-01 07:45:52 -07:00
|
|
|
solana_sdk::{
|
|
|
|
account::{AccountSharedData, ReadableAccount},
|
2022-11-22 15:36:57 -08:00
|
|
|
hash::Hash,
|
2022-11-01 07:45:52 -07:00
|
|
|
pubkey::Pubkey,
|
|
|
|
},
|
2022-05-10 14:41:04 -07:00
|
|
|
};
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_accounts_to_store_simple() {
|
2022-05-16 13:54:40 -07:00
|
|
|
let map = vec![];
|
2022-05-10 14:41:04 -07:00
|
|
|
let slot = 1;
|
2022-11-30 15:46:52 -08:00
|
|
|
let accounts_to_store = AccountsToStore::new(0, &map, 0, slot);
|
2022-05-10 14:41:04 -07:00
|
|
|
for selector in [StorageSelector::Primary, StorageSelector::Overflow] {
|
2022-11-22 15:36:57 -08:00
|
|
|
let accounts = accounts_to_store.get(selector);
|
2022-05-10 14:41:04 -07:00
|
|
|
assert!(accounts.is_empty());
|
|
|
|
}
|
2022-05-12 08:14:27 -07:00
|
|
|
assert!(!accounts_to_store.has_overflow());
|
2022-05-10 14:41:04 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_accounts_to_store_more() {
|
|
|
|
let pubkey = Pubkey::new(&[1; 32]);
|
|
|
|
let store_id = AppendVecId::default();
|
|
|
|
let account_size = 3;
|
|
|
|
|
|
|
|
let account = AccountSharedData::default();
|
|
|
|
|
|
|
|
let account_meta = AccountMeta {
|
|
|
|
lamports: 1,
|
|
|
|
owner: Pubkey::new(&[2; 32]),
|
|
|
|
executable: false,
|
|
|
|
rent_epoch: 0,
|
|
|
|
};
|
|
|
|
let offset = 3;
|
|
|
|
let hash = Hash::new(&[2; 32]);
|
|
|
|
let stored_meta = StoredMeta {
|
|
|
|
/// global write version
|
2022-12-14 07:43:40 -08:00
|
|
|
write_version_obsolete: 0,
|
2022-05-10 14:41:04 -07:00
|
|
|
/// key for the account
|
|
|
|
pubkey,
|
|
|
|
data_len: 43,
|
|
|
|
};
|
|
|
|
let account = StoredAccountMeta {
|
|
|
|
meta: &stored_meta,
|
|
|
|
/// account data
|
|
|
|
account_meta: &account_meta,
|
|
|
|
data: account.data(),
|
|
|
|
offset,
|
2022-07-07 16:37:14 -07:00
|
|
|
stored_size: account_size,
|
2022-05-10 14:41:04 -07:00
|
|
|
hash: &hash,
|
|
|
|
};
|
2022-07-07 16:37:14 -07:00
|
|
|
let found = FoundStoredAccount { account, store_id };
|
2022-11-01 07:45:52 -07:00
|
|
|
let map = vec![&found];
|
2022-05-10 14:41:04 -07:00
|
|
|
for (selector, available_bytes) in [
|
|
|
|
(StorageSelector::Primary, account_size),
|
|
|
|
(StorageSelector::Overflow, account_size - 1),
|
|
|
|
] {
|
|
|
|
let slot = 1;
|
2022-11-30 15:46:52 -08:00
|
|
|
let alive_total_bytes = account_size;
|
|
|
|
let accounts_to_store =
|
|
|
|
AccountsToStore::new(available_bytes as u64, &map, alive_total_bytes, slot);
|
2022-11-22 15:36:57 -08:00
|
|
|
let accounts = accounts_to_store.get(selector);
|
2022-05-10 14:41:04 -07:00
|
|
|
assert_eq!(
|
2022-11-30 10:09:23 -08:00
|
|
|
accounts.iter().map(|b| &b.account).collect::<Vec<_>>(),
|
2022-11-18 10:15:41 -08:00
|
|
|
map.iter().map(|b| &b.account).collect::<Vec<_>>(),
|
2022-05-10 14:41:04 -07:00
|
|
|
"mismatch"
|
|
|
|
);
|
2022-11-22 15:36:57 -08:00
|
|
|
let accounts = accounts_to_store.get(get_opposite(&selector));
|
2022-05-12 08:14:27 -07:00
|
|
|
assert_eq!(
|
|
|
|
selector == StorageSelector::Overflow,
|
|
|
|
accounts_to_store.has_overflow()
|
|
|
|
);
|
2022-05-10 14:41:04 -07:00
|
|
|
assert!(accounts.is_empty());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
fn get_opposite(selector: &StorageSelector) -> StorageSelector {
|
|
|
|
match selector {
|
|
|
|
StorageSelector::Overflow => StorageSelector::Primary,
|
|
|
|
StorageSelector::Primary => StorageSelector::Overflow,
|
|
|
|
}
|
|
|
|
}
|
2022-05-11 06:02:05 -07:00
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_get_ancient_append_vec_capacity() {
|
|
|
|
assert_eq!(
|
|
|
|
get_ancient_append_vec_capacity(),
|
2022-07-19 06:51:36 -07:00
|
|
|
crate::append_vec::MAXIMUM_APPEND_VEC_FILE_SIZE / 10 - 2048
|
2022-05-11 06:02:05 -07:00
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_is_ancient() {
|
|
|
|
for (size, expected_ancient) in [
|
|
|
|
(get_ancient_append_vec_capacity() + 1, true),
|
|
|
|
(get_ancient_append_vec_capacity(), true),
|
|
|
|
(get_ancient_append_vec_capacity() - 1, false),
|
|
|
|
] {
|
|
|
|
let tf = crate::append_vec::test_utils::get_append_vec_path("test_is_ancient");
|
|
|
|
let (_temp_dirs, _paths) = get_temp_accounts_paths(1).unwrap();
|
|
|
|
let av = AppendVec::new(&tf.path, true, size as usize);
|
|
|
|
|
|
|
|
assert_eq!(expected_ancient, is_ancient(&av));
|
|
|
|
}
|
|
|
|
}
|
2022-05-10 14:41:04 -07:00
|
|
|
}
|