StorableAccountsWithHashes uses callback for lifetime (#813)
* StorableAccountsWithHashes uses callback for lifetime * cleanup trait type
This commit is contained in:
parent
9ba0d6fabb
commit
9217cd476a
|
@ -61,8 +61,11 @@ impl<'a: 'b, 'b, U: StorableAccounts<'a>, V: Borrow<AccountHash>>
|
|||
}
|
||||
|
||||
/// get all account fields at 'index'
|
||||
pub fn get(&self, index: usize) -> (Option<AccountForStorage>, &Pubkey, &AccountHash) {
|
||||
let account = self.accounts.account_default_if_zero_lamport(index);
|
||||
pub fn get<Ret>(
|
||||
&self,
|
||||
index: usize,
|
||||
mut callback: impl FnMut((Option<AccountForStorage>, &Pubkey, &AccountHash)) -> Ret,
|
||||
) -> Ret {
|
||||
let pubkey = self.accounts.pubkey(index);
|
||||
let hash = if self.accounts.has_hash() {
|
||||
self.accounts.hash(index)
|
||||
|
@ -70,14 +73,19 @@ impl<'a: 'b, 'b, U: StorableAccounts<'a>, V: Borrow<AccountHash>>
|
|||
let item = self.hashes.as_ref().unwrap();
|
||||
item[index].borrow()
|
||||
};
|
||||
(account, pubkey, hash)
|
||||
let account = self.accounts.account_default_if_zero_lamport(index);
|
||||
callback((account, pubkey, hash))
|
||||
}
|
||||
|
||||
/// None if account at index has lamports == 0
|
||||
/// Otherwise, Some(account)
|
||||
/// This is the only way to access the account.
|
||||
pub fn account(&self, index: usize) -> Option<AccountForStorage<'a>> {
|
||||
self.accounts.account_default_if_zero_lamport(index)
|
||||
pub fn account<Ret>(
|
||||
&self,
|
||||
index: usize,
|
||||
mut callback: impl FnMut(Option<AccountForStorage<'a>>) -> Ret,
|
||||
) -> Ret {
|
||||
callback(self.accounts.account_default_if_zero_lamport(index))
|
||||
}
|
||||
|
||||
/// # accounts to write
|
||||
|
|
|
@ -5988,23 +5988,24 @@ impl AccountsDb {
|
|||
storage.set_status(AccountStorageStatus::Full);
|
||||
|
||||
// See if an account overflows the append vecs in the slot.
|
||||
let account = accounts_and_meta_to_store.account(infos.len());
|
||||
let data_len = account
|
||||
.map(|account| account.data().len())
|
||||
.unwrap_or_default();
|
||||
let data_len = (data_len + STORE_META_OVERHEAD) as u64;
|
||||
if !self.has_space_available(slot, data_len) {
|
||||
info!(
|
||||
"write_accounts_to_storage, no space: {}, {}, {}, {}, {}",
|
||||
storage.accounts.capacity(),
|
||||
storage.accounts.remaining_bytes(),
|
||||
data_len,
|
||||
infos.len(),
|
||||
accounts_and_meta_to_store.len()
|
||||
);
|
||||
let special_store_size = std::cmp::max(data_len * 2, self.file_size);
|
||||
self.create_and_insert_store(slot, special_store_size, "large create");
|
||||
}
|
||||
accounts_and_meta_to_store.account(infos.len(), |account| {
|
||||
let data_len = account
|
||||
.map(|account| account.data().len())
|
||||
.unwrap_or_default();
|
||||
let data_len = (data_len + STORE_META_OVERHEAD) as u64;
|
||||
if !self.has_space_available(slot, data_len) {
|
||||
info!(
|
||||
"write_accounts_to_storage, no space: {}, {}, {}, {}, {}",
|
||||
storage.accounts.capacity(),
|
||||
storage.accounts.remaining_bytes(),
|
||||
data_len,
|
||||
infos.len(),
|
||||
accounts_and_meta_to_store.len()
|
||||
);
|
||||
let special_store_size = std::cmp::max(data_len * 2, self.file_size);
|
||||
self.create_and_insert_store(slot, special_store_size, "large create");
|
||||
}
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -6014,10 +6015,11 @@ impl AccountsDb {
|
|||
|
||||
infos.push(AccountInfo::new(
|
||||
StorageLocation::AppendVec(store_id, stored_account_info.offset),
|
||||
accounts_and_meta_to_store
|
||||
.account(i)
|
||||
.map(|account| account.lamports())
|
||||
.unwrap_or_default(),
|
||||
accounts_and_meta_to_store.account(i, |account| {
|
||||
account
|
||||
.map(|account| account.lamports())
|
||||
.unwrap_or_default()
|
||||
}),
|
||||
));
|
||||
}
|
||||
// restore the state to available
|
||||
|
|
|
@ -747,44 +747,49 @@ impl AppendVec {
|
|||
// to compute the StoredAccountInfo of the last entry.
|
||||
let offsets_len = len - skip + 1;
|
||||
let mut offsets = Vec::with_capacity(offsets_len);
|
||||
let mut stop = false;
|
||||
for i in skip..len {
|
||||
let (account, pubkey, hash) = accounts.get(i);
|
||||
let account_meta = account
|
||||
.map(|account| AccountMeta {
|
||||
lamports: account.lamports(),
|
||||
owner: *account.owner(),
|
||||
rent_epoch: account.rent_epoch(),
|
||||
executable: account.executable(),
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let stored_meta = StoredMeta {
|
||||
pubkey: *pubkey,
|
||||
data_len: account
|
||||
.map(|account| account.data().len())
|
||||
.unwrap_or_default() as u64,
|
||||
write_version_obsolete: 0,
|
||||
};
|
||||
let meta_ptr = &stored_meta as *const StoredMeta;
|
||||
let account_meta_ptr = &account_meta as *const AccountMeta;
|
||||
let data_len = stored_meta.data_len as usize;
|
||||
let data_ptr = account
|
||||
.as_ref()
|
||||
.map(|account| account.data())
|
||||
.unwrap_or_default()
|
||||
.as_ptr();
|
||||
let hash_ptr = bytemuck::bytes_of(hash).as_ptr();
|
||||
let ptrs = [
|
||||
(meta_ptr as *const u8, mem::size_of::<StoredMeta>()),
|
||||
(account_meta_ptr as *const u8, mem::size_of::<AccountMeta>()),
|
||||
(hash_ptr, mem::size_of::<AccountHash>()),
|
||||
(data_ptr, data_len),
|
||||
];
|
||||
if let Some(res) = self.append_ptrs_locked(&mut offset, &ptrs) {
|
||||
offsets.push(res)
|
||||
} else {
|
||||
if stop {
|
||||
break;
|
||||
}
|
||||
accounts.get(i, |(account, pubkey, hash)| {
|
||||
let account_meta = account
|
||||
.map(|account| AccountMeta {
|
||||
lamports: account.lamports(),
|
||||
owner: *account.owner(),
|
||||
rent_epoch: account.rent_epoch(),
|
||||
executable: account.executable(),
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let stored_meta = StoredMeta {
|
||||
pubkey: *pubkey,
|
||||
data_len: account
|
||||
.map(|account| account.data().len())
|
||||
.unwrap_or_default() as u64,
|
||||
write_version_obsolete: 0,
|
||||
};
|
||||
let meta_ptr = &stored_meta as *const StoredMeta;
|
||||
let account_meta_ptr = &account_meta as *const AccountMeta;
|
||||
let data_len = stored_meta.data_len as usize;
|
||||
let data_ptr = account
|
||||
.as_ref()
|
||||
.map(|account| account.data())
|
||||
.unwrap_or_default()
|
||||
.as_ptr();
|
||||
let hash_ptr = bytemuck::bytes_of(hash).as_ptr();
|
||||
let ptrs = [
|
||||
(meta_ptr as *const u8, mem::size_of::<StoredMeta>()),
|
||||
(account_meta_ptr as *const u8, mem::size_of::<AccountMeta>()),
|
||||
(hash_ptr, mem::size_of::<AccountHash>()),
|
||||
(data_ptr, data_len),
|
||||
];
|
||||
if let Some(res) = self.append_ptrs_locked(&mut offset, &ptrs) {
|
||||
offsets.push(res)
|
||||
} else {
|
||||
stop = true;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if offsets.is_empty() {
|
||||
|
@ -955,9 +960,10 @@ pub mod tests {
|
|||
assert_eq!(storable.len(), pubkeys.len());
|
||||
assert!(!storable.is_empty());
|
||||
(0..2).for_each(|i| {
|
||||
let (_, pubkey, hash) = storable.get(i);
|
||||
assert_eq!(hash, &hashes[i]);
|
||||
assert_eq!(pubkey, &pubkeys[i]);
|
||||
storable.get(i, |(_, pubkey, hash)| {
|
||||
assert_eq!(hash, &hashes[i]);
|
||||
assert_eq!(pubkey, &pubkeys[i]);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -976,8 +982,9 @@ pub mod tests {
|
|||
let accounts = [(&pubkey, &account)];
|
||||
let accounts2 = (slot, &accounts[..]);
|
||||
let storable = StorableAccountsWithHashes::new_with_hashes(&accounts2, hashes.clone());
|
||||
let get_account = storable.account(0);
|
||||
assert!(get_account.is_none());
|
||||
storable.account(0, |get_account| {
|
||||
assert!(get_account.is_none());
|
||||
});
|
||||
|
||||
// non-zero lamports, data should be correct
|
||||
let account = Account {
|
||||
|
@ -990,8 +997,9 @@ pub mod tests {
|
|||
let accounts = [(&pubkey, &account)];
|
||||
let accounts2 = (slot, &accounts[..]);
|
||||
let storable = StorableAccountsWithHashes::new_with_hashes(&accounts2, hashes);
|
||||
let get_account = storable.account(0);
|
||||
assert!(accounts_equal(&account, &get_account.unwrap()));
|
||||
storable.account(0, |get_account| {
|
||||
assert!(accounts_equal(&account, &get_account.unwrap()));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
@ -178,7 +178,10 @@ mod tests {
|
|||
hot::HOT_FORMAT,
|
||||
index::IndexOffset,
|
||||
solana_sdk::{
|
||||
account::AccountSharedData, clock::Slot, hash::Hash, pubkey::Pubkey,
|
||||
account::{AccountSharedData, ReadableAccount},
|
||||
clock::Slot,
|
||||
hash::Hash,
|
||||
pubkey::Pubkey,
|
||||
system_instruction::MAX_PERMITTED_DATA_LENGTH,
|
||||
},
|
||||
std::{
|
||||
|
@ -352,8 +355,12 @@ mod tests {
|
|||
|
||||
let mut expected_accounts_map = HashMap::new();
|
||||
for i in 0..num_accounts {
|
||||
let (account, address, _account_hash) = storable_accounts.get(i);
|
||||
expected_accounts_map.insert(address, account);
|
||||
storable_accounts.get(i, |(account, address, _account_hash)| {
|
||||
expected_accounts_map.insert(
|
||||
*address,
|
||||
account.map(|account| account.to_account_shared_data()),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
let mut index_offset = IndexOffset(0);
|
||||
|
|
|
@ -747,48 +747,51 @@ impl HotStorageWriter {
|
|||
let total_input_accounts = len - skip;
|
||||
let mut stored_infos = Vec::with_capacity(total_input_accounts);
|
||||
for i in skip..len {
|
||||
let (account, address, _account_hash) = accounts.get(i);
|
||||
let index_entry = AccountIndexWriterEntry {
|
||||
address: *address,
|
||||
offset: HotAccountOffset::new(cursor)?,
|
||||
};
|
||||
address_range.update(address);
|
||||
accounts.get::<TieredStorageResult<()>>(i, |(account, address, _account_hash)| {
|
||||
let index_entry = AccountIndexWriterEntry {
|
||||
address: *address,
|
||||
offset: HotAccountOffset::new(cursor)?,
|
||||
};
|
||||
address_range.update(address);
|
||||
|
||||
// Obtain necessary fields from the account, or default fields
|
||||
// for a zero-lamport account in the None case.
|
||||
let (lamports, owner, data, executable, rent_epoch) = account
|
||||
.as_ref()
|
||||
.map(|acc| {
|
||||
(
|
||||
acc.lamports(),
|
||||
acc.owner(),
|
||||
acc.data(),
|
||||
acc.executable(),
|
||||
// only persist rent_epoch for those rent-paying accounts
|
||||
(acc.rent_epoch() != RENT_EXEMPT_RENT_EPOCH).then_some(acc.rent_epoch()),
|
||||
)
|
||||
})
|
||||
.unwrap_or((0, &OWNER_NO_OWNER, &[], false, None));
|
||||
let owner_offset = owners_table.insert(owner);
|
||||
let stored_size =
|
||||
self.write_account(lamports, owner_offset, data, executable, rent_epoch)?;
|
||||
cursor += stored_size;
|
||||
// Obtain necessary fields from the account, or default fields
|
||||
// for a zero-lamport account in the None case.
|
||||
let (lamports, owner, data, executable, rent_epoch) = account
|
||||
.as_ref()
|
||||
.map(|acc| {
|
||||
(
|
||||
acc.lamports(),
|
||||
acc.owner(),
|
||||
acc.data(),
|
||||
acc.executable(),
|
||||
// only persist rent_epoch for those rent-paying accounts
|
||||
(acc.rent_epoch() != RENT_EXEMPT_RENT_EPOCH)
|
||||
.then_some(acc.rent_epoch()),
|
||||
)
|
||||
})
|
||||
.unwrap_or((0, &OWNER_NO_OWNER, &[], false, None));
|
||||
let owner_offset = owners_table.insert(owner);
|
||||
let stored_size =
|
||||
self.write_account(lamports, owner_offset, data, executable, rent_epoch)?;
|
||||
cursor += stored_size;
|
||||
|
||||
stored_infos.push(StoredAccountInfo {
|
||||
// Here we pass the IndexOffset as the get_account() API
|
||||
// takes IndexOffset. Given the account address is also
|
||||
// maintained outside the TieredStorage, a potential optimization
|
||||
// is to store AccountOffset instead, which can further save
|
||||
// one jump from the index block to the accounts block.
|
||||
offset: index.len(),
|
||||
// Here we only include the stored size that the account directly
|
||||
// contribute (i.e., account entry + index entry that include the
|
||||
// account meta, data, optional fields, its address, and AccountOffset).
|
||||
// Storage size from those shared blocks like footer and owners block
|
||||
// is not included.
|
||||
size: stored_size + footer.index_block_format.entry_size::<HotAccountOffset>(),
|
||||
});
|
||||
index.push(index_entry);
|
||||
stored_infos.push(StoredAccountInfo {
|
||||
// Here we pass the IndexOffset as the get_account() API
|
||||
// takes IndexOffset. Given the account address is also
|
||||
// maintained outside the TieredStorage, a potential optimization
|
||||
// is to store AccountOffset instead, which can further save
|
||||
// one jump from the index block to the accounts block.
|
||||
offset: index.len(),
|
||||
// Here we only include the stored size that the account directly
|
||||
// contribute (i.e., account entry + index entry that include the
|
||||
// account meta, data, optional fields, its address, and AccountOffset).
|
||||
// Storage size from those shared blocks like footer and owners block
|
||||
// is not included.
|
||||
size: stored_size + footer.index_block_format.entry_size::<HotAccountOffset>(),
|
||||
});
|
||||
index.push(index_entry);
|
||||
Ok(())
|
||||
})?;
|
||||
}
|
||||
footer.account_entry_count = total_input_accounts as u32;
|
||||
|
||||
|
@ -1559,8 +1562,9 @@ mod tests {
|
|||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
let (account, address, _account_hash) = storable_accounts.get(i);
|
||||
verify_test_account(&stored_account_meta, account.as_ref(), address);
|
||||
storable_accounts.get(i, |(account, address, _account_hash)| {
|
||||
verify_test_account(&stored_account_meta, account.as_ref(), address);
|
||||
});
|
||||
|
||||
assert_eq!(i + 1, next.0 as usize);
|
||||
}
|
||||
|
@ -1577,8 +1581,9 @@ mod tests {
|
|||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
let (account, address, _account_hash) = storable_accounts.get(stored_info.offset);
|
||||
verify_test_account(&stored_account_meta, account.as_ref(), address);
|
||||
storable_accounts.get(stored_info.offset, |(account, address, _account_hash)| {
|
||||
verify_test_account(&stored_account_meta, account.as_ref(), address);
|
||||
});
|
||||
}
|
||||
|
||||
// verify get_accounts
|
||||
|
@ -1586,8 +1591,9 @@ mod tests {
|
|||
|
||||
// first, we verify everything
|
||||
for (i, stored_meta) in accounts.iter().enumerate() {
|
||||
let (account, address, _account_hash) = storable_accounts.get(i);
|
||||
verify_test_account(stored_meta, account.as_ref(), address);
|
||||
storable_accounts.get(i, |(account, address, _account_hash)| {
|
||||
verify_test_account(stored_meta, account.as_ref(), address);
|
||||
});
|
||||
}
|
||||
|
||||
// second, we verify various initial position
|
||||
|
|
Loading…
Reference in New Issue