blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Generalize the return value of append_accounts() (#30571) 

#### Problem
The existing append_accounts returns a vector of offsets, which accounts-db
later uses it to infer the storage size of each account.  However, such inference
does not apply to the new hot/cold storage format.

#### Summary of Changes
This PR changes the return value of append_accounts() to `Vec<StoredAccountInfo>`,
which includes both offset and storage size information of an account.
This commit is contained in:
Yueh-Hsuan Chiang 2023-03-06 09:52:05 -08:00 committed by GitHub
parent bba0ed702f
commit a78ef393bf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 40 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
runtime/benches/append_vec.rs
View File

@ -4,7 +4,9 @@ extern crate test;
use {
rand::{thread_rng, Rng},
solana_runtime::{
account_storage::meta::{StorableAccountsWithHashesAndWriteVersions, StoredMeta},
account_storage::meta::{
StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo, StoredMeta,
},
accounts_db::INCLUDE_SLOT_IN_HASH_TESTS,
append_vec::{
test_utils::{create_test_account, get_append_vec_path},
@ -32,7 +34,7 @@ fn append_account(
storage_meta: StoredMeta,
account: &AccountSharedData,
hash: Hash,
) -> Option<usize> {
) -> Option<StoredAccountInfo> {
let slot_ignored = Slot::MAX;
let accounts = [(&storage_meta.pubkey, account)];
let slice = &accounts[..];
@ -63,7 +65,7 @@ fn add_test_accounts(vec: &AppendVec, size: usize) -> Vec<(usize, usize)> {
(0..size)
.filter_map(|sample| {
let (meta, account) = create_test_account(sample);
append_account(vec, meta, &account, Hash::default()).map(|pos| (sample, pos))
append_account(vec, meta, &account, Hash::default()).map(|info| (sample, info.offset))
})
.collect()
}
@ -108,8 +110,8 @@ fn append_vec_concurrent_append_read(bencher: &mut Bencher) {
spawn(move || loop {
let sample = indexes1.lock().unwrap().len();
let (meta, account) = create_test_account(sample);
if let Some(pos) = append_account(&vec1, meta, &account, Hash::default()) {
indexes1.lock().unwrap().push((sample, pos))
if let Some(info) = append_account(&vec1, meta, &account, Hash::default()) {
indexes1.lock().unwrap().push((sample, info.offset))
} else {
break;
}
@ -148,8 +150,8 @@ fn append_vec_concurrent_read_append(bencher: &mut Bencher) {
bencher.iter(|| {
let sample: usize = thread_rng().gen_range(0, 256);
let (meta, account) = create_test_account(sample);
if let Some(pos) = append_account(&vec, meta, &account, Hash::default()) {
indexes.lock().unwrap().push((sample, pos))
if let Some(info) = append_account(&vec, meta, &account, Hash::default()) {
indexes.lock().unwrap().push((sample, info.offset))
}
});
}

6
runtime/src/account_storage/meta.rs
View File

@ -10,6 +10,12 @@ use {
};
pub type StoredMetaWriteVersion = u64;
// A tuple that stores offset and size respectively
#[derive(Debug, Clone)]
pub struct StoredAccountInfo {
pub offset: usize,
pub size: usize,
}
/// Goal is to eliminate copies and data reshaping given various code paths that store accounts.
/// This struct contains what is needed to store accounts to a storage

14
runtime/src/accounts_db.rs
View File

@ -6167,12 +6167,11 @@ impl AccountsDb {
}
let store_id = storage.append_vec_id();
for (i, offsets) in rvs.unwrap().windows(2).enumerate() {
let stored_size = offsets[1] - offsets[0];
storage.add_account(stored_size);
for (i, stored_account_info) in rvs.unwrap().into_iter().enumerate() {
storage.add_account(stored_account_info.size);
infos.push(AccountInfo::new(
StorageLocation::AppendVec(store_id, offsets[0]),
StorageLocation::AppendVec(store_id, stored_account_info.offset),
accounts_and_meta_to_store
.account(i)
.map(|account| account.lamports())
@ -10484,19 +10483,18 @@ pub mod tests {
vec![&hash],
vec![write_version],
);
let old_written = storage.written_bytes();
let offsets = storage
let stored_accounts_info = storage
.accounts
.append_accounts(&storable_accounts, 0)
.unwrap();
if mark_alive {
// updates 'alive_bytes' on the storage
storage.add_account((storage.written_bytes() - old_written) as usize);
storage.add_account(stored_accounts_info[0].size);
}
if let Some(index) = add_to_index {
let account_info = AccountInfo::new(
StorageLocation::AppendVec(storage.append_vec_id(), offsets[0]),
StorageLocation::AppendVec(storage.append_vec_id(), stored_accounts_info[0].offset),
account.lamports(),
);
index.upsert(

6
runtime/src/accounts_file.rs
View File

@ -1,6 +1,8 @@
use {
crate::{
account_storage::meta::{StorableAccountsWithHashesAndWriteVersions, StoredAccountMeta},
account_storage::meta::{
StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo, StoredAccountMeta,
},
append_vec::{AppendVec, MatchAccountOwnerError},
storable_accounts::StorableAccounts,
},
@ -120,7 +122,7 @@ impl AccountsFile {
&self,
accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>,
skip: usize,
) -> Option<Vec<usize>> {
) -> Option<Vec<StoredAccountInfo>> {
match self {
Self::AppendVec(av) => av.append_accounts(accounts, skip),
}

22
runtime/src/append_vec.rs
View File

@ -7,7 +7,8 @@
use {
crate::{
account_storage::meta::{
AccountMeta, StorableAccountsWithHashesAndWriteVersions, StoredAccountMeta, StoredMeta,
AccountMeta, StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo,
StoredAccountMeta, StoredMeta,
},
storable_accounts::StorableAccounts,
},
@ -489,12 +490,12 @@ impl AppendVec {
&self,
accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>,
skip: usize,
) -> Option<Vec<usize>> {
) -> Option<Vec<StoredAccountInfo>> {
let _lock = self.append_lock.lock().unwrap();
let mut offset = self.len();
let len = accounts.accounts.len();
let mut rv = Vec::with_capacity(len);
let mut offsets = Vec::with_capacity(len);
for i in skip..len {
let (account, pubkey, hash, write_version_obsolete) = accounts.get(i);
let account_meta = account
@ -528,18 +529,25 @@ impl AppendVec {
(data_ptr, data_len),
];
if let Some(res) = self.append_ptrs_locked(&mut offset, &ptrs) {
rv.push(res)
offsets.push(res)
} else {
break;
}
}
if rv.is_empty() {
if offsets.is_empty() {
None
} else {
// The last entry in this offset needs to be the u64 aligned offset, because that's
// where the *next* entry will begin to be stored.
rv.push(u64_align!(offset));
offsets.push(u64_align!(offset));
let mut rv = Vec::with_capacity(len);
for offsets in offsets.windows(2) {
rv.push(StoredAccountInfo {
offset: offsets[0],
size: offsets[1] - offsets[0],
});
}
Some(rv)
}
@ -580,7 +588,7 @@ pub mod tests {
);
self.append_accounts(&storable_accounts, 0)
.map(|res| res[0])
.map(|res| res[0].offset)
}
}