Remove serialization of future AppendVecs and serialize AccountStorage correctly (#5510)
This commit is contained in:
parent
1b6a200d6f
commit
58d4e32c97
|
@ -60,7 +60,12 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
|
|||
let snapshot_hard_links_dir = tempfile::tempdir_in(snapshot_path)?;
|
||||
|
||||
// Get a reference to all the relevant AccountStorageEntries
|
||||
let account_storage_entries = bank.rc.get_storage_entries();
|
||||
let account_storage_entries: Vec<_> = bank
|
||||
.rc
|
||||
.get_storage_entries()
|
||||
.into_iter()
|
||||
.filter(|x| x.fork_id() <= bank.slot())
|
||||
.collect();
|
||||
|
||||
// Create a snapshot package
|
||||
info!(
|
||||
|
|
|
@ -62,7 +62,7 @@ fn test_accounts_squash(bencher: &mut Bencher) {
|
|||
fn test_accounts_hash_internal_state(bencher: &mut Bencher) {
|
||||
let accounts = Accounts::new(Some("bench_accounts_hash_internal".to_string()));
|
||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||
create_test_accounts(&accounts, &mut pubkeys, 60000);
|
||||
create_test_accounts(&accounts, &mut pubkeys, 60000, 0);
|
||||
bencher.iter(|| {
|
||||
accounts.hash_internal_state(0);
|
||||
});
|
||||
|
|
|
@ -628,11 +628,11 @@ impl Accounts {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn create_test_accounts(accounts: &Accounts, pubkeys: &mut Vec<Pubkey>, num: usize) {
|
||||
pub fn create_test_accounts(accounts: &Accounts, pubkeys: &mut Vec<Pubkey>, num: usize, fork: u64) {
|
||||
for t in 0..num {
|
||||
let pubkey = Pubkey::new_rand();
|
||||
let account = Account::new((t + 1) as u64, 0, &Account::default().owner);
|
||||
accounts.store_slow(0, &pubkey, &account);
|
||||
accounts.store_slow(fork, &pubkey, &account);
|
||||
pubkeys.push(pubkey);
|
||||
}
|
||||
}
|
||||
|
@ -642,9 +642,9 @@ mod tests {
|
|||
// TODO: all the bank tests are bank specific, issue: 2194
|
||||
|
||||
use super::*;
|
||||
use crate::accounts_db::get_temp_accounts_paths;
|
||||
use crate::accounts_db::tests::copy_append_vecs;
|
||||
use bincode::{serialize_into, serialized_size};
|
||||
use crate::accounts_db::{get_temp_accounts_paths, AccountsDBSerialize};
|
||||
use bincode::serialize_into;
|
||||
use rand::{thread_rng, Rng};
|
||||
use solana_sdk::account::Account;
|
||||
use solana_sdk::fee_calculator::FeeCalculator;
|
||||
|
@ -1164,20 +1164,23 @@ mod tests {
|
|||
let accounts = Accounts::new(Some(paths));
|
||||
|
||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||
create_test_accounts(&accounts, &mut pubkeys, 100);
|
||||
create_test_accounts(&accounts, &mut pubkeys, 100, 0);
|
||||
check_accounts(&accounts, &pubkeys, 100);
|
||||
accounts.add_root(0);
|
||||
|
||||
let sz = serialized_size(&*accounts.accounts_db).unwrap();
|
||||
let mut buf = vec![0u8; sz as usize];
|
||||
let mut writer = Cursor::new(&mut buf[..]);
|
||||
serialize_into(&mut writer, &*accounts.accounts_db).unwrap();
|
||||
let mut writer = Cursor::new(vec![]);
|
||||
serialize_into(
|
||||
&mut writer,
|
||||
&AccountsDBSerialize::new(&*accounts.accounts_db, 0),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let copied_accounts = TempDir::new().unwrap();
|
||||
|
||||
// Simulate obtaining a copy of the AppendVecs from a tarball
|
||||
copy_append_vecs(&accounts.accounts_db, copied_accounts.path()).unwrap();
|
||||
|
||||
let buf = writer.into_inner();
|
||||
let mut reader = BufReader::new(&buf[..]);
|
||||
let (_accounts_dir, daccounts_paths) = get_temp_accounts_paths(2).unwrap();
|
||||
let daccounts = Accounts::new(Some(daccounts_paths.clone()));
|
||||
|
|
|
@ -85,7 +85,18 @@ type ForkStores = HashMap<usize, Arc<AccountStorageEntry>>;
|
|||
|
||||
#[derive(Clone, Default, Debug)]
|
||||
pub struct AccountStorage(pub HashMap<Fork, ForkStores>);
|
||||
|
||||
pub struct AccountStorageSerialize<'a> {
|
||||
account_storage: &'a AccountStorage,
|
||||
slot: u64,
|
||||
}
|
||||
impl<'a> AccountStorageSerialize<'a> {
|
||||
pub fn new(account_storage: &'a AccountStorage, slot: u64) -> Self {
|
||||
Self {
|
||||
account_storage,
|
||||
slot,
|
||||
}
|
||||
}
|
||||
}
|
||||
struct AccountStorageVisitor;
|
||||
|
||||
impl<'de> Visitor<'de> for AccountStorageVisitor {
|
||||
|
@ -113,22 +124,26 @@ impl<'de> Visitor<'de> for AccountStorageVisitor {
|
|||
}
|
||||
}
|
||||
|
||||
impl Serialize for AccountStorage {
|
||||
impl<'a> Serialize for AccountStorageSerialize<'a> {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
let mut len: usize = 0;
|
||||
for storage in self.0.values() {
|
||||
len += storage.len();
|
||||
for (fork_id, storage) in &self.account_storage.0 {
|
||||
if *fork_id <= self.slot {
|
||||
len += storage.len();
|
||||
}
|
||||
}
|
||||
let mut map = serializer.serialize_map(Some(len))?;
|
||||
let mut count = 0;
|
||||
let mut serialize_account_storage_timer = Measure::start("serialize_account_storage_ms");
|
||||
for fork_storage in self.0.values() {
|
||||
for fork_storage in self.account_storage.0.values() {
|
||||
for (storage_id, account_storage_entry) in fork_storage {
|
||||
map.serialize_entry(storage_id, &**account_storage_entry)?;
|
||||
count += 1;
|
||||
if account_storage_entry.fork_id <= self.slot {
|
||||
map.serialize_entry(storage_id, &**account_storage_entry)?;
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
serialize_account_storage_timer.stop();
|
||||
|
@ -298,6 +313,34 @@ pub fn get_temp_accounts_paths(count: u32) -> IOResult<(Vec<TempDir>, String)> {
|
|||
Ok((temp_dirs, paths.join(",")))
|
||||
}
|
||||
|
||||
pub struct AccountsDBSerialize<'a> {
|
||||
accounts_db: &'a AccountsDB,
|
||||
slot: u64,
|
||||
}
|
||||
|
||||
impl<'a> AccountsDBSerialize<'a> {
|
||||
pub fn new(accounts_db: &'a AccountsDB, slot: u64) -> Self {
|
||||
Self { accounts_db, slot }
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Serialize for AccountsDBSerialize<'a> {
|
||||
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::ser::Serializer,
|
||||
{
|
||||
use serde::ser::Error;
|
||||
let storage = self.accounts_db.storage.read().unwrap();
|
||||
let mut wr = Cursor::new(vec![]);
|
||||
let version: u64 = self.accounts_db.write_version.load(Ordering::Relaxed) as u64;
|
||||
let account_storage_serialize = AccountStorageSerialize::new(&*storage, self.slot);
|
||||
serialize_into(&mut wr, &account_storage_serialize).map_err(Error::custom)?;
|
||||
serialize_into(&mut wr, &version).map_err(Error::custom)?;
|
||||
let len = wr.position() as usize;
|
||||
serializer.serialize_bytes(&wr.into_inner()[..len])
|
||||
}
|
||||
}
|
||||
|
||||
// This structure handles the load/store of the accounts
|
||||
#[derive(Debug)]
|
||||
pub struct AccountsDB {
|
||||
|
@ -879,27 +922,11 @@ impl AccountsDB {
|
|||
}
|
||||
}
|
||||
|
||||
impl Serialize for AccountsDB {
|
||||
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::ser::Serializer,
|
||||
{
|
||||
use serde::ser::Error;
|
||||
let storage = self.storage.read().unwrap();
|
||||
let mut wr = Cursor::new(vec![]);
|
||||
let version: u64 = self.write_version.load(Ordering::Relaxed) as u64;
|
||||
serialize_into(&mut wr, &*storage).map_err(Error::custom)?;
|
||||
serialize_into(&mut wr, &version).map_err(Error::custom)?;
|
||||
let len = wr.position() as usize;
|
||||
serializer.serialize_bytes(&wr.into_inner()[..len])
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub mod tests {
|
||||
// TODO: all the bank tests are bank specific, issue: 2194
|
||||
use super::*;
|
||||
use bincode::{serialize_into, serialized_size};
|
||||
use bincode::serialize_into;
|
||||
use maplit::hashmap;
|
||||
use rand::{thread_rng, Rng};
|
||||
use solana_sdk::account::Account;
|
||||
|
@ -1387,12 +1414,12 @@ pub mod tests {
|
|||
let mut pubkeys1: Vec<Pubkey> = vec![];
|
||||
create_account(&accounts, &mut pubkeys1, 1, 10, 0, 0);
|
||||
|
||||
let mut buf = vec![0u8; serialized_size(&accounts).unwrap() as usize];
|
||||
let mut writer = Cursor::new(&mut buf[..]);
|
||||
serialize_into(&mut writer, &accounts).unwrap();
|
||||
let mut writer = Cursor::new(vec![]);
|
||||
serialize_into(&mut writer, &AccountsDBSerialize::new(&accounts, 1)).unwrap();
|
||||
assert!(check_storage(&accounts, 0, 100));
|
||||
assert!(check_storage(&accounts, 1, 10));
|
||||
|
||||
let buf = writer.into_inner();
|
||||
let mut reader = BufReader::new(&buf[..]);
|
||||
let daccounts = AccountsDB::new(None);
|
||||
let local_paths = daccounts.paths();
|
||||
|
|
|
@ -3,9 +3,9 @@
|
|||
//! on behalf of the caller, and a low-level API for when they have
|
||||
//! already been signed and verified.
|
||||
use crate::accounts::Accounts;
|
||||
use crate::accounts_db::AccountStorageEntry;
|
||||
use crate::accounts_db::{
|
||||
AppendVecId, ErrorCounters, InstructionAccounts, InstructionCredits, InstructionLoaders,
|
||||
AccountStorageEntry, AccountsDBSerialize, AppendVecId, ErrorCounters, InstructionAccounts,
|
||||
InstructionCredits, InstructionLoaders,
|
||||
};
|
||||
use crate::accounts_index::Fork;
|
||||
use crate::blockhash_queue::BlockhashQueue;
|
||||
|
@ -62,10 +62,13 @@ pub struct BankRc {
|
|||
|
||||
/// Previous checkpoint of this bank
|
||||
parent: RwLock<Option<Arc<Bank>>>,
|
||||
|
||||
/// Current slot
|
||||
slot: u64,
|
||||
}
|
||||
|
||||
impl BankRc {
|
||||
pub fn new(account_paths: String, id: AppendVecId) -> Self {
|
||||
pub fn new(account_paths: String, id: AppendVecId, slot: u64) -> Self {
|
||||
let accounts = Accounts::new(Some(account_paths));
|
||||
accounts
|
||||
.accounts_db
|
||||
|
@ -74,6 +77,7 @@ impl BankRc {
|
|||
BankRc {
|
||||
accounts: Arc::new(accounts),
|
||||
parent: RwLock::new(None),
|
||||
slot,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -108,7 +112,9 @@ impl Serialize for BankRc {
|
|||
{
|
||||
use serde::ser::Error;
|
||||
let mut wr = Cursor::new(Vec::new());
|
||||
serialize_into(&mut wr, &*self.accounts.accounts_db).map_err(Error::custom)?;
|
||||
let accounts_db_serialize =
|
||||
AccountsDBSerialize::new(&*self.accounts.accounts_db, self.slot);
|
||||
serialize_into(&mut wr, &accounts_db_serialize).map_err(Error::custom)?;
|
||||
let len = wr.position() as usize;
|
||||
serializer.serialize_bytes(&wr.into_inner()[..len])
|
||||
}
|
||||
|
@ -267,6 +273,7 @@ impl Bank {
|
|||
let rc = BankRc {
|
||||
accounts: Arc::new(Accounts::new_from_parent(&parent.rc.accounts)),
|
||||
parent: RwLock::new(Some(parent.clone())),
|
||||
slot,
|
||||
};
|
||||
let src = StatusCacheRc {
|
||||
status_cache: parent.src.status_cache.clone(),
|
||||
|
@ -350,7 +357,10 @@ impl Bank {
|
|||
id: AppendVecId,
|
||||
) -> Self {
|
||||
let mut bank = Self::default();
|
||||
bank.set_bank_rc(&BankRc::new(account_paths, id), &status_cache_rc);
|
||||
bank.set_bank_rc(
|
||||
&BankRc::new(account_paths, id, bank.slot()),
|
||||
&status_cache_rc,
|
||||
);
|
||||
bank.process_genesis_block(genesis_block);
|
||||
bank.ancestors.insert(0, 0);
|
||||
bank
|
||||
|
@ -2775,7 +2785,7 @@ mod tests {
|
|||
// Create a new set of directories for this bank's accounts
|
||||
let (_accounts_dir, dbank_paths) = get_temp_accounts_paths(4).unwrap();;
|
||||
dbank.set_bank_rc(
|
||||
&BankRc::new(dbank_paths.clone(), 0),
|
||||
&BankRc::new(dbank_paths.clone(), 0, dbank.slot()),
|
||||
&StatusCacheRc::default(),
|
||||
);
|
||||
// Create a directory to simulate AppendVecs unpackaged from a snapshot tar
|
||||
|
|
Loading…
Reference in New Issue