blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Remove serialization of future AppendVecs and serialize AccountStorage correctly (#5510)

This commit is contained in:
carllin 2019-08-13 16:05:37 -07:00 committed by GitHub
parent 1b6a200d6f
commit 58d4e32c97
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 89 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
core/src/snapshot_utils.rs
View File

@ -60,7 +60,12 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
let snapshot_hard_links_dir = tempfile::tempdir_in(snapshot_path)?;
// Get a reference to all the relevant AccountStorageEntries
let account_storage_entries = bank.rc.get_storage_entries();
let account_storage_entries: Vec<_> = bank
.rc
.get_storage_entries()
.into_iter()
.filter(|x| x.fork_id() <= bank.slot())
.collect();
// Create a snapshot package
info!(

2
runtime/benches/accounts.rs
View File

@ -62,7 +62,7 @@ fn test_accounts_squash(bencher: &mut Bencher) {
fn test_accounts_hash_internal_state(bencher: &mut Bencher) {
let accounts = Accounts::new(Some("bench_accounts_hash_internal".to_string()));
let mut pubkeys: Vec<Pubkey> = vec![];
create_test_accounts(&accounts, &mut pubkeys, 60000);
create_test_accounts(&accounts, &mut pubkeys, 60000, 0);
bencher.iter(|| {
accounts.hash_internal_state(0);
});

21
runtime/src/accounts.rs
View File

@ -628,11 +628,11 @@ impl Accounts {
}
}
pub fn create_test_accounts(accounts: &Accounts, pubkeys: &mut Vec<Pubkey>, num: usize) {
pub fn create_test_accounts(accounts: &Accounts, pubkeys: &mut Vec<Pubkey>, num: usize, fork: u64) {
for t in 0..num {
let pubkey = Pubkey::new_rand();
let account = Account::new((t + 1) as u64, 0, &Account::default().owner);
accounts.store_slow(0, &pubkey, &account);
accounts.store_slow(fork, &pubkey, &account);
pubkeys.push(pubkey);
}
}
@ -642,9 +642,9 @@ mod tests {
// TODO: all the bank tests are bank specific, issue: 2194
use super::*;
use crate::accounts_db::get_temp_accounts_paths;
use crate::accounts_db::tests::copy_append_vecs;
use bincode::{serialize_into, serialized_size};
use crate::accounts_db::{get_temp_accounts_paths, AccountsDBSerialize};
use bincode::serialize_into;
use rand::{thread_rng, Rng};
use solana_sdk::account::Account;
use solana_sdk::fee_calculator::FeeCalculator;
@ -1164,20 +1164,23 @@ mod tests {
let accounts = Accounts::new(Some(paths));
let mut pubkeys: Vec<Pubkey> = vec![];
create_test_accounts(&accounts, &mut pubkeys, 100);
create_test_accounts(&accounts, &mut pubkeys, 100, 0);
check_accounts(&accounts, &pubkeys, 100);
accounts.add_root(0);
let sz = serialized_size(&*accounts.accounts_db).unwrap();
let mut buf = vec![0u8; sz as usize];
let mut writer = Cursor::new(&mut buf[..]);
serialize_into(&mut writer, &*accounts.accounts_db).unwrap();
let mut writer = Cursor::new(vec![]);
serialize_into(
&mut writer,
&AccountsDBSerialize::new(&*accounts.accounts_db, 0),
)
.unwrap();
let copied_accounts = TempDir::new().unwrap();
// Simulate obtaining a copy of the AppendVecs from a tarball
copy_append_vecs(&accounts.accounts_db, copied_accounts.path()).unwrap();
let buf = writer.into_inner();
let mut reader = BufReader::new(&buf[..]);
let (_accounts_dir, daccounts_paths) = get_temp_accounts_paths(2).unwrap();
let daccounts = Accounts::new(Some(daccounts_paths.clone()));

81
runtime/src/accounts_db.rs
View File

@ -85,7 +85,18 @@ type ForkStores = HashMap<usize, Arc<AccountStorageEntry>>;
#[derive(Clone, Default, Debug)]
pub struct AccountStorage(pub HashMap<Fork, ForkStores>);
pub struct AccountStorageSerialize<'a> {
account_storage: &'a AccountStorage,
slot: u64,
}
impl<'a> AccountStorageSerialize<'a> {
pub fn new(account_storage: &'a AccountStorage, slot: u64) -> Self {
Self {
account_storage,
slot,
}
}
}
struct AccountStorageVisitor;
impl<'de> Visitor<'de> for AccountStorageVisitor {
@ -113,22 +124,26 @@ impl<'de> Visitor<'de> for AccountStorageVisitor {
}
}
impl Serialize for AccountStorage {
impl<'a> Serialize for AccountStorageSerialize<'a> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut len: usize = 0;
for storage in self.0.values() {
len += storage.len();
for (fork_id, storage) in &self.account_storage.0 {
if *fork_id <= self.slot {
len += storage.len();
}
}
let mut map = serializer.serialize_map(Some(len))?;
let mut count = 0;
let mut serialize_account_storage_timer = Measure::start("serialize_account_storage_ms");
for fork_storage in self.0.values() {
for fork_storage in self.account_storage.0.values() {
for (storage_id, account_storage_entry) in fork_storage {
map.serialize_entry(storage_id, &**account_storage_entry)?;
count += 1;
if account_storage_entry.fork_id <= self.slot {
map.serialize_entry(storage_id, &**account_storage_entry)?;
count += 1;
}
}
}
serialize_account_storage_timer.stop();
@ -298,6 +313,34 @@ pub fn get_temp_accounts_paths(count: u32) -> IOResult<(Vec<TempDir>, String)> {
Ok((temp_dirs, paths.join(",")))
}
pub struct AccountsDBSerialize<'a> {
accounts_db: &'a AccountsDB,
slot: u64,
}
impl<'a> AccountsDBSerialize<'a> {
pub fn new(accounts_db: &'a AccountsDB, slot: u64) -> Self {
Self { accounts_db, slot }
}
}
impl<'a> Serialize for AccountsDBSerialize<'a> {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::ser::Serializer,
{
use serde::ser::Error;
let storage = self.accounts_db.storage.read().unwrap();
let mut wr = Cursor::new(vec![]);
let version: u64 = self.accounts_db.write_version.load(Ordering::Relaxed) as u64;
let account_storage_serialize = AccountStorageSerialize::new(&*storage, self.slot);
serialize_into(&mut wr, &account_storage_serialize).map_err(Error::custom)?;
serialize_into(&mut wr, &version).map_err(Error::custom)?;
let len = wr.position() as usize;
serializer.serialize_bytes(&wr.into_inner()[..len])
}
}
// This structure handles the load/store of the accounts
#[derive(Debug)]
pub struct AccountsDB {
@ -879,27 +922,11 @@ impl AccountsDB {
}
}
impl Serialize for AccountsDB {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::ser::Serializer,
{
use serde::ser::Error;
let storage = self.storage.read().unwrap();
let mut wr = Cursor::new(vec![]);
let version: u64 = self.write_version.load(Ordering::Relaxed) as u64;
serialize_into(&mut wr, &*storage).map_err(Error::custom)?;
serialize_into(&mut wr, &version).map_err(Error::custom)?;
let len = wr.position() as usize;
serializer.serialize_bytes(&wr.into_inner()[..len])
}
}
#[cfg(test)]
pub mod tests {
// TODO: all the bank tests are bank specific, issue: 2194
use super::*;
use bincode::{serialize_into, serialized_size};
use bincode::serialize_into;
use maplit::hashmap;
use rand::{thread_rng, Rng};
use solana_sdk::account::Account;
@ -1387,12 +1414,12 @@ pub mod tests {
let mut pubkeys1: Vec<Pubkey> = vec![];
create_account(&accounts, &mut pubkeys1, 1, 10, 0, 0);
let mut buf = vec![0u8; serialized_size(&accounts).unwrap() as usize];
let mut writer = Cursor::new(&mut buf[..]);
serialize_into(&mut writer, &accounts).unwrap();
let mut writer = Cursor::new(vec![]);
serialize_into(&mut writer, &AccountsDBSerialize::new(&accounts, 1)).unwrap();
assert!(check_storage(&accounts, 0, 100));
assert!(check_storage(&accounts, 1, 10));
let buf = writer.into_inner();
let mut reader = BufReader::new(&buf[..]);
let daccounts = AccountsDB::new(None);
let local_paths = daccounts.paths();

22
runtime/src/bank.rs
View File

@ -3,9 +3,9 @@
//! on behalf of the caller, and a low-level API for when they have
//! already been signed and verified.
use crate::accounts::Accounts;
use crate::accounts_db::AccountStorageEntry;
use crate::accounts_db::{
AppendVecId, ErrorCounters, InstructionAccounts, InstructionCredits, InstructionLoaders,
AccountStorageEntry, AccountsDBSerialize, AppendVecId, ErrorCounters, InstructionAccounts,
InstructionCredits, InstructionLoaders,
};
use crate::accounts_index::Fork;
use crate::blockhash_queue::BlockhashQueue;
@ -62,10 +62,13 @@ pub struct BankRc {
/// Previous checkpoint of this bank
parent: RwLock<Option<Arc<Bank>>>,
/// Current slot
slot: u64,
}
impl BankRc {
pub fn new(account_paths: String, id: AppendVecId) -> Self {
pub fn new(account_paths: String, id: AppendVecId, slot: u64) -> Self {
let accounts = Accounts::new(Some(account_paths));
accounts
.accounts_db
@ -74,6 +77,7 @@ impl BankRc {
BankRc {
accounts: Arc::new(accounts),
parent: RwLock::new(None),
slot,
}
}
@ -108,7 +112,9 @@ impl Serialize for BankRc {
{
use serde::ser::Error;
let mut wr = Cursor::new(Vec::new());
serialize_into(&mut wr, &*self.accounts.accounts_db).map_err(Error::custom)?;
let accounts_db_serialize =
AccountsDBSerialize::new(&*self.accounts.accounts_db, self.slot);
serialize_into(&mut wr, &accounts_db_serialize).map_err(Error::custom)?;
let len = wr.position() as usize;
serializer.serialize_bytes(&wr.into_inner()[..len])
}
@ -267,6 +273,7 @@ impl Bank {
let rc = BankRc {
accounts: Arc::new(Accounts::new_from_parent(&parent.rc.accounts)),
parent: RwLock::new(Some(parent.clone())),
slot,
};
let src = StatusCacheRc {
status_cache: parent.src.status_cache.clone(),
@ -350,7 +357,10 @@ impl Bank {
id: AppendVecId,
) -> Self {
let mut bank = Self::default();
bank.set_bank_rc(&BankRc::new(account_paths, id), &status_cache_rc);
bank.set_bank_rc(
&BankRc::new(account_paths, id, bank.slot()),
&status_cache_rc,
);
bank.process_genesis_block(genesis_block);
bank.ancestors.insert(0, 0);
bank
@ -2775,7 +2785,7 @@ mod tests {
// Create a new set of directories for this bank's accounts
let (_accounts_dir, dbank_paths) = get_temp_accounts_paths(4).unwrap();;
dbank.set_bank_rc(
&BankRc::new(dbank_paths.clone(), 0),
&BankRc::new(dbank_paths.clone(), 0, dbank.slot()),
&StatusCacheRc::default(),
);
// Create a directory to simulate AppendVecs unpackaged from a snapshot tar