diff --git a/core/src/snapshot_utils.rs b/core/src/snapshot_utils.rs index b43e7a191b..684f4a804b 100644 --- a/core/src/snapshot_utils.rs +++ b/core/src/snapshot_utils.rs @@ -60,7 +60,12 @@ pub fn package_snapshot, Q: AsRef>( let snapshot_hard_links_dir = tempfile::tempdir_in(snapshot_path)?; // Get a reference to all the relevant AccountStorageEntries - let account_storage_entries = bank.rc.get_storage_entries(); + let account_storage_entries: Vec<_> = bank + .rc + .get_storage_entries() + .into_iter() + .filter(|x| x.fork_id() <= bank.slot()) + .collect(); // Create a snapshot package info!( diff --git a/runtime/benches/accounts.rs b/runtime/benches/accounts.rs index 8350e1a04e..c9e30aa6da 100644 --- a/runtime/benches/accounts.rs +++ b/runtime/benches/accounts.rs @@ -62,7 +62,7 @@ fn test_accounts_squash(bencher: &mut Bencher) { fn test_accounts_hash_internal_state(bencher: &mut Bencher) { let accounts = Accounts::new(Some("bench_accounts_hash_internal".to_string())); let mut pubkeys: Vec = vec![]; - create_test_accounts(&accounts, &mut pubkeys, 60000); + create_test_accounts(&accounts, &mut pubkeys, 60000, 0); bencher.iter(|| { accounts.hash_internal_state(0); }); diff --git a/runtime/src/accounts.rs b/runtime/src/accounts.rs index 4aba7dca73..18f1df5872 100644 --- a/runtime/src/accounts.rs +++ b/runtime/src/accounts.rs @@ -628,11 +628,11 @@ impl Accounts { } } -pub fn create_test_accounts(accounts: &Accounts, pubkeys: &mut Vec, num: usize) { +pub fn create_test_accounts(accounts: &Accounts, pubkeys: &mut Vec, num: usize, fork: u64) { for t in 0..num { let pubkey = Pubkey::new_rand(); let account = Account::new((t + 1) as u64, 0, &Account::default().owner); - accounts.store_slow(0, &pubkey, &account); + accounts.store_slow(fork, &pubkey, &account); pubkeys.push(pubkey); } } @@ -642,9 +642,9 @@ mod tests { // TODO: all the bank tests are bank specific, issue: 2194 use super::*; - use crate::accounts_db::get_temp_accounts_paths; use crate::accounts_db::tests::copy_append_vecs; - use bincode::{serialize_into, serialized_size}; + use crate::accounts_db::{get_temp_accounts_paths, AccountsDBSerialize}; + use bincode::serialize_into; use rand::{thread_rng, Rng}; use solana_sdk::account::Account; use solana_sdk::fee_calculator::FeeCalculator; @@ -1164,20 +1164,23 @@ mod tests { let accounts = Accounts::new(Some(paths)); let mut pubkeys: Vec = vec![]; - create_test_accounts(&accounts, &mut pubkeys, 100); + create_test_accounts(&accounts, &mut pubkeys, 100, 0); check_accounts(&accounts, &pubkeys, 100); accounts.add_root(0); - let sz = serialized_size(&*accounts.accounts_db).unwrap(); - let mut buf = vec![0u8; sz as usize]; - let mut writer = Cursor::new(&mut buf[..]); - serialize_into(&mut writer, &*accounts.accounts_db).unwrap(); + let mut writer = Cursor::new(vec![]); + serialize_into( + &mut writer, + &AccountsDBSerialize::new(&*accounts.accounts_db, 0), + ) + .unwrap(); let copied_accounts = TempDir::new().unwrap(); // Simulate obtaining a copy of the AppendVecs from a tarball copy_append_vecs(&accounts.accounts_db, copied_accounts.path()).unwrap(); + let buf = writer.into_inner(); let mut reader = BufReader::new(&buf[..]); let (_accounts_dir, daccounts_paths) = get_temp_accounts_paths(2).unwrap(); let daccounts = Accounts::new(Some(daccounts_paths.clone())); diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs index fa47c586e2..c3d7600e69 100644 --- a/runtime/src/accounts_db.rs +++ b/runtime/src/accounts_db.rs @@ -85,7 +85,18 @@ type ForkStores = HashMap>; #[derive(Clone, Default, Debug)] pub struct AccountStorage(pub HashMap); - +pub struct AccountStorageSerialize<'a> { + account_storage: &'a AccountStorage, + slot: u64, +} +impl<'a> AccountStorageSerialize<'a> { + pub fn new(account_storage: &'a AccountStorage, slot: u64) -> Self { + Self { + account_storage, + slot, + } + } +} struct AccountStorageVisitor; impl<'de> Visitor<'de> for AccountStorageVisitor { @@ -113,22 +124,26 @@ impl<'de> Visitor<'de> for AccountStorageVisitor { } } -impl Serialize for AccountStorage { +impl<'a> Serialize for AccountStorageSerialize<'a> { fn serialize(&self, serializer: S) -> Result where S: Serializer, { let mut len: usize = 0; - for storage in self.0.values() { - len += storage.len(); + for (fork_id, storage) in &self.account_storage.0 { + if *fork_id <= self.slot { + len += storage.len(); + } } let mut map = serializer.serialize_map(Some(len))?; let mut count = 0; let mut serialize_account_storage_timer = Measure::start("serialize_account_storage_ms"); - for fork_storage in self.0.values() { + for fork_storage in self.account_storage.0.values() { for (storage_id, account_storage_entry) in fork_storage { - map.serialize_entry(storage_id, &**account_storage_entry)?; - count += 1; + if account_storage_entry.fork_id <= self.slot { + map.serialize_entry(storage_id, &**account_storage_entry)?; + count += 1; + } } } serialize_account_storage_timer.stop(); @@ -298,6 +313,34 @@ pub fn get_temp_accounts_paths(count: u32) -> IOResult<(Vec, String)> { Ok((temp_dirs, paths.join(","))) } +pub struct AccountsDBSerialize<'a> { + accounts_db: &'a AccountsDB, + slot: u64, +} + +impl<'a> AccountsDBSerialize<'a> { + pub fn new(accounts_db: &'a AccountsDB, slot: u64) -> Self { + Self { accounts_db, slot } + } +} + +impl<'a> Serialize for AccountsDBSerialize<'a> { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::ser::Serializer, + { + use serde::ser::Error; + let storage = self.accounts_db.storage.read().unwrap(); + let mut wr = Cursor::new(vec![]); + let version: u64 = self.accounts_db.write_version.load(Ordering::Relaxed) as u64; + let account_storage_serialize = AccountStorageSerialize::new(&*storage, self.slot); + serialize_into(&mut wr, &account_storage_serialize).map_err(Error::custom)?; + serialize_into(&mut wr, &version).map_err(Error::custom)?; + let len = wr.position() as usize; + serializer.serialize_bytes(&wr.into_inner()[..len]) + } +} + // This structure handles the load/store of the accounts #[derive(Debug)] pub struct AccountsDB { @@ -879,27 +922,11 @@ impl AccountsDB { } } -impl Serialize for AccountsDB { - fn serialize(&self, serializer: S) -> std::result::Result - where - S: serde::ser::Serializer, - { - use serde::ser::Error; - let storage = self.storage.read().unwrap(); - let mut wr = Cursor::new(vec![]); - let version: u64 = self.write_version.load(Ordering::Relaxed) as u64; - serialize_into(&mut wr, &*storage).map_err(Error::custom)?; - serialize_into(&mut wr, &version).map_err(Error::custom)?; - let len = wr.position() as usize; - serializer.serialize_bytes(&wr.into_inner()[..len]) - } -} - #[cfg(test)] pub mod tests { // TODO: all the bank tests are bank specific, issue: 2194 use super::*; - use bincode::{serialize_into, serialized_size}; + use bincode::serialize_into; use maplit::hashmap; use rand::{thread_rng, Rng}; use solana_sdk::account::Account; @@ -1387,12 +1414,12 @@ pub mod tests { let mut pubkeys1: Vec = vec![]; create_account(&accounts, &mut pubkeys1, 1, 10, 0, 0); - let mut buf = vec![0u8; serialized_size(&accounts).unwrap() as usize]; - let mut writer = Cursor::new(&mut buf[..]); - serialize_into(&mut writer, &accounts).unwrap(); + let mut writer = Cursor::new(vec![]); + serialize_into(&mut writer, &AccountsDBSerialize::new(&accounts, 1)).unwrap(); assert!(check_storage(&accounts, 0, 100)); assert!(check_storage(&accounts, 1, 10)); + let buf = writer.into_inner(); let mut reader = BufReader::new(&buf[..]); let daccounts = AccountsDB::new(None); let local_paths = daccounts.paths(); diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 0225864ffc..0ddb9ab09e 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -3,9 +3,9 @@ //! on behalf of the caller, and a low-level API for when they have //! already been signed and verified. use crate::accounts::Accounts; -use crate::accounts_db::AccountStorageEntry; use crate::accounts_db::{ - AppendVecId, ErrorCounters, InstructionAccounts, InstructionCredits, InstructionLoaders, + AccountStorageEntry, AccountsDBSerialize, AppendVecId, ErrorCounters, InstructionAccounts, + InstructionCredits, InstructionLoaders, }; use crate::accounts_index::Fork; use crate::blockhash_queue::BlockhashQueue; @@ -62,10 +62,13 @@ pub struct BankRc { /// Previous checkpoint of this bank parent: RwLock>>, + + /// Current slot + slot: u64, } impl BankRc { - pub fn new(account_paths: String, id: AppendVecId) -> Self { + pub fn new(account_paths: String, id: AppendVecId, slot: u64) -> Self { let accounts = Accounts::new(Some(account_paths)); accounts .accounts_db @@ -74,6 +77,7 @@ impl BankRc { BankRc { accounts: Arc::new(accounts), parent: RwLock::new(None), + slot, } } @@ -108,7 +112,9 @@ impl Serialize for BankRc { { use serde::ser::Error; let mut wr = Cursor::new(Vec::new()); - serialize_into(&mut wr, &*self.accounts.accounts_db).map_err(Error::custom)?; + let accounts_db_serialize = + AccountsDBSerialize::new(&*self.accounts.accounts_db, self.slot); + serialize_into(&mut wr, &accounts_db_serialize).map_err(Error::custom)?; let len = wr.position() as usize; serializer.serialize_bytes(&wr.into_inner()[..len]) } @@ -267,6 +273,7 @@ impl Bank { let rc = BankRc { accounts: Arc::new(Accounts::new_from_parent(&parent.rc.accounts)), parent: RwLock::new(Some(parent.clone())), + slot, }; let src = StatusCacheRc { status_cache: parent.src.status_cache.clone(), @@ -350,7 +357,10 @@ impl Bank { id: AppendVecId, ) -> Self { let mut bank = Self::default(); - bank.set_bank_rc(&BankRc::new(account_paths, id), &status_cache_rc); + bank.set_bank_rc( + &BankRc::new(account_paths, id, bank.slot()), + &status_cache_rc, + ); bank.process_genesis_block(genesis_block); bank.ancestors.insert(0, 0); bank @@ -2775,7 +2785,7 @@ mod tests { // Create a new set of directories for this bank's accounts let (_accounts_dir, dbank_paths) = get_temp_accounts_paths(4).unwrap();; dbank.set_bank_rc( - &BankRc::new(dbank_paths.clone(), 0), + &BankRc::new(dbank_paths.clone(), 0, dbank.slot()), &StatusCacheRc::default(), ); // Create a directory to simulate AppendVecs unpackaged from a snapshot tar