solana/runtime/src/serde_snapshot/tests.rs

854 lines
34 KiB
Rust
Raw Normal View History

#[cfg(test)]
mod serde_snapshot_tests {
use {
crate::{
serde_snapshot::{
newer, reconstruct_accountsdb_from_fields, SerdeStyle, SerializableAccountsDb,
SnapshotAccountsDbFields, TypeContext,
},
snapshot_utils::{get_storages_to_serialize, StorageAndNextAppendVecId},
},
bincode::{serialize_into, Error},
log::info,
rand::{thread_rng, Rng},
solana_accounts_db::{
account_storage::{AccountStorageMap, AccountStorageReference},
accounts::Accounts,
accounts_db::{
get_temp_accounts_paths, test_utils::create_test_accounts, AccountShrinkThreshold,
AccountStorageEntry, AccountsDb, AtomicAppendVecId,
VerifyAccountsHashAndLamportsConfig,
},
accounts_file::{AccountsFile, AccountsFileError},
accounts_hash::AccountsHash,
accounts_index::AccountSecondaryIndexes,
ancestors::Ancestors,
rent_collector::RentCollector,
},
solana_sdk::{
account::{AccountSharedData, ReadableAccount},
clock::Slot,
epoch_schedule::EpochSchedule,
genesis_config::{ClusterType, GenesisConfig},
hash::Hash,
pubkey::Pubkey,
},
std::{
io::{BufReader, Cursor, Read, Write},
ops::RangeFull,
path::{Path, PathBuf},
sync::{atomic::Ordering, Arc},
},
tempfile::TempDir,
};
fn linear_ancestors(end_slot: u64) -> Ancestors {
let mut ancestors: Ancestors = vec![(0, 0)].into_iter().collect();
for i in 1..end_slot {
ancestors.insert(i, (i - 1) as usize);
}
ancestors
}
fn context_accountsdb_from_stream<'a, C, R>(
stream: &mut BufReader<R>,
account_paths: &[PathBuf],
storage_and_next_append_vec_id: StorageAndNextAppendVecId,
) -> Result<AccountsDb, Error>
where
C: TypeContext<'a>,
R: Read,
{
// read and deserialise the accounts database directly from the stream
let accounts_db_fields = C::deserialize_accounts_db_fields(stream)?;
let snapshot_accounts_db_fields = SnapshotAccountsDbFields {
full_snapshot_accounts_db_fields: accounts_db_fields,
incremental_snapshot_accounts_db_fields: None,
};
reconstruct_accountsdb_from_fields(
snapshot_accounts_db_fields,
account_paths,
storage_and_next_append_vec_id,
&GenesisConfig {
cluster_type: ClusterType::Development,
..GenesisConfig::default()
},
AccountSecondaryIndexes::default(),
None,
AccountShrinkThreshold::default(),
false,
Some(solana_accounts_db::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING),
None,
Arc::default(),
None,
(u64::default(), None),
None,
)
.map(|(accounts_db, _)| accounts_db)
Multi-version snapshot support (#9980) * Multi-version snapshot support * rustfmt * Remove CLI options and runtime support for selection output snapshot version. Address some clippy complaints. * Muzzle clippy type complexity warning. Despite clippy's suggestion, it is not currently possible to create type aliases for traits and so everything within the 'Box<...>' cannot be type aliased. This then leaves creating full blown traits, and either implementing said traits by closure (somehow) or moving the closures into new structs implementing said traits which seems a bit of a palaver. Alternatively it is possible to define and use the type alias 'type ResultBox<T> = Result<Box<T>>' which does seems rather pointless and not a great reduction in complexity but is enough to keep clippy happy. In the end I simply went with squelching the clippy warning. * Remove now unused Serialize/Deserialize trait implementations for AccountStorageEntry and AppendVec * refactor versioned de/serialisers * rename serde_utils to serde_snapshot * move call to accounts_db.generate_index() back down to context_accountsdb_from_stream() * update version 1.1.1 to 1.2.0 remove nested use of serialize_bytes * cleanups * Add back measurement of account storage entry serialization. Remove construction of Vec and HashMap temporaries during serialization. * consolidate serialisation test cases into serde_snapshot. clean up leakage of implementation details in serde_snapshot. * move short term / legacy snapshot code into child module * add serialize_iter_as_tuple * preliminary integration of following commit commit 6d58b73c47294bfb93465d5a83cd2175660b6e6d Author: Ryo Onodera <ryoqun@gmail.com> Date: Wed May 20 14:02:02 2020 +0900 Confine snapshot 1.1 relic to versioned codepath * refactored serde_snapshot, rustfmt legacy accounts_db format now "owns" both leading u64s, legacy bank_rc format has none * reduce type complexity (clippy)
2020-05-22 10:54:24 -07:00
}
fn accountsdb_from_stream<R>(
serde_style: SerdeStyle,
stream: &mut BufReader<R>,
account_paths: &[PathBuf],
storage_and_next_append_vec_id: StorageAndNextAppendVecId,
) -> Result<AccountsDb, Error>
where
R: Read,
{
match serde_style {
SerdeStyle::Newer => context_accountsdb_from_stream::<newer::Context, R>(
stream,
account_paths,
storage_and_next_append_vec_id,
),
}
}
Multi-version snapshot support (#9980) * Multi-version snapshot support * rustfmt * Remove CLI options and runtime support for selection output snapshot version. Address some clippy complaints. * Muzzle clippy type complexity warning. Despite clippy's suggestion, it is not currently possible to create type aliases for traits and so everything within the 'Box<...>' cannot be type aliased. This then leaves creating full blown traits, and either implementing said traits by closure (somehow) or moving the closures into new structs implementing said traits which seems a bit of a palaver. Alternatively it is possible to define and use the type alias 'type ResultBox<T> = Result<Box<T>>' which does seems rather pointless and not a great reduction in complexity but is enough to keep clippy happy. In the end I simply went with squelching the clippy warning. * Remove now unused Serialize/Deserialize trait implementations for AccountStorageEntry and AppendVec * refactor versioned de/serialisers * rename serde_utils to serde_snapshot * move call to accounts_db.generate_index() back down to context_accountsdb_from_stream() * update version 1.1.1 to 1.2.0 remove nested use of serialize_bytes * cleanups * Add back measurement of account storage entry serialization. Remove construction of Vec and HashMap temporaries during serialization. * consolidate serialisation test cases into serde_snapshot. clean up leakage of implementation details in serde_snapshot. * move short term / legacy snapshot code into child module * add serialize_iter_as_tuple * preliminary integration of following commit commit 6d58b73c47294bfb93465d5a83cd2175660b6e6d Author: Ryo Onodera <ryoqun@gmail.com> Date: Wed May 20 14:02:02 2020 +0900 Confine snapshot 1.1 relic to versioned codepath * refactored serde_snapshot, rustfmt legacy accounts_db format now "owns" both leading u64s, legacy bank_rc format has none * reduce type complexity (clippy)
2020-05-22 10:54:24 -07:00
fn accountsdb_to_stream<W>(
serde_style: SerdeStyle,
stream: &mut W,
accounts_db: &AccountsDb,
slot: Slot,
account_storage_entries: &[Vec<Arc<AccountStorageEntry>>],
) -> Result<(), Error>
where
W: Write,
{
match serde_style {
SerdeStyle::Newer => serialize_into(
stream,
&SerializableAccountsDb::<newer::Context> {
accounts_db,
slot,
account_storage_entries,
phantom: std::marker::PhantomData,
},
),
}
Multi-version snapshot support (#9980) * Multi-version snapshot support * rustfmt * Remove CLI options and runtime support for selection output snapshot version. Address some clippy complaints. * Muzzle clippy type complexity warning. Despite clippy's suggestion, it is not currently possible to create type aliases for traits and so everything within the 'Box<...>' cannot be type aliased. This then leaves creating full blown traits, and either implementing said traits by closure (somehow) or moving the closures into new structs implementing said traits which seems a bit of a palaver. Alternatively it is possible to define and use the type alias 'type ResultBox<T> = Result<Box<T>>' which does seems rather pointless and not a great reduction in complexity but is enough to keep clippy happy. In the end I simply went with squelching the clippy warning. * Remove now unused Serialize/Deserialize trait implementations for AccountStorageEntry and AppendVec * refactor versioned de/serialisers * rename serde_utils to serde_snapshot * move call to accounts_db.generate_index() back down to context_accountsdb_from_stream() * update version 1.1.1 to 1.2.0 remove nested use of serialize_bytes * cleanups * Add back measurement of account storage entry serialization. Remove construction of Vec and HashMap temporaries during serialization. * consolidate serialisation test cases into serde_snapshot. clean up leakage of implementation details in serde_snapshot. * move short term / legacy snapshot code into child module * add serialize_iter_as_tuple * preliminary integration of following commit commit 6d58b73c47294bfb93465d5a83cd2175660b6e6d Author: Ryo Onodera <ryoqun@gmail.com> Date: Wed May 20 14:02:02 2020 +0900 Confine snapshot 1.1 relic to versioned codepath * refactored serde_snapshot, rustfmt legacy accounts_db format now "owns" both leading u64s, legacy bank_rc format has none * reduce type complexity (clippy)
2020-05-22 10:54:24 -07:00
}
/// Simulates the unpacking & storage reconstruction done during snapshot unpacking
fn copy_append_vecs<P: AsRef<Path>>(
accounts_db: &AccountsDb,
output_dir: P,
) -> Result<StorageAndNextAppendVecId, AccountsFileError> {
let storage_entries = accounts_db.get_snapshot_storages(RangeFull).0;
let storage: AccountStorageMap = AccountStorageMap::with_capacity(storage_entries.len());
let mut next_append_vec_id = 0;
for storage_entry in storage_entries.into_iter() {
// Copy file to new directory
let storage_path = storage_entry.get_path();
let file_name =
AccountsFile::file_name(storage_entry.slot(), storage_entry.append_vec_id());
let output_path = output_dir.as_ref().join(file_name);
std::fs::copy(storage_path, &output_path)?;
Multi-version snapshot support (#9980) * Multi-version snapshot support * rustfmt * Remove CLI options and runtime support for selection output snapshot version. Address some clippy complaints. * Muzzle clippy type complexity warning. Despite clippy's suggestion, it is not currently possible to create type aliases for traits and so everything within the 'Box<...>' cannot be type aliased. This then leaves creating full blown traits, and either implementing said traits by closure (somehow) or moving the closures into new structs implementing said traits which seems a bit of a palaver. Alternatively it is possible to define and use the type alias 'type ResultBox<T> = Result<Box<T>>' which does seems rather pointless and not a great reduction in complexity but is enough to keep clippy happy. In the end I simply went with squelching the clippy warning. * Remove now unused Serialize/Deserialize trait implementations for AccountStorageEntry and AppendVec * refactor versioned de/serialisers * rename serde_utils to serde_snapshot * move call to accounts_db.generate_index() back down to context_accountsdb_from_stream() * update version 1.1.1 to 1.2.0 remove nested use of serialize_bytes * cleanups * Add back measurement of account storage entry serialization. Remove construction of Vec and HashMap temporaries during serialization. * consolidate serialisation test cases into serde_snapshot. clean up leakage of implementation details in serde_snapshot. * move short term / legacy snapshot code into child module * add serialize_iter_as_tuple * preliminary integration of following commit commit 6d58b73c47294bfb93465d5a83cd2175660b6e6d Author: Ryo Onodera <ryoqun@gmail.com> Date: Wed May 20 14:02:02 2020 +0900 Confine snapshot 1.1 relic to versioned codepath * refactored serde_snapshot, rustfmt legacy accounts_db format now "owns" both leading u64s, legacy bank_rc format has none * reduce type complexity (clippy)
2020-05-22 10:54:24 -07:00
// Read new file into append-vec and build new entry
let (accounts_file, num_accounts) =
AccountsFile::new_from_file(output_path, storage_entry.accounts.len())?;
let new_storage_entry = AccountStorageEntry::new_existing(
storage_entry.slot(),
storage_entry.append_vec_id(),
accounts_file,
num_accounts,
);
next_append_vec_id = next_append_vec_id.max(new_storage_entry.append_vec_id());
storage.insert(
new_storage_entry.slot(),
AccountStorageReference {
id: new_storage_entry.append_vec_id(),
storage: Arc::new(new_storage_entry),
},
);
}
Ok(StorageAndNextAppendVecId {
storage,
next_append_vec_id: AtomicAppendVecId::new(next_append_vec_id + 1),
})
}
fn reconstruct_accounts_db_via_serialization(accounts: &AccountsDb, slot: Slot) -> AccountsDb {
let mut writer = Cursor::new(vec![]);
let snapshot_storages = accounts.get_snapshot_storages(..=slot).0;
accountsdb_to_stream(
SerdeStyle::Newer,
&mut writer,
accounts,
slot,
&get_storages_to_serialize(&snapshot_storages),
)
.unwrap();
let buf = writer.into_inner();
let mut reader = BufReader::new(&buf[..]);
let copied_accounts = TempDir::new().unwrap();
// Simulate obtaining a copy of the AppendVecs from a tarball
let storage_and_next_append_vec_id =
copy_append_vecs(accounts, copied_accounts.path()).unwrap();
let mut accounts_db = accountsdb_from_stream(
SerdeStyle::Newer,
&mut reader,
&[],
storage_and_next_append_vec_id,
)
.unwrap();
// The append vecs will be used from `copied_accounts` directly by the new AccountsDb so keep
// its TempDir alive
accounts_db
.temp_paths
.as_mut()
.unwrap()
.push(copied_accounts);
accounts_db
Multi-version snapshot support (#9980) * Multi-version snapshot support * rustfmt * Remove CLI options and runtime support for selection output snapshot version. Address some clippy complaints. * Muzzle clippy type complexity warning. Despite clippy's suggestion, it is not currently possible to create type aliases for traits and so everything within the 'Box<...>' cannot be type aliased. This then leaves creating full blown traits, and either implementing said traits by closure (somehow) or moving the closures into new structs implementing said traits which seems a bit of a palaver. Alternatively it is possible to define and use the type alias 'type ResultBox<T> = Result<Box<T>>' which does seems rather pointless and not a great reduction in complexity but is enough to keep clippy happy. In the end I simply went with squelching the clippy warning. * Remove now unused Serialize/Deserialize trait implementations for AccountStorageEntry and AppendVec * refactor versioned de/serialisers * rename serde_utils to serde_snapshot * move call to accounts_db.generate_index() back down to context_accountsdb_from_stream() * update version 1.1.1 to 1.2.0 remove nested use of serialize_bytes * cleanups * Add back measurement of account storage entry serialization. Remove construction of Vec and HashMap temporaries during serialization. * consolidate serialisation test cases into serde_snapshot. clean up leakage of implementation details in serde_snapshot. * move short term / legacy snapshot code into child module * add serialize_iter_as_tuple * preliminary integration of following commit commit 6d58b73c47294bfb93465d5a83cd2175660b6e6d Author: Ryo Onodera <ryoqun@gmail.com> Date: Wed May 20 14:02:02 2020 +0900 Confine snapshot 1.1 relic to versioned codepath * refactored serde_snapshot, rustfmt legacy accounts_db format now "owns" both leading u64s, legacy bank_rc format has none * reduce type complexity (clippy)
2020-05-22 10:54:24 -07:00
}
fn check_accounts_local(accounts: &Accounts, pubkeys: &[Pubkey], num: usize) {
for _ in 1..num {
Bump rand to 0.8, rand_chacha to 0.3, getrandom to 0.2 (#32871) * sdk: Add concurrent support for rand 0.7 and 0.8 * Update rand, rand_chacha, and getrandom versions * Run command to replace `gen_range` Run `git grep -l gen_range | xargs sed -i'' -e 's/gen_range(\(\S*\), /gen_range(\1../' * sdk: Fix users of older `gen_range` * Replace `hash::new_rand` with `hash::new_with_thread_rng` Run: ``` git grep -l hash::new_rand | xargs sed -i'' -e 's/hash::new_rand([^)]*/hash::new_with_thread_rng(/' ``` * perf: Use `Keypair::new()` instead of `generate` * Use older rand version in zk-token-sdk * program-runtime: Inline random key generation * bloom: Fix clippy warnings in tests * streamer: Scope rng usage correctly * perf: Fix clippy warning * accounts-db: Map to char to generate a random string * Remove `from_secret_key_bytes`, it's just `keypair_from_seed` * ledger: Generate keypairs by hand * ed25519-tests: Use new rand * runtime: Use new rand in all tests * gossip: Clean up clippy and inline keypair generators * core: Inline keypair generation for tests * Push sbf lockfile change * sdk: Sort dependencies correctly * Remove `hash::new_with_thread_rng`, use `Hash::new_unique()` * Use Keypair::new where chacha isn't used * sdk: Fix build by marking rand 0.7 optional * Hardcode secret key length, add static assertion * Unify `getrandom` crate usage to fix linking errors * bloom: Fix tests that require a random hash * Remove some dependencies, try to unify others * Remove unnecessary uses of rand and rand_core * Update lockfiles * Add back some dependencies to reduce rebuilds * Increase max rebuilds from 14 to 15 * frozen-abi: Remove `getrandom` * Bump rebuilds to 17 * Remove getrandom from zk-token-proof
2023-08-21 10:11:21 -07:00
let idx = thread_rng().gen_range(0..num - 1);
let ancestors = vec![(0, 0)].into_iter().collect();
let account = accounts.load_without_fixed_root(&ancestors, &pubkeys[idx]);
let account1 = Some((
AccountSharedData::new((idx + 1) as u64, 0, AccountSharedData::default().owner()),
0,
));
assert_eq!(account, account1);
}
Multi-version snapshot support (#9980) * Multi-version snapshot support * rustfmt * Remove CLI options and runtime support for selection output snapshot version. Address some clippy complaints. * Muzzle clippy type complexity warning. Despite clippy's suggestion, it is not currently possible to create type aliases for traits and so everything within the 'Box<...>' cannot be type aliased. This then leaves creating full blown traits, and either implementing said traits by closure (somehow) or moving the closures into new structs implementing said traits which seems a bit of a palaver. Alternatively it is possible to define and use the type alias 'type ResultBox<T> = Result<Box<T>>' which does seems rather pointless and not a great reduction in complexity but is enough to keep clippy happy. In the end I simply went with squelching the clippy warning. * Remove now unused Serialize/Deserialize trait implementations for AccountStorageEntry and AppendVec * refactor versioned de/serialisers * rename serde_utils to serde_snapshot * move call to accounts_db.generate_index() back down to context_accountsdb_from_stream() * update version 1.1.1 to 1.2.0 remove nested use of serialize_bytes * cleanups * Add back measurement of account storage entry serialization. Remove construction of Vec and HashMap temporaries during serialization. * consolidate serialisation test cases into serde_snapshot. clean up leakage of implementation details in serde_snapshot. * move short term / legacy snapshot code into child module * add serialize_iter_as_tuple * preliminary integration of following commit commit 6d58b73c47294bfb93465d5a83cd2175660b6e6d Author: Ryo Onodera <ryoqun@gmail.com> Date: Wed May 20 14:02:02 2020 +0900 Confine snapshot 1.1 relic to versioned codepath * refactored serde_snapshot, rustfmt legacy accounts_db format now "owns" both leading u64s, legacy bank_rc format has none * reduce type complexity (clippy)
2020-05-22 10:54:24 -07:00
}
fn test_accounts_serialize_style(serde_style: SerdeStyle) {
solana_logger::setup();
let (_accounts_dir, paths) = get_temp_accounts_paths(4).unwrap();
let accounts_db = AccountsDb::new_with_config_for_tests(
paths,
&ClusterType::Development,
AccountSecondaryIndexes::default(),
AccountShrinkThreshold::default(),
);
let accounts = Accounts::new(Arc::new(accounts_db));
let slot = 0;
let mut pubkeys: Vec<Pubkey> = vec![];
create_test_accounts(&accounts, &mut pubkeys, 100, slot);
check_accounts_local(&accounts, &pubkeys, 100);
accounts.add_root(slot);
let accounts_delta_hash = accounts.accounts_db.calculate_accounts_delta_hash(slot);
let accounts_hash = AccountsHash(Hash::new_unique());
accounts
.accounts_db
2023-08-15 04:18:59 -07:00
.set_accounts_hash(slot, (accounts_hash, u64::default()));
let mut writer = Cursor::new(vec![]);
accountsdb_to_stream(
Multi-version snapshot support (#9980) * Multi-version snapshot support * rustfmt * Remove CLI options and runtime support for selection output snapshot version. Address some clippy complaints. * Muzzle clippy type complexity warning. Despite clippy's suggestion, it is not currently possible to create type aliases for traits and so everything within the 'Box<...>' cannot be type aliased. This then leaves creating full blown traits, and either implementing said traits by closure (somehow) or moving the closures into new structs implementing said traits which seems a bit of a palaver. Alternatively it is possible to define and use the type alias 'type ResultBox<T> = Result<Box<T>>' which does seems rather pointless and not a great reduction in complexity but is enough to keep clippy happy. In the end I simply went with squelching the clippy warning. * Remove now unused Serialize/Deserialize trait implementations for AccountStorageEntry and AppendVec * refactor versioned de/serialisers * rename serde_utils to serde_snapshot * move call to accounts_db.generate_index() back down to context_accountsdb_from_stream() * update version 1.1.1 to 1.2.0 remove nested use of serialize_bytes * cleanups * Add back measurement of account storage entry serialization. Remove construction of Vec and HashMap temporaries during serialization. * consolidate serialisation test cases into serde_snapshot. clean up leakage of implementation details in serde_snapshot. * move short term / legacy snapshot code into child module * add serialize_iter_as_tuple * preliminary integration of following commit commit 6d58b73c47294bfb93465d5a83cd2175660b6e6d Author: Ryo Onodera <ryoqun@gmail.com> Date: Wed May 20 14:02:02 2020 +0900 Confine snapshot 1.1 relic to versioned codepath * refactored serde_snapshot, rustfmt legacy accounts_db format now "owns" both leading u64s, legacy bank_rc format has none * reduce type complexity (clippy)
2020-05-22 10:54:24 -07:00
serde_style,
&mut writer,
&accounts.accounts_db,
slot,
&get_storages_to_serialize(&accounts.accounts_db.get_snapshot_storages(..=slot).0),
Multi-version snapshot support (#9980) * Multi-version snapshot support * rustfmt * Remove CLI options and runtime support for selection output snapshot version. Address some clippy complaints. * Muzzle clippy type complexity warning. Despite clippy's suggestion, it is not currently possible to create type aliases for traits and so everything within the 'Box<...>' cannot be type aliased. This then leaves creating full blown traits, and either implementing said traits by closure (somehow) or moving the closures into new structs implementing said traits which seems a bit of a palaver. Alternatively it is possible to define and use the type alias 'type ResultBox<T> = Result<Box<T>>' which does seems rather pointless and not a great reduction in complexity but is enough to keep clippy happy. In the end I simply went with squelching the clippy warning. * Remove now unused Serialize/Deserialize trait implementations for AccountStorageEntry and AppendVec * refactor versioned de/serialisers * rename serde_utils to serde_snapshot * move call to accounts_db.generate_index() back down to context_accountsdb_from_stream() * update version 1.1.1 to 1.2.0 remove nested use of serialize_bytes * cleanups * Add back measurement of account storage entry serialization. Remove construction of Vec and HashMap temporaries during serialization. * consolidate serialisation test cases into serde_snapshot. clean up leakage of implementation details in serde_snapshot. * move short term / legacy snapshot code into child module * add serialize_iter_as_tuple * preliminary integration of following commit commit 6d58b73c47294bfb93465d5a83cd2175660b6e6d Author: Ryo Onodera <ryoqun@gmail.com> Date: Wed May 20 14:02:02 2020 +0900 Confine snapshot 1.1 relic to versioned codepath * refactored serde_snapshot, rustfmt legacy accounts_db format now "owns" both leading u64s, legacy bank_rc format has none * reduce type complexity (clippy)
2020-05-22 10:54:24 -07:00
)
.unwrap();
Multi-version snapshot support (#9980) * Multi-version snapshot support * rustfmt * Remove CLI options and runtime support for selection output snapshot version. Address some clippy complaints. * Muzzle clippy type complexity warning. Despite clippy's suggestion, it is not currently possible to create type aliases for traits and so everything within the 'Box<...>' cannot be type aliased. This then leaves creating full blown traits, and either implementing said traits by closure (somehow) or moving the closures into new structs implementing said traits which seems a bit of a palaver. Alternatively it is possible to define and use the type alias 'type ResultBox<T> = Result<Box<T>>' which does seems rather pointless and not a great reduction in complexity but is enough to keep clippy happy. In the end I simply went with squelching the clippy warning. * Remove now unused Serialize/Deserialize trait implementations for AccountStorageEntry and AppendVec * refactor versioned de/serialisers * rename serde_utils to serde_snapshot * move call to accounts_db.generate_index() back down to context_accountsdb_from_stream() * update version 1.1.1 to 1.2.0 remove nested use of serialize_bytes * cleanups * Add back measurement of account storage entry serialization. Remove construction of Vec and HashMap temporaries during serialization. * consolidate serialisation test cases into serde_snapshot. clean up leakage of implementation details in serde_snapshot. * move short term / legacy snapshot code into child module * add serialize_iter_as_tuple * preliminary integration of following commit commit 6d58b73c47294bfb93465d5a83cd2175660b6e6d Author: Ryo Onodera <ryoqun@gmail.com> Date: Wed May 20 14:02:02 2020 +0900 Confine snapshot 1.1 relic to versioned codepath * refactored serde_snapshot, rustfmt legacy accounts_db format now "owns" both leading u64s, legacy bank_rc format has none * reduce type complexity (clippy)
2020-05-22 10:54:24 -07:00
let copied_accounts = TempDir::new().unwrap();
// Simulate obtaining a copy of the AppendVecs from a tarball
let storage_and_next_append_vec_id =
copy_append_vecs(&accounts.accounts_db, copied_accounts.path()).unwrap();
let buf = writer.into_inner();
let mut reader = BufReader::new(&buf[..]);
let (_accounts_dir, daccounts_paths) = get_temp_accounts_paths(2).unwrap();
let daccounts = Accounts::new(Arc::new(
accountsdb_from_stream(
serde_style,
&mut reader,
&daccounts_paths,
storage_and_next_append_vec_id,
)
.unwrap(),
));
check_accounts_local(&daccounts, &pubkeys, 100);
let daccounts_delta_hash = daccounts.accounts_db.calculate_accounts_delta_hash(slot);
assert_eq!(accounts_delta_hash, daccounts_delta_hash);
let daccounts_hash = daccounts.accounts_db.get_accounts_hash(slot).unwrap().0;
assert_eq!(accounts_hash, daccounts_hash);
}
#[test]
fn test_accounts_serialize_newer() {
test_accounts_serialize_style(SerdeStyle::Newer)
}
#[test]
fn test_remove_unrooted_slot_snapshot() {
solana_logger::setup();
let unrooted_slot = 9;
let unrooted_bank_id = 9;
let db = AccountsDb::new_single_for_tests();
let key = solana_sdk::pubkey::new_rand();
let account0 = AccountSharedData::new(1, 0, &key);
db.store_for_tests(unrooted_slot, &[(&key, &account0)]);
// Purge the slot
db.remove_unrooted_slots(&[(unrooted_slot, unrooted_bank_id)]);
// Add a new root
let key2 = solana_sdk::pubkey::new_rand();
let new_root = unrooted_slot + 1;
db.store_for_tests(new_root, &[(&key2, &account0)]);
db.add_root_and_flush_write_cache(new_root);
db.calculate_accounts_delta_hash(new_root);
db.update_accounts_hash_for_tests(new_root, &linear_ancestors(new_root), false, false);
// Simulate reconstruction from snapshot
let db = reconstruct_accounts_db_via_serialization(&db, new_root);
// Check root account exists
db.assert_load_account(new_root, key2, 1);
// Check purged account stays gone
let unrooted_slot_ancestors = vec![(unrooted_slot, 1)].into_iter().collect();
assert!(db
.load_without_fixed_root(&unrooted_slot_ancestors, &key)
.is_none());
}
#[test]
fn test_accounts_db_serialize1() {
for pass in 0..2 {
solana_logger::setup();
let accounts = AccountsDb::new_single_for_tests();
let mut pubkeys: Vec<Pubkey> = vec![];
// Create 100 accounts in slot 0
accounts.create_account(&mut pubkeys, 0, 100, 0, 0);
if pass == 0 {
accounts.add_root_and_flush_write_cache(0);
accounts.check_storage(0, 100);
accounts.clean_accounts_for_tests();
accounts.check_accounts(&pubkeys, 0, 100, 1);
// clean should have done nothing
continue;
}
// do some updates to those accounts and re-check
accounts.modify_accounts(&pubkeys, 0, 100, 2);
accounts.add_root_and_flush_write_cache(0);
accounts.check_storage(0, 100);
accounts.check_accounts(&pubkeys, 0, 100, 2);
accounts.calculate_accounts_delta_hash(0);
let mut pubkeys1: Vec<Pubkey> = vec![];
// CREATE SLOT 1
let latest_slot = 1;
// Modify the first 10 of the accounts from slot 0 in slot 1
accounts.modify_accounts(&pubkeys, latest_slot, 10, 3);
// Overwrite account 30 from slot 0 with lamports=0 into slot 1.
// Slot 1 should now have 10 + 1 = 11 accounts
let account = AccountSharedData::new(0, 0, AccountSharedData::default().owner());
accounts.store_for_tests(latest_slot, &[(&pubkeys[30], &account)]);
// Create 10 new accounts in slot 1, should now have 11 + 10 = 21
// accounts
accounts.create_account(&mut pubkeys1, latest_slot, 10, 0, 0);
accounts.calculate_accounts_delta_hash(latest_slot);
accounts.add_root_and_flush_write_cache(latest_slot);
accounts.check_storage(1, 21);
// CREATE SLOT 2
let latest_slot = 2;
let mut pubkeys2: Vec<Pubkey> = vec![];
// Modify first 20 of the accounts from slot 0 in slot 2
accounts.modify_accounts(&pubkeys, latest_slot, 20, 4);
accounts.clean_accounts_for_tests();
// Overwrite account 31 from slot 0 with lamports=0 into slot 2.
// Slot 2 should now have 20 + 1 = 21 accounts
let account = AccountSharedData::new(0, 0, AccountSharedData::default().owner());
accounts.store_for_tests(latest_slot, &[(&pubkeys[31], &account)]);
// Create 10 new accounts in slot 2. Slot 2 should now have
// 21 + 10 = 31 accounts
accounts.create_account(&mut pubkeys2, latest_slot, 10, 0, 0);
accounts.calculate_accounts_delta_hash(latest_slot);
accounts.add_root_and_flush_write_cache(latest_slot);
accounts.check_storage(2, 31);
let ancestors = linear_ancestors(latest_slot);
accounts.update_accounts_hash_for_tests(latest_slot, &ancestors, false, false);
accounts.clean_accounts_for_tests();
// The first 20 accounts of slot 0 have been updated in slot 2, as well as
// accounts 30 and 31 (overwritten with zero-lamport accounts in slot 1 and
// slot 2 respectively), so only 78 accounts are left in slot 0's storage entries.
accounts.check_storage(0, 78);
// 10 of the 21 accounts have been modified in slot 2, so only 11
// accounts left in slot 1.
accounts.check_storage(1, 11);
accounts.check_storage(2, 31);
let daccounts = reconstruct_accounts_db_via_serialization(&accounts, latest_slot);
assert_eq!(
daccounts.write_version.load(Ordering::Acquire),
accounts.write_version.load(Ordering::Acquire)
);
// Get the hashes for the latest slot, which should be the only hashes in the
// map on the deserialized AccountsDb
assert_eq!(daccounts.accounts_delta_hashes().lock().unwrap().len(), 1);
assert_eq!(daccounts.accounts_hashes().lock().unwrap().len(), 1);
assert_eq!(
daccounts.get_accounts_delta_hash(latest_slot).unwrap(),
accounts.get_accounts_delta_hash(latest_slot).unwrap(),
);
assert_eq!(
daccounts.get_accounts_hash(latest_slot).unwrap().0,
accounts.get_accounts_hash(latest_slot).unwrap().0,
);
daccounts.print_count_and_status("daccounts");
// Don't check the first 35 accounts which have not been modified on slot 0
daccounts.check_accounts(&pubkeys[35..], 0, 65, 37);
daccounts.check_accounts(&pubkeys1, 1, 10, 1);
daccounts.check_storage(0, 100);
daccounts.check_storage(1, 21);
daccounts.check_storage(2, 31);
assert_eq!(
daccounts.update_accounts_hash_for_tests(latest_slot, &ancestors, false, false,),
accounts.update_accounts_hash_for_tests(latest_slot, &ancestors, false, false,)
);
}
}
#[test]
fn test_accounts_db_serialize_zero_and_free() {
solana_logger::setup();
let some_lamport = 223;
let zero_lamport = 0;
let no_data = 0;
let owner = *AccountSharedData::default().owner();
let account = AccountSharedData::new(some_lamport, no_data, &owner);
let pubkey = solana_sdk::pubkey::new_rand();
let zero_lamport_account = AccountSharedData::new(zero_lamport, no_data, &owner);
let account2 = AccountSharedData::new(some_lamport + 1, no_data, &owner);
let pubkey2 = solana_sdk::pubkey::new_rand();
let accounts = AccountsDb::new_single_for_tests();
let mut current_slot = 1;
accounts.store_for_tests(current_slot, &[(&pubkey, &account)]);
accounts.add_root(current_slot);
current_slot += 1;
accounts.store_for_tests(current_slot, &[(&pubkey, &zero_lamport_account)]);
accounts.store_for_tests(current_slot, &[(&pubkey2, &account2)]);
Multi-version snapshot support (#9980) * Multi-version snapshot support * rustfmt * Remove CLI options and runtime support for selection output snapshot version. Address some clippy complaints. * Muzzle clippy type complexity warning. Despite clippy's suggestion, it is not currently possible to create type aliases for traits and so everything within the 'Box<...>' cannot be type aliased. This then leaves creating full blown traits, and either implementing said traits by closure (somehow) or moving the closures into new structs implementing said traits which seems a bit of a palaver. Alternatively it is possible to define and use the type alias 'type ResultBox<T> = Result<Box<T>>' which does seems rather pointless and not a great reduction in complexity but is enough to keep clippy happy. In the end I simply went with squelching the clippy warning. * Remove now unused Serialize/Deserialize trait implementations for AccountStorageEntry and AppendVec * refactor versioned de/serialisers * rename serde_utils to serde_snapshot * move call to accounts_db.generate_index() back down to context_accountsdb_from_stream() * update version 1.1.1 to 1.2.0 remove nested use of serialize_bytes * cleanups * Add back measurement of account storage entry serialization. Remove construction of Vec and HashMap temporaries during serialization. * consolidate serialisation test cases into serde_snapshot. clean up leakage of implementation details in serde_snapshot. * move short term / legacy snapshot code into child module * add serialize_iter_as_tuple * preliminary integration of following commit commit 6d58b73c47294bfb93465d5a83cd2175660b6e6d Author: Ryo Onodera <ryoqun@gmail.com> Date: Wed May 20 14:02:02 2020 +0900 Confine snapshot 1.1 relic to versioned codepath * refactored serde_snapshot, rustfmt legacy accounts_db format now "owns" both leading u64s, legacy bank_rc format has none * reduce type complexity (clippy)
2020-05-22 10:54:24 -07:00
accounts.add_root_and_flush_write_cache(current_slot);
accounts.assert_load_account(current_slot, pubkey, zero_lamport);
accounts.print_accounts_stats("accounts");
accounts.clean_accounts_for_tests();
accounts.print_accounts_stats("accounts_post_purge");
accounts.calculate_accounts_delta_hash(current_slot);
accounts.update_accounts_hash_for_tests(
current_slot,
&linear_ancestors(current_slot),
false,
false,
);
let accounts = reconstruct_accounts_db_via_serialization(&accounts, current_slot);
accounts.print_accounts_stats("reconstructed");
accounts.assert_load_account(current_slot, pubkey, zero_lamport);
}
fn with_chained_zero_lamport_accounts<F>(f: F)
where
F: Fn(AccountsDb, Slot) -> AccountsDb,
{
let some_lamport = 223;
let zero_lamport = 0;
let dummy_lamport = 999;
let no_data = 0;
let owner = *AccountSharedData::default().owner();
let account = AccountSharedData::new(some_lamport, no_data, &owner);
let account2 = AccountSharedData::new(some_lamport + 100_001, no_data, &owner);
let account3 = AccountSharedData::new(some_lamport + 100_002, no_data, &owner);
let zero_lamport_account = AccountSharedData::new(zero_lamport, no_data, &owner);
let pubkey = solana_sdk::pubkey::new_rand();
let purged_pubkey1 = solana_sdk::pubkey::new_rand();
let purged_pubkey2 = solana_sdk::pubkey::new_rand();
let dummy_account = AccountSharedData::new(dummy_lamport, no_data, &owner);
let dummy_pubkey = Pubkey::default();
let accounts = AccountsDb::new_single_for_tests();
let mut current_slot = 1;
accounts.store_for_tests(current_slot, &[(&pubkey, &account)]);
accounts.store_for_tests(current_slot, &[(&purged_pubkey1, &account2)]);
accounts.add_root_and_flush_write_cache(current_slot);
current_slot += 1;
accounts.store_for_tests(current_slot, &[(&purged_pubkey1, &zero_lamport_account)]);
accounts.store_for_tests(current_slot, &[(&purged_pubkey2, &account3)]);
accounts.add_root_and_flush_write_cache(current_slot);
current_slot += 1;
accounts.store_for_tests(current_slot, &[(&purged_pubkey2, &zero_lamport_account)]);
accounts.add_root_and_flush_write_cache(current_slot);
current_slot += 1;
accounts.store_for_tests(current_slot, &[(&dummy_pubkey, &dummy_account)]);
accounts.add_root_and_flush_write_cache(current_slot);
accounts.print_accounts_stats("pre_f");
accounts.calculate_accounts_delta_hash(current_slot);
accounts.update_accounts_hash_for_tests(4, &Ancestors::default(), false, false);
let accounts = f(accounts, current_slot);
accounts.print_accounts_stats("post_f");
accounts.assert_load_account(current_slot, pubkey, some_lamport);
accounts.assert_load_account(current_slot, purged_pubkey1, 0);
accounts.assert_load_account(current_slot, purged_pubkey2, 0);
accounts.assert_load_account(current_slot, dummy_pubkey, dummy_lamport);
let ancestors = Ancestors::default();
let epoch_schedule = EpochSchedule::default();
let rent_collector = RentCollector::default();
let config = VerifyAccountsHashAndLamportsConfig::new_for_test(
&ancestors,
&epoch_schedule,
&rent_collector,
);
accounts
.verify_accounts_hash_and_lamports(4, 1222, None, config)
.unwrap();
}
#[test]
fn test_accounts_purge_chained_purge_before_snapshot_restore() {
solana_logger::setup();
with_chained_zero_lamport_accounts(|accounts, current_slot| {
accounts.clean_accounts_for_tests();
reconstruct_accounts_db_via_serialization(&accounts, current_slot)
});
}
#[test]
fn test_accounts_purge_chained_purge_after_snapshot_restore() {
solana_logger::setup();
with_chained_zero_lamport_accounts(|accounts, current_slot| {
let accounts = reconstruct_accounts_db_via_serialization(&accounts, current_slot);
accounts.print_accounts_stats("after_reconstruct");
accounts.clean_accounts_for_tests();
reconstruct_accounts_db_via_serialization(&accounts, current_slot)
});
}
#[test]
fn test_accounts_purge_long_chained_after_snapshot_restore() {
solana_logger::setup();
let old_lamport = 223;
let zero_lamport = 0;
let no_data = 0;
let owner = *AccountSharedData::default().owner();
let account = AccountSharedData::new(old_lamport, no_data, &owner);
let account2 = AccountSharedData::new(old_lamport + 100_001, no_data, &owner);
let account3 = AccountSharedData::new(old_lamport + 100_002, no_data, &owner);
let dummy_account = AccountSharedData::new(99_999_999, no_data, &owner);
let zero_lamport_account = AccountSharedData::new(zero_lamport, no_data, &owner);
let pubkey = solana_sdk::pubkey::new_rand();
let dummy_pubkey = solana_sdk::pubkey::new_rand();
let purged_pubkey1 = solana_sdk::pubkey::new_rand();
let purged_pubkey2 = solana_sdk::pubkey::new_rand();
let mut current_slot = 0;
let accounts = AccountsDb::new_single_for_tests();
// create intermediate updates to purged_pubkey1 so that
// generate_index must add slots as root last at once
current_slot += 1;
accounts.store_for_tests(current_slot, &[(&pubkey, &account)]);
accounts.store_for_tests(current_slot, &[(&purged_pubkey1, &account2)]);
accounts.add_root_and_flush_write_cache(current_slot);
current_slot += 1;
accounts.store_for_tests(current_slot, &[(&purged_pubkey1, &account2)]);
accounts.add_root_and_flush_write_cache(current_slot);
current_slot += 1;
accounts.store_for_tests(current_slot, &[(&purged_pubkey1, &account2)]);
accounts.add_root_and_flush_write_cache(current_slot);
current_slot += 1;
accounts.store_for_tests(current_slot, &[(&purged_pubkey1, &zero_lamport_account)]);
accounts.store_for_tests(current_slot, &[(&purged_pubkey2, &account3)]);
accounts.add_root_and_flush_write_cache(current_slot);
current_slot += 1;
accounts.store_for_tests(current_slot, &[(&purged_pubkey2, &zero_lamport_account)]);
accounts.add_root_and_flush_write_cache(current_slot);
current_slot += 1;
accounts.store_for_tests(current_slot, &[(&dummy_pubkey, &dummy_account)]);
accounts.add_root_and_flush_write_cache(current_slot);
accounts.print_count_and_status("before reconstruct");
accounts.calculate_accounts_delta_hash(current_slot);
accounts.update_accounts_hash_for_tests(
current_slot,
&linear_ancestors(current_slot),
false,
false,
);
let accounts = reconstruct_accounts_db_via_serialization(&accounts, current_slot);
accounts.print_count_and_status("before purge zero");
accounts.clean_accounts_for_tests();
accounts.print_count_and_status("after purge zero");
accounts.assert_load_account(current_slot, pubkey, old_lamport);
accounts.assert_load_account(current_slot, purged_pubkey1, 0);
accounts.assert_load_account(current_slot, purged_pubkey2, 0);
}
#[test]
fn test_accounts_clean_after_snapshot_restore_then_old_revives() {
solana_logger::setup();
let old_lamport = 223;
let zero_lamport = 0;
let no_data = 0;
let dummy_lamport = 999_999;
let owner = *AccountSharedData::default().owner();
let account = AccountSharedData::new(old_lamport, no_data, &owner);
let account2 = AccountSharedData::new(old_lamport + 100_001, no_data, &owner);
let account3 = AccountSharedData::new(old_lamport + 100_002, no_data, &owner);
let dummy_account = AccountSharedData::new(dummy_lamport, no_data, &owner);
let zero_lamport_account = AccountSharedData::new(zero_lamport, no_data, &owner);
let pubkey1 = solana_sdk::pubkey::new_rand();
let pubkey2 = solana_sdk::pubkey::new_rand();
let dummy_pubkey = solana_sdk::pubkey::new_rand();
let mut current_slot = 0;
let accounts = AccountsDb::new_single_for_tests();
// A: Initialize AccountsDb with pubkey1 and pubkey2
current_slot += 1;
accounts.store_for_tests(current_slot, &[(&pubkey1, &account)]);
accounts.store_for_tests(current_slot, &[(&pubkey2, &account)]);
accounts.calculate_accounts_delta_hash(current_slot);
accounts.add_root(current_slot);
// B: Test multiple updates to pubkey1 in a single slot/storage
current_slot += 1;
assert_eq!(0, accounts.alive_account_count_in_slot(current_slot));
accounts.add_root_and_flush_write_cache(current_slot - 1);
assert_eq!(1, accounts.ref_count_for_pubkey(&pubkey1));
accounts.store_for_tests(current_slot, &[(&pubkey1, &account2)]);
accounts.store_for_tests(current_slot, &[(&pubkey1, &account2)]);
accounts.add_root_and_flush_write_cache(current_slot);
assert_eq!(1, accounts.alive_account_count_in_slot(current_slot));
// Stores to same pubkey, same slot only count once towards the
// ref count
assert_eq!(2, accounts.ref_count_for_pubkey(&pubkey1));
accounts.calculate_accounts_delta_hash(current_slot);
// C: Yet more update to trigger lazy clean of step A
current_slot += 1;
assert_eq!(2, accounts.ref_count_for_pubkey(&pubkey1));
accounts.store_for_tests(current_slot, &[(&pubkey1, &account3)]);
accounts.add_root_and_flush_write_cache(current_slot);
assert_eq!(3, accounts.ref_count_for_pubkey(&pubkey1));
accounts.calculate_accounts_delta_hash(current_slot);
accounts.add_root_and_flush_write_cache(current_slot);
// D: Make pubkey1 0-lamport; also triggers clean of step B
current_slot += 1;
assert_eq!(3, accounts.ref_count_for_pubkey(&pubkey1));
accounts.store_for_tests(current_slot, &[(&pubkey1, &zero_lamport_account)]);
accounts.add_root_and_flush_write_cache(current_slot);
// had to be a root to flush, but clean won't work as this test expects if it is a root
// so, remove the root from alive_roots, then restore it after clean
accounts
.accounts_index
.roots_tracker
.write()
.unwrap()
.alive_roots
.remove(&current_slot);
accounts.clean_accounts_for_tests();
accounts
.accounts_index
.roots_tracker
.write()
.unwrap()
.alive_roots
.insert(current_slot);
assert_eq!(
// Removed one reference from the dead slot (reference only counted once
// even though there were two stores to the pubkey in that slot)
3, /* == 3 - 1 + 1 */
accounts.ref_count_for_pubkey(&pubkey1)
);
accounts.calculate_accounts_delta_hash(current_slot);
accounts.add_root(current_slot);
// E: Avoid missing bank hash error
current_slot += 1;
accounts.store_for_tests(current_slot, &[(&dummy_pubkey, &dummy_account)]);
accounts.calculate_accounts_delta_hash(current_slot);
accounts.add_root(current_slot);
accounts.assert_load_account(current_slot, pubkey1, zero_lamport);
accounts.assert_load_account(current_slot, pubkey2, old_lamport);
accounts.assert_load_account(current_slot, dummy_pubkey, dummy_lamport);
// At this point, there is no index entries for A and B
// If step C and step D should be purged, snapshot restore would cause
// pubkey1 to be revived as the state of step A.
// So, prevent that from happening by introducing refcount
((current_slot - 1)..=current_slot).for_each(|slot| accounts.flush_root_write_cache(slot));
accounts.clean_accounts_for_tests();
accounts.update_accounts_hash_for_tests(
current_slot,
&linear_ancestors(current_slot),
false,
false,
);
let accounts = reconstruct_accounts_db_via_serialization(&accounts, current_slot);
accounts.clean_accounts_for_tests();
info!("pubkey: {}", pubkey1);
accounts.print_accounts_stats("pre_clean");
accounts.assert_load_account(current_slot, pubkey1, zero_lamport);
accounts.assert_load_account(current_slot, pubkey2, old_lamport);
accounts.assert_load_account(current_slot, dummy_pubkey, dummy_lamport);
// F: Finally, make Step A cleanable
current_slot += 1;
accounts.store_for_tests(current_slot, &[(&pubkey2, &account)]);
accounts.calculate_accounts_delta_hash(current_slot);
accounts.add_root(current_slot);
// Do clean
accounts.flush_root_write_cache(current_slot);
accounts.clean_accounts_for_tests();
// 2nd clean needed to clean-up pubkey1
accounts.clean_accounts_for_tests();
// Ensure pubkey2 is cleaned from the index finally
accounts.assert_not_load_account(current_slot, pubkey1);
accounts.assert_load_account(current_slot, pubkey2, old_lamport);
accounts.assert_load_account(current_slot, dummy_pubkey, dummy_lamport);
}
#[test]
fn test_shrink_stale_slots_processed() {
solana_logger::setup();
for startup in &[false, true] {
let accounts = AccountsDb::new_single_for_tests();
let pubkey_count = 100;
let pubkeys: Vec<_> = (0..pubkey_count)
.map(|_| solana_sdk::pubkey::new_rand())
.collect();
let some_lamport = 223;
let no_data = 0;
let owner = *AccountSharedData::default().owner();
let account = AccountSharedData::new(some_lamport, no_data, &owner);
let mut current_slot = 0;
current_slot += 1;
for pubkey in &pubkeys {
accounts.store_for_tests(current_slot, &[(pubkey, &account)]);
}
let shrink_slot = current_slot;
accounts.calculate_accounts_delta_hash(current_slot);
accounts.add_root_and_flush_write_cache(current_slot);
current_slot += 1;
let pubkey_count_after_shrink = 10;
let updated_pubkeys = &pubkeys[0..pubkey_count - pubkey_count_after_shrink];
for pubkey in updated_pubkeys {
accounts.store_for_tests(current_slot, &[(pubkey, &account)]);
}
accounts.calculate_accounts_delta_hash(current_slot);
accounts.add_root_and_flush_write_cache(current_slot);
accounts.clean_accounts_for_tests();
assert_eq!(
pubkey_count,
accounts.all_account_count_in_append_vec(shrink_slot)
);
accounts.shrink_all_slots(*startup, None, &EpochSchedule::default());
assert_eq!(
pubkey_count_after_shrink,
accounts.all_account_count_in_append_vec(shrink_slot)
);
let no_ancestors = Ancestors::default();
let epoch_schedule = EpochSchedule::default();
let rent_collector = RentCollector::default();
let config = VerifyAccountsHashAndLamportsConfig::new_for_test(
&no_ancestors,
&epoch_schedule,
&rent_collector,
);
accounts.update_accounts_hash_for_tests(current_slot, &no_ancestors, false, false);
accounts
.verify_accounts_hash_and_lamports(current_slot, 22300, None, config.clone())
.unwrap();
let accounts = reconstruct_accounts_db_via_serialization(&accounts, current_slot);
accounts
.verify_accounts_hash_and_lamports(current_slot, 22300, None, config)
.unwrap();
// repeating should be no-op
accounts.shrink_all_slots(*startup, None, &epoch_schedule);
assert_eq!(
pubkey_count_after_shrink,
accounts.all_account_count_in_append_vec(shrink_slot)
);
}
}
Multi-version snapshot support (#9980) * Multi-version snapshot support * rustfmt * Remove CLI options and runtime support for selection output snapshot version. Address some clippy complaints. * Muzzle clippy type complexity warning. Despite clippy's suggestion, it is not currently possible to create type aliases for traits and so everything within the 'Box<...>' cannot be type aliased. This then leaves creating full blown traits, and either implementing said traits by closure (somehow) or moving the closures into new structs implementing said traits which seems a bit of a palaver. Alternatively it is possible to define and use the type alias 'type ResultBox<T> = Result<Box<T>>' which does seems rather pointless and not a great reduction in complexity but is enough to keep clippy happy. In the end I simply went with squelching the clippy warning. * Remove now unused Serialize/Deserialize trait implementations for AccountStorageEntry and AppendVec * refactor versioned de/serialisers * rename serde_utils to serde_snapshot * move call to accounts_db.generate_index() back down to context_accountsdb_from_stream() * update version 1.1.1 to 1.2.0 remove nested use of serialize_bytes * cleanups * Add back measurement of account storage entry serialization. Remove construction of Vec and HashMap temporaries during serialization. * consolidate serialisation test cases into serde_snapshot. clean up leakage of implementation details in serde_snapshot. * move short term / legacy snapshot code into child module * add serialize_iter_as_tuple * preliminary integration of following commit commit 6d58b73c47294bfb93465d5a83cd2175660b6e6d Author: Ryo Onodera <ryoqun@gmail.com> Date: Wed May 20 14:02:02 2020 +0900 Confine snapshot 1.1 relic to versioned codepath * refactored serde_snapshot, rustfmt legacy accounts_db format now "owns" both leading u64s, legacy bank_rc format has none * reduce type complexity (clippy)
2020-05-22 10:54:24 -07:00
}